diff --git a/packages/adapters/cursor-local/src/server/execute.test.ts b/packages/adapters/cursor-local/src/server/execute.test.ts index 913d34cdc4..895b152e68 100644 --- a/packages/adapters/cursor-local/src/server/execute.test.ts +++ b/packages/adapters/cursor-local/src/server/execute.test.ts @@ -265,6 +265,7 @@ exit 7 await fs.mkdir(workspaceDir, { recursive: true }); await fs.mkdir(remoteWorkspace, { recursive: true }); const preferredAgentScript = `#!/bin/sh +cat >/dev/null printf '%s\\n' '{"type":"system","subtype":"init","session_id":"cursor-session-fresh-1","model":"auto"}' printf '%s\\n' '{"type":"assistant","message":{"content":[{"type":"output_text","text":"hello"}]}}' printf '%s\\n' '{"type":"result","subtype":"success","session_id":"cursor-session-fresh-1","result":"ok"}' @@ -288,7 +289,7 @@ printf '%s\\n' '{"type":"result","subtype":"success","session_id":"cursor-sessio finalPreparedCommand = preferredCommandPath; const runtimeEnv = { ...input.env, - PATH: `${path.join(systemHomeDir, ".local", "bin")}${path.delimiter}${input.env.PATH}`, + PATH: `${path.join(systemHomeDir, ".local", "bin")}${path.delimiter}${input.env.PATH ?? process.env.PATH ?? "/usr/bin:/bin"}`, }; await fs.mkdir(path.dirname(preferredCommandPath), { recursive: true }); await fs.writeFile(preferredCommandPath, preferredAgentScript); @@ -312,13 +313,14 @@ printf '%s\\n' '{"type":"result","subtype":"success","session_id":"cursor-sessio // shell seam faithful to that protocol instead of returning empty stdout // for every shell command. const runner = { - execute: async (input: { command: string; args?: string[]; env?: Record }) => { + execute: async (input: { command: string; args?: string[]; env?: Record; stdin?: string }) => { runnerState.commands.push(input.command); // Exercise actual bounded file reads during managed-home restoration; // reporting empty success for every shell command hides missing bytes. return runChildProcess(`cursor-fresh-lease-${runnerState.commands.length}`, input.command, input.args ?? [], { cwd: remoteWorkspace, env: { ...input.env, PATH: `${input.env?.PATH ?? ""}:/usr/bin:/bin` }, + stdin: input.stdin, timeoutSec: 30, graceSec: 5, onLog: async () => {}, diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs index ffe7fb2d0b..cad60a6cfc 100644 --- a/scripts/__tests__/run-vitest-stable-shard.test.mjs +++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs @@ -92,17 +92,37 @@ test("a route/authz suite never leaks into the general-server shards", () => { const shard = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", SHARD_COUNT.toString()]); for (const file of shard.selectedGeneralServerSuites) { assert.ok( - !/[^/]*(?:route|routes|authz)[^/]*\.test\.ts$/.test(file), + !(file.startsWith("server/src/__tests__/") && /[^/]*(?:route|routes|authz)[^/]*\.test\.ts$/.test(file)), `route/authz suite must stay in the serialized lane, not general-server: ${file}`, ); } }); +test("general shards retain scripts and route-named suites outside the serialized directory", () => { + const general = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"]); + for (const file of [ + "server/scripts/verify-runner-vendor-dependencies.test.mjs", + "server/src/services/openrouter-models.test.ts", + "server/src/routes/setup-token-route.test.ts", + ]) assert.ok(general.selectedGeneralServerSuites.includes(file), `missing configured server suite: ${file}`); +}); + test("shard flags are rejected for the workspaces-b group", () => { const result = dryRun(["--mode", "general", "--group", "general-workspaces-b", "--shard-index", "0", "--shard-count", "3"]); assert.notEqual(result.status, 0, "workspaces-b must not accept shard flags"); }); +test("workspace lanes cover every non-server project in the root Vitest configuration", () => { + const config = readFileSync(path.join(repoRoot, "vitest.config.ts"), "utf8"); + const roots = [...config.matchAll(/^\s+"([^"]+)",?\s*$/gm)].map(match => match[1]); + assert.ok(roots.includes("server"), "expected the explicit root Vitest project list"); + const expected = roots.filter(root => root !== "server") + .map(root => JSON.parse(readFileSync(path.join(repoRoot, root, "package.json"), "utf8")).name).sort(); + const actual = ["general-workspaces-a", "general-workspaces-b"] + .flatMap(group => dryRunJson(["--mode", "general", "--group", group]).workspaceProjects).sort(); + assert.deepEqual(actual, expected, "no configured project may be silently omitted or run twice"); +}); + test("workspaces-a shards map to Vitest native --shard slices over a stable project list", () => { const shards = [0, 1].map((index) => dryRunJson([ diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs index 3a9c12f477..b275680550 100644 --- a/scripts/run-vitest-stable.mjs +++ b/scripts/run-vitest-stable.mjs @@ -19,6 +19,7 @@ const serializedShardDurations = loadShardDurations( const serverRoot = path.join(repoRoot, "server"); const serverSrcDir = path.join(repoRoot, "server", "src"); const serverTestsDir = path.join(repoRoot, "server", "src", "__tests__"); +const serverScriptsDir = path.join(repoRoot, "server", "scripts"); const nonServerProjects = [ "@paperclipai/shared", "@paperclipai/skills-catalog", @@ -26,9 +27,14 @@ const nonServerProjects = [ "@paperclipai/adapter-utils", "@paperclipai/adapter-claude-local", "@paperclipai/adapter-codex-local", + "@paperclipai/adapter-cursor-cloud", + "@paperclipai/adapter-cursor-local", + "@paperclipai/adapter-gemini-local", "@paperclipai/adapter-grok-local", + "@paperclipai/adapter-kimi-local", "@paperclipai/adapter-openclaw-gateway", "@paperclipai/adapter-opencode-local", + "@paperclipai/adapter-pi-local", "@paperclipai/plugin-daytona", "@paperclipai/plugin-sdk", "@paperclipai/create-paperclip-plugin", @@ -345,6 +351,8 @@ function runVitest(args, label, testShard = null) { const env = { ...process.env, NODE_ENV: "test", + PAPERCLIP_TEST_HOST_HOME: process.env.PAPERCLIP_TEST_HOST_HOME + ?? (process.env.PAPERCLIP_HOME?.trim() || path.join(os.homedir(), ".paperclip")), PAPERCLIP_HOME: path.join(testRoot, "h"), // Config discovery otherwise prefers the checkout's .paperclip/config.json // over PAPERCLIP_HOME, importing preview scheduling policy into unit tests. @@ -539,10 +547,15 @@ const routeTests = walk(serverTestsDir) // config pins maxWorkers to 1, so the only way to parallelize is across jobs. // Suites are partitioned by recorded duration (scripts/general-server-shard.mjs) // rather than round-robin, so one slow suite cluster can't stretch a single shard. -const generalServerTestFiles = walk(serverSrcDir) +const serializedRepoPaths = new Set(routeTests.map(test => test.repoPath)); +const generalServerTestFiles = [ + ...walk(serverSrcDir).filter(file => file.endsWith(".test.ts")), + ...walk(serverScriptsDir).filter(file => file.endsWith(".test.mjs")), +] .map((file) => toRepoPath(file)) - .filter((repoPath) => repoPath.endsWith(".test.ts")) - .filter((repoPath) => !isRouteOrAuthzTest(repoPath)) + // Only exclude suites actually assigned to the serialized lane. A name + // such as services/openrouter-models.test.ts is not a serialized route. + .filter((repoPath) => !serializedRepoPaths.has(repoPath)) .sort((a, b) => a.localeCompare(b)); const options = parseCliOptions(process.argv.slice(2)); diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 6ffc53ff08..35ef8030ef 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -60633,19 +60633,23 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { db .select({ id: issueComments.id }) .from(issueComments) - .where(eq(issueComments.companyId, fixture.companyId)), + .where(eq(issueComments.companyId, fixture.companyId)) + .orderBy(issueComments.id), db .select({ id: issues.id }) .from(issues) - .where(eq(issues.companyId, fixture.companyId)), + .where(eq(issues.companyId, fixture.companyId)) + .orderBy(issues.id), db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) - .where(eq(heartbeatRuns.companyId, fixture.companyId)), + .where(eq(heartbeatRuns.companyId, fixture.companyId)) + .orderBy(heartbeatRuns.id), db .select({ id: chatPublications.id }) .from(chatPublications) - .where(eq(chatPublications.endpointId, endpoint.id)), + .where(eq(chatPublications.endpointId, endpoint.id)) + .orderBy(chatPublications.id), ]); const baseline = await unchangedRows(); const wakeupCount = wakeup.mock.calls.length; diff --git a/server/src/__tests__/cli-invocation-safety.test.ts b/server/src/__tests__/cli-invocation-safety.test.ts index 4fde7a8e13..e465e7a57e 100644 --- a/server/src/__tests__/cli-invocation-safety.test.ts +++ b/server/src/__tests__/cli-invocation-safety.test.ts @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -367,6 +368,13 @@ function isTestFile(relPath: string): boolean { function listGuidanceFiles(rootDir = repoRoot): string[] { const found: string[] = []; + // Include tracked files and new authored files, while respecting gitignored + // local settings. A tracked file remains scanned even if an ignore matches. + const authored = new Set(execFileSync( + "git", + ["ls-files", "--cached", "--others", "--exclude-standard", "-z"], + { cwd: rootDir, encoding: "utf8" }, + ).split("\0")); function walk(absDir: string, relDir: string): void { for (const entry of readdirSync(absDir, { withFileTypes: true })) { @@ -382,7 +390,7 @@ function listGuidanceFiles(rootDir = repoRoot): string[] { if (!SCAN_EXTENSIONS.has(path.extname(entry.name))) continue; if (isTestFile(relPath)) continue; if (SKIP_PATH_PREFIXES.some((prefix) => relPath.startsWith(prefix))) continue; - found.push(relPath); + if (authored.has(relPath)) found.push(relPath); } } @@ -486,6 +494,7 @@ describe("paperclipai CLI invocation safety", () => { "tests/runner-e2e/catalog.ts", ]; try { + execFileSync("git", ["init", "--quiet"], { cwd: root }); for (const relPath of [ ...sourcePaths, "tests/runner-e2e/results/campaign/attempt-1/snapshots/api-state.json", @@ -503,6 +512,7 @@ describe("paperclipai CLI invocation safety", () => { it("excludes root runtime recordings but still scans unsafe docs and source guidance", () => { const fixtureRoot = mkdtempSync(path.join(os.tmpdir(), "paperclip-cli-guidance-")); try { + execFileSync("git", ["init", "--quiet"], { cwd: fixtureRoot }); const fixtures = [ "doc/CLI.md", "src/guidance.ts", @@ -528,6 +538,26 @@ describe("paperclipai CLI invocation safety", () => { } }); + it("scans tracked and ordinary untracked guidance but excludes ignored untracked files", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "paperclip-cli-guidance-")); + try { + execFileSync("git", ["init", "--quiet"], { cwd: root }); + writeFileSync(path.join(root, ".gitignore"), "ignored.md\ntracked.md\n"); + for (const name of ["ignored.md", "untracked.md", "tracked.md"]) { + writeFileSync(path.join(root, name), "pnpm paperclipai issue get issue-1\n"); + } + execFileSync("git", ["add", "--force", "tracked.md"], { cwd: root }); + + const files = listGuidanceFiles(root).sort(); + expect(files).toEqual(["tracked.md", "untracked.md"]); + for (const name of files) { + expect(scanText(name, readFileSync(path.join(root, name), "utf8"))).toHaveLength(1); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it("allows only exact-allowlist pnpm paperclipai commands on every guidance surface", () => { const offenders = scanForOffenders(); expect( diff --git a/server/src/__tests__/document-annotation-routes.test.ts b/server/src/__tests__/document-annotation-routes.test.ts index b490c0c458..9afe2c1805 100644 --- a/server/src/__tests__/document-annotation-routes.test.ts +++ b/server/src/__tests__/document-annotation-routes.test.ts @@ -187,7 +187,7 @@ async function createApp(actor: "board" | "agent" = "board", actorCompanyId = co } describe("document annotation routes", () => { - beforeEach(() => { + beforeEach(async () => { vi.resetModules(); vi.doUnmock("../routes/issues.js"); vi.doUnmock("../middleware/index.js"); @@ -226,6 +226,11 @@ describe("document annotation routes", () => { mockAnnotationService.addComment.mockResolvedValue(annotationComment); mockAnnotationService.updateThread.mockResolvedValue({ ...annotationThread, status: "resolved" }); mockAnnotationService.remapOpenThreadsForDocument.mockResolvedValue([]); + // Cold route transforms belong to fixture setup, not the request timeout. + await Promise.all([ + vi.importActual("../routes/issues.js"), + vi.importActual("../middleware/index.js"), + ]); }); it("includes compact open annotations without comment bodies by default for agent document reads", async () => { diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 7886155e22..14a0327f11 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -11,6 +11,7 @@ import { stopSshEnvLabFixture, } from "@paperclipai/adapter-utils/ssh"; import { + costEvents, agents, builtInManagedResources, companies, @@ -219,6 +220,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { } await db.delete(environmentLeases); await db.delete(issues); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agents); await db.delete(environments); diff --git a/server/src/__tests__/execution-lock-orphan-cleanup.test.ts b/server/src/__tests__/execution-lock-orphan-cleanup.test.ts index b742862466..ae42a1e445 100644 --- a/server/src/__tests__/execution-lock-orphan-cleanup.test.ts +++ b/server/src/__tests__/execution-lock-orphan-cleanup.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { + costEvents, activityLog, agents, agentWakeupRequests, @@ -40,6 +41,7 @@ describeEmbeddedPostgres("execution lock orphan cleanup", () => { await db.delete(activityLog); await db.delete(heartbeatRunEvents); await db.delete(issues); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agents); await db.delete(companies); diff --git a/server/src/__tests__/heartbeat-accepted-plan-workspace-refresh.test.ts b/server/src/__tests__/heartbeat-accepted-plan-workspace-refresh.test.ts index 781937c13c..8156d8c7e6 100644 --- a/server/src/__tests__/heartbeat-accepted-plan-workspace-refresh.test.ts +++ b/server/src/__tests__/heartbeat-accepted-plan-workspace-refresh.test.ts @@ -7,6 +7,7 @@ import { promisify } from "node:util"; import { eq, ne } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, activityLog, agentRuntimeState, agentTaskSessions, @@ -136,6 +137,7 @@ describeEmbeddedPostgres("accepted plan workspace refresh", () => { await db.delete(activityLog); await db.delete(heartbeatRunEvents); try { + await db.delete(costEvents); await db.delete(heartbeatRuns); break; } catch (error) { diff --git a/server/src/__tests__/heartbeat-archived-company-guard.test.ts b/server/src/__tests__/heartbeat-archived-company-guard.test.ts index 085f8bfd21..bc9bdb40ac 100644 --- a/server/src/__tests__/heartbeat-archived-company-guard.test.ts +++ b/server/src/__tests__/heartbeat-archived-company-guard.test.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { + costEvents, agents, agentWakeupRequests, companies, @@ -35,6 +36,7 @@ describeEmbeddedPostgres("heartbeat archived-company guard", () => { afterEach(async () => { await db.delete(heartbeatRunEvents); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(issues); diff --git a/server/src/__tests__/heartbeat-dependency-scheduling.test.ts b/server/src/__tests__/heartbeat-dependency-scheduling.test.ts index 24cde3e745..cf4d7075d1 100644 --- a/server/src/__tests__/heartbeat-dependency-scheduling.test.ts +++ b/server/src/__tests__/heartbeat-dependency-scheduling.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, inArray, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, activityLog, agents, agentRuntimeState, @@ -143,6 +144,7 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () = await db.delete(issues); await db.delete(heartbeatRunEvents); await db.delete(activityLog); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(agentRuntimeState); diff --git a/server/src/__tests__/heartbeat-issue-rewake-throttle.test.ts b/server/src/__tests__/heartbeat-issue-rewake-throttle.test.ts index 38af962b6a..f7a7ccd0bc 100644 --- a/server/src/__tests__/heartbeat-issue-rewake-throttle.test.ts +++ b/server/src/__tests__/heartbeat-issue-rewake-throttle.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { and, desc, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, activityLog, agentRuntimeState, agentWakeupRequests, @@ -90,6 +91,7 @@ describeEmbeddedPostgres("heartbeat issue rewake throttle", () => { await db.delete(issues); await db.delete(heartbeatRunEvents); await db.delete(activityLog); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(agentRuntimeState); diff --git a/server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts b/server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts index 80dbae6822..c8b9cf500c 100644 --- a/server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts +++ b/server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, agents, agentRuntimeState, agentWakeupRequests, @@ -59,6 +60,7 @@ describeEmbeddedPostgres("heartbeat lock release on cross-agent reassignment", ( afterEach(async () => { await drainHeartbeatRunsToQuiescence(db, heartbeat); await db.delete(heartbeatRunEvents); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(issues); diff --git a/server/src/__tests__/heartbeat-orphaned-active-lease-sweep.test.ts b/server/src/__tests__/heartbeat-orphaned-active-lease-sweep.test.ts index 967f228a0a..6ff69f16f0 100644 --- a/server/src/__tests__/heartbeat-orphaned-active-lease-sweep.test.ts +++ b/server/src/__tests__/heartbeat-orphaned-active-lease-sweep.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { + costEvents, agents, companies, createDb, @@ -62,6 +63,7 @@ describeEmbeddedPostgres("heartbeat sweepOrphanedActiveLeases", () => { afterEach(async () => { await db.delete(environmentLeases); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(environments); await db.delete(agents); diff --git a/server/src/__tests__/heartbeat-pending-cleanup-sweep.test.ts b/server/src/__tests__/heartbeat-pending-cleanup-sweep.test.ts index aee3ccb40a..a118b6945c 100644 --- a/server/src/__tests__/heartbeat-pending-cleanup-sweep.test.ts +++ b/server/src/__tests__/heartbeat-pending-cleanup-sweep.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { + costEvents, activityLog, agents, heartbeatRuns, @@ -71,6 +72,7 @@ describeEmbeddedPostgres("heartbeat sweepPendingCleanupLeases", () => { afterEach(async () => { await db.delete(activityLog); await db.delete(environmentLeases); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(environments); await db.delete(agents); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index e936e13c57..bbc199d50b 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -357,6 +357,22 @@ async function waitForHeartbeatIdle( } } +async function stopRegisteredProcessesForCleanup() { + await Promise.all([...runningProcesses.values()].map(async ({ child, processGroupId }) => { + if (child.exitCode !== null || child.signalCode !== null || typeof child.kill !== "function") return; + await new Promise((resolve) => { + child.once("close", () => resolve()); + try { + if (processGroupId) process.kill(-processGroupId, "SIGKILL"); + else child.kill("SIGKILL"); + } catch { + child.kill("SIGKILL"); + } + }); + })); + runningProcesses.clear(); +} + async function cancelActiveRunsForCleanup( db: ReturnType, timeoutMs = 3_000, @@ -511,7 +527,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { provider: "test", model: "test-model", })); - runningProcesses.clear(); + await stopRegisteredProcessesForCleanup(); for (const child of childProcesses) { child.kill("SIGKILL"); } @@ -550,7 +566,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { } await new Promise((resolve) => setTimeout(resolve, 50)); await waitForHeartbeatIdle(db, 5_000); - await new Promise((resolve) => setTimeout(resolve, 100)); + // Terminal rows do not mean their background finalizers have returned. + // Drain tracked wakeups too: one can still create a run after the row poll. + await heartbeatService(db).drainActiveRunExecutions(); await db.delete(activityLog); await db.delete(agentRuntimeState); await db.delete(companySkills); @@ -669,7 +687,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { } } cleanupPids.clear(); - runningProcesses.clear(); + await stopRegisteredProcessesForCleanup(); if (externalTestDatabaseUrl) { await closeRegisteredClients(externalTestDatabaseUrl); } @@ -819,13 +837,12 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { } it("does not reap active adapter executions started by another heartbeat service instance", async () => { - let releaseAdapter: (() => void) | null = null; + let releaseAdapter!: () => void; + const adapterRelease = new Promise(resolve => { releaseAdapter = resolve; }); const adapterStarted = new Promise((resolve) => { mockAdapterExecute.mockImplementationOnce(async () => { resolve(); - await new Promise((release) => { - releaseAdapter = release; - }); + await adapterRelease; return { exitCode: 0, signal: null, @@ -849,52 +866,55 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const executorHeartbeat = heartbeatService(db); const reaperHeartbeat = heartbeatService(db); - await executorHeartbeat.resumeQueuedRuns(); - await Promise.race([ - adapterStarted, - new Promise((_, reject) => { - setTimeout( - () => - reject( - new Error("Timed out waiting for adapter execution to start"), - ), - 3_000, - ); - }), - ]); + try { + await executorHeartbeat.resumeQueuedRuns(); + await Promise.race([ + adapterStarted, + new Promise((_, reject) => { + setTimeout( + () => + reject( + new Error("Timed out waiting for adapter execution to start"), + ), + 3_000, + ); + }), + ]); - await db - .update(heartbeatRuns) - .set({ - updatedAt: new Date("2026-03-19T00:00:00.000Z"), - }) - .where(eq(heartbeatRuns.id, runId)); + await db + .update(heartbeatRuns) + .set({ + updatedAt: new Date("2026-03-19T00:00:00.000Z"), + }) + .where(eq(heartbeatRuns.id, runId)); - const result = await reaperHeartbeat.reapOrphanedRuns({ - staleThresholdMs: 1, - }); - expect(result).toEqual({ reaped: 0, runIds: [] }); + const result = await reaperHeartbeat.reapOrphanedRuns({ + staleThresholdMs: 1, + }); + expect(result).toEqual({ reaped: 0, runIds: [] }); - const activeRun = await reaperHeartbeat.getRun(runId); - expect(activeRun?.status).toBe("running"); - expect(activeRun?.errorCode).toBeNull(); + const activeRun = await reaperHeartbeat.getRun(runId); + expect(activeRun?.status).toBe("running"); + expect(activeRun?.errorCode).toBeNull(); - const wakeup = await db - .select() - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.id, wakeupRequestId)) - .then((rows) => rows[0] ?? null); - expect(wakeup?.status).toBe("claimed"); + const wakeup = await db + .select() + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, wakeupRequestId)) + .then((rows) => rows[0] ?? null); + expect(wakeup?.status).toBe("claimed"); - if (!releaseAdapter) - throw new Error("Adapter release handle was not captured"); - releaseAdapter(); - const settledRun = await waitForRunToSettle( - executorHeartbeat, - runId, - 5_000, - ); - expect(settledRun?.status).toBe("succeeded"); + releaseAdapter(); + const settledRun = await waitForRunToSettle( + executorHeartbeat, + runId, + 5_000, + ); + expect(settledRun?.status).toBe("succeeded"); + } finally { + releaseAdapter(); + await executorHeartbeat.drainActiveRunExecutions(); + } }); async function seedStrandedIssueFixture(input: { @@ -3608,7 +3628,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); it("persists codex_local spawn identity before hot restart and never loses the live run for missing metadata", async () => { - let releaseAdapter: (() => void) | null = null; + let releaseAdapter!: () => void; + const adapterRelease = new Promise(resolve => { releaseAdapter = resolve; }); let spawnedPid: number | null = null; const adapterStarted = new Promise((resolve) => { mockAdapterExecute.mockImplementationOnce(async (rawInput?: unknown) => { @@ -3630,9 +3651,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { startedAt: new Date("2026-07-30T07:00:00.000Z").toISOString(), }); resolve(); - await new Promise((release) => { - releaseAdapter = release; - }); + await adapterRelease; return { exitCode: 0, signal: null, @@ -3657,87 +3676,90 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { includeIssue: false, }); const heartbeat = heartbeatService(db); - await heartbeat.resumeQueuedRuns(); - await Promise.race([ - adapterStarted, - new Promise((_, reject) => { - setTimeout( - () => - reject( - new Error("Timed out waiting for codex_local spawn identity"), - ), - 3_000, - ); - }), - ]); - - const running = await waitForValue(async () => - db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)) - .then((rows) => { - const row = rows[0] ?? null; - return row?.status === "running" && row.processPid ? row : null; + try { + await heartbeat.resumeQueuedRuns(); + await Promise.race([ + adapterStarted, + new Promise((_, reject) => { + setTimeout( + () => + reject( + new Error("Timed out waiting for codex_local spawn identity"), + ), + 3_000, + ); }), - ); - const observedProcessStartedAt = await readProcessStartedAt(spawnedPid!); - expect(observedProcessStartedAt).not.toBeNull(); - expect(running).toMatchObject({ - id: runId, - status: "running", - processPid: spawnedPid, - processGroupId: null, - processStartedAt: new Date(observedProcessStartedAt!), - }); + ]); - await withTempPaperclipHome(async (home) => { - await writeHotRestartIntent({ - previousServerPid: process.pid, - previousServerVersion: "old-version", - requestedAt: new Date("2026-07-30T07:01:00.000Z"), - }); - await heartbeat.prepareHotRestartShutdown( - "SIGTERM", - new Date("2026-07-30T07:02:00.000Z"), - ); - - const adoption = await heartbeat.reconcileHotRestartAdoption( - new Date("2026-07-30T07:03:00.000Z"), - ); - expect(adoption).toMatchObject({ - mode: "reported", - adoptedRunIds: [runId], - finalizedWhileDownRunIds: [], - lostRunIds: [], - }); - const report = JSON.parse( - await fs.readFile(resolveHotRestartReportPath(home), "utf8"), - ) as { runs?: Array> }; - expect(report.runs).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - runId, - classification: "adopted", - reason: "process_pid_alive", + const running = await waitForValue(async () => + db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows) => { + const row = rows[0] ?? null; + return row?.status === "running" && row.processPid ? row : null; }), - ]), ); - expect(report.runs).not.toEqual( - expect.arrayContaining([ - expect.objectContaining({ - runId, - reason: "missing_process_metadata", - }), - ]), - ); - }); + const observedProcessStartedAt = await readProcessStartedAt(spawnedPid!); + expect(observedProcessStartedAt).not.toBeNull(); + expect(running).toMatchObject({ + id: runId, + status: "running", + processPid: spawnedPid, + processGroupId: null, + processStartedAt: new Date(observedProcessStartedAt!), + }); - if (!releaseAdapter) - throw new Error("Adapter release handle was not captured"); - releaseAdapter(); - const settled = await waitForRunToSettle(heartbeat, runId, 5_000); - expect(settled?.status).toBe("succeeded"); + await withTempPaperclipHome(async (home) => { + await writeHotRestartIntent({ + previousServerPid: process.pid, + previousServerVersion: "old-version", + requestedAt: new Date("2026-07-30T07:01:00.000Z"), + }); + await heartbeat.prepareHotRestartShutdown( + "SIGTERM", + new Date("2026-07-30T07:02:00.000Z"), + ); + + const adoption = await heartbeat.reconcileHotRestartAdoption( + new Date("2026-07-30T07:03:00.000Z"), + ); + expect(adoption).toMatchObject({ + mode: "reported", + adoptedRunIds: [runId], + finalizedWhileDownRunIds: [], + lostRunIds: [], + }); + const report = JSON.parse( + await fs.readFile(resolveHotRestartReportPath(home), "utf8"), + ) as { runs?: Array> }; + expect(report.runs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + runId, + classification: "adopted", + reason: "process_pid_alive", + }), + ]), + ); + expect(report.runs).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + runId, + reason: "missing_process_metadata", + }), + ]), + ); + }); + + releaseAdapter(); + const settled = await waitForRunToSettle(heartbeat, runId, 5_000); + expect(settled?.status).toBe("succeeded"); + } finally { + releaseAdapter(); + await heartbeat.drainActiveRunExecutions(); + } }); it("reports adopted hot-restart runs before startup reap can mark them process_lost", async () => { @@ -7479,7 +7501,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { } }); - it("dispatches interrupted CLI input after the executor releases its lease", async () => { + it.each([false, true])("dispatches interrupted CLI input and cleans up early fixture exits (%s)", async (earlyExit) => { const actualProcess = await vi.importActual("../adapters/process/execute.js"); const { companyId, agentId, issueId, runId } = await seedRunFixture({ runtimeMode: "legacy", adapterType: "codex_local", agentStatus: "idle", runStatus: "queued", @@ -7492,6 +7514,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const heartbeat = heartbeatService(db); await heartbeat.resumeQueuedRuns(); expect(await waitForValue(async () => runningProcesses.get(runId))).toBeTruthy(); + if (earlyExit) return; // Exercise afterEach while the unbounded CLI child is still live. const [comment] = await db.insert(issueComments).values({ companyId, issueId, authorUserId: "responsible-user", body: "continue" }).returning(); const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "automation", reason: "issue_commented", status: "deferred_issue_execution", @@ -10732,7 +10755,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(beforePromotion?.status).toBe("scheduled_retry"); // Simulate a server restart: no in-memory process/timer state carries over. - runningProcesses.clear(); + await stopRegisteredProcessesForCleanup(); const restarted = heartbeatService(db); const promotion = await restarted.promoteDueScheduledRetries( scheduled.dueAt, diff --git a/server/src/__tests__/heartbeat-responsible-user-invariant.test.ts b/server/src/__tests__/heartbeat-responsible-user-invariant.test.ts index 933468ad93..59c3cb2c41 100644 --- a/server/src/__tests__/heartbeat-responsible-user-invariant.test.ts +++ b/server/src/__tests__/heartbeat-responsible-user-invariant.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, activityLog, agents, agentRuntimeState, @@ -62,6 +63,7 @@ async function deleteHeartbeatRunsAfterEvents(db: ReturnType) { for (let attempt = 0; attempt < 5; attempt += 1) { await db.delete(heartbeatRunEvents); try { + await db.delete(costEvents); await db.delete(heartbeatRuns); return; } catch (error) { diff --git a/server/src/__tests__/heartbeat-retry-scheduling.test.ts b/server/src/__tests__/heartbeat-retry-scheduling.test.ts index 697b6e7fb5..a63113c2c0 100644 --- a/server/src/__tests__/heartbeat-retry-scheduling.test.ts +++ b/server/src/__tests__/heartbeat-retry-scheduling.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, inArray, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, agents, approvals, issueApprovals, @@ -164,6 +165,7 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { } async function cleanupRetryFixtureOnce() { + await db.delete(costEvents); await db.delete(activityLog); await db.delete(environmentLeases); await db.delete(issueRelations); diff --git a/server/src/__tests__/heartbeat-run-lease-release-terminalization.test.ts b/server/src/__tests__/heartbeat-run-lease-release-terminalization.test.ts index 3b015f5f3c..872da4596e 100644 --- a/server/src/__tests__/heartbeat-run-lease-release-terminalization.test.ts +++ b/server/src/__tests__/heartbeat-run-lease-release-terminalization.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, agents, companies, createDb, @@ -42,6 +43,7 @@ describeEmbeddedPostgres("heartbeat terminalizeRunOnLeaseRelease", () => { afterEach(async () => { await db.delete(heartbeatRunEvents); await db.delete(issues); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agents); await db.delete(companies); diff --git a/server/src/__tests__/heartbeat-run-terminalize-before-release.test.ts b/server/src/__tests__/heartbeat-run-terminalize-before-release.test.ts index 62555a2737..a291f5b9bc 100644 --- a/server/src/__tests__/heartbeat-run-terminalize-before-release.test.ts +++ b/server/src/__tests__/heartbeat-run-terminalize-before-release.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, agents, companies, createDb, @@ -71,6 +72,7 @@ describeEmbeddedPostgres("heartbeat teardown terminalizes the run before releasi afterEach(async () => { await db.delete(heartbeatRunEvents); await db.delete(issues); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agents); await db.delete(companies); diff --git a/server/src/__tests__/heartbeat-runtime-state.test.ts b/server/src/__tests__/heartbeat-runtime-state.test.ts index d66d54a094..fbc9c24f6a 100644 --- a/server/src/__tests__/heartbeat-runtime-state.test.ts +++ b/server/src/__tests__/heartbeat-runtime-state.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; import { + costEvents, agents, agentRuntimeState, agentWakeupRequests, @@ -52,6 +53,7 @@ describeEmbeddedPostgres("heartbeat runtime state deduplication", () => { afterEach(async () => { clearAllHeartbeatRunRuntimeStatuses(); await db.delete(heartbeatRunEvents); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(agentRuntimeState); diff --git a/server/src/__tests__/heartbeat-task-drain-admission-release.test.ts b/server/src/__tests__/heartbeat-task-drain-admission-release.test.ts index 8de38150f9..5f93b1ca9d 100644 --- a/server/src/__tests__/heartbeat-task-drain-admission-release.test.ts +++ b/server/src/__tests__/heartbeat-task-drain-admission-release.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { + costEvents, activityLog, agents, agentRuntimeState, @@ -51,6 +52,7 @@ describeEmbeddedPostgres("heartbeat task-drain admission release", () => { await db.delete(heartbeatRunEvents); await db.delete(activityLog); try { + await db.delete(costEvents); await db.delete(heartbeatRuns); return; } catch (error) { diff --git a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts index ef2ac9670f..6ccec0f02e 100644 --- a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts +++ b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts @@ -7,6 +7,7 @@ import { promisify } from "node:util"; import { and, eq, inArray } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, activityLog, agentRuntimeState, agentWakeupRequests, @@ -179,6 +180,7 @@ async function deleteHeartbeatRunsForCleanup(db: Db) { await db.delete(heartbeatRunEvents); await db.delete(activityLog); try { + await db.delete(costEvents); await db.delete(heartbeatRuns); return; } catch (error) { diff --git a/server/src/__tests__/heartbeat-workspace-busy.test.ts b/server/src/__tests__/heartbeat-workspace-busy.test.ts index b62f6f061c..0d2fc3f1be 100644 --- a/server/src/__tests__/heartbeat-workspace-busy.test.ts +++ b/server/src/__tests__/heartbeat-workspace-busy.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, agents, agentRuntimeState, agentWakeupRequests, @@ -151,6 +152,7 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => { } async function cleanupFixtureOnce() { + await db.delete(costEvents); await db.delete(activityLog); await db.delete(environmentLeases); await db.delete(issueComments); diff --git a/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts b/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts index ff6d564024..545b3a1630 100644 --- a/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts +++ b/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts @@ -7,6 +7,7 @@ import { promisify } from "node:util"; import { and, asc, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, activityLog, agentRuntimeState, agentTaskSessions, @@ -121,6 +122,7 @@ async function deleteHeartbeatRowsAfterActivityLogDrains(db: Db) { await db.delete(activityLog); await db.delete(heartbeatRunEvents); try { + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); return; diff --git a/server/src/__tests__/heartbeat-worktree-suppression.test.ts b/server/src/__tests__/heartbeat-worktree-suppression.test.ts index b007a83a00..ed2825711e 100644 --- a/server/src/__tests__/heartbeat-worktree-suppression.test.ts +++ b/server/src/__tests__/heartbeat-worktree-suppression.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { eq, sql } from "drizzle-orm"; import { + costEvents, activityLog, agents, agentWakeupRequests, @@ -52,6 +53,7 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => { await db.delete(heartbeatRunEvents); await db.delete(activityLog); try { + await db.delete(costEvents); await db.delete(heartbeatRuns); return; } catch (error) { diff --git a/server/src/__tests__/issue-monitor-scheduler.test.ts b/server/src/__tests__/issue-monitor-scheduler.test.ts index 84f62bb8d6..13389f7303 100644 --- a/server/src/__tests__/issue-monitor-scheduler.test.ts +++ b/server/src/__tests__/issue-monitor-scheduler.test.ts @@ -3,6 +3,7 @@ import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared"; import { + costEvents, activityLog, agentRuntimeState, agentWakeupRequests, @@ -111,6 +112,7 @@ describeEmbeddedPostgres("issue monitor scheduler", () => { await db.delete(environmentLeases); await db.delete(workspaceRuntimeServices); await db.delete(issues); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(agentRuntimeState); diff --git a/server/src/__tests__/issue-onboarding-first-task-routes.test.ts b/server/src/__tests__/issue-onboarding-first-task-routes.test.ts index 4e6015a802..0ae5a48a5e 100644 --- a/server/src/__tests__/issue-onboarding-first-task-routes.test.ts +++ b/server/src/__tests__/issue-onboarding-first-task-routes.test.ts @@ -4,6 +4,7 @@ import request from "supertest"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, activityLog, agentRuntimeState, agents, @@ -94,6 +95,7 @@ describeEmbeddedPostgres("issue create onboarding first-task routes", () => { await db.delete(issueThreadInteractions); await db.delete(issueComments); await db.delete(heartbeatRunEvents); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(agentRuntimeState); diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts index 357c3cf38f..9807310b3c 100644 --- a/server/src/__tests__/issue-recovery-actions.test.ts +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -7,6 +7,7 @@ import request from "supertest"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, agents, agentRuntimeState, authUsers, @@ -154,6 +155,7 @@ describeEmbeddedPostgres("issue recovery actions", () => { await db.delete(issueComments); await db.delete(environmentLeases); await db.delete(activityLog); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(environments); diff --git a/server/src/__tests__/issue-watchdogs-routes.test.ts b/server/src/__tests__/issue-watchdogs-routes.test.ts index dd5636d179..1106073b8f 100644 --- a/server/src/__tests__/issue-watchdogs-routes.test.ts +++ b/server/src/__tests__/issue-watchdogs-routes.test.ts @@ -4,6 +4,7 @@ import request from "supertest"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, activityLog, agentRuntimeState, agentWakeupRequests, @@ -80,6 +81,7 @@ describeEmbeddedPostgres("issue watchdog routes", () => { await db.delete(activityLog); await db.delete(issueComments); await db.delete(heartbeatRunEvents); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(agentRuntimeState); diff --git a/server/src/__tests__/low-trust-red-team-routes.test.ts b/server/src/__tests__/low-trust-red-team-routes.test.ts index 5de4504368..3b13d32de8 100644 --- a/server/src/__tests__/low-trust-red-team-routes.test.ts +++ b/server/src/__tests__/low-trust-red-team-routes.test.ts @@ -6,6 +6,7 @@ import { WebSocketServer } from "ws"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { + costEvents, activityLog, authUsers, agentWakeupRequests, @@ -103,6 +104,7 @@ async function deleteHeartbeatRunsAndWakeupsAfterActivityLogDrains(db: Db) { await db.delete(activityLog); try { await db.delete(heartbeatRunEvents); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); return; diff --git a/server/src/__tests__/native-workspace-sync-history.test.ts b/server/src/__tests__/native-workspace-sync-history.test.ts index 7ce201103a..d8c2e46f8a 100644 --- a/server/src/__tests__/native-workspace-sync-history.test.ts +++ b/server/src/__tests__/native-workspace-sync-history.test.ts @@ -11,7 +11,7 @@ import { runLocalGit } from "@paperclipai/adapter-utils/git-workspace-sync"; import { prepareNativeWorkspaceSync } from "../services/native-runtime/native-workspace-sync.js"; const runner: CommandManagedRuntimeRunner = { - execute: (input) => new Promise((resolve) => { + execute: (input) => new Promise((resolve, reject) => { const child = execFile(input.command, input.args ?? [], { cwd: input.cwd, env: { ...process.env, ...input.env }, timeout: input.timeoutMs, maxBuffer: 16 * 1024 * 1024, @@ -19,6 +19,11 @@ const runner: CommandManagedRuntimeRunner = { exitCode: error ? (typeof error.code === "number" ? error.code : 1) : 0, signal: error?.signal ?? null, timedOut: error?.killed ?? false, stdout, stderr, })); + // Git probes may exit without consuming stdin; their exit status remains + // authoritative. Surface unexpected pipe errors instead of masking them. + child.stdin?.on("error", (error: NodeJS.ErrnoException) => { + if (error.code !== "EPIPE") reject(error); + }); child.stdin?.end(input.stdin ?? ""); }), }; diff --git a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts index b2068ca567..c7e6a07f18 100644 --- a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts +++ b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + costEvents, activityLog, agents, companies, @@ -54,6 +55,7 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => { await db.delete(issueRelations); await db.delete(activityLog); await db.delete(heartbeatRunEvents); + await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(completionContracts); await db.delete(issues); diff --git a/server/src/__tests__/setup-supertest.ts b/server/src/__tests__/setup-supertest.ts index 3a34850d1d..9c7db2179d 100644 --- a/server/src/__tests__/setup-supertest.ts +++ b/server/src/__tests__/setup-supertest.ts @@ -21,6 +21,16 @@ type SupertestTestConstructor = { }; }; +// Keep the original home available only for read-only host port-lease checks. +process.env.PAPERCLIP_TEST_HOST_HOME ??= process.env.PAPERCLIP_HOME?.trim() || path.join(os.homedir(), ".paperclip"); + +// Receipt-spool recovery must never scan a developer's instance during tests. +const paperclipTestHome = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-vitest-home-")); +process.env.PAPERCLIP_HOME = paperclipTestHome; +// Setup-file afterAll hooks run before suites drain asynchronous heartbeats. +// Cleanup at worker exit, after all suite teardown, avoids deleting live state. +process.once("exit", () => fs.rmSync(paperclipTestHome, { recursive: true, force: true, maxRetries: 8, retryDelay: 50 })); + const require = createRequire(import.meta.url); const SupertestTest = require("supertest/lib/test.js") as SupertestTestConstructor; diff --git a/server/src/__tests__/vitest-chat-shards.test.ts b/server/src/__tests__/vitest-chat-shards.test.ts index dd929f1dd8..75de627551 100644 --- a/server/src/__tests__/vitest-chat-shards.test.ts +++ b/server/src/__tests__/vitest-chat-shards.test.ts @@ -12,6 +12,7 @@ it("runs every active nested/parameterized fixture case exactly once through the try { const tests = path.join(root, "server/src/__tests__"); mkdirSync(tests, { recursive: true }); + mkdirSync(path.join(root, "server/scripts"), { recursive: true }); symlinkSync(path.join(repoRoot, "node_modules"), path.join(root, "node_modules"), "junction"); writeFileSync(path.join(root, "package.json"), JSON.stringify({ private: true })); writeFileSync(path.join(root, "vitest.config.mjs"), `export default { diff --git a/server/src/__tests__/workspace-runtime-exposure-reservation.test.ts b/server/src/__tests__/workspace-runtime-exposure-reservation.test.ts index 08615ec352..ff08db371f 100644 --- a/server/src/__tests__/workspace-runtime-exposure-reservation.test.ts +++ b/server/src/__tests__/workspace-runtime-exposure-reservation.test.ts @@ -101,7 +101,9 @@ let NEXT_HMR_PORT: number; * must be read before that happens. Mirrors the default in * `resolvePaperclipHomeDir`. */ -const REAL_PAPERCLIP_HOME = process.env.PAPERCLIP_HOME?.trim() || path.join(os.homedir(), ".paperclip"); +const REAL_PAPERCLIP_HOME = process.env.PAPERCLIP_TEST_HOST_HOME?.trim() + || process.env.PAPERCLIP_HOME?.trim() + || path.join(os.homedir(), ".paperclip"); /** * Every port a local Paperclip instance's on-disk service registry currently diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index 66b9731dfa..cabbd2fdd5 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -43,6 +43,7 @@ import { releaseRuntimeServicesForRun, UnresolvedWorkspaceBaseRefError, resetRuntimeServicesForTests, + resetRuntimeServicePortReservationsForTests, MANAGED_RUNTIME_PUBLIC_URL_ENV, resolveManagedPaperclipRuntimePublicOrigin, resolveRuntimeProvisionCommand, @@ -454,6 +455,9 @@ afterEach(async () => { delete process.env.PAPERCLIP_WORKTREES_DIR; delete process.env.DATABASE_URL; await resetRuntimeServicesForTests(); + // Registry reset does not clear the process-local allocation claims. A + // failed-start fixture must not reserve another test's ephemeral port. + resetRuntimeServicePortReservationsForTests(); }); describe("sanitizeRuntimeServiceBaseEnv", () => { @@ -4099,6 +4103,7 @@ describe("realizeExecutionWorkspace", () => { "utf8", ); process.env.PAPERCLIP_WORKTREES_DIR = worktreesDir; + const canonicalInstanceRoot = await fs.realpath(instanceRoot); await cleanupExecutionWorkspaceArtifacts({ workspace: { @@ -4133,7 +4138,7 @@ describe("realizeExecutionWorkspace", () => { expect(operations[0]?.command).toBe("printf 'cleanup ok\\n'"); expect(operations[1]?.metadata).toMatchObject({ cleanupAction: "remove_worktree_instance", - instanceRoot, + instanceRoot: canonicalInstanceRoot, }); expect(operations[2]?.metadata).toMatchObject({ cleanupAction: "worktree_remove", @@ -7652,16 +7657,25 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => { process.env.PAPERCLIP_HOME = paperclipHome; process.env.PAPERCLIP_INSTANCE_ID = `runtime-desired-reconcile-${randomUUID()}`; - const reservePort = async () => { - const probe = net.createServer(); - await new Promise((resolve) => probe.listen(0, "127.0.0.1", resolve)); - const address = probe.address(); - const port = typeof address === "object" && address ? address.port : null; - await new Promise((resolve, reject) => { - probe.close((error) => error ? reject(error) : resolve()); - }); - if (!port) throw new Error("Failed to reserve runtime reconciliation test port"); - return port; + const reservePorts = async () => { + const probes = [net.createServer(), net.createServer()]; + try { + // Keep both sockets open until both allocations finish. Closing the + // first probe early lets the kernel return that same port again. + for (const probe of probes) await new Promise((resolve, reject) => { + probe.once("error", reject); + probe.listen(0, "127.0.0.1", resolve); + }); + return probes.map(probe => { + const address = probe.address(); + if (!address || typeof address !== "object") throw new Error("Failed to reserve runtime reconciliation test port"); + return address.port; + }); + } finally { + await Promise.all(probes.filter(probe => probe.listening).map(probe => new Promise((resolve, reject) => { + probe.close(error => error ? reject(error) : resolve()); + }))); + } }; const isLoopbackPortFree = async (port: number) => { const probe = net.createServer(); @@ -7682,8 +7696,8 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => { } throw new Error(`Port ${port} did not become free in time`); }; - const stoppedPort = await reservePort(); - const livePort = await reservePort(); + const [stoppedPort, livePort] = await reservePorts(); + expect(stoppedPort).not.toBe(livePort); const companyId = randomUUID(); const projectId = randomUUID(); const projectWorkspaceId = randomUUID(); @@ -9737,7 +9751,7 @@ describe("realizeExecutionWorkspace with an exact existing branch", () => { const workspace = await realizeExistingBranch(repoRoot, "feature/legacy-checkout"); - expect(workspace.cwd).toBe(path.resolve(legacyPath)); + expect(workspace.cwd).toBe(await fs.realpath(legacyPath)); expect(workspace.branchName).toBe("feature/legacy-checkout"); expect(workspace.created).toBe(false); expect(await readGit(workspace.cwd, ["rev-parse", "HEAD"])).toBe(branchTip); diff --git a/tests/e2e/ai-personal-connection-recovery.spec.ts b/tests/e2e/ai-personal-connection-recovery.spec.ts index 84b3864bcc..3da4fb68ba 100644 --- a/tests/e2e/ai-personal-connection-recovery.spec.ts +++ b/tests/e2e/ai-personal-connection-recovery.spec.ts @@ -61,7 +61,7 @@ test("missing onboarding key offers personal Claude setup in chat and resumes on const after = await json(await request.get(`/api/companies/${company.id}/heartbeat-runs`)); const resumed = await json(await request.get(`/api/heartbeat-runs/${after.find((run: { id: string }) => run.id !== sourceRunId).id}`)); expect(resumed.contextSnapshot.aiConnection).toMatchObject({ connectionId: accounts[0].id, responsibleUserId: "local-board" }); - expect((await json(await request.get(`/api/issues/${chat.id}`))).status).toBe("in_review"); + await expect.poll(async () => (await json(await request.get(`/api/issues/${chat.id}`))).status).toBe("in_review"); await page.reload(); await expect(page.getByText("Answered the pending follow-up once.", { exact: false })).toBeVisible(); await expect(page.getByRole("button", { name: "Connect Claude", exact: true })).toHaveCount(0); diff --git a/ui/src/features/connections/ConnectionIntentInteractionBody.test.tsx b/ui/src/features/connections/ConnectionIntentInteractionBody.test.tsx index a30d15906c..a776a26bd2 100644 --- a/ui/src/features/connections/ConnectionIntentInteractionBody.test.tsx +++ b/ui/src/features/connections/ConnectionIntentInteractionBody.test.tsx @@ -509,6 +509,7 @@ describe("AI repair inside the card", () => { if (fails) adoptMock.mockRejectedValue(new Error("Connection test failed")); else adoptMock.mockResolvedValue({ ...interaction, status: "accepted" }); renderBody(interaction); await flush(); + await waitForAssertion(() => expect(button("Connect OpenAI")).toBeDefined()); await act(() => button("Connect OpenAI")!.click()); await act(() => button("Reconnect selected account")!.click()); expect(adoptMock).not.toHaveBeenCalled(); @@ -528,7 +529,7 @@ describe("AI repair inside the card", () => { completeMock.mockResolvedValue({ ...interaction, status: "accepted" }); renderBody(interaction); await flush(); const providerName = provider === "anthropic" ? "Claude" : "OpenAI"; - expect(document.body.textContent).toContain(`Connect your ${providerName} account`); + await waitForAssertion(() => expect(document.body.textContent).toContain(`Connect your ${providerName} account`)); expect(document.body.textContent).toContain("needs your own AI connection"); await act(() => button(`Connect ${providerName}`)!.click()); expect(document.querySelector('[role="dialog"]')).toBeNull();