test: isolate accounting fixtures and include all adapter suites (#14989)

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip authored and GitHub committed 2026-10-06 18:13:46 -07:00
1 parent ac8f3eb143
commit 3ebd7bc0c9
39 files changed
+340 -160

No files matched your search

@@ -265,6 +265,7 @@ exit 7
await fs.mkdir(workspaceDir, { recursive: true });
await fs.mkdir(remoteWorkspace, { recursive: true });
const preferredAgentScript = `#!/bin/sh
cat >/dev/null
printf '%s\\n' '{"type":"system","subtype":"init","session_id":"cursor-session-fresh-1","model":"auto"}'
printf '%s\\n' '{"type":"assistant","message":{"content":[{"type":"output_text","text":"hello"}]}}'
printf '%s\\n' '{"type":"result","subtype":"success","session_id":"cursor-session-fresh-1","result":"ok"}'
@@ -288,7 +289,7 @@ printf '%s\\n' '{"type":"result","subtype":"success","session_id":"cursor-sessio
finalPreparedCommand = preferredCommandPath;
const runtimeEnv = {
...input.env,
PATH: `${path.join(systemHomeDir, ".local", "bin")}${path.delimiter}${input.env.PATH}`,
PATH: `${path.join(systemHomeDir, ".local", "bin")}${path.delimiter}${input.env.PATH ?? process.env.PATH ?? "/usr/bin:/bin"}`,
};
await fs.mkdir(path.dirname(preferredCommandPath), { recursive: true });
await fs.writeFile(preferredCommandPath, preferredAgentScript);
@@ -312,13 +313,14 @@ printf '%s\\n' '{"type":"result","subtype":"success","session_id":"cursor-sessio
// shell seam faithful to that protocol instead of returning empty stdout
// for every shell command.
const runner = {
execute: async (input: { command: string; args?: string[]; env?: Record<string, string> }) => {
execute: async (input: { command: string; args?: string[]; env?: Record<string, string>; stdin?: string }) => {
runnerState.commands.push(input.command);
// Exercise actual bounded file reads during managed-home restoration;
// reporting empty success for every shell command hides missing bytes.
return runChildProcess(`cursor-fresh-lease-${runnerState.commands.length}`, input.command, input.args ?? [], {
cwd: remoteWorkspace,
env: { ...input.env, PATH: `${input.env?.PATH ?? ""}:/usr/bin:/bin` },
stdin: input.stdin,
timeoutSec: 30,
graceSec: 5,
onLog: async () => {},
@@ -92,17 +92,37 @@ test("a route/authz suite never leaks into the general-server shards", () => {
const shard = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", SHARD_COUNT.toString()]);
for (const file of shard.selectedGeneralServerSuites) {
assert.ok(
!/[^/]*(?:route|routes|authz)[^/]*\.test\.ts$/.test(file),
!(file.startsWith("server/src/__tests__/") && /[^/]*(?:route|routes|authz)[^/]*\.test\.ts$/.test(file)),
`route/authz suite must stay in the serialized lane, not general-server: ${file}`,
);
}
});
test("general shards retain scripts and route-named suites outside the serialized directory", () => {
const general = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"]);
for (const file of [
"server/scripts/verify-runner-vendor-dependencies.test.mjs",
"server/src/services/openrouter-models.test.ts",
"server/src/routes/setup-token-route.test.ts",
]) assert.ok(general.selectedGeneralServerSuites.includes(file), `missing configured server suite: ${file}`);
});
test("shard flags are rejected for the workspaces-b group", () => {
const result = dryRun(["--mode", "general", "--group", "general-workspaces-b", "--shard-index", "0", "--shard-count", "3"]);
assert.notEqual(result.status, 0, "workspaces-b must not accept shard flags");
});
test("workspace lanes cover every non-server project in the root Vitest configuration", () => {
const config = readFileSync(path.join(repoRoot, "vitest.config.ts"), "utf8");
const roots = [...config.matchAll(/^\s+"([^"]+)",?\s*$/gm)].map(match => match[1]);
assert.ok(roots.includes("server"), "expected the explicit root Vitest project list");
const expected = roots.filter(root => root !== "server")
.map(root => JSON.parse(readFileSync(path.join(repoRoot, root, "package.json"), "utf8")).name).sort();
const actual = ["general-workspaces-a", "general-workspaces-b"]
.flatMap(group => dryRunJson(["--mode", "general", "--group", group]).workspaceProjects).sort();
assert.deepEqual(actual, expected, "no configured project may be silently omitted or run twice");
});
test("workspaces-a shards map to Vitest native --shard slices over a stable project list", () => {
const shards = [0, 1].map((index) =>
dryRunJson([
+16 -3
View File
@@ -19,6 +19,7 @@ const serializedShardDurations = loadShardDurations(
const serverRoot = path.join(repoRoot, "server");
const serverSrcDir = path.join(repoRoot, "server", "src");
const serverTestsDir = path.join(repoRoot, "server", "src", "__tests__");
const serverScriptsDir = path.join(repoRoot, "server", "scripts");
const nonServerProjects = [
"@paperclipai/shared",
"@paperclipai/skills-catalog",
@@ -26,9 +27,14 @@ const nonServerProjects = [
"@paperclipai/adapter-utils",
"@paperclipai/adapter-claude-local",
"@paperclipai/adapter-codex-local",
"@paperclipai/adapter-cursor-cloud",
"@paperclipai/adapter-cursor-local",
"@paperclipai/adapter-gemini-local",
"@paperclipai/adapter-grok-local",
"@paperclipai/adapter-kimi-local",
"@paperclipai/adapter-openclaw-gateway",
"@paperclipai/adapter-opencode-local",
"@paperclipai/adapter-pi-local",
"@paperclipai/plugin-daytona",
"@paperclipai/plugin-sdk",
"@paperclipai/create-paperclip-plugin",
@@ -345,6 +351,8 @@ function runVitest(args, label, testShard = null) {
const env = {
...process.env,
NODE_ENV: "test",
PAPERCLIP_TEST_HOST_HOME: process.env.PAPERCLIP_TEST_HOST_HOME
?? (process.env.PAPERCLIP_HOME?.trim() || path.join(os.homedir(), ".paperclip")),
PAPERCLIP_HOME: path.join(testRoot, "h"),
// Config discovery otherwise prefers the checkout's .paperclip/config.json
// over PAPERCLIP_HOME, importing preview scheduling policy into unit tests.
@@ -539,10 +547,15 @@ const routeTests = walk(serverTestsDir)
// config pins maxWorkers to 1, so the only way to parallelize is across jobs.
// Suites are partitioned by recorded duration (scripts/general-server-shard.mjs)
// rather than round-robin, so one slow suite cluster can't stretch a single shard.
const generalServerTestFiles = walk(serverSrcDir)
const serializedRepoPaths = new Set(routeTests.map(test => test.repoPath));
const generalServerTestFiles = [
...walk(serverSrcDir).filter(file => file.endsWith(".test.ts")),
...walk(serverScriptsDir).filter(file => file.endsWith(".test.mjs")),
]
.map((file) => toRepoPath(file))
.filter((repoPath) => repoPath.endsWith(".test.ts"))
.filter((repoPath) => !isRouteOrAuthzTest(repoPath))
// Only exclude suites actually assigned to the serialized lane. A name
// such as services/openrouter-models.test.ts is not a serialized route.
.filter((repoPath) => !serializedRepoPaths.has(repoPath))
.sort((a, b) => a.localeCompare(b));
const options = parseCliOptions(process.argv.slice(2));
@@ -60633,19 +60633,23 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
db
.select({ id: issueComments.id })
.from(issueComments)
.where(eq(issueComments.companyId, fixture.companyId)),
.where(eq(issueComments.companyId, fixture.companyId))
.orderBy(issueComments.id),
db
.select({ id: issues.id })
.from(issues)
.where(eq(issues.companyId, fixture.companyId)),
.where(eq(issues.companyId, fixture.companyId))
.orderBy(issues.id),
db
.select({ id: heartbeatRuns.id })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.companyId, fixture.companyId)),
.where(eq(heartbeatRuns.companyId, fixture.companyId))
.orderBy(heartbeatRuns.id),
db
.select({ id: chatPublications.id })
.from(chatPublications)
.where(eq(chatPublications.endpointId, endpoint.id)),
.where(eq(chatPublications.endpointId, endpoint.id))
.orderBy(chatPublications.id),
]);
const baseline = await unchangedRows();
const wakeupCount = wakeup.mock.calls.length;
@@ -1,3 +1,4 @@
import { execFileSync } from "node:child_process";
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import os from "node:os";
import path from "node:path";
@@ -367,6 +368,13 @@ function isTestFile(relPath: string): boolean {
function listGuidanceFiles(rootDir = repoRoot): string[] {
const found: string[] = [];
// Include tracked files and new authored files, while respecting gitignored
// local settings. A tracked file remains scanned even if an ignore matches.
const authored = new Set(execFileSync(
"git",
["ls-files", "--cached", "--others", "--exclude-standard", "-z"],
{ cwd: rootDir, encoding: "utf8" },
).split("\0"));
function walk(absDir: string, relDir: string): void {
for (const entry of readdirSync(absDir, { withFileTypes: true })) {
@@ -382,7 +390,7 @@ function listGuidanceFiles(rootDir = repoRoot): string[] {
if (!SCAN_EXTENSIONS.has(path.extname(entry.name))) continue;
if (isTestFile(relPath)) continue;
if (SKIP_PATH_PREFIXES.some((prefix) => relPath.startsWith(prefix))) continue;
found.push(relPath);
if (authored.has(relPath)) found.push(relPath);
}
}
@@ -486,6 +494,7 @@ describe("paperclipai CLI invocation safety", () => {
"tests/runner-e2e/catalog.ts",
];
try {
execFileSync("git", ["init", "--quiet"], { cwd: root });
for (const relPath of [
...sourcePaths,
"tests/runner-e2e/results/campaign/attempt-1/snapshots/api-state.json",
@@ -503,6 +512,7 @@ describe("paperclipai CLI invocation safety", () => {
it("excludes root runtime recordings but still scans unsafe docs and source guidance", () => {
const fixtureRoot = mkdtempSync(path.join(os.tmpdir(), "paperclip-cli-guidance-"));
try {
execFileSync("git", ["init", "--quiet"], { cwd: fixtureRoot });
const fixtures = [
"doc/CLI.md",
"src/guidance.ts",
@@ -528,6 +538,26 @@ describe("paperclipai CLI invocation safety", () => {
}
});
it("scans tracked and ordinary untracked guidance but excludes ignored untracked files", () => {
const root = mkdtempSync(path.join(os.tmpdir(), "paperclip-cli-guidance-"));
try {
execFileSync("git", ["init", "--quiet"], { cwd: root });
writeFileSync(path.join(root, ".gitignore"), "ignored.md\ntracked.md\n");
for (const name of ["ignored.md", "untracked.md", "tracked.md"]) {
writeFileSync(path.join(root, name), "pnpm paperclipai issue get issue-1\n");
}
execFileSync("git", ["add", "--force", "tracked.md"], { cwd: root });
const files = listGuidanceFiles(root).sort();
expect(files).toEqual(["tracked.md", "untracked.md"]);
for (const name of files) {
expect(scanText(name, readFileSync(path.join(root, name), "utf8"))).toHaveLength(1);
}
} finally {
rmSync(root, { recursive: true, force: true });
}
});
it("allows only exact-allowlist pnpm paperclipai commands on every guidance surface", () => {
const offenders = scanForOffenders();
expect(
@@ -187,7 +187,7 @@ async function createApp(actor: "board" | "agent" = "board", actorCompanyId = co
}
describe("document annotation routes", () => {
beforeEach(() => {
beforeEach(async () => {
vi.resetModules();
vi.doUnmock("../routes/issues.js");
vi.doUnmock("../middleware/index.js");
@@ -226,6 +226,11 @@ describe("document annotation routes", () => {
mockAnnotationService.addComment.mockResolvedValue(annotationComment);
mockAnnotationService.updateThread.mockResolvedValue({ ...annotationThread, status: "resolved" });
mockAnnotationService.remapOpenThreadsForDocument.mockResolvedValue([]);
// Cold route transforms belong to fixture setup, not the request timeout.
await Promise.all([
vi.importActual("../routes/issues.js"),
vi.importActual("../middleware/index.js"),
]);
});
it("includes compact open annotations without comment bodies by default for agent document reads", async () => {
@@ -11,6 +11,7 @@ import {
stopSshEnvLabFixture,
} from "@paperclipai/adapter-utils/ssh";
import {
costEvents,
agents,
builtInManagedResources,
companies,
@@ -219,6 +220,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
}
await db.delete(environmentLeases);
await db.delete(issues);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agents);
await db.delete(environments);
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
costEvents,
activityLog,
agents,
agentWakeupRequests,
@@ -40,6 +41,7 @@ describeEmbeddedPostgres("execution lock orphan cleanup", () => {
await db.delete(activityLog);
await db.delete(heartbeatRunEvents);
await db.delete(issues);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agents);
await db.delete(companies);
@@ -7,6 +7,7 @@ import { promisify } from "node:util";
import { eq, ne } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
activityLog,
agentRuntimeState,
agentTaskSessions,
@@ -136,6 +137,7 @@ describeEmbeddedPostgres("accepted plan workspace refresh", () => {
await db.delete(activityLog);
await db.delete(heartbeatRunEvents);
try {
await db.delete(costEvents);
await db.delete(heartbeatRuns);
break;
} catch (error) {
@@ -1,6 +1,7 @@
import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
costEvents,
agents,
agentWakeupRequests,
companies,
@@ -35,6 +36,7 @@ describeEmbeddedPostgres("heartbeat archived-company guard", () => {
afterEach(async () => {
await db.delete(heartbeatRunEvents);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
await db.delete(issues);
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { and, eq, inArray, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
activityLog,
agents,
agentRuntimeState,
@@ -143,6 +144,7 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
await db.delete(issues);
await db.delete(heartbeatRunEvents);
await db.delete(activityLog);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
await db.delete(agentRuntimeState);
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { and, desc, eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
activityLog,
agentRuntimeState,
agentWakeupRequests,
@@ -90,6 +91,7 @@ describeEmbeddedPostgres("heartbeat issue rewake throttle", () => {
await db.delete(issues);
await db.delete(heartbeatRunEvents);
await db.delete(activityLog);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
await db.delete(agentRuntimeState);
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
agents,
agentRuntimeState,
agentWakeupRequests,
@@ -59,6 +60,7 @@ describeEmbeddedPostgres("heartbeat lock release on cross-agent reassignment", (
afterEach(async () => {
await drainHeartbeatRunsToQuiescence(db, heartbeat);
await db.delete(heartbeatRunEvents);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
await db.delete(issues);
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import {
costEvents,
agents,
companies,
createDb,
@@ -62,6 +63,7 @@ describeEmbeddedPostgres("heartbeat sweepOrphanedActiveLeases", () => {
afterEach(async () => {
await db.delete(environmentLeases);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(environments);
await db.delete(agents);
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import {
costEvents,
activityLog,
agents,
heartbeatRuns,
@@ -71,6 +72,7 @@ describeEmbeddedPostgres("heartbeat sweepPendingCleanupLeases", () => {
afterEach(async () => {
await db.delete(activityLog);
await db.delete(environmentLeases);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(environments);
await db.delete(agents);
@@ -357,6 +357,22 @@ async function waitForHeartbeatIdle(
}
}
async function stopRegisteredProcessesForCleanup() {
await Promise.all([...runningProcesses.values()].map(async ({ child, processGroupId }) => {
if (child.exitCode !== null || child.signalCode !== null || typeof child.kill !== "function") return;
await new Promise<void>((resolve) => {
child.once("close", () => resolve());
try {
if (processGroupId) process.kill(-processGroupId, "SIGKILL");
else child.kill("SIGKILL");
} catch {
child.kill("SIGKILL");
}
});
}));
runningProcesses.clear();
}
async function cancelActiveRunsForCleanup(
db: ReturnType<typeof createDb>,
timeoutMs = 3_000,
@@ -511,7 +527,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
provider: "test",
model: "test-model",
}));
runningProcesses.clear();
await stopRegisteredProcessesForCleanup();
for (const child of childProcesses) {
child.kill("SIGKILL");
}
@@ -550,7 +566,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
}
await new Promise((resolve) => setTimeout(resolve, 50));
await waitForHeartbeatIdle(db, 5_000);
await new Promise((resolve) => setTimeout(resolve, 100));
// Terminal rows do not mean their background finalizers have returned.
// Drain tracked wakeups too: one can still create a run after the row poll.
await heartbeatService(db).drainActiveRunExecutions();
await db.delete(activityLog);
await db.delete(agentRuntimeState);
await db.delete(companySkills);
@@ -669,7 +687,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
}
}
cleanupPids.clear();
runningProcesses.clear();
await stopRegisteredProcessesForCleanup();
if (externalTestDatabaseUrl) {
await closeRegisteredClients(externalTestDatabaseUrl);
}
@@ -819,13 +837,12 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
}
it("does not reap active adapter executions started by another heartbeat service instance", async () => {
let releaseAdapter: (() => void) | null = null;
let releaseAdapter!: () => void;
const adapterRelease = new Promise<void>(resolve => { releaseAdapter = resolve; });
const adapterStarted = new Promise<void>((resolve) => {
mockAdapterExecute.mockImplementationOnce(async () => {
resolve();
await new Promise<void>((release) => {
releaseAdapter = release;
});
await adapterRelease;
return {
exitCode: 0,
signal: null,
@@ -849,52 +866,55 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
const executorHeartbeat = heartbeatService(db);
const reaperHeartbeat = heartbeatService(db);
await executorHeartbeat.resumeQueuedRuns();
await Promise.race([
adapterStarted,
new Promise<never>((_, reject) => {
setTimeout(
() =>
reject(
new Error("Timed out waiting for adapter execution to start"),
),
3_000,
);
}),
]);
try {
await executorHeartbeat.resumeQueuedRuns();
await Promise.race([
adapterStarted,
new Promise<never>((_, reject) => {
setTimeout(
() =>
reject(
new Error("Timed out waiting for adapter execution to start"),
),
3_000,
);
}),
]);
await db
.update(heartbeatRuns)
.set({
updatedAt: new Date("2026-03-19T00:00:00.000Z"),
})
.where(eq(heartbeatRuns.id, runId));
await db
.update(heartbeatRuns)
.set({
updatedAt: new Date("2026-03-19T00:00:00.000Z"),
})
.where(eq(heartbeatRuns.id, runId));
const result = await reaperHeartbeat.reapOrphanedRuns({
staleThresholdMs: 1,
});
expect(result).toEqual({ reaped: 0, runIds: [] });
const result = await reaperHeartbeat.reapOrphanedRuns({
staleThresholdMs: 1,
});
expect(result).toEqual({ reaped: 0, runIds: [] });
const activeRun = await reaperHeartbeat.getRun(runId);
expect(activeRun?.status).toBe("running");
expect(activeRun?.errorCode).toBeNull();
const activeRun = await reaperHeartbeat.getRun(runId);
expect(activeRun?.status).toBe("running");
expect(activeRun?.errorCode).toBeNull();
const wakeup = await db
.select()
.from(agentWakeupRequests)
.where(eq(agentWakeupRequests.id, wakeupRequestId))
.then((rows) => rows[0] ?? null);
expect(wakeup?.status).toBe("claimed");
const wakeup = await db
.select()
.from(agentWakeupRequests)
.where(eq(agentWakeupRequests.id, wakeupRequestId))
.then((rows) => rows[0] ?? null);
expect(wakeup?.status).toBe("claimed");
if (!releaseAdapter)
throw new Error("Adapter release handle was not captured");
releaseAdapter();
const settledRun = await waitForRunToSettle(
executorHeartbeat,
runId,
5_000,
);
expect(settledRun?.status).toBe("succeeded");
releaseAdapter();
const settledRun = await waitForRunToSettle(
executorHeartbeat,
runId,
5_000,
);
expect(settledRun?.status).toBe("succeeded");
} finally {
releaseAdapter();
await executorHeartbeat.drainActiveRunExecutions();
}
});
async function seedStrandedIssueFixture(input: {
@@ -3608,7 +3628,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
});
it("persists codex_local spawn identity before hot restart and never loses the live run for missing metadata", async () => {
let releaseAdapter: (() => void) | null = null;
let releaseAdapter!: () => void;
const adapterRelease = new Promise<void>(resolve => { releaseAdapter = resolve; });
let spawnedPid: number | null = null;
const adapterStarted = new Promise<void>((resolve) => {
mockAdapterExecute.mockImplementationOnce(async (rawInput?: unknown) => {
@@ -3630,9 +3651,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
startedAt: new Date("2026-07-30T07:00:00.000Z").toISOString(),
});
resolve();
await new Promise<void>((release) => {
releaseAdapter = release;
});
await adapterRelease;
return {
exitCode: 0,
signal: null,
@@ -3657,87 +3676,90 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
includeIssue: false,
});
const heartbeat = heartbeatService(db);
await heartbeat.resumeQueuedRuns();
await Promise.race([
adapterStarted,
new Promise<never>((_, reject) => {
setTimeout(
() =>
reject(
new Error("Timed out waiting for codex_local spawn identity"),
),
3_000,
);
}),
]);
const running = await waitForValue(async () =>
db
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.then((rows) => {
const row = rows[0] ?? null;
return row?.status === "running" && row.processPid ? row : null;
try {
await heartbeat.resumeQueuedRuns();
await Promise.race([
adapterStarted,
new Promise<never>((_, reject) => {
setTimeout(
() =>
reject(
new Error("Timed out waiting for codex_local spawn identity"),
),
3_000,
);
}),
);
const observedProcessStartedAt = await readProcessStartedAt(spawnedPid!);
expect(observedProcessStartedAt).not.toBeNull();
expect(running).toMatchObject({
id: runId,
status: "running",
processPid: spawnedPid,
processGroupId: null,
processStartedAt: new Date(observedProcessStartedAt!),
});
]);
await withTempPaperclipHome(async (home) => {
await writeHotRestartIntent({
previousServerPid: process.pid,
previousServerVersion: "old-version",
requestedAt: new Date("2026-07-30T07:01:00.000Z"),
});
await heartbeat.prepareHotRestartShutdown(
"SIGTERM",
new Date("2026-07-30T07:02:00.000Z"),
);
const adoption = await heartbeat.reconcileHotRestartAdoption(
new Date("2026-07-30T07:03:00.000Z"),
);
expect(adoption).toMatchObject({
mode: "reported",
adoptedRunIds: [runId],
finalizedWhileDownRunIds: [],
lostRunIds: [],
});
const report = JSON.parse(
await fs.readFile(resolveHotRestartReportPath(home), "utf8"),
) as { runs?: Array<Record<string, unknown>> };
expect(report.runs).toEqual(
expect.arrayContaining([
expect.objectContaining({
runId,
classification: "adopted",
reason: "process_pid_alive",
const running = await waitForValue(async () =>
db
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.then((rows) => {
const row = rows[0] ?? null;
return row?.status === "running" && row.processPid ? row : null;
}),
]),
);
expect(report.runs).not.toEqual(
expect.arrayContaining([
expect.objectContaining({
runId,
reason: "missing_process_metadata",
}),
]),
);
});
const observedProcessStartedAt = await readProcessStartedAt(spawnedPid!);
expect(observedProcessStartedAt).not.toBeNull();
expect(running).toMatchObject({
id: runId,
status: "running",
processPid: spawnedPid,
processGroupId: null,
processStartedAt: new Date(observedProcessStartedAt!),
});
if (!releaseAdapter)
throw new Error("Adapter release handle was not captured");
releaseAdapter();
const settled = await waitForRunToSettle(heartbeat, runId, 5_000);
expect(settled?.status).toBe("succeeded");
await withTempPaperclipHome(async (home) => {
await writeHotRestartIntent({
previousServerPid: process.pid,
previousServerVersion: "old-version",
requestedAt: new Date("2026-07-30T07:01:00.000Z"),
});
await heartbeat.prepareHotRestartShutdown(
"SIGTERM",
new Date("2026-07-30T07:02:00.000Z"),
);
const adoption = await heartbeat.reconcileHotRestartAdoption(
new Date("2026-07-30T07:03:00.000Z"),
);
expect(adoption).toMatchObject({
mode: "reported",
adoptedRunIds: [runId],
finalizedWhileDownRunIds: [],
lostRunIds: [],
});
const report = JSON.parse(
await fs.readFile(resolveHotRestartReportPath(home), "utf8"),
) as { runs?: Array<Record<string, unknown>> };
expect(report.runs).toEqual(
expect.arrayContaining([
expect.objectContaining({
runId,
classification: "adopted",
reason: "process_pid_alive",
}),
]),
);
expect(report.runs).not.toEqual(
expect.arrayContaining([
expect.objectContaining({
runId,
reason: "missing_process_metadata",
}),
]),
);
});
releaseAdapter();
const settled = await waitForRunToSettle(heartbeat, runId, 5_000);
expect(settled?.status).toBe("succeeded");
} finally {
releaseAdapter();
await heartbeat.drainActiveRunExecutions();
}
});
it("reports adopted hot-restart runs before startup reap can mark them process_lost", async () => {
@@ -7479,7 +7501,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
}
});
it("dispatches interrupted CLI input after the executor releases its lease", async () => {
it.each([false, true])("dispatches interrupted CLI input and cleans up early fixture exits (%s)", async (earlyExit) => {
const actualProcess = await vi.importActual<typeof import("../adapters/process/execute.js")>("../adapters/process/execute.js");
const { companyId, agentId, issueId, runId } = await seedRunFixture({
runtimeMode: "legacy", adapterType: "codex_local", agentStatus: "idle", runStatus: "queued",
@@ -7492,6 +7514,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
const heartbeat = heartbeatService(db);
await heartbeat.resumeQueuedRuns();
expect(await waitForValue(async () => runningProcesses.get(runId))).toBeTruthy();
if (earlyExit) return; // Exercise afterEach while the unbounded CLI child is still live.
const [comment] = await db.insert(issueComments).values({ companyId, issueId, authorUserId: "responsible-user", body: "continue" }).returning();
const [wake] = await db.insert(agentWakeupRequests).values({
companyId, agentId, source: "automation", reason: "issue_commented", status: "deferred_issue_execution",
@@ -10732,7 +10755,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(beforePromotion?.status).toBe("scheduled_retry");
// Simulate a server restart: no in-memory process/timer state carries over.
runningProcesses.clear();
await stopRegisteredProcessesForCleanup();
const restarted = heartbeatService(db);
const promotion = await restarted.promoteDueScheduledRetries(
scheduled.dueAt,
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
activityLog,
agents,
agentRuntimeState,
@@ -62,6 +63,7 @@ async function deleteHeartbeatRunsAfterEvents(db: ReturnType<typeof createDb>) {
for (let attempt = 0; attempt < 5; attempt += 1) {
await db.delete(heartbeatRunEvents);
try {
await db.delete(costEvents);
await db.delete(heartbeatRuns);
return;
} catch (error) {
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { and, eq, inArray, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
agents,
approvals,
issueApprovals,
@@ -164,6 +165,7 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
}
async function cleanupRetryFixtureOnce() {
await db.delete(costEvents);
await db.delete(activityLog);
await db.delete(environmentLeases);
await db.delete(issueRelations);
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
agents,
companies,
createDb,
@@ -42,6 +43,7 @@ describeEmbeddedPostgres("heartbeat terminalizeRunOnLeaseRelease", () => {
afterEach(async () => {
await db.delete(heartbeatRunEvents);
await db.delete(issues);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agents);
await db.delete(companies);
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
agents,
companies,
createDb,
@@ -71,6 +72,7 @@ describeEmbeddedPostgres("heartbeat teardown terminalizes the run before releasi
afterEach(async () => {
await db.delete(heartbeatRunEvents);
await db.delete(issues);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agents);
await db.delete(companies);
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import {
costEvents,
agents,
agentRuntimeState,
agentWakeupRequests,
@@ -52,6 +53,7 @@ describeEmbeddedPostgres("heartbeat runtime state deduplication", () => {
afterEach(async () => {
clearAllHeartbeatRunRuntimeStatuses();
await db.delete(heartbeatRunEvents);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
await db.delete(agentRuntimeState);
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
costEvents,
activityLog,
agents,
agentRuntimeState,
@@ -51,6 +52,7 @@ describeEmbeddedPostgres("heartbeat task-drain admission release", () => {
await db.delete(heartbeatRunEvents);
await db.delete(activityLog);
try {
await db.delete(costEvents);
await db.delete(heartbeatRuns);
return;
} catch (error) {
@@ -7,6 +7,7 @@ import { promisify } from "node:util";
import { and, eq, inArray } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
activityLog,
agentRuntimeState,
agentWakeupRequests,
@@ -179,6 +180,7 @@ async function deleteHeartbeatRunsForCleanup(db: Db) {
await db.delete(heartbeatRunEvents);
await db.delete(activityLog);
try {
await db.delete(costEvents);
await db.delete(heartbeatRuns);
return;
} catch (error) {
@@ -5,6 +5,7 @@ import path from "node:path";
import { and, eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
agents,
agentRuntimeState,
agentWakeupRequests,
@@ -151,6 +152,7 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
}
async function cleanupFixtureOnce() {
await db.delete(costEvents);
await db.delete(activityLog);
await db.delete(environmentLeases);
await db.delete(issueComments);
@@ -7,6 +7,7 @@ import { promisify } from "node:util";
import { and, asc, eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
activityLog,
agentRuntimeState,
agentTaskSessions,
@@ -121,6 +122,7 @@ async function deleteHeartbeatRowsAfterActivityLogDrains(db: Db) {
await db.delete(activityLog);
await db.delete(heartbeatRunEvents);
try {
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
return;
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { eq, sql } from "drizzle-orm";
import {
costEvents,
activityLog,
agents,
agentWakeupRequests,
@@ -52,6 +53,7 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => {
await db.delete(heartbeatRunEvents);
await db.delete(activityLog);
try {
await db.delete(costEvents);
await db.delete(heartbeatRuns);
return;
} catch (error) {
@@ -3,6 +3,7 @@ import { eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared";
import {
costEvents,
activityLog,
agentRuntimeState,
agentWakeupRequests,
@@ -111,6 +112,7 @@ describeEmbeddedPostgres("issue monitor scheduler", () => {
await db.delete(environmentLeases);
await db.delete(workspaceRuntimeServices);
await db.delete(issues);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
await db.delete(agentRuntimeState);
@@ -4,6 +4,7 @@ import request from "supertest";
import { and, eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
activityLog,
agentRuntimeState,
agents,
@@ -94,6 +95,7 @@ describeEmbeddedPostgres("issue create onboarding first-task routes", () => {
await db.delete(issueThreadInteractions);
await db.delete(issueComments);
await db.delete(heartbeatRunEvents);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
await db.delete(agentRuntimeState);
@@ -7,6 +7,7 @@ import request from "supertest";
import { and, eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
agents,
agentRuntimeState,
authUsers,
@@ -154,6 +155,7 @@ describeEmbeddedPostgres("issue recovery actions", () => {
await db.delete(issueComments);
await db.delete(environmentLeases);
await db.delete(activityLog);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
await db.delete(environments);
@@ -4,6 +4,7 @@ import request from "supertest";
import { and, eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
activityLog,
agentRuntimeState,
agentWakeupRequests,
@@ -80,6 +81,7 @@ describeEmbeddedPostgres("issue watchdog routes", () => {
await db.delete(activityLog);
await db.delete(issueComments);
await db.delete(heartbeatRunEvents);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
await db.delete(agentRuntimeState);
@@ -6,6 +6,7 @@ import { WebSocketServer } from "ws";
import { and, eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
costEvents,
activityLog,
authUsers,
agentWakeupRequests,
@@ -103,6 +104,7 @@ async function deleteHeartbeatRunsAndWakeupsAfterActivityLogDrains(db: Db) {
await db.delete(activityLog);
try {
await db.delete(heartbeatRunEvents);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(agentWakeupRequests);
return;
@@ -11,7 +11,7 @@ import { runLocalGit } from "@paperclipai/adapter-utils/git-workspace-sync";
import { prepareNativeWorkspaceSync } from "../services/native-runtime/native-workspace-sync.js";
const runner: CommandManagedRuntimeRunner = {
execute: (input) => new Promise((resolve) => {
execute: (input) => new Promise((resolve, reject) => {
const child = execFile(input.command, input.args ?? [], {
cwd: input.cwd, env: { ...process.env, ...input.env },
timeout: input.timeoutMs, maxBuffer: 16 * 1024 * 1024,
@@ -19,6 +19,11 @@ const runner: CommandManagedRuntimeRunner = {
exitCode: error ? (typeof error.code === "number" ? error.code : 1) : 0,
signal: error?.signal ?? null, timedOut: error?.killed ?? false, stdout, stderr,
}));
// Git probes may exit without consuming stdin; their exit status remains
// authoritative. Surface unexpected pipe errors instead of masking them.
child.stdin?.on("error", (error: NodeJS.ErrnoException) => {
if (error.code !== "EPIPE") reject(error);
});
child.stdin?.end(input.stdin ?? "");
}),
};
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
costEvents,
activityLog,
agents,
companies,
@@ -54,6 +55,7 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => {
await db.delete(issueRelations);
await db.delete(activityLog);
await db.delete(heartbeatRunEvents);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(completionContracts);
await db.delete(issues);
+10
View File
@@ -21,6 +21,16 @@ type SupertestTestConstructor = {
};
};
// Keep the original home available only for read-only host port-lease checks.
process.env.PAPERCLIP_TEST_HOST_HOME ??= process.env.PAPERCLIP_HOME?.trim() || path.join(os.homedir(), ".paperclip");
// Receipt-spool recovery must never scan a developer's instance during tests.
const paperclipTestHome = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-vitest-home-"));
process.env.PAPERCLIP_HOME = paperclipTestHome;
// Setup-file afterAll hooks run before suites drain asynchronous heartbeats.
// Cleanup at worker exit, after all suite teardown, avoids deleting live state.
process.once("exit", () => fs.rmSync(paperclipTestHome, { recursive: true, force: true, maxRetries: 8, retryDelay: 50 }));
const require = createRequire(import.meta.url);
const SupertestTest = require("supertest/lib/test.js") as SupertestTestConstructor;
@@ -12,6 +12,7 @@ it("runs every active nested/parameterized fixture case exactly once through the
try {
const tests = path.join(root, "server/src/__tests__");
mkdirSync(tests, { recursive: true });
mkdirSync(path.join(root, "server/scripts"), { recursive: true });
symlinkSync(path.join(repoRoot, "node_modules"), path.join(root, "node_modules"), "junction");
writeFileSync(path.join(root, "package.json"), JSON.stringify({ private: true }));
writeFileSync(path.join(root, "vitest.config.mjs"), `export default {
@@ -101,7 +101,9 @@ let NEXT_HMR_PORT: number;
* must be read before that happens. Mirrors the default in
* `resolvePaperclipHomeDir`.
*/
const REAL_PAPERCLIP_HOME = process.env.PAPERCLIP_HOME?.trim() || path.join(os.homedir(), ".paperclip");
const REAL_PAPERCLIP_HOME = process.env.PAPERCLIP_TEST_HOST_HOME?.trim()
|| process.env.PAPERCLIP_HOME?.trim()
|| path.join(os.homedir(), ".paperclip");
/**
* Every port a local Paperclip instance's on-disk service registry currently
+28 -14
View File
@@ -43,6 +43,7 @@ import {
releaseRuntimeServicesForRun,
UnresolvedWorkspaceBaseRefError,
resetRuntimeServicesForTests,
resetRuntimeServicePortReservationsForTests,
MANAGED_RUNTIME_PUBLIC_URL_ENV,
resolveManagedPaperclipRuntimePublicOrigin,
resolveRuntimeProvisionCommand,
@@ -454,6 +455,9 @@ afterEach(async () => {
delete process.env.PAPERCLIP_WORKTREES_DIR;
delete process.env.DATABASE_URL;
await resetRuntimeServicesForTests();
// Registry reset does not clear the process-local allocation claims. A
// failed-start fixture must not reserve another test's ephemeral port.
resetRuntimeServicePortReservationsForTests();
});
describe("sanitizeRuntimeServiceBaseEnv", () => {
@@ -4099,6 +4103,7 @@ describe("realizeExecutionWorkspace", () => {
"utf8",
);
process.env.PAPERCLIP_WORKTREES_DIR = worktreesDir;
const canonicalInstanceRoot = await fs.realpath(instanceRoot);
await cleanupExecutionWorkspaceArtifacts({
workspace: {
@@ -4133,7 +4138,7 @@ describe("realizeExecutionWorkspace", () => {
expect(operations[0]?.command).toBe("printf 'cleanup ok\\n'");
expect(operations[1]?.metadata).toMatchObject({
cleanupAction: "remove_worktree_instance",
instanceRoot,
instanceRoot: canonicalInstanceRoot,
});
expect(operations[2]?.metadata).toMatchObject({
cleanupAction: "worktree_remove",
@@ -7652,16 +7657,25 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
process.env.PAPERCLIP_HOME = paperclipHome;
process.env.PAPERCLIP_INSTANCE_ID = `runtime-desired-reconcile-${randomUUID()}`;
const reservePort = async () => {
const probe = net.createServer();
await new Promise<void>((resolve) => probe.listen(0, "127.0.0.1", resolve));
const address = probe.address();
const port = typeof address === "object" && address ? address.port : null;
await new Promise<void>((resolve, reject) => {
probe.close((error) => error ? reject(error) : resolve());
});
if (!port) throw new Error("Failed to reserve runtime reconciliation test port");
return port;
const reservePorts = async () => {
const probes = [net.createServer(), net.createServer()];
try {
// Keep both sockets open until both allocations finish. Closing the
// first probe early lets the kernel return that same port again.
for (const probe of probes) await new Promise<void>((resolve, reject) => {
probe.once("error", reject);
probe.listen(0, "127.0.0.1", resolve);
});
return probes.map(probe => {
const address = probe.address();
if (!address || typeof address !== "object") throw new Error("Failed to reserve runtime reconciliation test port");
return address.port;
});
} finally {
await Promise.all(probes.filter(probe => probe.listening).map(probe => new Promise<void>((resolve, reject) => {
probe.close(error => error ? reject(error) : resolve());
})));
}
};
const isLoopbackPortFree = async (port: number) => {
const probe = net.createServer();
@@ -7682,8 +7696,8 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
}
throw new Error(`Port ${port} did not become free in time`);
};
const stoppedPort = await reservePort();
const livePort = await reservePort();
const [stoppedPort, livePort] = await reservePorts();
expect(stoppedPort).not.toBe(livePort);
const companyId = randomUUID();
const projectId = randomUUID();
const projectWorkspaceId = randomUUID();
@@ -9737,7 +9751,7 @@ describe("realizeExecutionWorkspace with an exact existing branch", () => {
const workspace = await realizeExistingBranch(repoRoot, "feature/legacy-checkout");
expect(workspace.cwd).toBe(path.resolve(legacyPath));
expect(workspace.cwd).toBe(await fs.realpath(legacyPath));
expect(workspace.branchName).toBe("feature/legacy-checkout");
expect(workspace.created).toBe(false);
expect(await readGit(workspace.cwd, ["rev-parse", "HEAD"])).toBe(branchTip);
@@ -61,7 +61,7 @@ test("missing onboarding key offers personal Claude setup in chat and resumes on
const after = await json(await request.get(`/api/companies/${company.id}/heartbeat-runs`));
const resumed = await json(await request.get(`/api/heartbeat-runs/${after.find((run: { id: string }) => run.id !== sourceRunId).id}`));
expect(resumed.contextSnapshot.aiConnection).toMatchObject({ connectionId: accounts[0].id, responsibleUserId: "local-board" });
expect((await json(await request.get(`/api/issues/${chat.id}`))).status).toBe("in_review");
await expect.poll(async () => (await json(await request.get(`/api/issues/${chat.id}`))).status).toBe("in_review");
await page.reload();
await expect(page.getByText("Answered the pending follow-up once.", { exact: false })).toBeVisible();
await expect(page.getByRole("button", { name: "Connect Claude", exact: true })).toHaveCount(0);
@@ -509,6 +509,7 @@ describe("AI repair inside the card", () => {
if (fails) adoptMock.mockRejectedValue(new Error("Connection test failed"));
else adoptMock.mockResolvedValue({ ...interaction, status: "accepted" });
renderBody(interaction); await flush();
await waitForAssertion(() => expect(button("Connect OpenAI")).toBeDefined());
await act(() => button("Connect OpenAI")!.click());
await act(() => button("Reconnect selected account")!.click());
expect(adoptMock).not.toHaveBeenCalled();
@@ -528,7 +529,7 @@ describe("AI repair inside the card", () => {
completeMock.mockResolvedValue({ ...interaction, status: "accepted" });
renderBody(interaction); await flush();
const providerName = provider === "anthropic" ? "Claude" : "OpenAI";
expect(document.body.textContent).toContain(`Connect your ${providerName} account`);
await waitForAssertion(() => expect(document.body.textContent).toContain(`Connect your ${providerName} account`));
expect(document.body.textContent).toContain("needs your own AI connection");
await act(() => button(`Connect ${providerName}`)!.click());
expect(document.querySelector('[role="dialog"]')).toBeNull();