diff --git a/doc/test-infrastructure.md b/doc/test-infrastructure.md index 60c663fa66f..16b11bb6cbe 100644 --- a/doc/test-infrastructure.md +++ b/doc/test-infrastructure.md @@ -96,6 +96,25 @@ falls back to the fresh path automatically, so the cache can make a suite slower but never fail it or skip it: a migration that cannot apply still fails the suite on the fresh path. +## Seeded database statistics and teardown + +After seeding a large population, run `ANALYZE` on the populated tables before +measuring or exercising it. The empty template has no statistics for those rows, +and waiting for automatic analysis can cause slow query plans and test timeouts. +The census and review-attention equivalence suites retain their full populations +and original deadlines while analyzing their fixture tables before the checks. + +Tests that exercise route-triggered heartbeats must await +`drainActiveRunExecutions()` before deleting fixture rows. A completed HTTP +response can still have wake admission in flight; draining only run executions +would leave that writer racing with deletion. + +Setup destination admission uses `NO KEY UPDATE` on the endpoint. It serializes +competing admissions while permitting `KEY SHARE` from resource foreign-key +checks. The concurrency regression uses the real `ensureResource` write and +connector invalidation triggers, with barriers to force the endpoint/agent lock +order. Restoring `FOR UPDATE` produces PostgreSQL deadlock code `40P01`. + ## Typecheck memory TypeScript 7 is a native binary, so `NODE_OPTIONS=--max-old-space-size` does not diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts index 18dbd995d89..6408a0e33d1 100644 --- a/server/src/__tests__/issue-recovery-actions.test.ts +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -189,6 +189,7 @@ describeEmbeddedPostgres("issue recovery actions", () => { }, 30_000); afterEach(async () => { + await heartbeatService(db).drainActiveRunExecutions(); await db.delete(caseHandoffs); await db.delete(cases); await db.delete(documentRevisions); diff --git a/server/src/__tests__/issue-recovery-live-run-agent-census.test.ts b/server/src/__tests__/issue-recovery-live-run-agent-census.test.ts index e727a08eaf6..8da7b884595 100644 --- a/server/src/__tests__/issue-recovery-live-run-agent-census.test.ts +++ b/server/src/__tests__/issue-recovery-live-run-agent-census.test.ts @@ -137,6 +137,7 @@ describeIssueRecovery("stranded-task checks beside one long live run (census)", async function boot() { const f = await bounded(seed(), () => "the fixture to be seeded"); + await state.db.execute(sql`ANALYZE agents, issues, heartbeat_runs`); const events = record(); useCensusClock(); await bounded(state.start({}), () => "the recovery workers to start"); diff --git a/server/src/__tests__/review-attention-classification-equivalence.test.ts b/server/src/__tests__/review-attention-classification-equivalence.test.ts index 305e88d30dd..f38c8a913ca 100644 --- a/server/src/__tests__/review-attention-classification-equivalence.test.ts +++ b/server/src/__tests__/review-attention-classification-equivalence.test.ts @@ -1,5 +1,6 @@ import express from "express"; import request from "supertest"; +import { sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { createDb, inboxDismissals } from "@paperclipai/db"; import type { AttentionFeed, AttentionSummary, Issue } from "@paperclipai/shared"; @@ -76,6 +77,11 @@ describeEmbeddedPostgres("review attention through the routes: indexed classific // Two companies of the same shapes. Each request reads one of them. fixture = await seedReviewAttentionFixture(db, { tasks: 640, agents: 24, prefix: "RAE" }); other = await seedReviewAttentionFixture(db, { tasks: 320, agents: 12, prefix: "OTH" }); + await db.execute(sql`ANALYZE agents, issues, heartbeat_runs, issue_relations, + issue_thread_interactions, issue_recovery_actions, agent_wakeup_requests, + approvals, issue_approvals, chat_endpoints, chat_endpoint_resources, + chat_conversations, chat_github_reviews, chat_message_links, chat_deliveries, + chat_actions, run_identity_contexts, status_decisions`); }, 120_000); beforeEach(() => { diff --git a/server/src/services/chat-channels/inbound/resource-admission.test.ts b/server/src/services/chat-channels/inbound/resource-admission.test.ts new file mode 100644 index 00000000000..e4656676573 --- /dev/null +++ b/server/src/services/chat-channels/inbound/resource-admission.test.ts @@ -0,0 +1,73 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, chatEndpointResources, chatEndpoints, companies, createDb, toolApplications, toolConnections } from "@paperclipai/db"; +import type { Thread } from "chat"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; +import { createMessageResourceAdmission } from "./resource-admission.js"; +import { createChatInboundAuthority } from "./authorization.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const describePostgres = support.supported ? describe : describe.skip; + +describePostgres("setup destination admission with concurrent runtime invalidation", () => { + let database: Awaited>; + let db: ReturnType; + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("chat-resource-admission-"); + db = createDb(database.connectionString); + }, 90_000); + + afterAll(async () => { + await db?.$client.end({ timeout: 1 }); + await database?.cleanup(); + }); + + it("admits a destination while an agent-locked resource insert checks its endpoint foreign key", async () => { + const [company] = await db.insert(companies).values({ name: "Admission" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company!.id, name: "Worker", role: "engineer", + status: "idle", adapterType: "process" }).returning(); + const [application] = await db.insert(toolApplications).values({ companyId: company!.id, name: "GitHub", type: "chat" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId: company!.id, applicationId: application!.id, + name: "GitHub", uid: randomUUID(), transport: "chat_sdk", connectionPurpose: "channel", status: "active" }).returning(); + const [endpoint] = await db.insert(chatEndpoints).values({ companyId: company!.id, connectionId: connection!.id, + provider: "github", publicId: randomUUID(), assignedAgentId: agent!.id, status: "verifying" }).returning(); + const [resource] = await db.insert(chatEndpointResources).values({ companyId: company!.id, endpointId: endpoint!.id, + type: "repository", providerResourceId: "example/first", label: "First", enabled: false }).returning(); + let releaseEndpoint!: () => void; + let releaseAgent!: () => void; + const endpointLocked = new Promise(resolve => { releaseEndpoint = resolve; }); + const agentLocked = new Promise(resolve => { releaseAgent = resolve; }); + const authority = createChatInboundAuthority(db, { + voiceStore: { authorizePrincipal: async () => { throw new Error("Unexpected voice authorization"); } }, + runtimeCallbackEndpoint: async () => { throw new Error("Unexpected runtime callback"); }, + }); + const admission = createMessageResourceAdmission(db, { + ensureResource: async (...args) => { + releaseEndpoint(); + await agentLocked; + return authority.ensureResource(...args); + }, + }); + + const thread = { isDM: false, id: "github:example/first:1", channelId: "example/first", + channel: { name: "example/first" } } as Thread; + const admitting = admission.admitMessageResource({ endpoint: endpoint!, thread, + addressed: true, suppressSetupDestinationActivation: false, surfaceKind: "native_thread" }); + const inserting = (async () => { + await endpointLocked; + await db.transaction(async tx => { + await tx.select({ id: agents.id }).from(agents).where(eq(agents.id, agent!.id)).for("no key update"); + releaseAgent(); + await tx.insert(chatEndpointResources).values({ companyId: company!.id, endpointId: endpoint!.id, + type: "repository", providerResourceId: "example/second", label: "Second", enabled: false }); + }); + })(); + const results = await Promise.allSettled([admitting, inserting]); + for (const result of results) if (result.status === "rejected") throw result.reason; + const resources = await db.select().from(chatEndpointResources).where(eq(chatEndpointResources.endpointId, endpoint!.id)); + expect(resources).toHaveLength(2); + expect(resources.filter(row => row.enabled).map(row => row.id)).toEqual([resource!.id]); + }); +}); diff --git a/server/src/services/chat-channels/inbound/resource-admission.ts b/server/src/services/chat-channels/inbound/resource-admission.ts index e83ba1c9749..df67f932e8f 100644 --- a/server/src/services/chat-channels/inbound/resource-admission.ts +++ b/server/src/services/chat-channels/inbound/resource-admission.ts @@ -56,7 +56,7 @@ export function createMessageResourceAdmission(db: Db, options: { ensureResource .select({ status: chatEndpoints.status }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)) - .for("update") + .for("no key update") .then((rows) => rows[0] ?? null); if (!currentEndpoint) throw notFound("Chat endpoint not found"); const enabledResourceCount = await tx