diff --git a/packages/db/src/migrations/0076_company_search_document_indexes.sql b/packages/db/src/migrations/0076_company_search_document_indexes.sql new file mode 100644 index 00000000000..5f31d42cecb --- /dev/null +++ b/packages/db/src/migrations/0076_company_search_document_indexes.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS "documents_title_search_idx" ON "documents" USING gin ("title" gin_trgm_ops);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "documents_latest_body_search_idx" ON "documents" USING gin ("latest_body" gin_trgm_ops); diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index f0d21b99889..5af87edf582 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -533,6 +533,13 @@ "when": 1777572332006, "tag": "0075_cultured_sebastian_shaw", "breakpoints": true + }, + { + "idx": 76, + "version": "7", + "when": 1777821410992, + "tag": "0076_company_search_document_indexes", + "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/packages/db/src/schema/documents.ts b/packages/db/src/schema/documents.ts index 53d5f358526..1dae7e1d74a 100644 --- a/packages/db/src/schema/documents.ts +++ b/packages/db/src/schema/documents.ts @@ -22,5 +22,7 @@ export const documents = pgTable( (table) => ({ companyUpdatedIdx: index("documents_company_updated_idx").on(table.companyId, table.updatedAt), companyCreatedIdx: index("documents_company_created_idx").on(table.companyId, table.createdAt), + titleSearchIdx: index("documents_title_search_idx").using("gin", table.title.op("gin_trgm_ops")), + bodySearchIdx: index("documents_latest_body_search_idx").using("gin", table.latestBody.op("gin_trgm_ops")), }), ); diff --git a/packages/shared/src/validators/search.ts b/packages/shared/src/validators/search.ts index 08f446339cb..4f5419c8203 100644 --- a/packages/shared/src/validators/search.ts +++ b/packages/shared/src/validators/search.ts @@ -5,7 +5,7 @@ export const COMPANY_SEARCH_MAX_QUERY_LENGTH = 200; export const COMPANY_SEARCH_MAX_TOKENS = 8; export const COMPANY_SEARCH_DEFAULT_LIMIT = 20; export const COMPANY_SEARCH_MAX_LIMIT = 50; -export const COMPANY_SEARCH_MAX_OFFSET = 5_000; +export const COMPANY_SEARCH_MAX_OFFSET = 200; function firstQueryValue(value: unknown): unknown { return Array.isArray(value) ? value[0] : value; diff --git a/server/src/__tests__/company-search-rate-limit-routes.test.ts b/server/src/__tests__/company-search-rate-limit-routes.test.ts new file mode 100644 index 00000000000..1c52c42b6c7 --- /dev/null +++ b/server/src/__tests__/company-search-rate-limit-routes.test.ts @@ -0,0 +1,53 @@ +import express from "express"; +import request from "supertest"; +import { describe, expect, it, vi } from "vitest"; +import { issueRoutes } from "../routes/issues.js"; +import { createCompanySearchRateLimiter } from "../services/company-search-rate-limit.js"; +import type { CompanySearchQuery, CompanySearchResponse } from "@paperclipai/shared"; + +function createSearchResponse(query: CompanySearchQuery): CompanySearchResponse { + return { + query: query.q, + normalizedQuery: query.q.trim().toLowerCase(), + scope: query.scope, + limit: query.limit, + offset: query.offset, + results: [], + countsByType: { issue: 0, agent: 0, project: 0 }, + hasMore: false, + }; +} + +describe("company search route rate limiting", () => { + it("rejects repeated same-actor search calls before invoking search", async () => { + const search = vi.fn(async (_companyId: string, query: CompanySearchQuery) => createSearchResponse(query)); + const app = express(); + app.use((req, _res, next) => { + req.actor = { + type: "agent", + agentId: "agent-1", + companyId: "company-1", + source: "agent_key", + }; + next(); + }); + app.use("/api", issueRoutes({} as never, {} as never, { + searchService: { search }, + searchRateLimiter: createCompanySearchRateLimiter({ + maxRequests: 1, + windowMs: 60_000, + now: () => 1_000, + }), + })); + + await request(app).get("/api/companies/company-1/search?q=wizard").expect(200); + const limited = await request(app).get("/api/companies/company-1/search?q=wizard").expect(429); + + expect(search).toHaveBeenCalledTimes(1); + expect(limited.body).toMatchObject({ + error: "Search rate limit exceeded", + retryAfterSeconds: 60, + }); + expect(limited.headers["retry-after"]).toBe("60"); + }); +}); diff --git a/server/src/__tests__/company-search-service.test.ts b/server/src/__tests__/company-search-service.test.ts index 312375fdd1f..fe1cb00d24c 100644 --- a/server/src/__tests__/company-search-service.test.ts +++ b/server/src/__tests__/company-search-service.test.ts @@ -16,7 +16,11 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; -import { companySearchService } from "../services/company-search.js"; +import { + COMPANY_SEARCH_BRANCH_FETCH_LIMIT, + companySearchBranchFetchLimit, + companySearchService, +} from "../services/company-search.js"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; @@ -38,9 +42,17 @@ describe("company search query validation", () => { expect(parsed.q).toHaveLength(COMPANY_SEARCH_MAX_QUERY_LENGTH); expect(parsed.limit).toBe(50); - expect(parsed.offset).toBe(5000); + expect(parsed.offset).toBe(200); expect(parsed.scope).toBe("all"); }); + + it("keeps the internal per-branch fetch limit independent of high offsets", () => { + const lowOffset = companySearchQuerySchema.parse({ q: "needle", limit: "50", offset: "0" }); + const highOffset = companySearchQuerySchema.parse({ q: "needle", limit: "50", offset: "9000" }); + + expect(companySearchBranchFetchLimit(lowOffset.limit)).toBe(COMPANY_SEARCH_BRANCH_FETCH_LIMIT); + expect(companySearchBranchFetchLimit(highOffset.limit)).toBe(COMPANY_SEARCH_BRANCH_FETCH_LIMIT); + }); }); describeEmbeddedPostgres("companySearchService", () => { diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 4e08eab5231..a65d6d9221d 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -31,6 +31,8 @@ import { updateIssueSchema, getClosedIsolatedExecutionWorkspaceMessage, isClosedIsolatedExecutionWorkspace, + type CompanySearchQuery, + type CompanySearchResponse, type ExecutionWorkspace, } from "@paperclipai/shared"; import { trackAgentTaskCompleted } from "@paperclipai/shared/telemetry"; @@ -79,6 +81,10 @@ import { executionWorkspaceService as executionWorkspaceServiceDirect } from ".. import { feedbackService } from "../services/feedback.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { environmentService } from "../services/environments.js"; +import { + createCompanySearchRateLimiter, + type CompanySearchRateLimiter, +} from "../services/company-search-rate-limit.js"; import { applyIssueExecutionPolicyTransition, normalizeIssueExecutionPolicy, @@ -95,6 +101,9 @@ const updateIssueRouteSchema = updateIssueSchema.extend({ type ParsedExecutionState = NonNullable>; type NormalizedExecutionPolicy = NonNullable>; +type CompanySearchService = { + search(companyId: string, query: CompanySearchQuery): Promise; +}; type ActivityIssueRelationSummary = { id: string; identifier: string | null; @@ -152,6 +161,23 @@ function summarizeIssueRelationForActivity(relation: { }; } +const defaultCompanySearchRateLimiter = createCompanySearchRateLimiter(); + +function companySearchRateLimitActor(req: Request, companyId: string) { + if (req.actor.type === "agent") { + return { + companyId, + actorType: "agent" as const, + actorId: req.actor.agentId ?? req.actor.keyId ?? "unknown-agent", + }; + } + return { + companyId, + actorType: "board" as const, + actorId: req.actor.userId ?? req.actor.source ?? "board", + }; +} + function summarizeIssueReferenceActivityDetails(input: | { addedReferencedIssues: ActivityIssueRelationSummary[]; @@ -447,6 +473,8 @@ export function issueRoutes( now?: Date; }): Promise; }; + searchService?: CompanySearchService; + searchRateLimiter?: CompanySearchRateLimiter; pluginWorkerManager?: PluginWorkerManager; } = {}, ) { @@ -458,7 +486,8 @@ export function issueRoutes( }); const feedback = feedbackService(db); const companiesSvc = companyService(db); - const searchSvc = companySearchService(db); + const searchSvc = opts.searchService ?? companySearchService(db); + const searchRateLimiter = opts.searchRateLimiter ?? defaultCompanySearchRateLimiter; const instanceSettings = instanceSettingsService(db); const agentsSvc = agentService(db); const projectsSvc = projectService(db); @@ -951,6 +980,17 @@ export function issueRoutes( const companyId = req.params.companyId as string; assertCompanyAccess(req, companyId); const query = companySearchQuerySchema.parse(req.query); + const rateLimit = searchRateLimiter.consume(companySearchRateLimitActor(req, companyId)); + res.setHeader("X-RateLimit-Limit", String(rateLimit.limit)); + res.setHeader("X-RateLimit-Remaining", String(rateLimit.remaining)); + if (!rateLimit.allowed) { + res.setHeader("Retry-After", String(rateLimit.retryAfterSeconds)); + res.status(429).json({ + error: "Search rate limit exceeded", + retryAfterSeconds: rateLimit.retryAfterSeconds, + }); + return; + } const result = await searchSvc.search(companyId, query); res.json(result); }); diff --git a/server/src/services/company-search-rate-limit.ts b/server/src/services/company-search-rate-limit.ts new file mode 100644 index 00000000000..4ac32e80d67 --- /dev/null +++ b/server/src/services/company-search-rate-limit.ts @@ -0,0 +1,63 @@ +export const COMPANY_SEARCH_RATE_LIMIT_WINDOW_MS = 60_000; +export const COMPANY_SEARCH_RATE_LIMIT_MAX_REQUESTS = 60; + +export type CompanySearchRateLimitActor = { + companyId: string; + actorType: "agent" | "board"; + actorId: string; +}; + +export type CompanySearchRateLimitResult = { + allowed: boolean; + limit: number; + remaining: number; + retryAfterSeconds: number; +}; + +export type CompanySearchRateLimiter = { + consume(actor: CompanySearchRateLimitActor): CompanySearchRateLimitResult; +}; + +export function createCompanySearchRateLimiter(options: { + windowMs?: number; + maxRequests?: number; + now?: () => number; +} = {}): CompanySearchRateLimiter { + const windowMs = options.windowMs ?? COMPANY_SEARCH_RATE_LIMIT_WINDOW_MS; + const maxRequests = options.maxRequests ?? COMPANY_SEARCH_RATE_LIMIT_MAX_REQUESTS; + const now = options.now ?? Date.now; + const hitsByKey = new Map(); + + function key(actor: CompanySearchRateLimitActor) { + return `${actor.companyId}:${actor.actorType}:${actor.actorId}`; + } + + return { + consume(actor) { + const currentTime = now(); + const cutoff = currentTime - windowMs; + const actorKey = key(actor); + const recentHits = (hitsByKey.get(actorKey) ?? []).filter((hit) => hit > cutoff); + + if (recentHits.length >= maxRequests) { + const oldestHit = recentHits[0] ?? currentTime; + hitsByKey.set(actorKey, recentHits); + return { + allowed: false, + limit: maxRequests, + remaining: 0, + retryAfterSeconds: Math.max(1, Math.ceil((oldestHit + windowMs - currentTime) / 1000)), + }; + } + + recentHits.push(currentTime); + hitsByKey.set(actorKey, recentHits); + return { + allowed: true, + limit: maxRequests, + remaining: Math.max(0, maxRequests - recentHits.length), + retryAfterSeconds: 0, + }; + }, + }; +} diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.ts index 9ba67f0b5f3..99b3c4e2867 100644 --- a/server/src/services/company-search.ts +++ b/server/src/services/company-search.ts @@ -3,6 +3,7 @@ import type { SQL } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, companies, issues, projects } from "@paperclipai/db"; import { + COMPANY_SEARCH_MAX_LIMIT, COMPANY_SEARCH_MAX_TOKENS, type CompanySearchIssueSummary, type CompanySearchQuery, @@ -16,6 +17,7 @@ import { const MIN_TOKEN_LENGTH = 2; const MIN_FUZZY_QUERY_LENGTH = 4; const SNIPPET_MAX_CHARS = 240; +export const COMPANY_SEARCH_BRANCH_FETCH_LIMIT = COMPANY_SEARCH_MAX_LIMIT + 1; type IssueSearchRow = { id: string; @@ -259,6 +261,11 @@ function simpleTextCondition(fields: SQL[], containsPattern: string, tokenArray: return sql`(${sql.join([...phraseConditions, ...tokenConditions], sql` OR `)})`; } +export function companySearchBranchFetchLimit(limit: number) { + const normalizedLimit = Number.isFinite(limit) ? Math.max(1, Math.floor(limit)) : COMPANY_SEARCH_MAX_LIMIT; + return Math.min(COMPANY_SEARCH_BRANCH_FETCH_LIMIT, normalizedLimit + 1); +} + export function companySearchService(db: Db) { return { search: async (companyId: string, query: CompanySearchQuery): Promise => { @@ -287,7 +294,7 @@ export function companySearchService(db: Db) { .where(eq(companies.id, companyId)) .then((rows) => rows[0] ?? null); const prefix = routePrefix(company?.issuePrefix); - const fetchLimit = offset + limit + 1; + const fetchLimit = companySearchBranchFetchLimit(limit); const tokenArray = sqlTextArray(tokens); const containsPattern = `%${normalizedQuery}%`; const startsWithPattern = `${normalizedQuery}%`; @@ -526,6 +533,7 @@ export function companySearchService(db: Db) { )) .orderBy(desc(score), desc(issues.updatedAt), desc(issues.id)) .limit(fetchLimit) + .offset(offset) : []; const simpleCondition = simpleTextCondition([ @@ -547,6 +555,7 @@ export function companySearchService(db: Db) { .where(and(eq(agents.companyId, companyId), simpleCondition)) .orderBy(desc(agents.updatedAt), desc(agents.id)) .limit(fetchLimit) + .offset(offset) : []; const projectCondition = simpleTextCondition([ @@ -565,6 +574,7 @@ export function companySearchService(db: Db) { .where(and(eq(projects.companyId, companyId), isNull(projects.archivedAt), projectCondition)) .orderBy(desc(projects.updatedAt), desc(projects.id)) .limit(fetchLimit) + .offset(offset) : []; const results: CompanySearchResult[] = [ @@ -606,7 +616,7 @@ export function companySearchService(db: Db) { return (right.updatedAt ?? "").localeCompare(left.updatedAt ?? ""); }); - const paged = results.slice(offset, offset + limit); + const paged = results.slice(0, limit); return { query: query.q, normalizedQuery, @@ -615,7 +625,7 @@ export function companySearchService(db: Db) { offset, results: paged, countsByType: makeCounts(results), - hasMore: results.length > offset + limit, + hasMore: results.length > limit, }; }, };