From a8281c6c71be7d0db882da4967cf9941c89d0235 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 16 Jul 2026 00:47:25 +0000 Subject: [PATCH] fix(search): honor extract match limits --- doc/SPEC-implementation.md | 4 +-- packages/shared/src/index.ts | 1 + packages/shared/src/types/search.ts | 1 + packages/shared/src/validators/index.ts | 1 + packages/shared/src/validators/search.ts | 13 +++++++- .../company-search-extract-routes.test.ts | 10 +++++- .../company-search-extract-service.test.ts | 31 +++++++++++++++++-- server/src/services/company-search-extract.ts | 4 +-- 8 files changed, 56 insertions(+), 9 deletions(-) diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index 89d6ec41ecb..ff496115870 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -810,8 +810,8 @@ The current app also exposes V1-supporting surfaces for: - issue approvals, issue references/search, labels, read state, inbox/archive state, and work products - company search through `GET /companies/:companyId/search` plus agent-oriented bulk extraction through `GET /companies/:companyId/search/extract`; extraction accepts a server-escaped literal `contains`, optional - server-owned URL expansion, issue/comment/document scopes, status/date filters, issue-level pagination, and - explicit issue/match truncation flags + server-owned URL expansion, issue/comment/document scopes, status/date filters, issue-level pagination, a + bounded `matchesPerIssue` override for machine consumers, and explicit issue/match truncation flags - execution workspaces, project workspaces, workspace runtime services, and workspace operations - task watchdog configuration and reusable watchdog issue orchestration for explicitly watched issue subtrees - routines and scheduled/API/webhook triggers diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 111d82f893a..4f609042fa5 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1269,6 +1269,7 @@ export { companySearchExtractQuerySchema, companySearchQuerySchema, COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT, + COMPANY_SEARCH_EXTRACT_DEFAULT_MATCHES_PER_ISSUE, COMPANY_SEARCH_EXTRACT_MAX_LIMIT, COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE, COMPANY_SEARCH_EXTRACT_MAX_OFFSET, diff --git a/packages/shared/src/types/search.ts b/packages/shared/src/types/search.ts index 1b812bba5de..481766eca57 100644 --- a/packages/shared/src/types/search.ts +++ b/packages/shared/src/types/search.ts @@ -146,6 +146,7 @@ export interface CompanySearchExtractResponse { scope: CompanySearchExtractScope; limit: number; offset: number; + matchesPerIssue: number; results: CompanySearchExtractIssueResult[]; hasMore: boolean; truncated: boolean; diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.ts index 45dc8b20f08..b1bcaa96dab 100644 --- a/packages/shared/src/validators/index.ts +++ b/packages/shared/src/validators/index.ts @@ -408,6 +408,7 @@ export { export { COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT, + COMPANY_SEARCH_EXTRACT_DEFAULT_MATCHES_PER_ISSUE, COMPANY_SEARCH_EXTRACT_MAX_LIMIT, COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE, COMPANY_SEARCH_EXTRACT_MAX_OFFSET, diff --git a/packages/shared/src/validators/search.ts b/packages/shared/src/validators/search.ts index b54922c1e6b..839f4cb5296 100644 --- a/packages/shared/src/validators/search.ts +++ b/packages/shared/src/validators/search.ts @@ -16,7 +16,8 @@ export const COMPANY_SEARCH_MAX_OFFSET = 200; export const COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT = 100; export const COMPANY_SEARCH_EXTRACT_MAX_LIMIT = 200; export const COMPANY_SEARCH_EXTRACT_MAX_OFFSET = 5_000; -export const COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE = 20; +export const COMPANY_SEARCH_EXTRACT_DEFAULT_MATCHES_PER_ISSUE = 20; +export const COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE = 200; const UPDATED_WITHIN_RE = /^[1-9]\d{0,2}(h|d|w|m)$/; @@ -238,6 +239,16 @@ export const companySearchExtractQuerySchema = z.object({ offset: z.unknown() .optional() .transform((value, ctx) => parseIntegerQuery(value, ctx, "offset", 0, 0, COMPANY_SEARCH_EXTRACT_MAX_OFFSET)), + matchesPerIssue: z.unknown() + .optional() + .transform((value, ctx) => parseIntegerQuery( + value, + ctx, + "matchesPerIssue", + COMPANY_SEARCH_EXTRACT_DEFAULT_MATCHES_PER_ISSUE, + 1, + COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE, + )), status: z.unknown() .optional() .transform((value, ctx) => parseEnumList(value, ctx, "status", ISSUE_STATUSES)), diff --git a/server/src/__tests__/company-search-extract-routes.test.ts b/server/src/__tests__/company-search-extract-routes.test.ts index 67a7144c96c..375247858b6 100644 --- a/server/src/__tests__/company-search-extract-routes.test.ts +++ b/server/src/__tests__/company-search-extract-routes.test.ts @@ -17,6 +17,7 @@ function extractResponse(query: CompanySearchExtractQuery): CompanySearchExtract scope: query.scope, limit: query.limit, offset: query.offset, + matchesPerIssue: query.matchesPerIssue, results: [], hasMore: false, truncated: false, @@ -57,7 +58,13 @@ describe("company extract-search route", () => { const response = await request(app) .get("/api/companies/company-1/search/extract") - .query({ contains: "github.com/example/repo/pull", kind: "url", scope: "comments", limit: "200" }) + .query({ + contains: "github.com/example/repo/pull", + kind: "url", + scope: "comments", + limit: "200", + matchesPerIssue: "200", + }) .expect(200); expect(extract).toHaveBeenCalledWith("company-1", expect.objectContaining({ @@ -65,6 +72,7 @@ describe("company extract-search route", () => { kind: "url", scope: "comments", limit: 200, + matchesPerIssue: 200, })); expect(response.body).toMatchObject({ kind: "url", scope: "comments", truncated: false }); }); diff --git a/server/src/__tests__/company-search-extract-service.test.ts b/server/src/__tests__/company-search-extract-service.test.ts index e03b7c6de6e..d50c0e149af 100644 --- a/server/src/__tests__/company-search-extract-service.test.ts +++ b/server/src/__tests__/company-search-extract-service.test.ts @@ -9,6 +9,7 @@ import { issues, } from "@paperclipai/db"; import { + COMPANY_SEARCH_EXTRACT_DEFAULT_MATCHES_PER_ISSUE, COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE, companySearchExtractQuerySchema, } from "@paperclipai/shared"; @@ -36,15 +37,18 @@ describe("extract-search query validation", () => { status: "in_progress,in_review", limit: "200", offset: "5000", + matchesPerIssue: "200", updatedWithin: "30d", }); expect(parsed.kind).toBe("url"); expect(parsed.scope).toBe("comments"); expect(parsed.status).toEqual(["in_progress", "in_review"]); + expect(parsed.matchesPerIssue).toBe(COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE); expect(() => companySearchExtractQuerySchema.parse({ contains: ".*", kind: "regex" })).toThrow(); expect(() => companySearchExtractQuerySchema.parse({ contains: "x" })).toThrow(); expect(() => companySearchExtractQuerySchema.parse({ contains: "needle", limit: "201" })).toThrow(); + expect(() => companySearchExtractQuerySchema.parse({ contains: "needle", matchesPerIssue: "201" })).toThrow(); expect(() => companySearchExtractQuerySchema.parse({ contains: "needle", updatedWithin: "30d", @@ -214,10 +218,10 @@ describeEmbeddedPostgres("companySearchExtractService", () => { expect(result.results.map((row) => row.issueId)).toEqual([recentId]); }); - it("caps distinct matches per issue and marks truncation explicitly", async () => { + it("uses the default distinct-match cap and marks truncation explicitly", async () => { const companyId = await createCompany(); const urls = Array.from( - { length: COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE + 1 }, + { length: COMPANY_SEARCH_EXTRACT_DEFAULT_MATCHES_PER_ISSUE + 1 }, (_, index) => `https://github.com/paperclipai/paperclip/pull/${index + 1}`, ); await createIssue(companyId, { description: urls.join(" ") }); @@ -227,11 +231,32 @@ describeEmbeddedPostgres("companySearchExtractService", () => { kind: "url", })); - expect(result.results[0]?.matches).toHaveLength(COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE); + expect(result.matchesPerIssue).toBe(COMPANY_SEARCH_EXTRACT_DEFAULT_MATCHES_PER_ISSUE); + expect(result.results[0]?.matches).toHaveLength(COMPANY_SEARCH_EXTRACT_DEFAULT_MATCHES_PER_ISSUE); expect(result.results[0]?.matchesTruncated).toBe(true); expect(result.truncated).toBe(true); }); + it("supports a bounded per-issue match cap for complete machine extraction", async () => { + const companyId = await createCompany(); + const urls = Array.from( + { length: COMPANY_SEARCH_EXTRACT_DEFAULT_MATCHES_PER_ISSUE + 1 }, + (_, index) => `https://github.com/paperclipai/paperclip/pull/${index + 1}`, + ); + await createIssue(companyId, { description: urls.join(" ") }); + + const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({ + contains: "github.com/paperclipai/paperclip/pull", + kind: "url", + matchesPerIssue: COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE, + })); + + expect(result.matchesPerIssue).toBe(COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE); + expect(result.results[0]?.matches).toHaveLength(urls.length); + expect(result.results[0]?.matchesTruncated).toBe(false); + expect(result.truncated).toBe(false); + }); + it("does not return matching issues from another company", async () => { const companyId = await createCompany(); const otherCompanyId = await createCompany("Other"); diff --git a/server/src/services/company-search-extract.ts b/server/src/services/company-search-extract.ts index 6462b33445e..e8c59d17f15 100644 --- a/server/src/services/company-search-extract.ts +++ b/server/src/services/company-search-extract.ts @@ -3,7 +3,6 @@ import type { SQL, SQLWrapper } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { documents, issueComments, issueDocuments, issues } from "@paperclipai/db"; import { - COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE, type CompanySearchExtractIssueResult, type CompanySearchExtractMatch, type CompanySearchExtractQuery, @@ -131,7 +130,7 @@ function extractMatches(sources: ExtractSource[], query: CompanySearchExtractQue const dedupeKey = occurrence.value.toLowerCase(); if (seen.has(dedupeKey)) continue; seen.add(dedupeKey); - if (matches.length >= COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE) { + if (matches.length >= query.matchesPerIssue) { matchesTruncated = true; continue; } @@ -340,6 +339,7 @@ export function companySearchExtractService(db: Db) { scope: query.scope, limit: query.limit, offset: query.offset, + matchesPerIssue: query.matchesPerIssue, results, hasMore, truncated: hasMore || results.some((result) => result.matchesTruncated),