Fix literal wildcards in company search

This commit is contained in:
Dotta committed 2026-05-03 18:14:18 -05:00
1 parent 3a6ff11602
commit 4f1ba08baf
2 files changed
+230 -33

No files matched your search

@@ -108,6 +108,32 @@ describeEmbeddedPostgres("companySearchService", () => {
return id;
}
async function createAgent(companyId: string, values: Partial<typeof agents.$inferInsert> = {}) {
const id = values.id ?? randomUUID();
await db.insert(agents).values({
id,
companyId,
name: values.name ?? "Search agent",
role: values.role ?? "engineer",
title: values.title ?? null,
capabilities: values.capabilities ?? null,
...values,
});
return id;
}
async function createProject(companyId: string, values: Partial<typeof projects.$inferInsert> = {}) {
const id = values.id ?? randomUUID();
await db.insert(projects).values({
id,
companyId,
name: values.name ?? "Search project",
description: values.description ?? null,
...values,
});
return id;
}
it("ranks exact issue identifiers before weaker title matches", async () => {
const companyId = await createCompany();
const exactId = await createIssue(companyId, {
@@ -198,6 +224,171 @@ describeEmbeddedPostgres("companySearchService", () => {
expect(result.results.map((item) => item.id)).toEqual([visibleId]);
});
it("treats bare SQL wildcard characters as literals instead of match-all queries", async () => {
const companyId = await createCompany();
const issueId = await createIssue(companyId, {
identifier: "TST-20",
title: "Plain issue target",
description: "Plain issue description",
});
await db.insert(issueComments).values({
companyId,
issueId,
body: "Plain comment body",
});
const documentId = randomUUID();
await db.insert(documents).values({
id: documentId,
companyId,
title: "Plain document",
latestBody: "Plain document body",
format: "markdown",
});
await db.insert(issueDocuments).values({
companyId,
issueId,
documentId,
key: "plain",
});
await createAgent(companyId, {
name: "Plain Agent",
role: "engineer",
capabilities: "Plain agent capabilities",
});
await createProject(companyId, {
name: "Plain Project",
description: "Plain project description",
});
for (const q of ["%", "_", "\\"]) {
const result = await svc.search(companyId, companySearchQuerySchema.parse({ q }));
expect(result.results, `q=${q}`).toEqual([]);
}
});
it("matches percent characters literally across issue, comment, document, agent, and project results", async () => {
const companyId = await createCompany();
const issueMatchId = await createIssue(companyId, {
identifier: "TST-21",
title: "Release 100% checklist",
});
const issueDecoyId = await createIssue(companyId, {
identifier: "TST-22",
title: "Release 1000 checklist",
});
const commentMatchId = await createIssue(companyId, {
identifier: "TST-23",
title: "Comment literal holder",
});
const commentDecoyId = await createIssue(companyId, {
identifier: "TST-24",
title: "Comment decoy holder",
});
await db.insert(issueComments).values([
{
companyId,
issueId: commentMatchId,
body: "QA is 100% confident in this result.",
},
{
companyId,
issueId: commentDecoyId,
body: "QA is 1000 confident in this result.",
},
]);
const documentMatchIssueId = await createIssue(companyId, {
identifier: "TST-25",
title: "Document literal holder",
});
const documentDecoyIssueId = await createIssue(companyId, {
identifier: "TST-26",
title: "Document decoy holder",
});
const documentMatchId = randomUUID();
const documentDecoyId = randomUUID();
await db.insert(documents).values([
{
id: documentMatchId,
companyId,
title: "Literal rollout",
latestBody: "Ship 100% complete adapter support.",
format: "markdown",
},
{
id: documentDecoyId,
companyId,
title: "Decoy rollout",
latestBody: "Ship 1000 complete adapter support.",
format: "markdown",
},
]);
await db.insert(issueDocuments).values([
{
companyId,
issueId: documentMatchIssueId,
documentId: documentMatchId,
key: "literal",
},
{
companyId,
issueId: documentDecoyIssueId,
documentId: documentDecoyId,
key: "decoy",
},
]);
const agentMatchId = await createAgent(companyId, {
name: "100% Specialist",
role: "engineer",
});
const agentDecoyId = await createAgent(companyId, {
name: "1000 Specialist",
role: "engineer",
});
const projectMatchId = await createProject(companyId, {
name: "100% Launch Plan",
});
const projectDecoyId = await createProject(companyId, {
name: "1000 Launch Plan",
});
const result = await svc.search(companyId, companySearchQuerySchema.parse({ q: "100%" }));
const ids = result.results.map((row) => row.id);
expect(ids).toEqual(expect.arrayContaining([
issueMatchId,
commentMatchId,
documentMatchIssueId,
agentMatchId,
projectMatchId,
]));
expect(ids).not.toEqual(expect.arrayContaining([
issueDecoyId,
commentDecoyId,
documentDecoyIssueId,
agentDecoyId,
projectDecoyId,
]));
});
it("escapes underscore and backslash characters in issue phrase and token patterns", async () => {
const companyId = await createCompany();
const literalId = await createIssue(companyId, {
identifier: "TST-27",
title: "Literal foo_bar path c:\\tmp",
});
const decoyId = await createIssue(companyId, {
identifier: "TST-28",
title: "Decoy fooXbar path c:tmp",
});
for (const q of ["foo_bar", "c:\\tmp"]) {
const result = await svc.search(companyId, companySearchQuerySchema.parse({ q, scope: "issues" }));
const ids = result.results.map((row) => row.id);
expect(ids, `q=${q}`).toContain(literalId);
expect(ids, `q=${q}`).not.toContain(decoyId);
}
});
it("uses pg_trgm for conservative fuzzy title matches", async () => {
const companyId = await createCompany();
const issueId = await createIssue(companyId, {
+39 -33
View File
@@ -60,11 +60,15 @@ function normalizeQuery(query: string) {
return query.trim().replace(/\s+/g, " ").toLowerCase();
}
function escapeLikePattern(value: string): string {
return value.replace(/[\\%_]/g, "\\$&");
}
function tokenizeQuery(normalizedQuery: string) {
const matches = normalizedQuery.match(/"[^"]+"|[^\s]+/g) ?? [];
const tokens: string[] = [];
for (const match of matches) {
const token = match.replace(/^"|"$/g, "").replace(/^[^\p{L}\p{N}-]+|[^\p{L}\p{N}-]+$/gu, "");
const token = match.replace(/^"|"$/g, "").replace(/^[^\p{L}\p{N}%_\\-]+|[^\p{L}\p{N}%_\\-]+$/gu, "");
if (token.length < MIN_TOKEN_LENGTH) continue;
if (!tokens.includes(token)) tokens.push(token);
if (tokens.length >= COMPANY_SEARCH_MAX_TOKENS) break;
@@ -86,7 +90,7 @@ function tokenMatchExpression(textExpression: SQL, tokenArray: SQL) {
EXISTS (
SELECT 1
FROM unnest(${tokenArray}) AS search_token(value)
WHERE lower(coalesce(${textExpression}, '')) LIKE '%' || search_token.value || '%'
WHERE lower(coalesce(${textExpression}, '')) LIKE '%' || search_token.value || '%' ESCAPE '\\'
)
`;
}
@@ -269,7 +273,7 @@ function scoreSimpleRow(row: SimpleSearchRow, normalizedQuery: string, tokens: s
}
function simpleTextCondition(fields: SQL[], containsPattern: string, tokenArray: SQL) {
const phraseConditions = fields.map((field) => sql<boolean>`lower(coalesce(${field}, '')) LIKE ${containsPattern}`);
const phraseConditions = fields.map((field) => sql<boolean>`lower(coalesce(${field}, '')) LIKE ${containsPattern} ESCAPE '\\'`);
const tokenConditions = fields.map((field) => tokenMatchExpression(field, tokenArray));
return sql<boolean>`(${sql.join([...phraseConditions, ...tokenConditions], sql` OR `)})`;
}
@@ -308,19 +312,21 @@ export function companySearchService(db: Db) {
.then((rows) => rows[0] ?? null);
const prefix = routePrefix(company?.issuePrefix);
const fetchLimit = companySearchBranchFetchLimit(limit);
const tokenArray = sqlTextArray(tokens);
const escapedTokens = tokens.map(escapeLikePattern);
const tokenArray = sqlTextArray(escapedTokens);
const fuzzyTokens = fuzzyEligibleTokens(tokens);
const fuzzyTokenArray = sqlTextArray(fuzzyTokens);
const containsPattern = `%${normalizedQuery}%`;
const startsWithPattern = `${normalizedQuery}%`;
const fuzzyEnabled = normalizedQuery.length >= MIN_FUZZY_QUERY_LENGTH;
const escapedQuery = escapeLikePattern(normalizedQuery);
const containsPattern = `%${escapedQuery}%`;
const startsWithPattern = `${escapedQuery}%`;
const fuzzyEnabled = normalizedQuery.length >= MIN_FUZZY_QUERY_LENGTH && !/[\\%_]/.test(normalizedQuery);
const fuzzyTokensEnabled = fuzzyEnabled && fuzzyTokens.length > 0;
const titlePhraseMatch = sql<boolean>`lower(${issues.title}) LIKE ${containsPattern}`;
const titleStartsWith = sql<boolean>`lower(${issues.title}) LIKE ${startsWithPattern}`;
const identifierPhraseMatch = sql<boolean>`lower(coalesce(${issues.identifier}, '')) LIKE ${containsPattern}`;
const identifierStartsWith = sql<boolean>`lower(coalesce(${issues.identifier}, '')) LIKE ${startsWithPattern}`;
const descriptionPhraseMatch = sql<boolean>`lower(coalesce(${issues.description}, '')) LIKE ${containsPattern}`;
const titlePhraseMatch = sql<boolean>`lower(${issues.title}) LIKE ${containsPattern} ESCAPE '\\'`;
const titleStartsWith = sql<boolean>`lower(${issues.title}) LIKE ${startsWithPattern} ESCAPE '\\'`;
const identifierPhraseMatch = sql<boolean>`lower(coalesce(${issues.identifier}, '')) LIKE ${containsPattern} ESCAPE '\\'`;
const identifierStartsWith = sql<boolean>`lower(coalesce(${issues.identifier}, '')) LIKE ${startsWithPattern} ESCAPE '\\'`;
const descriptionPhraseMatch = sql<boolean>`lower(coalesce(${issues.description}, '')) LIKE ${containsPattern} ESCAPE '\\'`;
const titleTokenMatch = tokenMatchExpression(sql`${issues.title}`, tokenArray);
const identifierTokenMatch = tokenMatchExpression(sql`${issues.identifier}`, tokenArray);
const descriptionTokenMatch = tokenMatchExpression(sql`${issues.description}`, tokenArray);
@@ -339,7 +345,7 @@ export function companySearchService(db: Db) {
WHERE search_comments.company_id = ${companyId}
AND search_comments.issue_id = issues.id
AND (
lower(search_comments.body) LIKE ${containsPattern}
lower(search_comments.body) LIKE ${containsPattern} ESCAPE '\\'
OR ${tokenMatchExpression(sql`search_comments.body`, tokenArray)}
)
)
@@ -354,8 +360,8 @@ export function companySearchService(db: Db) {
AND search_documents.company_id = ${companyId}
AND search_issue_documents.issue_id = issues.id
AND (
lower(coalesce(search_documents.title, '')) LIKE ${containsPattern}
OR lower(search_documents.latest_body) LIKE ${containsPattern}
lower(coalesce(search_documents.title, '')) LIKE ${containsPattern} ESCAPE '\\'
OR lower(search_documents.latest_body) LIKE ${containsPattern} ESCAPE '\\'
OR ${tokenMatchExpression(sql`search_documents.title`, tokenArray)}
OR ${tokenMatchExpression(sql`search_documents.latest_body`, tokenArray)}
)
@@ -399,15 +405,15 @@ export function companySearchService(db: Db) {
(
SELECT count(*)::int
FROM unnest(${tokenArray}) AS search_token(value)
WHERE lower(${issues.title}) LIKE '%' || search_token.value || '%'
OR lower(coalesce(${issues.identifier}, '')) LIKE '%' || search_token.value || '%'
OR lower(coalesce(${issues.description}, '')) LIKE '%' || search_token.value || '%'
WHERE lower(${issues.title}) LIKE '%' || search_token.value || '%' ESCAPE '\\'
OR lower(coalesce(${issues.identifier}, '')) LIKE '%' || search_token.value || '%' ESCAPE '\\'
OR lower(coalesce(${issues.description}, '')) LIKE '%' || search_token.value || '%' ESCAPE '\\'
OR EXISTS (
SELECT 1
FROM issue_comments coverage_comments
WHERE coverage_comments.company_id = ${companyId}
AND coverage_comments.issue_id = issues.id
AND lower(coverage_comments.body) LIKE '%' || search_token.value || '%'
AND lower(coverage_comments.body) LIKE '%' || search_token.value || '%' ESCAPE '\\'
)
OR EXISTS (
SELECT 1
@@ -418,8 +424,8 @@ export function companySearchService(db: Db) {
AND coverage_documents.company_id = ${companyId}
AND coverage_issue_documents.issue_id = issues.id
AND (
lower(coalesce(coverage_documents.title, '')) LIKE '%' || search_token.value || '%'
OR lower(coverage_documents.latest_body) LIKE '%' || search_token.value || '%'
lower(coalesce(coverage_documents.title, '')) LIKE '%' || search_token.value || '%' ESCAPE '\\'
OR lower(coverage_documents.latest_body) LIKE '%' || search_token.value || '%' ESCAPE '\\'
)
)
)
@@ -477,11 +483,11 @@ export function companySearchService(db: Db) {
WHERE search_comments.company_id = ${companyId}
AND search_comments.issue_id = issues.id
AND (
lower(search_comments.body) LIKE ${containsPattern}
lower(search_comments.body) LIKE ${containsPattern} ESCAPE '\\'
OR ${tokenMatchExpression(sql`search_comments.body`, tokenArray)}
)
ORDER BY
CASE WHEN lower(search_comments.body) LIKE ${containsPattern} THEN 0 ELSE 1 END,
CASE WHEN lower(search_comments.body) LIKE ${containsPattern} ESCAPE '\\' THEN 0 ELSE 1 END,
search_comments.updated_at DESC,
search_comments.id DESC
LIMIT 1
@@ -494,11 +500,11 @@ export function companySearchService(db: Db) {
WHERE search_comments.company_id = ${companyId}
AND search_comments.issue_id = issues.id
AND (
lower(search_comments.body) LIKE ${containsPattern}
lower(search_comments.body) LIKE ${containsPattern} ESCAPE '\\'
OR ${tokenMatchExpression(sql`search_comments.body`, tokenArray)}
)
ORDER BY
CASE WHEN lower(search_comments.body) LIKE ${containsPattern} THEN 0 ELSE 1 END,
CASE WHEN lower(search_comments.body) LIKE ${containsPattern} ESCAPE '\\' THEN 0 ELSE 1 END,
search_comments.updated_at DESC,
search_comments.id DESC
LIMIT 1
@@ -514,15 +520,15 @@ export function companySearchService(db: Db) {
AND search_documents.company_id = ${companyId}
AND search_issue_documents.issue_id = issues.id
AND (
lower(coalesce(search_documents.title, '')) LIKE ${containsPattern}
OR lower(search_documents.latest_body) LIKE ${containsPattern}
lower(coalesce(search_documents.title, '')) LIKE ${containsPattern} ESCAPE '\\'
OR lower(search_documents.latest_body) LIKE ${containsPattern} ESCAPE '\\'
OR ${tokenMatchExpression(sql`search_documents.title`, tokenArray)}
OR ${tokenMatchExpression(sql`search_documents.latest_body`, tokenArray)}
)
ORDER BY
CASE
WHEN lower(coalesce(search_documents.title, '')) LIKE ${containsPattern} THEN 0
WHEN lower(search_documents.latest_body) LIKE ${containsPattern} THEN 1
WHEN lower(coalesce(search_documents.title, '')) LIKE ${containsPattern} ESCAPE '\\' THEN 0
WHEN lower(search_documents.latest_body) LIKE ${containsPattern} ESCAPE '\\' THEN 1
ELSE 2
END,
search_documents.updated_at DESC,
@@ -540,8 +546,8 @@ export function companySearchService(db: Db) {
AND search_documents.company_id = ${companyId}
AND search_issue_documents.issue_id = issues.id
AND (
lower(coalesce(search_documents.title, '')) LIKE ${containsPattern}
OR lower(search_documents.latest_body) LIKE ${containsPattern}
lower(coalesce(search_documents.title, '')) LIKE ${containsPattern} ESCAPE '\\'
OR lower(search_documents.latest_body) LIKE ${containsPattern} ESCAPE '\\'
OR ${tokenMatchExpression(sql`search_documents.title`, tokenArray)}
OR ${tokenMatchExpression(sql`search_documents.latest_body`, tokenArray)}
)
@@ -559,8 +565,8 @@ export function companySearchService(db: Db) {
AND search_documents.company_id = ${companyId}
AND search_issue_documents.issue_id = issues.id
AND (
lower(coalesce(search_documents.title, '')) LIKE ${containsPattern}
OR lower(search_documents.latest_body) LIKE ${containsPattern}
lower(coalesce(search_documents.title, '')) LIKE ${containsPattern} ESCAPE '\\'
OR lower(search_documents.latest_body) LIKE ${containsPattern} ESCAPE '\\'
OR ${tokenMatchExpression(sql`search_documents.title`, tokenArray)}
OR ${tokenMatchExpression(sql`search_documents.latest_body`, tokenArray)}
)