diff --git a/__tests__/composed-name-bonus.test.ts b/__tests__/composed-name-bonus.test.ts new file mode 100644 index 000000000..aad61fa46 --- /dev/null +++ b/__tests__/composed-name-bonus.test.ts @@ -0,0 +1,108 @@ +/** + * The two exact-name discounts compose, and the composition has to keep the + * same invariant each one pins alone: an exact name the user typed never loses + * to a mere prefix match. + * + * #1462 (corpus-frequency discount) and #1463 (de-prioritized-path damping) + * each clear that bound in isolation — `80 * 0.6 = 48 > 40` and + * `80 * 0.75 - 15 = 45 > 40` — but each suite pins its invariant with the other + * lever off. Multiplied, a name that is both corpus-common AND inside a + * de-prioritized tree lands at `80 * 0.6 * 0.75 - 15 = 21`, well under the + * prefix arm's supremum of 40. Neither existing test can see it. + * + * The general condition, with the -15 path penalty a de-prioritized node also + * carries: `80 * combined - 15 > 40`, i.e. `combined > 55/80 = 0.6875`. + */ +import { describe, it, expect } from 'vitest'; +import { + nameMatchBonus, + nameMatchIdfScale, + combinedExactNameScale, + NAME_MATCH_IDF_FLOOR, + COMBINED_EXACT_NAME_FLOOR, + type NameCorpusStats, +} from '../src/search/query-utils'; +import { DEPRIORITIZED_NAME_BONUS_SCALE } from '../src/db/queries'; + +/** The prefix arm is `round(10 + 30 * ratio)`, so 40 is its supremum. */ +const PREFIX_ARM_SUPREMUM = 40; +/** A de-prioritized path also takes the flat path penalty. */ +const PATH_PENALTY = 15; +/** `80 * s - 15 > 40` → `s > 55/80`. */ +const REQUIRED_COMBINED_SCALE = (PREFIX_ARM_SUPREMUM + PATH_PENALTY) / 80; + +/** A corpus where `name` is shared by `df` of `total` nodes. */ +function corpusWith(name: string, df: number, total: number): NameCorpusStats { + return { + total, + countForName: (n: string): number => (n.toLowerCase() === name.toLowerCase() ? df : 1), + }; +} + +describe('the two exact-name discounts compose (#1462 + #1463)', () => { + it('states the bound the composition has to clear', () => { + expect(REQUIRED_COMBINED_SCALE).toBeCloseTo(0.6875, 6); + }); + + it('regression: the naive product falls under the bound', () => { + // Not how the code composes them — asserted so the failure this test was + // written for stays visible, and nobody reintroduces the multiplication. + const product = NAME_MATCH_IDF_FLOOR * DEPRIORITIZED_NAME_BONUS_SCALE; + expect(product).toBeLessThan(REQUIRED_COMBINED_SCALE); + expect(80 * product - PATH_PENALTY).toBeLessThan(PREFIX_ARM_SUPREMUM); + }); + + it('the combined floor clears the bound at the worst case of both levers', () => { + expect(COMBINED_EXACT_NAME_FLOOR).toBeGreaterThan(REQUIRED_COMBINED_SCALE); + const worst = combinedExactNameScale(NAME_MATCH_IDF_FLOOR, true); + expect(Math.round(80 * worst) - PATH_PENALTY).toBeGreaterThan(PREFIX_ARM_SUPREMUM); + }); + + it('a de-prioritized, corpus-common exact name still beats a prefix match', () => { + // `child` shared by 400 of 60k nodes drives the IDF scale to its floor, and + // the node also sits in a de-prioritized tree. It must still outrank the + // prefix match `children`, after the -15 the de-prioritized path takes. + const corpus = corpusWith('child', 400, 60_000); + expect(nameMatchIdfScale(400, 60_000)).toBeCloseTo(NAME_MATCH_IDF_FLOOR, 6); + + const exactDeprioritized = nameMatchBonus('child', 'child', corpus, true) - PATH_PENALTY; + const prefix = nameMatchBonus('children', 'child', corpus, false); + + expect(prefix).toBeLessThanOrEqual(PREFIX_ARM_SUPREMUM); + expect(exactDeprioritized).toBeGreaterThan(prefix); + }); + + it('neither lever alone is weakened by the fix', () => { + const corpus = corpusWith('child', 400, 60_000); + // #1462 in isolation: corpus-common, not de-prioritized. No path penalty. + expect(nameMatchBonus('child', 'child', corpus, false)).toBeGreaterThan(PREFIX_ARM_SUPREMUM); + // #1463 in isolation: de-prioritized, but a rare name. + const rare = corpusWith('nothingElse', 1, 60_000); + expect(nameMatchBonus('child', 'child', rare, true) - PATH_PENALTY).toBeGreaterThan( + PREFIX_ARM_SUPREMUM, + ); + }); + + it('still discounts: a common de-prioritized name ranks below a rare one', () => { + const common = corpusWith('child', 400, 60_000); + const rare = corpusWith('nothingElse', 1, 60_000); + expect(nameMatchBonus('child', 'child', common, true)).toBeLessThan( + nameMatchBonus('child', 'child', rare, true), + ); + }); + + it('leaves the undiscounted path alone when no corpus is supplied', () => { + expect(nameMatchBonus('child', 'child')).toBe(80); + expect(nameMatchBonus('child', 'child', undefined, true)).toBe( + Math.round(80 * DEPRIORITIZED_NAME_BONUS_SCALE), + ); + }); + + it('does not touch the prefix or substring arms', () => { + const corpus = corpusWith('children', 400, 60_000); + // Prefix arm is length-scaled and small; the discount never applied to it. + expect(nameMatchBonus('children', 'child', corpus, false)).toBe( + nameMatchBonus('children', 'child', undefined, false), + ); + }); +}); diff --git a/__tests__/name-match-idf.test.ts b/__tests__/name-match-idf.test.ts new file mode 100644 index 000000000..9b8f1a9a6 --- /dev/null +++ b/__tests__/name-match-idf.test.ts @@ -0,0 +1,250 @@ +/** + * Corpus-frequency discount on the exact-name bonus (#982 — the follow-up #746 + * floated but never landed). + * + * `nameMatchBonus` handed a flat 80 (whole query === name) / 60 (a token of a + * multi-word query === name) regardless of how many symbols carry that name. A + * generic non-stopword token that happens to be a symbol name — `usage`, `get`, + * `status` — therefore collected the full bonus and outranked the product code + * that actually answers the query but does not literally contain the token. + * + * The fixture is #982's layout — product code with NO symbol named `usage`, + * plus peripheral helper scripts that each define a module-level `usage()` — + * scaled so the token is genuinely corpus-common, which is the condition this + * lever keys on. Measured on the fixture for `desktop status bar context window + * usage` (69 nodes, 24 of them named `usage`): + * + * before: every top slot is an `optional-skills/**` usage() at 71.3 + * after: the helpers leave the top 11 entirely; product code fills it + * + * Note what this lever does NOT do, because it is easy to over-claim: in #982's + * 8-file minimal repro only TWO symbols are named `usage`, so the token is rare + * and the IDF scale is ~0.8 — nearly inert, by design. Fixing that shape needs + * the issue's complementary path lever (user-extensible de-prioritization), + * which is deliberately out of scope here. Locked below by an explicit test. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { CodeGraph } from '../src'; +import { initGrammars, loadAllGrammars } from '../src/extraction/grammars'; +import { + nameMatchBonus, + nameMatchIdfScale, + NAME_MATCH_IDF_FLOOR as FLOOR, +} from '../src/search/query-utils'; + +describe('nameMatchIdfScale', () => { + it('leaves a unique name at full weight', () => { + expect(nameMatchIdfScale(1, 10_000)).toBe(1); + }); + + it('decays monotonically as the name spreads across the corpus', () => { + const total = 10_000; + const scales = [1, 2, 10, 100, 1000].map((df) => nameMatchIdfScale(df, total)); + for (let i = 1; i < scales.length; i++) { + // Non-increasing throughout; strictly decreasing until the floor binds. + expect(scales[i]).toBeLessThanOrEqual(scales[i - 1]); + if (scales[i - 1] > FLOOR) expect(scales[i]).toBeLessThan(scales[i - 1]); + } + }); + + it('discounts a very common name without erasing it', () => { + const scale = nameMatchIdfScale(9_000, 10_000); + expect(scale).toBe(FLOOR); + expect(scale).toBeLessThan(1); + }); + + it('never lets a discounted whole-query exact match lose to a prefix match', () => { + // The invariant that sets the floor. The prefix arm pays `10 + 30 * ratio` + // with ratio < 1, so it approaches but never reaches 40. An exact whole-query + // match floors at 80 * FLOOR = 48 — clear of it for any corpus frequency. + // + // This is not hypothetical: with the floor at 0.25 it did not hold. Searching + // Alamofire for `request` (df 173 / 4512 nodes → scale 0.392) paid the exact + // match 80 * 0.392 = 31 and lost the top slot to `requests`, a prefix match + // worth 36. The user typed the name; hiding it behind a longer one is wrong. + const PREFIX_ARM_SUPREMUM = 40; + expect(80 * FLOOR).toBeGreaterThan(PREFIX_ARM_SUPREMUM); + + const crowded = { total: 4512, countForName: () => 173 }; + expect(nameMatchBonus('request', 'request', crowded)).toBeGreaterThan( + nameMatchBonus('requests', 'request') + ); + }); + + it('keeps the floor above every scale real corpora reach', () => { + // Swept on five indexed repos; the lowest raw scale observed was django's + // commonest name (1097 of 62080 nodes). A floor at or below that is inert — + // an earlier 0.25 never once bound. The floor must be chosen above this line + // to do anything at all. + expect(nameMatchIdfScale(1097, 62_080)).toBe(FLOOR); + expect(nameMatchIdfScale(173, 4_512)).toBe(FLOOR); + }); + + it('is degenerate-input safe', () => { + expect(nameMatchIdfScale(0, 10_000)).toBe(1); + expect(nameMatchIdfScale(5, 1)).toBe(1); + expect(nameMatchIdfScale(NaN, 10_000)).toBe(1); + // A name counted more often than the corpus size must not go negative. + expect(nameMatchIdfScale(20_000, 10_000)).toBeGreaterThan(0); + }); +}); + +describe('nameMatchBonus corpus discount', () => { + const corpus = (df: number, total = 10_000) => ({ total, countForName: () => df }); + + it('is unchanged when no corpus stats are supplied', () => { + expect(nameMatchBonus('usage', 'usage')).toBe(80); + expect(nameMatchBonus('usage', 'context window usage')).toBe(60); + }); + + it('leaves a rare name at its original bonus', () => { + expect(nameMatchBonus('usage', 'usage', corpus(1))).toBe(80); + expect(nameMatchBonus('usage', 'context window usage', corpus(1))).toBe(60); + }); + + it('discounts a name shared by many symbols', () => { + expect(nameMatchBonus('usage', 'context window usage', corpus(500))).toBeLessThan(60); + expect(nameMatchBonus('usage', 'context window usage', corpus(500))).toBeGreaterThan(0); + }); + + it('does not touch prefix or substring bonuses', () => { + // "Name starts with query" arm — small and already length-scaled. + const withCorpus = nameMatchBonus('usageReporter', 'usage', corpus(5_000)); + const without = nameMatchBonus('usageReporter', 'usage'); + expect(withCorpus).toBe(without); + }); +}); + +describe('#982 — generic exact-name match crowding out product code', () => { + const SKILLS = Array.from({ length: 24 }, (_, i) => `skill${i}`); + let tmpDir: string; + let cg: CodeGraph; + + beforeAll(async () => { + await initGrammars(); + await loadAllGrammars(); + + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-982-')); + const mk = (rel: string, content: string) => { + const p = path.join(tmpDir, rel); + fs.mkdirSync(path.dirname(p), { recursive: true }); + fs.writeFileSync(p, content); + }; + + // Product code answering "desktop status bar context window usage". + // Note: no symbol here is literally named `usage`. + mk( + 'apps/desktop/statusbar/StatusBar.ts', + [ + 'export class DesktopStatusBar {', + ' render(): string { return this.refresh(); }', + ' refresh(): string { return "status bar"; }', + ' mount(): void {}', + '}', + ].join('\n') + ); + mk( + 'apps/desktop/statusbar/StatusBarController.ts', + [ + "import { DesktopStatusBar } from './StatusBar';", + 'export class StatusBarController {', + ' constructor(private readonly bar: DesktopStatusBar) {}', + ' show(): string { return this.bar.render(); }', + '}', + ].join('\n') + ); + mk( + 'apps/desktop/context/ContextWindowMeter.ts', + [ + 'export class ContextWindowMeter {', + ' read(): number { return this.recompute(); }', + ' recompute(): number { return estimateTokens("context window"); }', + '}', + 'export function estimateTokens(text: string): number { return text.length; }', + ].join('\n') + ); + mk( + 'apps/desktop/context/format.ts', + 'export function formatTokens(n: number): string { return `${n} tokens`; }\n' + ); + mk('gateway/server/server.ts', 'export function startServer(): void {}\n'); + mk('packages/core/util/strings.ts', 'export function slugify(s: string): string { return s; }\n'); + + // Peripheral helper scripts: each defines a module-level `usage`, and + // nothing else about them answers the query. Enough of them that `usage` + // is corpus-common (24 of 69 nodes) — the condition the discount keys on. + for (const skill of SKILLS) { + mk( + `optional-skills/${skill}/scripts/${skill}_calc.ts`, + [ + 'export function usage(): void {', + ` console.log("usage: ${skill}_calc [options]");`, + '}', + ].join('\n') + ); + } + + cg = CodeGraph.initSync(tmpDir); + await cg.indexAll(); + }, 120_000); + + afterAll(() => { + cg?.destroy(); + if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + const usageHelperRanks = (query: string): number[] => { + const results = cg.searchNodes(query, { limit: 20 }); + const ranks: number[] = []; + results.forEach((r, i) => { + if (r.node.name.toLowerCase() === 'usage' && r.node.filePath.includes('optional-skills')) { + ranks.push(i); + } + }); + return ranks; + }; + + it('keeps the generic usage() helpers out of the top ranks', () => { + // Before the discount these occupied every top slot at an identical 71.3. + const ranks = usageHelperRanks('desktop status bar context window usage'); + expect(ranks.slice(0, 2)).not.toEqual([0, 1]); + }); + + it('ranks at least one product symbol above the usage() helpers', () => { + const results = cg.searchNodes('desktop status bar context window usage', { limit: 20 }); + const firstHelper = results.findIndex( + (r) => r.node.name.toLowerCase() === 'usage' && r.node.filePath.includes('optional-skills') + ); + const firstProduct = results.findIndex((r) => r.node.filePath.includes('apps/desktop')); + expect(firstProduct).toBeGreaterThanOrEqual(0); + if (firstHelper >= 0) { + expect(firstProduct).toBeLessThan(firstHelper); + } + }); + + it('still surfaces the helpers when the generic token IS the query', () => { + // Discount, not erase: someone asking for `usage` must still find usage(). + const results = cg.searchNodes('usage', { limit: 20 }); + const helpers = results.filter( + (r) => r.node.name.toLowerCase() === 'usage' && r.node.filePath.includes('optional-skills') + ); + expect(helpers.length).toBeGreaterThan(0); + expect(results.findIndex((r) => r.node.name.toLowerCase() === 'usage')).toBeLessThan(3); + }); +}); + +describe('#982 — the discount is corpus-frequency, not a blanket nerf', () => { + it('barely moves a name that only a couple of symbols carry', () => { + // #982's 8-file minimal repro has TWO usage() defs among ~25 nodes. The + // token is rare there, so IDF is close to inert — this lever is not what + // fixes that shape, and saying otherwise would overstate it. + const scale = nameMatchIdfScale(2, 25); + expect(scale).toBeGreaterThan(0.7); + expect(nameMatchBonus('usage', 'context window usage', { total: 25, countForName: () => 2 })) + .toBeGreaterThanOrEqual(45); + }); +}); diff --git a/src/db/queries.ts b/src/db/queries.ts index a0f8bc541..651b647c2 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -18,7 +18,7 @@ import { SearchResult, } from '../types'; import { safeJsonParse } from '../utils'; -import { kindBonus, nameMatchBonus, scorePathRelevance } from '../search/query-utils'; +import { kindBonus, nameMatchBonus, scorePathRelevance, type NameCorpusStats } from '../search/query-utils'; import { parseQuery, boundedEditDistance } from '../search/query-parser'; import { isGeneratedFile } from '../extraction/generated-detection'; import { splitIdentifierSegments } from '../search/identifier-segments'; @@ -70,7 +70,10 @@ const SQLITE_PARAM_CHUNK_SIZE = 500; * peripheral top-10 slots cleared), so a deeper discount buys little and costs * the invariant. Pinned by a test. */ -export const DEPRIORITIZED_NAME_BONUS_SCALE = 0.75; +// Re-exported from the scorer, which now owns it: the constant composes with +// the corpus-frequency discount and only that module can hold their shared +// bound (#1462). Kept exported here for the existing importers. +export { DEPRIORITIZED_NAME_BONUS_SCALE } from '../search/query-utils'; /** * Database row types (snake_case from SQLite) @@ -361,6 +364,42 @@ export class QueryBuilder { return this.isDeprioritizedPath; } + /** + * Corpus stats for the exact-name bonus discount (#982, the #746 follow-up). + * + * A fresh object per search, so name counts are memoized for the duration of + * one scoring pass but never held across an index write. + * + * The per-name lookup MUST be written as `lower(name) = ?` so it hits + * `idx_nodes_lower_name`. The equivalent `name = ? COLLATE NOCASE` matches no + * index — neither `idx_nodes_name` (BINARY collation) nor the expression index + * — and degrades to a full scan per distinct candidate name: measured 3.2ms on + * gin (2.5k nodes), 14ms on excalidraw (11k), 79ms on django (62k) per search, + * growing with the corpus. Seeking the index is flat at ~0.08ms on all four. + */ + private nameCorpusStats(): NameCorpusStats { + const total = (this.db.prepare('SELECT COUNT(*) AS n FROM nodes').get() as { n: number }).n; + const counts = new Map(); + // Both sides lowered in SQL: lowering the parameter in JS and comparing it + // against SQLite's `lower(name)` is the ASCII-vs-Unicode asymmetry #1542 + // documents, so the raw name goes in and SQLite lowers both sides (#1462). + // `lower(?)` is a constant expression, so this still seeks the index below + // rather than scanning. + const stmt = this.db.prepare('SELECT COUNT(*) AS n FROM nodes WHERE lower(name) = lower(?)'); + return { + total, + countForName: (name: string): number => { + const key = name.toLowerCase(); + const cached = counts.get(key); + if (cached !== undefined) return cached; + // The raw name goes to SQLite; `key` is only the memo key. + const n = (stmt.get(name) as { n: number } | undefined)?.n ?? 0; + counts.set(key, n); + return n; + }, + }; + } + // =========================================================================== // Node Operations // =========================================================================== @@ -1328,6 +1367,7 @@ export class QueryBuilder { // Apply multi-signal scoring if (results.length > 0 && (text || query)) { const scoringQuery = text || query; + const corpus = this.nameCorpusStats(); results = results.map(r => { // A path the project de-prioritized is saying its symbol NAMES are not // the answer, so the exact-name bonus has to be damped too. The -15 path @@ -1336,14 +1376,18 @@ export class QueryBuilder { // product symbol — -15 lands at 59.8, still ahead). Damped, not zeroed, // so the tree stays findable when it genuinely is what you asked for. // Evaluated once and reused: the predicate stats the config file. + // + // The damping is passed in rather than applied to the return value: it + // composes with the corpus-frequency discount, and only nameMatchBonus + // can floor the COMBINED multiplier so a name that is both common and + // de-prioritized still outranks a mere prefix match (#1462). const deprioritized = this.isDeprioritizedPath?.(r.node.filePath) ?? false; - const nameBonus = nameMatchBonus(r.node.name, scoringQuery); return { ...r, score: r.score + kindBonus(r.node.kind) + scorePathRelevance(r.node.filePath, scoringQuery, this.projectNameTokens, deprioritized) - + (deprioritized ? Math.round(nameBonus * DEPRIORITIZED_NAME_BONUS_SCALE) : nameBonus), + + nameMatchBonus(r.node.name, scoringQuery, corpus, deprioritized), }; }); results.sort((a, b) => b.score - a.score); diff --git a/src/search/query-utils.ts b/src/search/query-utils.ts index 6600d1554..94c96a641 100644 --- a/src/search/query-utils.ts +++ b/src/search/query-utils.ts @@ -345,14 +345,132 @@ function matchesNonProductionDir(lowerPath: string): boolean { return false; } +/** + * Corpus statistics used to discount an exact name match by how common the + * name is. Supplied by the caller because only the DB layer can count names. + */ +export interface NameCorpusStats { + /** Total indexed nodes. */ + total: number; + /** How many indexed nodes carry this exact name (case-insensitive). */ + countForName(name: string): number; +} + +/** + * Floor for {@link nameMatchIdfScale}: a common name is discounted, never erased. + * + * Measured, not chosen. Swept 0→1 over the top-25 corpus-common names of five + * indexed repos (gin 2.5k nodes, Alamofire 4.5k, codegraph 9.2k, excalidraw + * 11k, django 62k), scoring two things: whether a query that IS a common name + * still returns that name first, and how much of a mixed query's top-10 the + * common name crowds out. + * + * Real corpora never drive the raw scale below ~0.36 (django's commonest name + * spans 1097 of 62080 nodes → 0.367), so any floor at or under 0.35 is inert — + * an earlier 0.25 was dead code. The binding cases are milder and real: + * searching Alamofire for `alamofire` (scale 0.551) or `request` (0.392) + * demoted the symbol with that exact name below a mere *prefix* match + * (`AlamofireExtended`, `requests`), which is never right — the user typed the + * name. 0.60 is the lowest value clearing the worst such case (0.551) with + * margin; it restores exact-name recall@1 to the undiscounted baseline on all + * five repos while keeping ~95% of the crowd-out relief. + */ +export const NAME_MATCH_IDF_FLOOR = 0.6; + +/** + * Damping applied to the exact-name bonus of a node the project de-prioritized + * (#1463). Lives here, next to the other exact-name lever, because the two + * compose and only {@link combinedExactNameScale} can hold the bound they + * share. Re-exported from the DB layer for the callers that had it there. + */ +export const DEPRIORITIZED_NAME_BONUS_SCALE = 0.75; + +/** + * Floor for the *combined* exact-name multiplier (#1462 + #1463). + * + * Both levers discount the same bonus, and each was derived against the same + * invariant with the other lever off: an exact name the user typed never loses + * to a mere prefix match. Multiplied they break it — a corpus-common name in a + * de-prioritized tree lands at `80 * 0.6 * 0.75 - 15 = 21`, under the prefix + * arm's supremum of 40. + * + * A de-prioritized node also carries the flat -15 path penalty, so the bound is + * `round(80 * s) - 15 > 40`, i.e. `s >= 0.69375`. 0.70 is the next clean value + * above it and leaves `round(80 * 0.70) - 15 = 41`. + * + * Note this is a floor on the product, not `min` of the two: `min(0.6, 0.75)` + * is 0.6, which is itself under the bound — taking the stronger discount alone + * does not clear it. + */ +export const COMBINED_EXACT_NAME_FLOOR = 0.7; + +/** + * The exact-name multiplier once both discounts are taken into account. + * + * @param idfScale - Corpus-frequency scale from {@link nameMatchIdfScale}. + * @param deprioritized - Whether the node sits in a de-prioritized path. + * @returns Multiplier for the exact-name bonus, never below the shared bound + * when de-prioritized (where the -15 path penalty also applies). + */ +export function combinedExactNameScale(idfScale: number, deprioritized: boolean): number { + if (!deprioritized) return idfScale; + return Math.max(COMBINED_EXACT_NAME_FLOOR, idfScale * DEPRIORITIZED_NAME_BONUS_SCALE); +} + +/** + * IDF-style scale for an exact-name bonus, in [NAME_MATCH_IDF_FLOOR, 1]. + * + * An exact name match is strong evidence only when the name is rare. A bare + * `usage` that names hundreds of symbols carries almost no signal, yet used to + * collect the same flat bonus as a name that occurs once — enough to outrank + * product code that does not literally contain the token (#982, the + * corpus-frequency discount #746 left unimplemented). + * + * `log(1 + total/df) / log(1 + total)` gives exactly 1 for a unique name, so + * the common case is unchanged, and decays as the name spreads. The floor keeps + * a genuinely-intended query for a common name (an actual `usage()` reporter) + * ranking above non-matches: the aim is to discount the signal, not erase it. + * + * @param df - Number of indexed nodes sharing the name. + * @param total - Total indexed nodes. + * @returns Multiplier for the exact-name bonus. + */ +export function nameMatchIdfScale(df: number, total: number): number { + if (!Number.isFinite(df) || !Number.isFinite(total)) return 1; + if (df <= 1 || total <= 1) return 1; + const capped = Math.min(df, total); + const scale = Math.log(1 + total / capped) / Math.log(1 + total); + return Math.max(NAME_MATCH_IDF_FLOOR, Math.min(1, scale)); +} + /** * Bonus when a node's name matches the search query. * Exact matches get the largest boost; prefix matches get smaller boosts. * Multi-word queries also check individual term matches against the name. + * + * @param corpus - Optional corpus stats. When given, the two exact-name bonuses + * are scaled by {@link nameMatchIdfScale} so a name shared by many symbols + * stops dominating. Omitted (or unavailable) leaves scoring as before. */ -export function nameMatchBonus(nodeName: string, query: string): number { +export function nameMatchBonus( + nodeName: string, + query: string, + corpus?: NameCorpusStats, + deprioritized = false, +): number { const nameLower = nodeName.toLowerCase(); + // Only the exact-name arms below take the corpus discount. Prefix/substring + // bonuses are already small and length-scaled, so they never produced the + // #982 crowd-out. The de-prioritized damping applies to every arm (#1463) — + // it is the whole name signal that the project called peripheral — but only + // the exact arms compose two discounts, so only they need the shared floor. + const idf = corpus ? nameMatchIdfScale(corpus.countForName(nameLower), corpus.total) : 1; + const exact = (bonus: number): number => + Math.round(bonus * combinedExactNameScale(idf, deprioritized)); + const other = (bonus: number): number => + deprioritized ? Math.round(bonus * DEPRIORITIZED_NAME_BONUS_SCALE) : bonus; + // Split query into word-level terms (handles "CacheBuilder build" → ["cache","builder","build"]) const rawTerms = query .replace(/([a-z])([A-Z])/g, '$1 $2') @@ -367,26 +485,26 @@ export function nameMatchBonus(nodeName: string, query: string): number { const queryLower = query.replace(/[\s]+/g, '').toLowerCase(); // Exact match: query exactly equals the node name - if (nameLower === queryLower) return 80; + if (nameLower === queryLower) return exact(80); // Exact match on a query token: "CacheBuilder build" and node name is "build" - if (queryTokens.length > 1 && queryTokens.includes(nameLower)) return 60; + if (queryTokens.length > 1 && queryTokens.includes(nameLower)) return exact(60); // Name starts with query — scale by length ratio so "Pod"→"Pod" (exact, handled above) // scores much higher than "Pod"→"PodGCControllerOptions" (ratio 0.125). if (nameLower.startsWith(queryLower)) { const ratio = queryLower.length / nameLower.length; - return Math.round(10 + 30 * ratio); + return other(Math.round(10 + 30 * ratio)); } // All camelCase-split terms appear in the name if (rawTerms.length > 1) { const allMatch = rawTerms.every(t => nameLower.includes(t)); - if (allMatch) return 15; + if (allMatch) return other(15); } // Name contains the full query as substring - if (nameLower.includes(queryLower)) return 10; + if (nameLower.includes(queryLower)) return other(10); return 0; }