feat(search): discount the exact-name bonus by corpus frequency (#982)

__tests__/composed-name-bonus.test.ts

@@ -0,0 +1,108 @@
+/**
+ * The two exact-name discounts compose, and the composition has to keep the
+ * same invariant each one pins alone: an exact name the user typed never loses
+ * to a mere prefix match.
+ *
+ * #1462 (corpus-frequency discount) and #1463 (de-prioritized-path damping)
+ * each clear that bound in isolation — `80 * 0.6 = 48 > 40` and
+ * `80 * 0.75 - 15 = 45 > 40` — but each suite pins its invariant with the other
+ * lever off. Multiplied, a name that is both corpus-common AND inside a
+ * de-prioritized tree lands at `80 * 0.6 * 0.75 - 15 = 21`, well under the
+ * prefix arm's supremum of 40. Neither existing test can see it.
+ *
+ * The general condition, with the -15 path penalty a de-prioritized node also
+ * carries: `80 * combined - 15 > 40`, i.e. `combined > 55/80 = 0.6875`.
+ */
+import { describe, it, expect } from 'vitest';
+import {
+  nameMatchBonus,
+  nameMatchIdfScale,
+  combinedExactNameScale,
+  NAME_MATCH_IDF_FLOOR,
+  COMBINED_EXACT_NAME_FLOOR,
+  type NameCorpusStats,
+} from '../src/search/query-utils';
+import { DEPRIORITIZED_NAME_BONUS_SCALE } from '../src/db/queries';
+
+/** The prefix arm is `round(10 + 30 * ratio)`, so 40 is its supremum. */
+const PREFIX_ARM_SUPREMUM = 40;
+/** A de-prioritized path also takes the flat path penalty. */
+const PATH_PENALTY = 15;
+/** `80 * s - 15 > 40` → `s > 55/80`. */
+const REQUIRED_COMBINED_SCALE = (PREFIX_ARM_SUPREMUM + PATH_PENALTY) / 80;
+
+/** A corpus where `name` is shared by `df` of `total` nodes. */
+function corpusWith(name: string, df: number, total: number): NameCorpusStats {
+  return {
+    total,
+    countForName: (n: string): number => (n.toLowerCase() === name.toLowerCase() ? df : 1),
+  };
+}
+
+describe('the two exact-name discounts compose (#1462 + #1463)', () => {
+  it('states the bound the composition has to clear', () => {
+    expect(REQUIRED_COMBINED_SCALE).toBeCloseTo(0.6875, 6);
+  });
+
+  it('regression: the naive product falls under the bound', () => {
+    // Not how the code composes them — asserted so the failure this test was
+    // written for stays visible, and nobody reintroduces the multiplication.
+    const product = NAME_MATCH_IDF_FLOOR * DEPRIORITIZED_NAME_BONUS_SCALE;
+    expect(product).toBeLessThan(REQUIRED_COMBINED_SCALE);
+    expect(80 * product - PATH_PENALTY).toBeLessThan(PREFIX_ARM_SUPREMUM);
+  });
+
+  it('the combined floor clears the bound at the worst case of both levers', () => {
+    expect(COMBINED_EXACT_NAME_FLOOR).toBeGreaterThan(REQUIRED_COMBINED_SCALE);
+    const worst = combinedExactNameScale(NAME_MATCH_IDF_FLOOR, true);
+    expect(Math.round(80 * worst) - PATH_PENALTY).toBeGreaterThan(PREFIX_ARM_SUPREMUM);
+  });
+
+  it('a de-prioritized, corpus-common exact name still beats a prefix match', () => {
+    // `child` shared by 400 of 60k nodes drives the IDF scale to its floor, and
+    // the node also sits in a de-prioritized tree. It must still outrank the
+    // prefix match `children`, after the -15 the de-prioritized path takes.
+    const corpus = corpusWith('child', 400, 60_000);
+    expect(nameMatchIdfScale(400, 60_000)).toBeCloseTo(NAME_MATCH_IDF_FLOOR, 6);
+
+    const exactDeprioritized = nameMatchBonus('child', 'child', corpus, true) - PATH_PENALTY;
+    const prefix = nameMatchBonus('children', 'child', corpus, false);
+
+    expect(prefix).toBeLessThanOrEqual(PREFIX_ARM_SUPREMUM);
+    expect(exactDeprioritized).toBeGreaterThan(prefix);
+  });
+
+  it('neither lever alone is weakened by the fix', () => {
+    const corpus = corpusWith('child', 400, 60_000);
+    // #1462 in isolation: corpus-common, not de-prioritized. No path penalty.
+    expect(nameMatchBonus('child', 'child', corpus, false)).toBeGreaterThan(PREFIX_ARM_SUPREMUM);
+    // #1463 in isolation: de-prioritized, but a rare name.
+    const rare = corpusWith('nothingElse', 1, 60_000);
+    expect(nameMatchBonus('child', 'child', rare, true) - PATH_PENALTY).toBeGreaterThan(
+      PREFIX_ARM_SUPREMUM,
+    );
+  });
+
+  it('still discounts: a common de-prioritized name ranks below a rare one', () => {
+    const common = corpusWith('child', 400, 60_000);
+    const rare = corpusWith('nothingElse', 1, 60_000);
+    expect(nameMatchBonus('child', 'child', common, true)).toBeLessThan(
+      nameMatchBonus('child', 'child', rare, true),
+    );
+  });
+
+  it('leaves the undiscounted path alone when no corpus is supplied', () => {
+    expect(nameMatchBonus('child', 'child')).toBe(80);
+    expect(nameMatchBonus('child', 'child', undefined, true)).toBe(
+      Math.round(80 * DEPRIORITIZED_NAME_BONUS_SCALE),
+    );
+  });
+
+  it('does not touch the prefix or substring arms', () => {
+    const corpus = corpusWith('children', 400, 60_000);
+    // Prefix arm is length-scaled and small; the discount never applied to it.
+    expect(nameMatchBonus('children', 'child', corpus, false)).toBe(
+      nameMatchBonus('children', 'child', undefined, false),
+    );
+  });
+});

__tests__/name-match-idf.test.ts

@@ -0,0 +1,250 @@
+/**
+ * Corpus-frequency discount on the exact-name bonus (#982 — the follow-up #746
+ * floated but never landed).
+ *
+ * `nameMatchBonus` handed a flat 80 (whole query === name) / 60 (a token of a
+ * multi-word query === name) regardless of how many symbols carry that name. A
+ * generic non-stopword token that happens to be a symbol name — `usage`, `get`,
+ * `status` — therefore collected the full bonus and outranked the product code
+ * that actually answers the query but does not literally contain the token.
+ *
+ * The fixture is #982's layout — product code with NO symbol named `usage`,
+ * plus peripheral helper scripts that each define a module-level `usage()` —
+ * scaled so the token is genuinely corpus-common, which is the condition this
+ * lever keys on. Measured on the fixture for `desktop status bar context window
+ * usage` (69 nodes, 24 of them named `usage`):
+ *
+ *   before: every top slot is an `optional-skills/**` usage() at 71.3
+ *   after:  the helpers leave the top 11 entirely; product code fills it
+ *
+ * Note what this lever does NOT do, because it is easy to over-claim: in #982's
+ * 8-file minimal repro only TWO symbols are named `usage`, so the token is rare
+ * and the IDF scale is ~0.8 — nearly inert, by design. Fixing that shape needs
+ * the issue's complementary path lever (user-extensible de-prioritization),
+ * which is deliberately out of scope here. Locked below by an explicit test.
+ */
+
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { CodeGraph } from '../src';
+import { initGrammars, loadAllGrammars } from '../src/extraction/grammars';
+import {
+  nameMatchBonus,
+  nameMatchIdfScale,
+  NAME_MATCH_IDF_FLOOR as FLOOR,
+} from '../src/search/query-utils';
+
+describe('nameMatchIdfScale', () => {
+  it('leaves a unique name at full weight', () => {
+    expect(nameMatchIdfScale(1, 10_000)).toBe(1);
+  });
+
+  it('decays monotonically as the name spreads across the corpus', () => {
+    const total = 10_000;
+    const scales = [1, 2, 10, 100, 1000].map((df) => nameMatchIdfScale(df, total));
+    for (let i = 1; i < scales.length; i++) {
+      // Non-increasing throughout; strictly decreasing until the floor binds.
+      expect(scales[i]).toBeLessThanOrEqual(scales[i - 1]);
+      if (scales[i - 1] > FLOOR) expect(scales[i]).toBeLessThan(scales[i - 1]);
+    }
+  });
+
+  it('discounts a very common name without erasing it', () => {
+    const scale = nameMatchIdfScale(9_000, 10_000);
+    expect(scale).toBe(FLOOR);
+    expect(scale).toBeLessThan(1);
+  });
+
+  it('never lets a discounted whole-query exact match lose to a prefix match', () => {
+    // The invariant that sets the floor. The prefix arm pays `10 + 30 * ratio`
+    // with ratio < 1, so it approaches but never reaches 40. An exact whole-query
+    // match floors at 80 * FLOOR = 48 — clear of it for any corpus frequency.
+    //
+    // This is not hypothetical: with the floor at 0.25 it did not hold. Searching
+    // Alamofire for `request` (df 173 / 4512 nodes → scale 0.392) paid the exact
+    // match 80 * 0.392 = 31 and lost the top slot to `requests`, a prefix match
+    // worth 36. The user typed the name; hiding it behind a longer one is wrong.
+    const PREFIX_ARM_SUPREMUM = 40;
+    expect(80 * FLOOR).toBeGreaterThan(PREFIX_ARM_SUPREMUM);
+
+    const crowded = { total: 4512, countForName: () => 173 };
+    expect(nameMatchBonus('request', 'request', crowded)).toBeGreaterThan(
+      nameMatchBonus('requests', 'request')
+    );
+  });
+
+  it('keeps the floor above every scale real corpora reach', () => {
+    // Swept on five indexed repos; the lowest raw scale observed was django's
+    // commonest name (1097 of 62080 nodes). A floor at or below that is inert —
+    // an earlier 0.25 never once bound. The floor must be chosen above this line
+    // to do anything at all.
+    expect(nameMatchIdfScale(1097, 62_080)).toBe(FLOOR);
+    expect(nameMatchIdfScale(173, 4_512)).toBe(FLOOR);
+  });
+
+  it('is degenerate-input safe', () => {
+    expect(nameMatchIdfScale(0, 10_000)).toBe(1);
+    expect(nameMatchIdfScale(5, 1)).toBe(1);
+    expect(nameMatchIdfScale(NaN, 10_000)).toBe(1);
+    // A name counted more often than the corpus size must not go negative.
+    expect(nameMatchIdfScale(20_000, 10_000)).toBeGreaterThan(0);
+  });
+});
+
+describe('nameMatchBonus corpus discount', () => {
+  const corpus = (df: number, total = 10_000) => ({ total, countForName: () => df });
+
+  it('is unchanged when no corpus stats are supplied', () => {
+    expect(nameMatchBonus('usage', 'usage')).toBe(80);
+    expect(nameMatchBonus('usage', 'context window usage')).toBe(60);
+  });
+
+  it('leaves a rare name at its original bonus', () => {
+    expect(nameMatchBonus('usage', 'usage', corpus(1))).toBe(80);
+    expect(nameMatchBonus('usage', 'context window usage', corpus(1))).toBe(60);
+  });
+
+  it('discounts a name shared by many symbols', () => {
+    expect(nameMatchBonus('usage', 'context window usage', corpus(500))).toBeLessThan(60);
+    expect(nameMatchBonus('usage', 'context window usage', corpus(500))).toBeGreaterThan(0);
+  });
+
+  it('does not touch prefix or substring bonuses', () => {
+    // "Name starts with query" arm — small and already length-scaled.
+    const withCorpus = nameMatchBonus('usageReporter', 'usage', corpus(5_000));
+    const without = nameMatchBonus('usageReporter', 'usage');
+    expect(withCorpus).toBe(without);
+  });
+});
+
+describe('#982 — generic exact-name match crowding out product code', () => {
+  const SKILLS = Array.from({ length: 24 }, (_, i) => `skill${i}`);
+  let tmpDir: string;
+  let cg: CodeGraph;
+
+  beforeAll(async () => {
+    await initGrammars();
+    await loadAllGrammars();
+
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-982-'));
+    const mk = (rel: string, content: string) => {
+      const p = path.join(tmpDir, rel);
+      fs.mkdirSync(path.dirname(p), { recursive: true });
+      fs.writeFileSync(p, content);
+    };
+
+    // Product code answering "desktop status bar context window usage".
+    // Note: no symbol here is literally named `usage`.
+    mk(
+      'apps/desktop/statusbar/StatusBar.ts',
+      [
+        'export class DesktopStatusBar {',
+        '  render(): string { return this.refresh(); }',
+        '  refresh(): string { return "status bar"; }',
+        '  mount(): void {}',
+        '}',
+      ].join('\n')
+    );
+    mk(
+      'apps/desktop/statusbar/StatusBarController.ts',
+      [
+        "import { DesktopStatusBar } from './StatusBar';",
+        'export class StatusBarController {',
+        '  constructor(private readonly bar: DesktopStatusBar) {}',
+        '  show(): string { return this.bar.render(); }',
+        '}',
+      ].join('\n')
+    );
+    mk(
+      'apps/desktop/context/ContextWindowMeter.ts',
+      [
+        'export class ContextWindowMeter {',
+        '  read(): number { return this.recompute(); }',
+        '  recompute(): number { return estimateTokens("context window"); }',
+        '}',
+        'export function estimateTokens(text: string): number { return text.length; }',
+      ].join('\n')
+    );
+    mk(
+      'apps/desktop/context/format.ts',
+      'export function formatTokens(n: number): string { return `${n} tokens`; }\n'
+    );
+    mk('gateway/server/server.ts', 'export function startServer(): void {}\n');
+    mk('packages/core/util/strings.ts', 'export function slugify(s: string): string { return s; }\n');
+
+    // Peripheral helper scripts: each defines a module-level `usage`, and
+    // nothing else about them answers the query. Enough of them that `usage`
+    // is corpus-common (24 of 69 nodes) — the condition the discount keys on.
+    for (const skill of SKILLS) {
+      mk(
+        `optional-skills/${skill}/scripts/${skill}_calc.ts`,
+        [
+          'export function usage(): void {',
+          `  console.log("usage: ${skill}_calc [options]");`,
+          '}',
+        ].join('\n')
+      );
+    }
+
+    cg = CodeGraph.initSync(tmpDir);
+    await cg.indexAll();
+  }, 120_000);
+
+  afterAll(() => {
+    cg?.destroy();
+    if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  const usageHelperRanks = (query: string): number[] => {
+    const results = cg.searchNodes(query, { limit: 20 });
+    const ranks: number[] = [];
+    results.forEach((r, i) => {
+      if (r.node.name.toLowerCase() === 'usage' && r.node.filePath.includes('optional-skills')) {
+        ranks.push(i);
+      }
+    });
+    return ranks;
+  };
+
+  it('keeps the generic usage() helpers out of the top ranks', () => {
+    // Before the discount these occupied every top slot at an identical 71.3.
+    const ranks = usageHelperRanks('desktop status bar context window usage');
+    expect(ranks.slice(0, 2)).not.toEqual([0, 1]);
+  });
+
+  it('ranks at least one product symbol above the usage() helpers', () => {
+    const results = cg.searchNodes('desktop status bar context window usage', { limit: 20 });
+    const firstHelper = results.findIndex(
+      (r) => r.node.name.toLowerCase() === 'usage' && r.node.filePath.includes('optional-skills')
+    );
+    const firstProduct = results.findIndex((r) => r.node.filePath.includes('apps/desktop'));
+    expect(firstProduct).toBeGreaterThanOrEqual(0);
+    if (firstHelper >= 0) {
+      expect(firstProduct).toBeLessThan(firstHelper);
+    }
+  });
+
+  it('still surfaces the helpers when the generic token IS the query', () => {
+    // Discount, not erase: someone asking for `usage` must still find usage().
+    const results = cg.searchNodes('usage', { limit: 20 });
+    const helpers = results.filter(
+      (r) => r.node.name.toLowerCase() === 'usage' && r.node.filePath.includes('optional-skills')
+    );
+    expect(helpers.length).toBeGreaterThan(0);
+    expect(results.findIndex((r) => r.node.name.toLowerCase() === 'usage')).toBeLessThan(3);
+  });
+});
+
+describe('#982 — the discount is corpus-frequency, not a blanket nerf', () => {
+  it('barely moves a name that only a couple of symbols carry', () => {
+    // #982's 8-file minimal repro has TWO usage() defs among ~25 nodes. The
+    // token is rare there, so IDF is close to inert — this lever is not what
+    // fixes that shape, and saying otherwise would overstate it.
+    const scale = nameMatchIdfScale(2, 25);
+    expect(scale).toBeGreaterThan(0.7);
+    expect(nameMatchBonus('usage', 'context window usage', { total: 25, countForName: () => 2 }))
+      .toBeGreaterThanOrEqual(45);
+  });
+});