From a102b68f1e70246953055bd61621ebffc033607f Mon Sep 17 00:00:00 2001 From: Arsalan Ahmed Date: Wed, 12 Aug 2026 17:33:31 +0530 Subject: [PATCH 1/2] fix(CommandPalette): keep astral characters intact when inserting highlight marks Fuse's indices are UTF-16 code-unit offsets, so a region boundary can land between the two surrogates of an astral character; substring() there put inside the character and orphaned both halves, rendering as U+FFFD. Snap such a boundary outward before slicing, so the highlight always covers whole characters. The single-character region skip had the same unit inconsistency: measured in code units, a lone astral character (two units) was highlighted while a lone BMP character was skipped. It is now measured in code points. Closes #362 --- src/runtime/utils/search.ts | 46 +++++++++++++--- test/utils/search.spec.ts | 101 ++++++++++++++++++++++++++++++++---- 2 files changed, 130 insertions(+), 17 deletions(-) diff --git a/src/runtime/utils/search.ts b/src/runtime/utils/search.ts index 1a5bd4f5..11df6607 100644 --- a/src/runtime/utils/search.ts +++ b/src/runtime/utils/search.ts @@ -70,6 +70,22 @@ export function sanitizeSnippet(snippet: string): string { .replaceAll(tagClose, '') } +// Fuse's `indices` are UTF-16 code-unit offsets, so a region boundary can land +// between the two surrogates of an astral character (emoji, most CJK extension +// blocks). Slicing there puts the `` inside the character and orphans +// both halves, which render as `�`. True when `index` points at a low +// surrogate whose predecessor is a high surrogate. +function splitsSurrogatePair(value: string, index: number): boolean { + if (index <= 0 || index >= value.length) { + return false + } + + const low = value.charCodeAt(index) + const high = value.charCodeAt(index - 1) + + return low >= 0xDC00 && low <= 0xDFFF && high >= 0xD800 && high <= 0xDBFF +} + export function highlight(item: T & { matches?: FuseResult['matches'] }, searchTerm: string, forceKey?: GetItemKeys, omitKeys?: GetItemKeys[], useTokenSearch?: boolean) { const tokens = useTokenSearch ? (searchTerm.match(/[\p{L}\p{M}\p{N}_]+/gu) || []) : [] const minTokenLength = tokens.length > 0 ? Math.min(...tokens.map(t => t.length)) : searchTerm.length @@ -80,22 +96,38 @@ export function highlight(item: T & { matches?: FuseResult['matches'] }, s let nextUnhighlightedRegionStartingIndex = 0 indices.forEach((region) => { - // skip if region is a single character - if (region.length === 2 && region[0] === region[1]) { + // Snap a boundary that lands inside an astral character outward, so the + // highlight always covers whole characters. + let start = region[0] + let end = region[1] + 1 + + if (splitsSurrogatePair(value, start)) { + start-- + } + if (splitsSurrogatePair(value, end)) { + end++ + } + + // A widened start must not reach back into a region that has already been + // emitted — `substring` swaps reversed arguments and would duplicate it. + start = Math.max(start, nextUnhighlightedRegionStartingIndex) + + // skip if region is a single character — one code point, so a lone astral + // character (two code units) is skipped the same as a lone BMP one + if (end - start <= 1 || (end - start === 2 && splitsSurrogatePair(value, start + 1))) { return } - const lastIndiceNextIndex = region[1] + 1 - const isMatched = (lastIndiceNextIndex - region[0]) >= minTokenLength + const isMatched = (region[1] + 1 - region[0]) >= minTokenLength content += [ - escapeHTML(value.substring(nextUnhighlightedRegionStartingIndex, region[0])), + escapeHTML(value.substring(nextUnhighlightedRegionStartingIndex, start)), isMatched && ``, - escapeHTML(value.substring(region[0], lastIndiceNextIndex)), + escapeHTML(value.substring(start, end)), isMatched && '' ].filter(Boolean).join('') - nextUnhighlightedRegionStartingIndex = lastIndiceNextIndex + nextUnhighlightedRegionStartingIndex = end }) content += escapeHTML(value.substring(nextUnhighlightedRegionStartingIndex)) diff --git a/test/utils/search.spec.ts b/test/utils/search.spec.ts index ffc1503f..5c54cb47 100644 --- a/test/utils/search.spec.ts +++ b/test/utils/search.spec.ts @@ -1,4 +1,5 @@ import { describe, it, expect } from 'vitest' +import Fuse from 'fuse.js' import { highlight, sanitizeSnippet } from '../../src/runtime/utils/search' describe('sanitizeSnippet', () => { @@ -31,6 +32,16 @@ describe('sanitizeSnippet', () => { }) describe('highlight', () => { + // Matches a high surrogate not followed by a low one, or a low surrogate not + // preceded by a high one — i.e. half of an astral character. + const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/ + + // One character from each corner of the surrogate ranges. A fixture built only + // from characters sitting comfortably inside them hides an implementation whose + // range bounds are off by one: U+20000 encodes to a low surrogate of U+DC00 and + // U+1F3FF to U+DFFF — the two edges — while U+1F600 sits between them. + const ASTRAL = ['\u{10000}', '\u{10FFFF}', '\u{1F600}', '\u{1F3FF}', '\u{20000}'] + it('escapes an injected tag in the unmatched tail while keeping the match highlighted', () => { const value = 'zzzzz &' const result = highlight({ label: value, matches: [{ key: 'label', value, indices: [[0, 4]] }] }, 'zzzzz', 'label') @@ -62,22 +73,12 @@ describe('highlight', () => { }) describe('truncation from the start', () => { - // Matches a high surrogate not followed by a low one, or a low surrogate not - // preceded by a high one — i.e. half of an astral character. - const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/ - // `maxLength` counts the tag characters that the counter inside // `truncateHTMLFromStart` skips, so the two cancel and the surviving prefix is // always `''.length + ''.length` characters — whatever the match // is, and whether the content is BMP or astral. const RETAINED = ''.length + ''.length - // One character from each corner of the surrogate ranges. A fixture built only - // from characters sitting comfortably inside them hides an implementation whose - // range bounds are off by one: U+20000 encodes to a low surrogate of U+DC00 and - // U+1F3FF to U+DFFF — the two edges — while U+1F600 sits between them. - const ASTRAL = ['\u{10000}', '\u{10FFFF}', '\u{1F600}', '\u{1F3FF}', '\u{20000}'] - function highlightAfterFiller(filler: string, count: number) { const value = filler.repeat(count) + 'match' const index = value.indexOf('match') @@ -123,4 +124,84 @@ describe('highlight', () => { expect(result).toBe(`...${'b'.repeat(RETAINED)}match${'\u{1F600}'.repeat(10)}`) }) }) + + describe('insertion boundaries', () => { + // Fuse's `indices` are UTF-16 code-unit offsets, so a region boundary can + // land between the surrogates of an astral character. The `` must + // snap outward to cover the whole character instead of splitting it. + + it('closes the mark after the whole character when the region ends inside it', () => { + // a b c d e \uD83D \uDE00 x y z — the region's exclusive end (6) points + // at the low surrogate. + const value = 'abcde\u{1F600}xyz' + const result = highlight({ label: value, matches: [{ key: 'label', value, indices: [[0, 4]] }] }, 'abcde', 'label') + + expect(result).toBe('abcde\u{1F600}xyz') + + const split = highlight({ label: value, matches: [{ key: 'label', value, indices: [[0, 5]] }] }, 'abcde', 'label') + + expect(split).toBe('abcde\u{1F600}xyz') + expect(LONE_SURROGATE.test(split ?? '')).toBe(false) + }) + + it('opens the mark before the whole character when the region starts inside it', () => { + // a b c \uD83D \uDE00 d e f — the region start (4) points at the low + // surrogate. + const value = 'abc\u{1F600}def' + const result = highlight({ label: value, matches: [{ key: 'label', value, indices: [[4, 6]] }] }, 'def', 'label') + + expect(result).toBe('abc\u{1F600}def') + expect(LONE_SURROGATE.test(result ?? '')).toBe(false) + }) + + it.each(ASTRAL)('snaps both boundaries when each lands inside %s', (astral) => { + // a b c d e f — start (3) and exclusive end (7) each + // point at a low surrogate. + const value = `ab${astral}cd${astral}ef` + const result = highlight({ label: value, matches: [{ key: 'label', value, indices: [[3, 6]] }] }, 'abcd', 'label') + + expect(result).toBe(`ab${astral}cd${astral}ef`) + expect(LONE_SURROGATE.test(result ?? '')).toBe(false) + }) + + it('skips a region covering a single astral character, like a single BMP one', () => { + const value = 'ab\u{1F600}cd' + + // [1, 1] is one BMP character and was always skipped; [2, 3] is one astral + // character — two code units, but still a single-character region. + expect(highlight({ label: value, matches: [{ key: 'label', value, indices: [[1, 1]] }] }, 'b', 'label')).toBe(value) + expect(highlight({ label: value, matches: [{ key: 'label', value, indices: [[2, 3]] }] }, 'b', 'label')).toBe(value) + }) + + it.each(ASTRAL)('keeps every %s intact wherever the region boundary lands', (astral) => { + const value = astral.repeat(5) + + for (let end = 0; end < value.length; end++) { + const result = highlight({ label: value, matches: [{ key: 'label', value, indices: [[0, end]] }] }, 'aa', 'label') ?? '' + + // Count what survived, not only lone surrogates: deleting the split + // character would otherwise pass. + expect(result.split(astral).length - 1).toBe(5) + expect(LONE_SURROGATE.test(result)).toBe(false) + } + }) + + it('does not split characters at the boundaries fuse reports (repro from #362)', () => { + // Real fuse.js at ContentSearch's shipped defaults, with well-formed input + // on both sides: the user searched with the wrong emoji. + const label = 'deployment \u{1F600} pipeline' + const fuse = new Fuse([{ label }], { ignoreLocation: true, includeMatches: true, threshold: 0.1, keys: ['label'] }) + const matches = fuse.search('deployment \u{1F680}')[0]?.matches + + // Pin fuse's offsets: the first region's exclusive end (12) falls between + // the surrogates of the emoji. If a fuse upgrade stops producing a + // mid-pair boundary, fail loudly here instead of passing vacuously. + expect(matches?.[0]?.indices).toEqual([[0, 11], [13, 13]]) + + const result = highlight({ label, matches: [...matches!] }, 'deployment \u{1F680}', 'label', undefined, true) + + expect(result).toBe('deployment \u{1F600} pipeline') + expect(LONE_SURROGATE.test(result ?? '')).toBe(false) + }) + }) }) From be3b5c560157339f8038078eab40f02d7a4df2af Mon Sep 17 00:00:00 2001 From: Arsalan Ahmed Date: Thu, 13 Aug 2026 13:47:50 +0530 Subject: [PATCH 2/2] fix(CommandPalette): clamp highlight regions to the value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A region lying past the end of the value slipped through the single-character skip: the comparison runs on raw numbers while substring() clamps its own arguments, so the region sliced to nothing and emitted a bare — then pushed the next-region cursor past the end, collapsing every legitimate region after it. Clamp both boundaries to the value before the skip. --- src/runtime/utils/search.ts | 7 ++++++- test/utils/search.spec.ts | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/runtime/utils/search.ts b/src/runtime/utils/search.ts index 11df6607..224358c6 100644 --- a/src/runtime/utils/search.ts +++ b/src/runtime/utils/search.ts @@ -110,7 +110,12 @@ export function highlight(item: T & { matches?: FuseResult['matches'] }, s // A widened start must not reach back into a region that has already been // emitted — `substring` swaps reversed arguments and would duplicate it. - start = Math.max(start, nextUnhighlightedRegionStartingIndex) + // Both boundaries are also clamped to the value: `end - start` is compared + // below on the raw numbers while `substring` clamps its own arguments, so + // a region lying past the end of the value would otherwise compare as + // non-empty, slice to nothing and emit a bare ``. + start = Math.min(Math.max(start, nextUnhighlightedRegionStartingIndex), value.length) + end = Math.min(end, value.length) // skip if region is a single character — one code point, so a lone astral // character (two code units) is skipped the same as a lone BMP one diff --git a/test/utils/search.spec.ts b/test/utils/search.spec.ts index 5c54cb47..11216d7e 100644 --- a/test/utils/search.spec.ts +++ b/test/utils/search.spec.ts @@ -186,6 +186,29 @@ describe('highlight', () => { } }) + it('drops a region lying past the end of the value instead of emitting an empty mark', () => { + // `highlight()` is a published export and `CommandPaletteGroup.postFilter` + // lets a caller supply its own matches, so offsets computed against an + // older revision of the text can point past its end. `substring` clamps + // its arguments, so such a region sliced to nothing and emitted a bare + // `` — and pushed the next-region cursor past the end, + // collapsing every legitimate region after it. + const value = 'The quarterly revenue report is available' + const outOfRange = highlight({ label: value, matches: [{ key: 'label', value, indices: [[126, 132]] }] }, 'report', 'label') + + expect(outOfRange).toBe(value) + + const afterValid = highlight({ label: value, matches: [{ key: 'label', value, indices: [[4, 12], [126, 132]] }] }, 'quarterly', 'label') + + expect(afterValid).toBe('The quarterly revenue report is available') + + // A region straddling the end keeps its in-range half; the usual + // 13-character retained window applies in front of the mark. + const straddling = highlight({ label: value, matches: [{ key: 'label', value, indices: [[32, 132]] }] }, 'available', 'label') + + expect(straddling).toBe('...ue report is available') + }) + it('does not split characters at the boundaries fuse reports (repro from #362)', () => { // Real fuse.js at ContentSearch's shipped defaults, with well-formed input // on both sides: the user searched with the wrong emoji.