Multiple Highlighter Support (#273)

### Description

Adds support for multiple highlight providers.

> [!NOTE]
> For reviewers: You may notice that this uses an underscored module
`_RopeModule`. This module is safe, as in it has tests and is used in
production (it backs AttributedString and BigString Foundation types).
It's underscored because the API may change in the future, and the
swift-collections devs consider it to be the incorrect collection type
to use for most applications. However, this application meets every
requirement for using a Rope.

### Related Issues

* #40

### Checklist

- [x] I read and understood the [contributing
guide](https://github.com/CodeEditApp/CodeEdit/blob/main/CONTRIBUTING.md)
as well as the [code of
conduct](https://github.com/CodeEditApp/CodeEdit/blob/main/CODE_OF_CONDUCT.md)
- [x] The issues this PR addresses are related to each other
- [x] My changes generate no new warnings
- [x] My code builds and runs on my machine
- [x] My changes are all related to the related issue above
- [x] I documented my code

### Screenshots

N/A

---------

Co-authored-by: Tom Ludwig <[email protected]>

Author: thecoolwinter

.swiftpm/xcode/xcshareddata/xcschemes/CodeEditTextView.xcscheme

@@ -57,7 +57,7 @@ public struct CodeEditSourceEditor: NSViewControllerRepresentable {
         editorOverscroll: CGFloat = 0,
         cursorPositions: Binding<[CursorPosition]>,
         useThemeBackground: Bool = true,
-        highlightProvider: HighlightProviding? = nil,
+        highlightProviders: [HighlightProviding] = [TreeSitterClient()],
         contentInsets: NSEdgeInsets? = nil,
         isEditable: Bool = true,
         isSelectable: Bool = true,
@@ -78,7 +78,7 @@ public struct CodeEditSourceEditor: NSViewControllerRepresentable {
         self.wrapLines = wrapLines
         self.editorOverscroll = editorOverscroll
         self.cursorPositions = cursorPositions
-        self.highlightProvider = highlightProvider
+        self.highlightProviders = highlightProviders
         self.contentInsets = contentInsets
         self.isEditable = isEditable
         self.isSelectable = isSelectable
@@ -132,7 +132,7 @@ public struct CodeEditSourceEditor: NSViewControllerRepresentable {
         editorOverscroll: CGFloat = 0,
         cursorPositions: Binding<[CursorPosition]>,
         useThemeBackground: Bool = true,
-        highlightProvider: HighlightProviding? = nil,
+        highlightProviders: [HighlightProviding] = [TreeSitterClient()],
         contentInsets: NSEdgeInsets? = nil,
         isEditable: Bool = true,
         isSelectable: Bool = true,
@@ -153,7 +153,7 @@ public struct CodeEditSourceEditor: NSViewControllerRepresentable {
         self.wrapLines = wrapLines
         self.editorOverscroll = editorOverscroll
         self.cursorPositions = cursorPositions
-        self.highlightProvider = highlightProvider
+        self.highlightProviders = highlightProviders
         self.contentInsets = contentInsets
         self.isEditable = isEditable
         self.isSelectable = isSelectable
@@ -179,7 +179,7 @@ public struct CodeEditSourceEditor: NSViewControllerRepresentable {
     private var editorOverscroll: CGFloat
     package var cursorPositions: Binding<[CursorPosition]>
     private var useThemeBackground: Bool
-    private var highlightProvider: HighlightProviding?
+    private var highlightProviders: [HighlightProviding]
     private var contentInsets: NSEdgeInsets?
     private var isEditable: Bool
     private var isSelectable: Bool
@@ -204,7 +204,7 @@ public struct CodeEditSourceEditor: NSViewControllerRepresentable {
             cursorPositions: cursorPositions.wrappedValue,
             editorOverscroll: editorOverscroll,
             useThemeBackground: useThemeBackground,
-            highlightProvider: highlightProvider,
+            highlightProviders: highlightProviders,
             contentInsets: contentInsets,
             isEditable: isEditable,
             isSelectable: isSelectable,

Package.resolved

@@ -15,31 +15,14 @@ extension TextViewController {
             self.highlighter = nil
         }
 
-        self.highlighter = Highlighter(
+        let highlighter = Highlighter(
             textView: textView,
-            highlightProvider: highlightProvider,
-            theme: theme,
+            providers: highlightProviders,
             attributeProvider: self,
             language: language
         )
-        textView.addStorageDelegate(highlighter!)
-        setHighlightProvider(self.highlightProvider)
-    }
-
-    internal func setHighlightProvider(_ highlightProvider: HighlightProviding? = nil) {
-        var provider: HighlightProviding?
-
-        if let highlightProvider = highlightProvider {
-            provider = highlightProvider
-        } else {
-            self.treeSitterClient = TreeSitterClient()
-            provider = self.treeSitterClient!
-        }
-
-        if let provider = provider {
-            self.highlightProvider = provider
-            highlighter?.setHighlightProvider(provider)
-        }
+        textView.addStorageDelegate(highlighter)
+        self.highlighter = highlighter
     }
 }
 

Sources/CodeEditSourceEditor/CodeEditSourceEditor/CodeEditSourceEditor.swift

@@ -113,7 +113,7 @@ public class TextViewController: NSViewController {
     public var useThemeBackground: Bool
 
     /// The provided highlight provider.
-    public var highlightProvider: HighlightProviding?
+    public var highlightProviders: [HighlightProviding]
 
     /// Optional insets to offset the text view in the scroll view by.
     public var contentInsets: NSEdgeInsets?
@@ -217,7 +217,7 @@ public class TextViewController: NSViewController {
         cursorPositions: [CursorPosition],
         editorOverscroll: CGFloat,
         useThemeBackground: Bool,
-        highlightProvider: HighlightProviding?,
+        highlightProviders: [HighlightProviding] = [TreeSitterClient()],
         contentInsets: NSEdgeInsets?,
         isEditable: Bool,
         isSelectable: Bool,
@@ -237,7 +237,7 @@ public class TextViewController: NSViewController {
         self.cursorPositions = cursorPositions
         self.editorOverscroll = editorOverscroll
         self.useThemeBackground = useThemeBackground
-        self.highlightProvider = highlightProvider
+        self.highlightProviders = highlightProviders
         self.contentInsets = contentInsets
         self.isEditable = isEditable
         self.isSelectable = isSelectable
@@ -307,7 +307,7 @@ public class TextViewController: NSViewController {
             textView.removeStorageDelegate(highlighter)
         }
         highlighter = nil
-        highlightProvider = nil
+        highlightProviders.removeAll()
         textCoordinators.values().forEach {
             $0.destroy()
         }

Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift

@@ -0,0 +1,133 @@
+//
+//  CaptureModifiers.swift
+//  CodeEditSourceEditor
+//
+//  Created by Khan Winter on 10/24/24.
+//
+
+// https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#semanticTokenModifiers
+
+/// A collection of possible syntax capture modifiers. Represented by an integer for memory efficiency, and with the
+/// ability to convert to and from strings for ease of use with tools.
+///
+/// These are useful for helping differentiate between similar types of syntax. Eg two variables may be declared like
+/// ```swift
+/// var a = 1
+/// let b = 1
+/// ```
+/// ``CaptureName`` will represent both these later in code, but combined ``CaptureModifier`` themes can differentiate
+/// between constants (`b` in the example) and regular variables (`a` in the example).
+///
+/// This is `Int8` raw representable for memory considerations. In large documents there can be *lots* of these created
+/// and passed around, so representing them with a single integer is preferable to a string to save memory.
+///
+public enum CaptureModifier: Int8, CaseIterable, Sendable {
+    case declaration
+    case definition
+    case readonly
+    case `static`
+    case deprecated
+    case abstract
+    case async
+    case modification
+    case documentation
+    case defaultLibrary
+
+    public var stringValue: String {
+        switch self {
+        case .declaration:
+            return "declaration"
+        case .definition:
+            return "definition"
+        case .readonly:
+            return "readonly"
+        case .static:
+            return "static"
+        case .deprecated:
+            return "deprecated"
+        case .abstract:
+            return "abstract"
+        case .async:
+            return "async"
+        case .modification:
+            return "modification"
+        case .documentation:
+            return "documentation"
+        case .defaultLibrary:
+            return "defaultLibrary"
+        }
+    }
+
+    // swiftlint:disable:next cyclomatic_complexity
+    public static func fromString(_ string: String) -> CaptureModifier? {
+        switch string {
+        case "declaration":
+            return .declaration
+        case "definition":
+            return .definition
+        case "readonly":
+            return .readonly
+        case "static`":
+            return .static
+        case "deprecated":
+            return .deprecated
+        case "abstract":
+            return .abstract
+        case "async":
+            return .async
+        case "modification":
+            return .modification
+        case "documentation":
+            return .documentation
+        case "defaultLibrary":
+            return .defaultLibrary
+        default:
+            return nil
+        }
+    }
+}
+
+extension CaptureModifier: CustomDebugStringConvertible {
+    public var debugDescription: String { stringValue }
+}
+
+/// A set of capture modifiers, efficiently represented by a single integer.
+public struct CaptureModifierSet: OptionSet, Equatable, Hashable, Sendable {
+    public var rawValue: UInt
+
+    public init(rawValue: UInt) {
+        self.rawValue = rawValue
+    }
+
+    public static let declaration = CaptureModifierSet(rawValue: 1 << CaptureModifier.declaration.rawValue)
+    public static let definition = CaptureModifierSet(rawValue: 1 << CaptureModifier.definition.rawValue)
+    public static let readonly = CaptureModifierSet(rawValue: 1 << CaptureModifier.readonly.rawValue)
+    public static let `static` = CaptureModifierSet(rawValue: 1 << CaptureModifier.static.rawValue)
+    public static let deprecated = CaptureModifierSet(rawValue: 1 << CaptureModifier.deprecated.rawValue)
+    public static let abstract = CaptureModifierSet(rawValue: 1 << CaptureModifier.abstract.rawValue)
+    public static let async = CaptureModifierSet(rawValue: 1 << CaptureModifier.async.rawValue)
+    public static let modification = CaptureModifierSet(rawValue: 1 << CaptureModifier.modification.rawValue)
+    public static let documentation = CaptureModifierSet(rawValue: 1 << CaptureModifier.documentation.rawValue)
+    public static let defaultLibrary = CaptureModifierSet(rawValue: 1 << CaptureModifier.defaultLibrary.rawValue)
+
+    /// All values in the set.
+    public var values: [CaptureModifier] {
+        var rawValue = self.rawValue
+
+        // This set is represented by an integer, where each `1` in the binary number represents a value.
+        // We can interpret the index of the `1` as the raw value of a ``CaptureModifier`` (the index in 0b0100 would
+        // be 2). This loops through each `1` in the `rawValue`, finds the represented modifier, and 0's out the `1` so
+        // we can get the next one using the binary & operator (0b0110 -> 0b0100 -> 0b0000 -> finish).
+        var values: [Int8] = []
+        while rawValue > 0 {
+            values.append(Int8(rawValue.trailingZeroBitCount))
+            // Clears the bit at the desired index (eg: 0b110 if clearing index 0)
+            rawValue &= ~UInt(1 << rawValue.trailingZeroBitCount)
+        }
+        return values.compactMap({ CaptureModifier(rawValue: $0) })
+    }
+
+    public mutating func insert(_ value: CaptureModifier) {
+        rawValue &= 1 << value.rawValue
+    }
+}