feat: implement string interning for file_path and module_path

pylight/benches/search.rs

@@ -1,6 +1,5 @@
 use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
 use pylight::{SearchEngine, Symbol, SymbolKind};
-use std::path::PathBuf;
 use std::sync::Arc;
 
 fn generate_symbols(count: usize) -> Vec<Arc<Symbol>> {
@@ -11,15 +10,15 @@ fn generate_symbols(count: usize) -> Vec<Arc<Symbol>> {
             symbols.push(Arc::new(Symbol::new(
                 format!("function_{i}"),
                 SymbolKind::Function,
-                PathBuf::from(format!("file{}.py", i % 10)),
+                format!("file{}.py", i % 10),
                 i,
                 0,
             )));
         } else if i % 3 == 1 {
             symbols.push(Arc::new(Symbol::new(
                 format!("Class_{i}"),
                 SymbolKind::Class,
-                PathBuf::from(format!("file{}.py", i % 10)),
+                format!("file{}.py", i % 10),
                 i,
                 0,
             )));
@@ -28,7 +27,7 @@ fn generate_symbols(count: usize) -> Vec<Arc<Symbol>> {
                 Symbol::new(
                     format!("method_{i}"),
                     SymbolKind::Method,
-                    PathBuf::from(format!("file{}.py", i % 10)),
+                    format!("file{}.py", i % 10),
                     i,
                     4,
                 )

pylight/src/bin/pylight.rs

@@ -103,7 +103,7 @@ fn run_standalone(
                 "{:2}. {} ({}:{})",
                 i + 1,
                 result.symbol.name,
-                result.symbol.file_path.display(),
+                result.symbol.file_path.as_ref(),
                 result.symbol.line
             );
         }

pylight/src/index/mod.rs

@@ -4,4 +4,4 @@ pub mod files;
 pub mod symbol_index;
 pub mod updater;
 
-pub use symbol_index::{FileMetadata, SymbolIndex};
+pub use symbol_index::SymbolIndex;

pylight/src/index/symbol_index.rs

@@ -1,6 +1,7 @@
 //! Symbol index implementation
 
 use crate::parser::{create_parser, ParserBackend};
+use crate::string_cache::StringCache;
 use crate::{Result, Symbol};
 use parking_lot::RwLock;
 use rayon::prelude::*;
@@ -13,23 +14,17 @@ use super::files;
 pub struct SymbolIndex {
     symbols: Arc<RwLock<HashMap<PathBuf, Vec<Arc<Symbol>>>>>,
     all_symbols: Arc<RwLock<Vec<Arc<Symbol>>>>,
-    file_metadata: Arc<RwLock<HashMap<PathBuf, FileMetadata>>>,
     parser_backend: ParserBackend,
-}
-
-#[derive(Debug, Clone)]
-pub struct FileMetadata {
-    pub last_modified: std::time::SystemTime,
-    pub symbol_count: usize,
+    string_cache: StringCache,
 }
 
 impl SymbolIndex {
     pub fn new(parser_backend: ParserBackend) -> Self {
         Self {
             symbols: Arc::new(RwLock::new(HashMap::new())),
             all_symbols: Arc::new(RwLock::new(Vec::new())),
-            file_metadata: Arc::new(RwLock::new(HashMap::new())),
             parser_backend,
+            string_cache: StringCache::new(),
         }
     }
 
@@ -44,28 +39,22 @@ impl SymbolIndex {
 
         let mut file_symbols = self.symbols.write();
         let mut all = self.all_symbols.write();
-        let mut metadata = self.file_metadata.write();
 
         // Remove old symbols for this file if any
         if let Some(_old_symbols) = file_symbols.get(&canonical_path) {
-            all.retain(|s| s.file_path != canonical_path);
+            all.retain(|s| s.file_path.as_ref() != canonical_path.to_string_lossy().as_ref());
         }
 
-        // Update metadata
-        if let Ok(file_metadata) = std::fs::metadata(&canonical_path) {
-            if let Ok(modified) = file_metadata.modified() {
-                metadata.insert(
-                    canonical_path.clone(),
-                    FileMetadata {
-                        last_modified: modified,
-                        symbol_count: symbols.len(),
-                    },
-                );
-            }
-        }
-
-        // Convert symbols to Arc
-        let arc_symbols: Vec<Arc<Symbol>> = symbols.into_iter().map(Arc::new).collect();
+        // Convert symbols to Arc and intern strings
+        let arc_symbols: Vec<Arc<Symbol>> = symbols
+            .into_iter()
+            .map(|mut symbol| {
+                // Intern file_path and module_path
+                symbol.file_path = self.string_cache.intern(symbol.file_path.as_ref());
+                symbol.module_path = self.string_cache.intern(symbol.module_path.as_ref());
+                Arc::new(symbol)
+            })
+            .collect();
 
         // Add new symbols
         all.extend(arc_symbols.clone());
@@ -80,11 +69,9 @@ impl SymbolIndex {
 
         let mut file_symbols = self.symbols.write();
         let mut all = self.all_symbols.write();
-        let mut metadata = self.file_metadata.write();
 
         file_symbols.remove(&canonical_path);
-        all.retain(|s| s.file_path != canonical_path);
-        metadata.remove(&canonical_path);
+        all.retain(|s| s.file_path.as_ref() != canonical_path.to_string_lossy().as_ref());
 
         Ok(())
     }
@@ -110,7 +97,6 @@ impl SymbolIndex {
     pub fn clear(&self) {
         self.symbols.write().clear();
         self.all_symbols.write().clear();
-        self.file_metadata.write().clear();
     }
 
     /// Get the total number of indexed files
@@ -124,45 +110,33 @@ impl SymbolIndex {
         self.symbols.read().contains_key(&canonical_path)
     }
 
-    /// Get metadata for a file
-    pub fn get_file_metadata(&self, path: &Path) -> Option<FileMetadata> {
-        let canonical_path = path.canonicalize().unwrap_or_else(|_| path.to_path_buf());
-        self.file_metadata.read().get(&canonical_path).cloned()
-    }
-
     /// Update specific files without full re-index
     pub fn update_files_batch(
         &self,
         updates: Vec<(PathBuf, Vec<Symbol>)>,
     ) -> Result<(usize, usize)> {
         let mut file_symbols = self.symbols.write();
         let mut all = self.all_symbols.write();
-        let mut metadata = self.file_metadata.write();
 
         let mut updated_files = 0;
         let mut total_symbols = 0;
 
         for (path, symbols) in updates {
             // Remove old symbols for this file if any
             if file_symbols.contains_key(&path) {
-                all.retain(|s| s.file_path != path);
+                all.retain(|s| s.file_path.as_ref() != path.to_string_lossy().as_ref());
             }
 
-            // Update metadata
-            if let Ok(file_metadata) = std::fs::metadata(&path) {
-                if let Ok(modified) = file_metadata.modified() {
-                    metadata.insert(
-                        path.clone(),
-                        FileMetadata {
-                            last_modified: modified,
-                            symbol_count: symbols.len(),
-                        },
-                    );
-                }
-            }
-
-            // Convert symbols to Arc
-            let arc_symbols: Vec<Arc<Symbol>> = symbols.into_iter().map(Arc::new).collect();
+            // Convert symbols to Arc and intern strings
+            let arc_symbols: Vec<Arc<Symbol>> = symbols
+                .into_iter()
+                .map(|mut symbol| {
+                    // Intern file_path and module_path
+                    symbol.file_path = self.string_cache.intern(symbol.file_path.as_ref());
+                    symbol.module_path = self.string_cache.intern(symbol.module_path.as_ref());
+                    Arc::new(symbol)
+                })
+                .collect();
             total_symbols += arc_symbols.len();
 
             // Add new symbols
@@ -179,45 +153,36 @@ impl SymbolIndex {
         // Acquire all locks in a consistent order to avoid deadlocks
         let mut symbols = self.symbols.write();
         let mut all_symbols = self.all_symbols.write();
-        let mut metadata = self.file_metadata.write();
 
         let new_symbols = new_index.symbols.read();
         let new_all = new_index.all_symbols.read();
-        let new_metadata = new_index.file_metadata.read();
 
         // Swap the contents
         *symbols = new_symbols.clone();
         *all_symbols = new_all.clone();
-        *metadata = new_metadata.clone();
     }
 
     /// Add multiple files in a single batch operation to minimize lock contention
     pub fn add_files_batch(&self, files: Vec<(PathBuf, Vec<Symbol>)>) -> Result<()> {
         let mut file_symbols = self.symbols.write();
         let mut all = self.all_symbols.write();
-        let mut metadata = self.file_metadata.write();
 
         for (path, symbols) in files {
             // Remove old symbols for this file if any
             if file_symbols.contains_key(&path) {
-                all.retain(|s| s.file_path != path);
-            }
-
-            // Update metadata
-            if let Ok(file_metadata) = std::fs::metadata(&path) {
-                if let Ok(modified) = file_metadata.modified() {
-                    metadata.insert(
-                        path.clone(),
-                        FileMetadata {
-                            last_modified: modified,
-                            symbol_count: symbols.len(),
-                        },
-                    );
-                }
+                all.retain(|s| s.file_path.as_ref() != path.to_string_lossy().as_ref());
             }
 
-            // Convert symbols to Arc
-            let arc_symbols: Vec<Arc<Symbol>> = symbols.into_iter().map(Arc::new).collect();
+            // Convert symbols to Arc and intern strings
+            let arc_symbols: Vec<Arc<Symbol>> = symbols
+                .into_iter()
+                .map(|mut symbol| {
+                    // Intern file_path and module_path
+                    symbol.file_path = self.string_cache.intern(symbol.file_path.as_ref());
+                    symbol.module_path = self.string_cache.intern(symbol.module_path.as_ref());
+                    Arc::new(symbol)
+                })
+                .collect();
 
             // Add new symbols
             all.extend(arc_symbols.clone());

pylight/src/lib.rs

@@ -9,6 +9,7 @@ pub mod index;
 pub mod lsp;
 pub mod parser;
 pub mod search;
+pub mod string_cache;
 pub mod symbols;
 pub mod watcher;
 

pylight/src/lsp/handlers.rs

@@ -53,7 +53,7 @@ pub fn handle_workspace_symbol(
             .take(200) // Limit results
             .filter_map(|result| {
                 let symbol = &result.symbol;
-                let uri = url::Url::from_file_path(&symbol.file_path).ok()?;
+                let uri = url::Url::from_file_path(symbol.file_path.as_ref()).ok()?;
 
                 #[allow(deprecated)]
                 Some(SymbolInformation {

pylight/src/parser/extractor.rs

@@ -72,7 +72,13 @@ impl<'a> SymbolExtractor<'a> {
             _ => SymbolKind::Function,
         };
 
-        let mut symbol = Symbol::new(name.clone(), kind, self.path.clone(), line, column);
+        let mut symbol = Symbol::new(
+            name.clone(),
+            kind,
+            self.path.to_string_lossy().into_owned(),
+            line,
+            column,
+        );
 
         // Set container name if we're inside another context
         if !self.context_stack.is_empty() {
@@ -131,7 +137,13 @@ impl<'a> SymbolExtractor<'a> {
             SymbolKind::Class
         };
 
-        let mut symbol = Symbol::new(name.clone(), kind, self.path.clone(), line, column);
+        let mut symbol = Symbol::new(
+            name.clone(),
+            kind,
+            self.path.to_string_lossy().into_owned(),
+            line,
+            column,
+        );
 
         // Set container name if we're inside another context
         if !self.context_stack.is_empty() {

pylight/src/parser/ruff.rs

@@ -7,7 +7,7 @@ use ruff_python_ast::{
 };
 use ruff_python_parser::{parse, Mode};
 use ruff_source_file::{LineIndex, SourceCode};
-use std::path::{Path, PathBuf};
+use std::path::Path;
 
 use super::r#trait::Parser;
 
@@ -33,15 +33,15 @@ enum Context {
 
 struct SymbolExtractor<'a> {
     symbols: &'a mut Vec<Symbol>,
-    file_path: PathBuf,
+    file_path: String,
     context_stack: Vec<Context>,
     source_code: SourceCode<'a, 'a>,
 }
 
 impl<'a> SymbolExtractor<'a> {
     fn new(
         symbols: &'a mut Vec<Symbol>,
-        file_path: PathBuf,
+        file_path: String,
         source: &'a str,
         line_index: &'a LineIndex,
     ) -> Self {
@@ -118,8 +118,7 @@ impl<'a> Visitor<'a> for SymbolExtractor<'a> {
                 let (line, column) = self.get_line_column(func_def.name.range.start().to_u32());
 
                 // Get module name from file path
-                let module_path = self
-                    .file_path
+                let module_path = std::path::Path::new(&self.file_path)
                     .file_stem()
                     .and_then(|s| s.to_str())
                     .unwrap_or("unknown")
@@ -148,8 +147,7 @@ impl<'a> Visitor<'a> for SymbolExtractor<'a> {
                 let (line, column) = self.get_line_column(class_def.name.range.start().to_u32());
 
                 // Get module name from file path
-                let module_path = self
-                    .file_path
+                let module_path = std::path::Path::new(&self.file_path)
                     .file_stem()
                     .and_then(|s| s.to_str())
                     .unwrap_or("unknown")
@@ -183,8 +181,12 @@ impl Parser for RuffParser {
             .map_err(|e| Error::Parse(format!("Ruff parse error: {e:?}")))?;
 
         let mut symbols = Vec::new();
-        let mut extractor =
-            SymbolExtractor::new(&mut symbols, file_path.to_path_buf(), source, &line_index);
+        let mut extractor = SymbolExtractor::new(
+            &mut symbols,
+            file_path.to_string_lossy().into_owned(),
+            source,
+            &line_index,
+        );
 
         match parsed.syntax() {
             Mod::Module(module) => {

pylight/src/parser/tests.rs

@@ -142,8 +142,7 @@ class MyClass:
             assert_eq!(symbols[0].line, 1);
             assert_eq!(
                 symbols[0].column, 4,
-                "Function name should start at column 4 (0-based) for parser {:?}",
-                backend
+                "Function name should start at column 4 (0-based) for parser {backend:?}"
             );
 
             // Test class column position
@@ -154,8 +153,7 @@ class MyClass:
             assert_eq!(symbols[0].line, 1);
             assert_eq!(
                 symbols[0].column, 6,
-                "Class name should start at column 6 (0-based) for parser {:?}",
-                backend
+                "Class name should start at column 6 (0-based) for parser {backend:?}"
             );
 
             // Test indented method column position
@@ -165,8 +163,7 @@ class MyClass:
             assert_eq!(method.line, 2);
             assert_eq!(
                 method.column, 8,
-                "Method name should start at column 8 (0-based) for parser {:?}",
-                backend
+                "Method name should start at column 8 (0-based) for parser {backend:?}"
             );
         }
     }