Keep source file filtering independent of parser grammars

Author: anhnh2002

codewiki/src/be/dependency_analyzer/analysis/analysis_service.py

@@ -11,10 +11,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
-from codewiki.src.be.dependency_analyzer.analysis.call_graph_analyzer import (
-    CallGraphAnalyzer,
-    is_skipped_source_file,
-)
+from codewiki.src.be.dependency_analyzer.analysis.call_graph_analyzer import CallGraphAnalyzer
 from codewiki.src.be.dependency_analyzer.analysis.cloning import (
     cleanup_repository,
     clone_repository,
@@ -23,6 +20,7 @@
 from codewiki.src.be.dependency_analyzer.analysis.repo_analyzer import RepoAnalyzer
 from codewiki.src.be.dependency_analyzer.models.analysis import AnalysisResult
 from codewiki.src.be.dependency_analyzer.models.core import Repository
+from codewiki.src.be.dependency_analyzer.utils.file_selection import is_skipped_source_file
 from codewiki.src.be.dependency_analyzer.utils.security import assert_safe_path, safe_open_text
 
 if TYPE_CHECKING:  # pragma: no cover

codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py

@@ -28,24 +28,6 @@
 logger = logging.getLogger(__name__)
 
 
-def is_skipped_source_file(file_info: dict, base_dir: str) -> bool:
-    """Reuse explicit analyzer exclusions when retaining symbol-free files."""
-    language = file_info.get("language")
-    if language == "go":
-        from codewiki.src.be.dependency_analyzer.analyzers.go import is_skipped_go_file
-
-        return is_skipped_go_file(file_info["path"])
-    if language == "zig":
-        from codewiki.src.be.dependency_analyzer.analyzers.zig import is_skipped_zig_file
-
-        return is_skipped_zig_file(file_info["path"])
-    if language == "php":
-        from codewiki.src.be.dependency_analyzer.analyzers.php import is_template_file
-
-        return is_template_file(str(Path(base_dir) / file_info["path"]))
-    return False
-
-
 class TimeoutError(BaseException):
     """Raised when file parsing exceeds timeout.
 

codewiki/src/be/dependency_analyzer/analyzers/go.py

@@ -51,6 +51,7 @@
 from tree_sitter_language_pack import get_parser
 
 from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node
+from codewiki.src.be.dependency_analyzer.utils.file_selection import is_skipped_go_file
 
 logger = logging.getLogger(__name__)
 
@@ -222,12 +223,6 @@ def split_go_package_ref(callee: str) -> tuple[str, str] | None:
     return None
 
 
-def is_skipped_go_file(relative_path: str) -> bool:
-    """`_test.go` files and vendored dependencies yield no components."""
-    posix = Path(relative_path).as_posix()
-    return posix.endswith("_test.go") or "vendor" in posix.split("/")[:-1]
-
-
 def package_dir_of(relative_path: str) -> str:
     return posixpath.dirname(Path(relative_path).as_posix())
 

codewiki/src/be/dependency_analyzer/analyzers/php.py

@@ -13,6 +13,11 @@
 from tree_sitter import Parser, Language
 import tree_sitter_php
 from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship
+from codewiki.src.be.dependency_analyzer.utils.file_selection import (
+    PHP_TEMPLATE_DIRECTORIES,
+    PHP_TEMPLATE_PATTERNS,
+    is_php_template_file,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -63,22 +68,13 @@
 }
 
 # Template file patterns to skip
-TEMPLATE_PATTERNS: Set[str] = {".blade.php", ".phtml", ".twig.php"}
-TEMPLATE_DIRECTORIES: Set[str] = {"views", "templates", "resources/views"}
+TEMPLATE_PATTERNS = PHP_TEMPLATE_PATTERNS
+TEMPLATE_DIRECTORIES = PHP_TEMPLATE_DIRECTORIES
 
 # Maximum recursion depth to prevent stack overflow
 MAX_RECURSION_DEPTH = 100
 
 
-def is_template_file(file_path: str) -> bool:
-    """The same exclusion policy for parsing and the source-file inventory."""
-    file_str = str(file_path)
-    return any(file_str.endswith(pattern) for pattern in TEMPLATE_PATTERNS) or any(
-        f"/{directory}/" in file_str or f"\\{directory}\\" in file_str
-        for directory in TEMPLATE_DIRECTORIES
-    )
-
-
 class NamespaceResolver:
     """Resolves PHP class names to fully qualified names using use statements."""
 
@@ -147,7 +143,7 @@ def __init__(self, file_path: str, content: str, repo_path: str = None):
 
     def _is_template_file(self) -> bool:
         """Check if file is a PHP template that should be skipped."""
-        return is_template_file(str(self.file_path))
+        return is_php_template_file(str(self.file_path))
 
     def _get_module_path(self) -> str:
         """Get module path for the file."""

codewiki/src/be/dependency_analyzer/analyzers/zig.py

@@ -50,6 +50,7 @@
 from tree_sitter_language_pack import get_parser
 
 from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node
+from codewiki.src.be.dependency_analyzer.utils.file_selection import is_skipped_zig_file
 
 logger = logging.getLogger(__name__)
 
@@ -61,7 +62,6 @@
     "opaque": ("class", "opaque"),
 }
 
-_SKIPPED_DIRS = {"zig-cache", ".zig-cache", "zig-out"}
 _EXTERNAL_IMPORTS = {"std", "builtin", "root"}
 # Constructors by convention: their result is a value of the owning type.
 _CONSTRUCTOR_NAMES = {"init", "initCapacity", "create", "new", "open", "parse"}
@@ -84,10 +84,6 @@
 _UNKNOWN = ("unknown",)
 
 
-def is_skipped_zig_file(relative_path: str) -> bool:
-    return any(part in _SKIPPED_DIRS for part in Path(relative_path).parts[:-1])
-
-
 @lru_cache(maxsize=64)
 def _parse_build_zig(path: str, mtime_ns: int) -> tuple[tuple[str, str], ...]:
     try:

codewiki/src/be/dependency_analyzer/utils/file_selection.py

@@ -0,0 +1,37 @@
+"""Explicit source-file exclusions, without importing optional parser grammars."""
+
+from pathlib import Path
+
+PHP_TEMPLATE_PATTERNS = {".blade.php", ".phtml", ".twig.php"}
+PHP_TEMPLATE_DIRECTORIES = {"views", "templates", "resources/views"}
+ZIG_SKIPPED_DIRS = {"zig-cache", ".zig-cache", "zig-out"}
+
+
+def is_skipped_go_file(relative_path: str) -> bool:
+    """`_test.go` files and vendored dependencies yield no components."""
+    posix = Path(relative_path).as_posix()
+    return posix.endswith("_test.go") or "vendor" in posix.split("/")[:-1]
+
+
+def is_skipped_zig_file(relative_path: str) -> bool:
+    return any(part in ZIG_SKIPPED_DIRS for part in Path(relative_path).parts[:-1])
+
+
+def is_php_template_file(file_path: str) -> bool:
+    file_str = str(file_path)
+    return any(file_str.endswith(pattern) for pattern in PHP_TEMPLATE_PATTERNS) or any(
+        f"/{directory}/" in file_str or f"\\{directory}\\" in file_str
+        for directory in PHP_TEMPLATE_DIRECTORIES
+    )
+
+
+def is_skipped_source_file(file_info: dict, base_dir: str) -> bool:
+    """Apply the same exclusions for parsing and symbol-free file retention."""
+    language = file_info.get("language")
+    if language == "go":
+        return is_skipped_go_file(file_info["path"])
+    if language == "zig":
+        return is_skipped_zig_file(file_info["path"])
+    if language == "php":
+        return is_php_template_file(str(Path(base_dir) / file_info["path"]))
+    return False

tests/test_module_scope.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import asyncio
+import builtins
 import json
 from pathlib import Path
 from types import SimpleNamespace
@@ -191,6 +192,53 @@ def test_capped_artifact_does_not_hide_symbol_free_source_code(tmp_path):
     assert get_module_token_count(components) >= count_tokens(source)
 
 
+@pytest.mark.parametrize(
+    ("language", "included", "excluded"),
+    [
+        ("go", "source.go", "vendor/source.go"),
+        ("zig", "source.zig", "zig-out/source.zig"),
+        ("php", "source.php", "templates/source.php"),
+    ],
+)
+def test_file_exclusions_do_not_import_parsers(monkeypatch, language, included, excluded):
+    from codewiki.src.be.dependency_analyzer.utils.file_selection import is_skipped_source_file
+
+    original_import = builtins.__import__
+
+    def without_grammars(name, *args, **kwargs):
+        if name.startswith(("tree_sitter", "codewiki.src.be.dependency_analyzer.analyzers")):
+            raise ModuleNotFoundError(name)
+        return original_import(name, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "__import__", without_grammars)
+    assert not is_skipped_source_file({"language": language, "path": included}, "/repo")
+    assert is_skipped_source_file({"language": language, "path": excluded}, "/repo")
+
+
+def test_symbol_free_file_survives_missing_php_grammar(tmp_path, monkeypatch):
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    source = "<?php\nconst CAPACITY = 16;\n"
+    (repo / "source.php").write_text(source)
+    templates = repo / "templates"
+    templates.mkdir()
+    (templates / "view.php").write_text("<?php echo 'view';\n")
+    original_import = builtins.__import__
+
+    def without_php(name, *args, **kwargs):
+        if name in ("tree_sitter_php", "codewiki.src.be.dependency_analyzer.analyzers.php"):
+            raise ModuleNotFoundError(name)
+        return original_import(name, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "__import__", without_php)
+    components, leaves = DependencyGraphBuilder(
+        _config(repo, tmp_path / "out")
+    ).build_dependency_graph()
+    assert component_files(list(components), components) == {"source.php"}
+    assert components["source.php::__file__"].source_code == source
+    assert complete_file_candidates(leaves, components) == ["source.php::__file__"]
+
+
 def test_recursive_size_uses_child_files_not_whole_graph(tmp_path):
     components = {
         cid: _node(cid, source)