diff --git a/api/poetry.lock b/api/poetry.lock index af86aa3f9..63d50530e 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -958,6 +958,40 @@ files = [ {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, ] +[[package]] +name = "gitdb" +version = "4.0.12" +description = "Git Object Database" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, + {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.58" +description = "GitPython is a Python library used to interact with Git repositories" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "gitpython-3.1.58-py3-none-any.whl", hash = "sha256:d331e722577f0fd7fc1f857419b3ecc07af66282b933d2a4d95f84a042fdd50f"}, + {file = "gitpython-3.1.58.tar.gz", hash = "sha256:621416df10ef3fd0e19fabf9172ddeed0fa704d353d04f194eec56a625a95b22"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" + +[package.extras] +doc = ["sphinx (>=7.4.7,<8)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] +test = ["basedpyright (==1.39.9) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy (==1.18.2) ; python_version >= \"3.9\"", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] + [[package]] name = "google-ai-generativelanguage" version = "0.6.15" @@ -2825,6 +2859,18 @@ files = [ {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] +[[package]] +name = "smmap" +version = "5.0.3" +description = "A pure Python implementation of a sliding window memory map manager" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f"}, + {file = "smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c"}, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -3458,4 +3504,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "86ee62820be3180e14bcd1bb48607994e24ba6648e13a346b8cee26fcdc89a06" +content-hash = "530d6dc9f317228da098c01ccf1e0fee602cb197b115a272cfa171fb8ad3f2f7" diff --git a/api/pyproject.toml b/api/pyproject.toml index ad1f5d32c..5cdf552b7 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -29,6 +29,7 @@ websockets = ">=11.0.3" azure-identity = ">=1.12.0" azure-core = ">=1.24.0" anthropic = "^0.117.1" +gitpython = "^3.1.58" [build-system] diff --git a/api/repository.py b/api/repository.py index 3ef46db91..b87b8ff5b 100644 --- a/api/repository.py +++ b/api/repository.py @@ -1,7 +1,11 @@ import os import subprocess +from functools import wraps +from collections.abc import Callable from urllib.parse import quote, urlparse, urlunparse +from git import Repo as GitRepo, GIT_OK, GitCommandError + from api.logger import get_logger from api.utils import deepwiki_root @@ -11,120 +15,107 @@ CLONE_REPO_ROOT = os.path.join(deepwiki_root(), "repo") -def download_repo( - repo_url: str, local_path: str, repo_type: str = None, access_token: str = None -) -> str: - """ - Downloads a Git repository (GitHub, GitLab, or Bitbucket) to a specified local path. - - Args: - repo_type(str): Type of repository - repo_url (str): The URL of the Git repository to clone. - local_path (str): The local directory where the repository will be cloned. - access_token (str, optional): Access token for private repositories. - - Returns: - str: The output message from the `git` command. - """ - try: - # Check if Git is installed - logger.info(f"Preparing to clone repository to {local_path}") - subprocess.run( - ["git", "--version"], - check=True, - capture_output=True, +def _exception_cleanup(func: Callable) -> Callable: + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except (subprocess.CalledProcessError, GitCommandError) as e: + err_msg: str | bytes = e.stderr + if isinstance(err_msg, bytes): + err_msg = err_msg.decode("utf-8") + token = kwargs.get("access_token", None) + if token: + token_mask = "***TOKEN***" + err_msg = err_msg.replace(token, token_mask) + encoded_token = quote(token, safe="") + err_msg = err_msg.replace(encoded_token, token_mask) + raise ValueError(err_msg) + + return wrapper + + +@_exception_cleanup +def _clone_from_gitlab( + remote_url: str, + local_path: str, + *, + access_token: str | None = None, + **kwargs, +) -> GitRepo: + if access_token: + parsed = urlparse(remote_url) + access_token = quote(access_token, safe="") + + remote_url = urlunparse( + ( + parsed.scheme, + f"oauth2:{access_token}@{parsed.netloc}", + parsed.path, + "", + "", + "", + ) ) - - # Check if repository already exists - if os.path.exists(local_path) and os.listdir(local_path): - # Directory exists and is not empty - logger.warning( - f"Repository already exists at {local_path}. Using existing repository." + return GitRepo.clone_from(url=remote_url, to_path=local_path, **kwargs) + + +@_exception_cleanup +def _clone_from_github( + remote_url: str, + local_path: str, + *, + access_token: str | None = None, + **kwargs, +) -> GitRepo: + if access_token: + parsed = urlparse(remote_url) + + remote_url = urlunparse( + ( + parsed.scheme, + f"{access_token}@{parsed.netloc}", + parsed.path, + "", + "", + "", ) - return f"Using existing repository at {local_path}" - - # Ensure the local path exists - os.makedirs(local_path, exist_ok=True) - - # Prepare the clone URL with access token if provided - clone_url = repo_url - if access_token: - parsed = urlparse(repo_url) - # URL-encode the token to handle special characters - encoded_token = quote(access_token, safe="") - # Determine the repository type and format the URL accordingly - if repo_type == "github": - # Format: https://{token}@{domain}/owner/repo.git - # Works for both github.com and enterprise GitHub domains - clone_url = urlunparse( - ( - parsed.scheme, - f"{encoded_token}@{parsed.netloc}", - parsed.path, - "", - "", - "", - ) - ) - elif repo_type == "gitlab": - # Format: https://oauth2:{token}@gitlab.com/owner/repo.git - clone_url = urlunparse( - ( - parsed.scheme, - f"oauth2:{encoded_token}@{parsed.netloc}", - parsed.path, - "", - "", - "", - ) - ) - elif repo_type == "bitbucket": - # Bitbucket has two token formats with different auth schemes: - # - HTTP access tokens (prefix "ATCTT") use x-bitbucket-api-token-auth - # - App passwords (deprecated, EOL June 2026) use x-token-auth - # Detect by token prefix so existing app password users keep working. - if access_token.startswith("ATCTT"): - auth_scheme = "x-bitbucket-api-token-auth" - else: - auth_scheme = "x-token-auth" - # Format: https://{auth_scheme}:{token}@bitbucket.org/owner/repo.git - clone_url = urlunparse( - ( - parsed.scheme, - f"{auth_scheme}:{encoded_token}@{parsed.netloc}", - parsed.path, - "", - "", - "", - ) - ) - - logger.info("Using access token for authentication") - - # Clone the repository - logger.info(f"Cloning repository from {repo_url} to {local_path}") - # We use repo_url in the log to avoid exposing the token in logs - result = subprocess.run( - ["git", "clone", "--depth=1", "--single-branch", clone_url, local_path], - check=True, - capture_output=True, ) - - logger.info("Repository cloned successfully") - return result.stdout.decode("utf-8") - - except subprocess.CalledProcessError as e: - error_msg = e.stderr.decode("utf-8") - # Sanitize error message to remove any tokens (both raw and URL-encoded) - if access_token: - # Remove raw token - error_msg = error_msg.replace(access_token, "***TOKEN***") - # Also remove URL-encoded token to prevent leaking encoded version - encoded_token = quote(access_token, safe="") - error_msg = error_msg.replace(encoded_token, "***TOKEN***") - raise ValueError(f"Error during cloning: {error_msg}") - except Exception as e: - raise ValueError(f"An unexpected error occurred: {str(e)}") + return GitRepo.clone_from(url=remote_url, to_path=local_path, **kwargs) + + +@_exception_cleanup +def _clone_from_bitbucket( + remote_url: str, + local_path: str, + *, + access_token: str | None = None, + **kwargs, +) -> GitRepo: + if access_token: + parsed = urlparse(remote_url) + # Bitbucket has two token formats with different auth schemes: + # - HTTP access tokens (prefix "ATCTT") use x-bitbucket-api-token-auth + # - App passwords (deprecated, EOL June 2026) use x-token-auth + # Detect by token prefix so existing app password users keep working. + auth_scheme = ( + "x-bitbucket-api-token-auth" + if access_token.startswith("ATCTT") + else "x-token-auth" + ) + access_token = quote(access_token, safe="") + + remote_url = urlunparse( + ( + parsed.scheme, + f"{auth_scheme}:{access_token}@{parsed.netloc}", + parsed.path, + "", + "", + "", + ) + ) + return GitRepo.clone_from(url=remote_url, to_path=local_path, **kwargs) def _path_is_url(path: str) -> bool: @@ -200,9 +191,29 @@ def _extract_repo_name(repo_url: str, repo_type: str | None) -> str: def download(self, force: bool = False) -> None: if force or (not self.downloaded and not self.is_local): os.makedirs(self.save_path, exist_ok=True) - download_repo( - self.repo_url, self.save_path, self.repo_type, self.access_token - ) + + if not GIT_OK: + raise RuntimeError("Missing `git` in current environment") + + kwargs = { + "remote_url": self.repo_url, + "local_path": self.save_path, + "access_token": self.access_token, + "multi_options": ["--depth=1", "--single-branch"], + } + + if self.repo_type == "github": + _clone_from_github(**kwargs) + + elif self.repo_type == "gitlab": + _clone_from_gitlab(**kwargs) + + elif self.repo_type == "bitbucket": + _clone_from_bitbucket(**kwargs) + else: + raise NotImplementedError(f"Unknown repo type: {self.repo_type}") + + logger.info("Repository %s cloned successfully", self.name) @property def save_path(self) -> str: @@ -213,3 +224,6 @@ def save_path(self) -> str: @property def downloaded(self) -> bool: return os.path.exists(self.save_path) and bool(os.listdir(self.save_path)) + + def __repr__(self) -> str: + return f"{self.repo_type}: {self.name}" diff --git a/tests/test_repository.py b/tests/test_repository.py new file mode 100644 index 000000000..c36cdfa5d --- /dev/null +++ b/tests/test_repository.py @@ -0,0 +1,85 @@ +import pytest +import re +import os + +import git + +from api.repository import Repo + + +def test_repo_is_local(): + repo = Repo(repo_url="./", repo_type="local") + assert repo.is_local + + +def test_repo_is_remote(tmpdir): + repo = Repo( + repo_url="https://github.com/AsyncFuncAI/deepwiki-open", + repo_type="github", + root_path=tmpdir, + ) + assert not repo.is_local + assert not repo.downloaded + + +def test_repo_download_no_git(tmpdir, monkeypatch): + repo = Repo( + repo_url="https://github.com/AsyncFuncAI/deepwiki-open", + repo_type="github", + root_path=tmpdir, + ) + from api import repository + monkeypatch.setattr(repository, "GIT_OK", value=False) + + with pytest.raises(RuntimeError, match="Missing `git` in current environment"): + repo.download() + + +def test_repo_download_path_exists(tmpdir, mocker): + repo = Repo( + repo_url="https://github.com/AsyncFuncAI/deepwiki-open", + repo_type="github", + root_path=tmpdir, + ) + + def touch_file(*args, **kwargs): + tmp_file = os.path.join(repo.save_path, "touch") + with open(tmp_file, "w") as f: + f.write("") + mocker.patch.object(git.Repo, "clone_from", return_value=None, side_effect=touch_file) + + repo.download() + assert repo.downloaded + assert os.path.exists(repo.save_path) + + +def test_repo_git_clone_message_masking(tmpdir, mocker): + repo = Repo( + repo_url="https://github.com/AsyncFuncAI/deepwiki-open", + repo_type="github", + root_path=tmpdir, + access_token="123456789" + ) + + def raise_error(*args, **kwargs): + raise git.GitCommandError(command="git clone", stderr="123456789 is not a valid token") + + mocker.patch.object(git.Repo, "clone_from", return_value=None, side_effect=raise_error) + + with pytest.raises(ValueError, match=re.escape("***TOKEN*** is not a valid token")): + repo.download() + + +@pytest.mark.network +@pytest.mark.parametrize( + "repo_url, repo_type", + [ + ("https://github.com/AsyncFuncAI/deepwiki-open", "github"), + ("https://gitlab.com/gitlab-org/gitlab-pages", "gitlab"), + ] +) +def test_repo_download(repo_url, repo_type, tmpdir): + repo = Repo(repo_url, repo_type, root_path=tmpdir) + repo.download() + + assert repo.downloaded