chore: improve max download restrictions for malicious metadata tutorial (#1188)

art1f1c3R · web-flow · commit a9a1ecf397db · 2025-09-26T20:24:51.000+10:00
Maximum file download size is now stored in the [downloads] section. UNKNOWN is no longer returned when the source code is larger than the file limit.

Signed-off-by: Carl Flottmann &lt;carl.flottmann@oracle.com&gt;
diff --git a/docs/source/pages/tutorials/detect_malicious_package.rst b/docs/source/pages/tutorials/detect_malicious_package.rst
@@ -136,6 +136,8 @@ By default, the source code analyzer is run in conjunction with the other metada
 
   ./run_macaron.sh analyze -purl pkg:pypi/django@5.0.6 --python-venv "/tmp/.django_venv" --force-analyze-source
 
+.. note:: Some packages source code, like ``django@5.0.6``, will be larger than the default download limit of 10 megabytes. This is controlled using the ``max_download_size`` configuration under ``downloads`` in ``defaults.ini``, and can be increased by either modifying that value in ``defaults.ini`` or by passing in a configuration file using ``-dp`` with this value increased.
+
 If any suspicious patterns are triggered, this will be identified in the ``mcn_detect_malicious_metadata_1`` result for the heuristic named ``suspicious_patterns``. The output database ``output/macaron.db`` can be used to get the specific results of the analysis by querying the :class:`detect_malicious_metadata_check.result field <macaron.database>`. This will provide detailed JSON information about all data collected by the ``mcn_detect_malicious_metadata_1`` check, including, for source code analysis, any malicious code patterns detected, what Semgrep rule detected it, the file in which it was detected, and the line number for the detection.
 
 +++++++++++++++++++++++++++++++++++++++
diff --git a/docs/source/pages/tutorials/provenance.rst b/docs/source/pages/tutorials/provenance.rst
@@ -204,7 +204,7 @@ Build Types
 File Download Limit
 *******************
 
-To prevent analyses from taking too long, Macaron imposes a configurable size limit for downloads. This includes files being downloaded for provenance verification. In cases where the limit is being reached and you wish to continue analysis regardless, you can specify a new download size in the default configuration file. This value can be found under the ``slsa.verifier`` section, listed as ``max_download_size`` with a default limit of 10 megabytes. See :ref:`How to change the default configuration <change-config>` for more details on configuring values like these.
+To prevent analyses from taking too long, Macaron imposes a configurable size limit for downloads. This includes files being downloaded for provenance verification. In cases where the limit is being reached and you wish to continue analysis regardless, you can specify a new download size in the default configuration file. This value can be found under the ``downloads`` section, listed as ``max_download_size`` with a default limit of 10 megabytes. See :ref:`How to change the default configuration <change-config>` for more details on configuring values like these.
 
 **************************************
 Run ``verify-policy`` command (semver)
diff --git a/src/macaron/config/defaults.ini b/src/macaron/config/defaults.ini
@@ -10,6 +10,8 @@ error_retries = 5
 [downloads]
 # The default timeout in seconds for downloading assets.
 timeout = 120
+# This is the acceptable maximum size (in bytes) to download an asset.
+max_download_size = 10000000
 
 # This is the database to store Macaron's results.
 [database]
@@ -486,8 +488,6 @@ provenance_extensions =
     intoto.jsonl.gz
     intoto.jsonl.url
     intoto.jsonl.gz.url
-# This is the acceptable maximum size (in bytes) to download an asset.
-max_download_size = 10000000
 # This is the timeout (in seconds) to run the SLSA verifier.
 timeout = 120
 # The allowed hostnames for URL file links for provenance download
diff --git a/src/macaron/malware_analyzer/pypi_heuristics/sourcecode/suspicious_setup.py b/src/macaron/malware_analyzer/pypi_heuristics/sourcecode/suspicious_setup.py
@@ -56,7 +56,7 @@ def _get_setup_source_code(self, pypi_package_json: PyPIPackageJsonAsset) -> str
         with tempfile.TemporaryDirectory() as temp_dir:
             source_file = os.path.join(temp_dir, file_name)
             timeout = defaults.getint("downloads", "timeout", fallback=120)
-            size_limit = defaults.getint("slsa.verifier", "max_download_size", fallback=10000000)
+            size_limit = defaults.getint("downloads", "max_download_size", fallback=10000000)
             if not download_file_with_size_limit(sourcecode_url, {}, source_file, timeout, size_limit):
                 return None
 
diff --git a/src/macaron/provenance/provenance_finder.py b/src/macaron/provenance/provenance_finder.py
@@ -255,7 +255,7 @@ def find_gav_provenance(purl: PackageURL, registry: JFrogMavenRegistry) -> list[
         return []
 
     max_valid_provenance_size = defaults.getint(
-        "slsa.verifier",
+        "downloads",
         "max_download_size",
         fallback=1000000,
     )
@@ -458,7 +458,7 @@ def download_provenances_from_ci_service(ci_info: CIInfo, download_path: str) ->
         for prov_asset in prov_assets:
             # Check the size before downloading.
             if prov_asset.size_in_bytes > defaults.getint(
-                "slsa.verifier",
+                "downloads",
                 "max_download_size",
                 fallback=1000000,
             ):
diff --git a/src/macaron/provenance/provenance_verifier.py b/src/macaron/provenance/provenance_verifier.py
@@ -190,7 +190,7 @@ def verify_ci_provenance(analyze_ctx: AnalyzeContext, ci_info: CIInfo, download_
                 return False
             if not Path(download_path, sub_asset["name"]).is_file():
                 if "size" in sub_asset and sub_asset["size"] > defaults.getint(
-                    "slsa.verifier", "max_download_size", fallback=1000000
+                    "downloads", "max_download_size", fallback=1000000
                 ):
                     logger.debug("Sub asset too large to verify: %s", sub_asset["name"])
                     return False
diff --git a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py
@@ -148,6 +148,10 @@ def analyze_source(
         if not force and analyzer.depends_on and self._should_skip(results, analyzer.depends_on):
             return {analyzer.heuristic: HeuristicResult.SKIP}, {}
 
+        if not pypi_package_json.can_download_sourcecode():
+            logger.debug("Source code will exceed download limits. Please increase the download size limit to analyze.")
+            return {analyzer.heuristic: HeuristicResult.SKIP}, {}
+
         try:
             with pypi_package_json.sourcecode():
                 result, detail_info = analyzer.analyze(pypi_package_json)
diff --git a/src/macaron/slsa_analyzer/git_service/api_client.py b/src/macaron/slsa_analyzer/git_service/api_client.py
@@ -643,7 +643,7 @@ def download_asset(self, url: str, download_path: str) -> bool:
         logger.debug("Download assets from %s at %s.", url, download_path)
 
         timeout = defaults.getint("downloads", "timeout", fallback=120)
-        size_limit = defaults.getint("slsa.verifier", "max_download_size", fallback=10000000)
+        size_limit = defaults.getint("downloads", "max_download_size", fallback=10000000)
         headers = {"Accept": "application/octet-stream", "Authorization": self.headers["Authorization"]}
 
         return download_file_with_size_limit(url, headers, download_path, timeout, size_limit)
diff --git a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py
@@ -286,7 +286,7 @@ def get_artifact_hash(self, purl: PackageURL) -> str | None:
 
         hash_algorithm = hashlib.sha256()
         timeout = defaults.getint("downloads", "timeout", fallback=120)
-        size_limit = defaults.getint("slsa.verifier", "max_download_size", fallback=10000000)
+        size_limit = defaults.getint("downloads", "max_download_size", fallback=10000000)
         if not stream_file_with_size_limit(artifact_url, {}, hash_algorithm.update, timeout, size_limit):
             return None
 
diff --git a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py
@@ -26,7 +26,12 @@
 from macaron.json_tools import json_extract
 from macaron.malware_analyzer.datetime_parser import parse_datetime
 from macaron.slsa_analyzer.package_registry.package_registry import PackageRegistry
-from macaron.util import download_file_with_size_limit, send_get_http_raw, stream_file_with_size_limit
+from macaron.util import (
+    can_download_file,
+    download_file_with_size_limit,
+    send_get_http_raw,
+    stream_file_with_size_limit,
+)
 
 if TYPE_CHECKING:
     from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo
@@ -209,6 +214,23 @@ def cleanup_sourcecode_directory(
             raise InvalidHTTPResponseError(error_message) from error
         raise InvalidHTTPResponseError(error_message)
 
+    def can_download_package_sourcecode(self, url: str) -> bool:
+        """Check if the package source code can be downloaded within the default file limits.
+
+        Parameters
+        ----------
+        url: str
+            The package source code url.
+
+        Returns
+        -------
+        bool
+            True if it can be downloaded within the size limits, otherwise False.
+        """
+        size_limit = defaults.getint("downloads", "max_download_size", fallback=10000000)
+        timeout = defaults.getint("downloads", "timeout", fallback=120)
+        return can_download_file(url, size_limit, timeout=timeout)
+
     def download_package_sourcecode(self, url: str) -> str:
         """Download the package source code from pypi registry.
 
@@ -235,7 +257,7 @@ def download_package_sourcecode(self, url: str) -> str:
         temp_dir = tempfile.mkdtemp(prefix=f"{package_name}_")
         source_file = os.path.join(temp_dir, file_name)
         timeout = defaults.getint("downloads", "timeout", fallback=120)
-        size_limit = defaults.getint("slsa.verifier", "max_download_size", fallback=10000000)
+        size_limit = defaults.getint("downloads", "max_download_size", fallback=10000000)
         if not download_file_with_size_limit(url, {}, source_file, timeout, size_limit):
             self.cleanup_sourcecode_directory(temp_dir, "Could not download the file.")
 
@@ -273,7 +295,7 @@ def get_artifact_hash(self, artifact_url: str) -> str | None:
         """
         hash_algorithm = hashlib.sha256()
         timeout = defaults.getint("downloads", "timeout", fallback=120)
-        size_limit = defaults.getint("slsa.verifier", "max_download_size", fallback=10000000)
+        size_limit = defaults.getint("downloads", "max_download_size", fallback=10000000)
         if not stream_file_with_size_limit(artifact_url, {}, hash_algorithm.update, timeout, size_limit):
             return None
 
@@ -624,6 +646,19 @@ def download_sourcecode(self) -> bool:
                 logger.debug(error)
         return False
 
+    def can_download_sourcecode(self) -> bool:
+        """Return whether the package source code can be downloaded within the download file size limits.
+
+        Returns
+        -------
+        bool
+            ``True`` if the source code can be downloaded; ``False`` if not.
+        """
+        url = self.get_sourcecode_url()
+        if url:
+            return self.pypi_registry.can_download_package_sourcecode(url)
+        return False
+
     def get_sourcecode_file_contents(self, path: str) -> bytes:
         """
         Get the contents of a single source code file specified by the path.
diff --git a/src/macaron/util.py b/src/macaron/util.py
@@ -286,6 +286,35 @@ def chunk_function(self, chunk: bytes) -> None:
         self.file.write(chunk)
 
 
+def can_download_file(url: str, size_limit: int, timeout: int | None = None) -> bool:
+    """Send a head request to check if the file provided at url can be downloaded within the size limit.
+
+    It expects a URL to a file, and checks the "Content-Length" field of the response.
+
+    Parameters
+    ----------
+    url: str
+        The target of the request.
+    size_limit: int
+        The size limit in bytes of the file.
+    timeout: int | None
+        The request timeout (optional).
+
+    Returns
+    -------
+    bool
+        True if the file can be downloaded within the size limit, False otherwise.
+    """
+    response = send_head_http_raw(url, timeout=timeout, allow_redirects=True)
+    if not response:
+        return False
+
+    size = response.headers.get("Content-Length")
+    if size and int(size) <= size_limit:
+        return True
+    return False
+
+
 def download_file_with_size_limit(
     url: str, headers: dict, file_path: str, timeout: int = 40, size_limit: int = 0
 ) -> bool:
diff --git a/tests/integration/cases/django_with_dep_resolution_virtual_env_as_input/config.ini b/tests/integration/cases/django_with_dep_resolution_virtual_env_as_input/config.ini
@@ -1,5 +1,5 @@
 # Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
-[slsa.verifier]
+[downloads]
 max_download_size = 15000000
diff --git a/tests/integration/cases/ossf_scorecard/config.ini b/tests/integration/cases/ossf_scorecard/config.ini
@@ -1,12 +1,12 @@
 # Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
+[downloads]
+max_download_size = 15000000
+
 [analysis.checks]
 exclude =
 include =
     mcn_provenance_expectation_1
     mcn_provenance_verified_1
     mcn_trusted_builder_level_three_1
-
-[slsa.verifier]
-max_download_size = 15000000

-Original file line number
+Diff line change
   ./run_macaron.sh analyze -purl pkg:pypi/[email protected] --python-venv "/tmp/.django_venv" --force-analyze-source
 +.. note:: Some packages source code, like ``[email protected]``, will be larger than the default download limit of 10 megabytes. This is controlled using the ``max_download_size`` configuration under ``downloads`` in ``defaults.ini``, and can be increased by either modifying that value in ``defaults.ini`` or by passing in a configuration file using ``-dp`` with this value increased.
++
 If any suspicious patterns are triggered, this will be identified in the ``mcn_detect_malicious_metadata_1`` result for the heuristic named ``suspicious_patterns``. The output database ``output/macaron.db`` can be used to get the specific results of the analysis by querying the :class:`detect_malicious_metadata_check.result field <macaron.database>`. This will provide detailed JSON information about all data collected by the ``mcn_detect_malicious_metadata_1`` check, including, for source code analysis, any malicious code patterns detected, what Semgrep rule detected it, the file in which it was detected, and the line number for the detection.
 +++++++++++++++++++++++++++++++++++++++