Make list comparison accept lists in documents and support matching networks (#885)

* Make list comparison accept lists in documents and support matching networks

* Fix black formatting

* Split out network comparison from list_comparison processor to network_comparison processor

* Make network comparison work with getter callbacks and add tests

* Refactor list_comparison and network_comparison processor

* Refactor network_comparison processor testdata and fix name in docstring

* Make network_comparison handle warning errors

* Add missing line endings

* Fix mypy

* Separate network_comparison config from list_comparison config

* Make network_comparison rule loading fail if network is invalid

Author: ppcad

CHANGELOG.md

@@ -26,6 +26,8 @@
 * make generic adder processor be refreshable with http getter
 * make generic resolver processor be refreshable with http getter
 * add option for refreshable getters to return default values if no value could be obtained
+* list comparison processor can now also match fields that contain lists in documents
+* add network comparison processor that can match IPs with networks in CIDR notation
 
 ### Improvements
 

logprep/ng/processor/list_comparison/processor.py

@@ -54,9 +54,9 @@ def setup(self) -> None:
         for rule in self.rules:
             rule.init_list_comparison(self._config.list_search_base_path)
 
-    def _apply_rules(self, event: dict, rule: ListComparisonRule) -> None:
-        """
-        Apply matching rule to given log event.
+    def _apply_rules(self, event, rule):
+        """Apply matching rule to given log event.
+
         In the process of doing so, add the result of comparing
         the log with a given list to the specified subfield. This subfield will contain
         a list of list comparison results that might be based on multiple rules.
@@ -75,20 +75,38 @@ def _apply_rules(self, event: dict, rule: ListComparisonRule) -> None:
             add_fields_to(event, fields, rule=rule, merge_with_target=True)
 
     def _list_comparison(self, rule: ListComparisonRule, event: dict) -> tuple[list[str], str]:
-        """
-        Check if field value violates block or allow list.
-        Returns the result of the comparison (res_key), as well as a dictionary containing
-        the result (key) and a list of filenames pertaining to said result (value).
+        """Check if field value violates block or allow list.
+
+        Returns
+        -------
+        tuple[list[str], str]
+            The result of the comparison, as well as a dictionary containing the result and a list
+            of filenames pertaining to said result.
+
         """
 
-        field_value = get_dotted_field_value(event, rule.source_fields[0])
+        field_value_to_be_checked = get_dotted_field_value(event, rule.source_fields[0])
+        value_list = (
+            field_value_to_be_checked
+            if isinstance(field_value_to_be_checked, list)
+            else [field_value_to_be_checked]
+        )
 
-        list_matches = [
-            compare_list
-            for compare_list in rule.compare_sets
-            if field_value in rule.compare_sets[compare_list]
-        ]
+        list_matches = self._get_lists_matching_with_values(rule, value_list, event)
 
-        if not list_matches:
+        if len(list_matches) == 0:
             return list(rule.compare_sets.keys()), "not_in_list"
         return list_matches, "in_list"
+
+    def _get_lists_matching_with_values(
+        self, rule: ListComparisonRule, value_list: list, _: dict
+    ) -> list:
+        """Iterate over string lists, check if element is in any."""
+        list_matches = []
+        for value in value_list:
+            for compare_list in rule.compare_sets:
+                if compare_list in list_matches:
+                    continue
+                if value in rule.compare_sets[compare_list]:
+                    list_matches.append(compare_list)
+        return list_matches

logprep/ng/processor/network_comparison/__init__.py

@@ -0,0 +1,58 @@
+"""
+NetworkComparison
+=================
+
+The `ng_network_comparison` processor allows to compare values of source fields against lists provided
+as files.
+
+
+Processor Configuration
+^^^^^^^^^^^^^^^^^^^^^^^
+..  code-block:: yaml
+    :linenos:
+
+    - networkcomparisonname:
+        type: ng_network_comparison
+        rules:
+            - tests/testdata/rules/rules
+        list_search_base_path: /path/to/list/dir
+
+.. autoclass:: logprep.ng.processor.network_comparison.processor.NetworkComparison.Config
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :noindex:
+
+.. automodule:: logprep.ng.processor.network_comparison.rule
+"""
+
+from ipaddress import ip_address
+
+from logprep.ng.processor.list_comparison.processor import ListComparison
+from logprep.processor.network_comparison.rule import NetworkComparisonRule
+
+
+class NetworkComparison(ListComparison):
+    """Resolve values in documents by referencing a mapping list."""
+
+    rule_class = NetworkComparisonRule
+
+    def _get_lists_matching_with_values(
+        self, rule: NetworkComparisonRule, value_list: list, event: dict
+    ) -> list:
+        """Iterate over network lists, check if element is in any."""
+        list_matches: list = []
+        for value in value_list:
+            try:
+                ip_address_object = ip_address(value)
+            except ValueError as error:
+                self._handle_warning_error(event, rule, error)
+                continue
+
+            for compare_list in rule.compare_sets:
+                for network in rule.compare_sets[compare_list]:
+                    if compare_list in list_matches:
+                        continue
+                    if ip_address_object in network:
+                        list_matches.append(compare_list)
+        return list_matches

logprep/ng/processor/network_comparison/processor.py

@@ -74,23 +74,39 @@ def _apply_rules(self, event, rule):
             fields = {f"{rule.target_field}.{comparison_key}": comparison_result}
             add_fields_to(event, fields, rule=rule, merge_with_target=True)
 
-    def _list_comparison(self, rule: ListComparisonRule, event: dict):
-        """
-        Check if field value violates block or allow list.
-        Returns the result of the comparison (res_key), as well as a dictionary containing
-        the result (key) and a list of filenames pertaining to said result (value).
+    def _list_comparison(self, rule: ListComparisonRule, event: dict) -> tuple[list, str]:
+        """Check if field value violates block or allow list.
+
+        Returns
+        -------
+        tuple[list[str], str]
+            The result of the comparison, as well as a dictionary containing the result and a list
+            of filenames pertaining to said result.
+
         """
 
-        # get value that should be checked in the lists
-        field_value = get_dotted_field_value(event, rule.source_fields[0])
+        field_value_to_be_checked = get_dotted_field_value(event, rule.source_fields[0])
+        value_list = (
+            field_value_to_be_checked
+            if isinstance(field_value_to_be_checked, list)
+            else [field_value_to_be_checked]
+        )
 
-        # iterate over lists and check if element is in any
-        list_matches = []
-        for compare_list in rule.compare_sets:
-            if field_value in rule.compare_sets[compare_list]:
-                list_matches.append(compare_list)
+        list_matches = self._get_lists_matching_with_values(rule, value_list, event)
 
-        # if matching list was found return it, otherwise return all list names
         if len(list_matches) == 0:
             return list(rule.compare_sets.keys()), "not_in_list"
         return list_matches, "in_list"
+
+    def _get_lists_matching_with_values(
+        self, rule: ListComparisonRule, value_list: list, _: dict
+    ) -> list:
+        """Iterate over string lists, check if element is in any."""
+        list_matches = []
+        for value in value_list:
+            for compare_list in rule.compare_sets:
+                if compare_list in list_matches:
+                    continue
+                if value in rule.compare_sets[compare_list]:
+                    list_matches.append(compare_list)
+        return list_matches

logprep/processor/list_comparison/processor.py

@@ -0,0 +1,59 @@
+"""
+NetworkComparison
+=================
+
+The `network_comparison` processor allows to compare values of source fields against lists provided
+as files.
+
+
+Processor Configuration
+^^^^^^^^^^^^^^^^^^^^^^^
+..  code-block:: yaml
+    :linenos:
+
+    - networkcomparisonname:
+        type: network_comparison
+        rules:
+            - tests/testdata/rules/rules
+        list_search_base_path: /path/to/list/dir
+
+.. autoclass:: logprep.processor.network_comparison.processor.NetworkComparison.Config
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :noindex:
+
+.. automodule:: logprep.processor.network_comparison.rule
+"""
+
+from ipaddress import ip_address
+
+from logprep.processor.list_comparison.processor import ListComparison
+from logprep.processor.network_comparison.rule import NetworkComparisonRule
+from logprep.util.helper import get_dotted_field_value
+
+
+class NetworkComparison(ListComparison):
+    """Resolve values in documents by referencing a mapping list."""
+
+    rule_class = NetworkComparisonRule
+
+    def _get_lists_matching_with_values(
+        self, rule: NetworkComparisonRule, value_list: list, event: dict
+    ) -> list:
+        """Iterate over network lists, check if element is in any."""
+        list_matches: list = []
+        for value in value_list:
+            try:
+                ip_address_object = ip_address(value)
+            except ValueError as error:
+                self._handle_warning_error(event, rule, error)
+                continue
+
+            for compare_list in rule.compare_sets:
+                for network in rule.compare_sets[compare_list]:
+                    if compare_list in list_matches:
+                        continue
+                    if ip_address_object in network:
+                        list_matches.append(compare_list)
+        return list_matches

logprep/processor/network_comparison/__init__.py

@@ -0,0 +1,108 @@
+"""
+Rule Configuration
+^^^^^^^^^^^^^^^^^^
+
+The network comparison enricher can match IPs to IP strings and networks in CIDR notation.
+
+The network comparison enricher requires the additional field :code:`network_comparison`.
+The mandatory keys under :code:`network_comparison` are :code:`source_fields`
+(as list with one element) and :code:`target_field`. Former
+is used to identify the field which is to be checked against the provided lists.
+And the latter is used to define the parent field where the results should
+be written to. Both fields can be dotted subfields.
+
+Additionally, a list or array of lists can be provided underneath the
+required field :code:`list_file_paths`.
+
+In the following example, the field :code:`ip` will be checked against the provided list
+(:code:`networks.txt`).
+Assuming that the value :code:`127.0.0.1` will match the provided list,
+the result of the network comparison (:code:`in_list`) will be added to the
+target field :code:`network_comparison.example`.
+
+..  code-block:: yaml
+    :linenos:
+    :caption: Example Rule to compare a single field against a provided list.
+
+    filter: 'ip'
+    network_comparison:
+        source_fields: ['ip']
+        target_field: 'network_comparison.example'
+        list_file_paths:
+            - lists/networks.txt
+    description: '...'
+
+.. note::
+
+    Currently, it is not possible to check in more than one :code:`source_field` per rule.
+
+.. autoclass:: logprep.processor.network_comparison.rule.NetworkComparisonRule.Config
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :noindex:
+"""
+
+from ipaddress import ip_network
+from typing import Optional, List
+
+from attrs import define, field, validators
+
+from logprep.processor.field_manager.rule import FieldManagerRule
+from logprep.processor.list_comparison.rule import ListComparisonRule
+from logprep.util.getter import HttpGetter
+
+
+class NetworkComparisonRule(ListComparisonRule):
+    """Check if documents match a filter."""
+
+    _compare_sets: dict
+
+    @define(kw_only=True)
+    class Config(FieldManagerRule.Config):
+        """RuleConfig for NetworkComparisonRule"""
+
+        list_file_paths: List[str] = field(
+            validator=validators.deep_iterable(member_validator=validators.instance_of(str))
+        )
+        """List of files. For string format see :ref:`getters`.
+
+        .. security-best-practice::
+           :title: Processor - Network Comparison list file paths Memory Consumption
+
+           Be aware that all values of the remote files were loaded into memory. Consider to avoid
+           dynamic increasing lists without setting limits for Memory consumption. Additionally
+           avoid loading large files all at once to avoid exceeding http body limits.
+
+        .. security-best-practice::
+           :title: Processor - Network Comparison list file paths Authenticity and Integrity
+
+           Consider to use TLS protocol with authentication via mTLS or Oauth to ensure
+           authenticity and integrity of the loaded values.
+
+        """
+        list_search_base_path: str = field(validator=validators.instance_of(str), factory=str)
+        """Base Path from where to find relative files from :code:`list_file_paths`.
+        You can also pass a template with keys from environment,
+        e.g.,  :code:`${<your environment variable>}`. The special key :code:`${LOGPREP_LIST}`
+        will be filled by this processor. """
+        mapping: dict = field(default="", init=False, repr=False, eq=False)
+        ignore_missing_fields: bool = field(default=False, init=False, repr=False, eq=False)
+
+    def init_list_comparison(self, list_search_base_path: Optional[str] = None) -> None:
+        """init method for list_comparison lists"""
+        super().init_list_comparison(list_search_base_path)
+        self._convert_compare_sets_to_networks()
+
+    def _update_compare_sets_via_http(self, http_getter: HttpGetter, list_path: str) -> None:
+        super()._update_compare_sets_via_http(http_getter, list_path)
+        self._convert_compare_sets_to_networks()
+
+    def _convert_compare_sets_to_networks(self) -> None:
+        network_comparison: dict = {}
+        for list_name, compare_strings in self._compare_sets.items():
+            if compare_strings:
+                network_comparison[list_name] = set()
+                for compare_string in compare_strings:
+                    network_comparison[list_name].add(ip_network(compare_string))
+        self._compare_sets = network_comparison

logprep/processor/network_comparison/processor.py

@@ -69,6 +69,9 @@
 from logprep.ng.processor.list_comparison.processor import (
     ListComparison as NgListComparison,
 )
+from logprep.ng.processor.network_comparison.processor import (
+    NetworkComparison as NgNetworkComparison,
+)
 from logprep.ng.processor.pre_detector.processor import PreDetector as NgPreDetector
 from logprep.ng.processor.pseudonymizer.processor import (
     Pseudonymizer as NgPseudonymizer,
@@ -108,6 +111,7 @@
 from logprep.processor.key_checker.processor import KeyChecker
 from logprep.processor.labeler.processor import Labeler
 from logprep.processor.list_comparison.processor import ListComparison
+from logprep.processor.network_comparison.processor import NetworkComparison
 from logprep.processor.pre_detector.processor import PreDetector
 from logprep.processor.pseudonymizer.processor import Pseudonymizer
 from logprep.processor.replacer.processor import Replacer
@@ -143,6 +147,7 @@ class Registry:
         "key_checker": KeyChecker,
         "labeler": Labeler,
         "list_comparison": ListComparison,
+        "network_comparison": NetworkComparison,
         "ng_amides": NgAmides,
         "ng_calculator": NGCalculator,
         "ng_clusterer": NgClusterer,
@@ -162,6 +167,7 @@ class Registry:
         "ng_key_checker": NgKeyChecker,
         "ng_labeler": NgLabeler,
         "ng_list_comparison": NgListComparison,
+        "ng_network_comparison": NgNetworkComparison,
         "ng_replacer": NgReplacer,
         "ng_requester": NgRequester,
         "ng_string_splitter": NgStringSplitter,