Merge pull request #2 from PacketTotal/feature/analyst_tools

removed test print statement

Author: JaminB

CHANGELOG.md

@@ -0,0 +1,11 @@
+## 0.1.0
+
+- API Wrapper via the `packettotal_sdk/packettotal_api.py` wrapper module
+- packettotal cmdline added
+
+## 0.2.0
+- Bug fixes:
+  - `packettotal_sdk/packettotal_api.PacketTotalAPI.deep_search` now returns a proper `request.Response` obj
+ 
+- Added `packettotal_sdk/packettotal_api.search_tools` module
+  - provides the ability to search by a list of IOCs

docs/.buildinfo

@@ -83,7 +83,6 @@ <h1>Source code for packettotal_sdk.packettotal_api</h1><div class="highlight"><
             <span class="n">body</span><span class="p">[</span><span class="s1">&#39;pcap_name&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">pcap_name</span>
 
         <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">pcap_sources</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
-            <span class="nb">print</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">pcap_sources</span><span class="p">),</span> <span class="n">pcap_sources</span><span class="p">)</span>
             <span class="n">body</span><span class="p">[</span><span class="s1">&#39;sources&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">pcap_sources</span>
 
         <span class="n">response</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">post</span><span class="p">(</span>
@@ -127,7 +126,7 @@ <h1>Source code for packettotal_sdk.packettotal_api</h1><div class="highlight"><
         <span class="n">response</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">post</span><span class="p">(</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">version</span> <span class="o">+</span> <span class="s1">&#39;/search/deep&#39;</span><span class="p">,</span>
             <span class="n">headers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="p">,</span>
-            <span class="n">body</span><span class="o">=</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">body</span><span class="p">)</span>
+            <span class="n">data</span><span class="o">=</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">body</span><span class="p">)</span>
         <span class="p">)</span>
 
         <span class="k">return</span> <span class="n">response</span></div>
@@ -147,7 +146,7 @@ <h1>Source code for packettotal_sdk.packettotal_api</h1><div class="highlight"><
             <span class="n">headers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">headers</span>
         <span class="p">)</span>
 
-        <span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">text</span></div>
+        <span class="k">return</span> <span class="n">response</span></div>
 
 <div class="viewcode-block" id="PacketTotalApi.pcap_analysis"><a class="viewcode-back" href="../../packettotal_sdk/packettotal_sdk.html#packettotal_sdk.packettotal_api.PacketTotalApi.pcap_analysis">[docs]</a>    <span class="k">def</span> <span class="nf">pcap_analysis</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pcap_md5</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">requests</span><span class="o">.</span><span class="n">Response</span><span class="p">:</span>
         <span class="sd">&quot;&quot;&quot;</span>

docs/_modules/packettotal_sdk/packettotal_api.html

@@ -93,7 +93,7 @@ def deep_search_create(self, query: str) -> requests.Response:
         response = requests.post(
             self.base_url + self.version + '/search/deep',
             headers=self.headers,
-            body=json.dumps(body)
+            data=json.dumps(body)
         )
 
         return response
@@ -113,7 +113,7 @@ def deep_search_get(self, search_id: str, pretty=False) -> requests.Response:
             headers=self.headers
         )
 
-        return response.text
+        return response
 
     def pcap_analysis(self, pcap_md5: str) -> requests.Response:
         """

packettotal_sdk/packettotal_api.py

@@ -0,0 +1,68 @@
+import time
+import typing
+import requests
+from sys import stderr
+from datetime import datetime
+
+
+from packettotal_sdk import packettotal_api
+
+
+class SearchTools(packettotal_api.PacketTotalApi):
+
+    def __init__(self, api_key: str):
+        """
+        :param api_key: An API authentication token
+        """
+        super().__init__(api_key)
+
+    def search_by_pcap(self, pcap_file_obj: typing.BinaryIO) -> requests.Response:
+        """
+        Search by a pcap/pcapng file, get list list of similar packet captures
+
+        :param pcap_file_obj: A file like object that provides a .read() interface (E.G open('path_to_pcap.pcap, 'rb') )
+        :return: A request.Response instance, containing a graph of similar pcaps with matched terms
+        """
+        response = super().analyze(pcap_file_obj)
+        if response.status_code == 200:
+            sim_response = super().pcap_similar(response.json()['pcap_metadata']['md5'])
+        elif response.status_code == 202:
+            pcap_id = response.json()['id']
+            info_response = super().pcap_info(pcap_id)
+            while info_response.status_code == 404:
+                print('[{}] Waiting for {} to finish analyzing.'.format(datetime.utcnow(), pcap_id))
+                info_response = super().pcap_info(response.json()['id'])
+                time.sleep(10)
+            sim_response = super().pcap_similar(response.json()['id'])
+        else:
+            return response
+        return sim_response
+
+    def search_by_iocs(self, ioc_file: typing.TextIO) -> requests.Response:
+        """
+        Search up to 1000 IOC terms at once, and get matching packet captures
+
+        :param ioc_file: A file like object that provides a .read() interface (E.G open('path_to_iocs.txt, 'r')
+                         contents are line delim
+        :return: A request.Response instance containing the search results containing at least one matching IOC
+        """
+        text = ioc_file.read()
+        delim = '\n'
+        if '\r\n' in text[0:2048]:
+            delim = '\r\n'
+        elif '\r' in text[0:2048]:
+            delim = '\r'
+        elif ',' in text[0:2048]:
+            delim = ','
+        elif '\t' in text[0:2048]:
+            delim = '\t'
+        text_delimd = text.split(delim)
+        search_str = ''
+        for i, ioc in enumerate(text_delimd[0: -2]):
+            search_str += '"{}" OR '.format(ioc.strip())
+            if i > 1000:
+                print('Warning searching only the first 1000 IOC terms of {}.'.format(len(text_delimd)), file=stderr)
+                break
+        search_str += '"{}"'.format(text_delimd[-1].strip())
+        response = super().search(search_str)
+        return response

packettotal_sdk/search_tools.py

@@ -6,7 +6,7 @@ import getpass
 import argparse
 
 import requests
-from packettotal_sdk import packettotal_api
+from packettotal_sdk import search_tools
 
 
 def _output_text_response(response: requests.Response, output_file=None) -> None:
@@ -36,7 +36,7 @@ def _output_binary_response(response: requests.Response, pcap_id: str) -> None:
     :param response: A request.Response instance containing a populated .content variable
     :param pcap_id: An md5 hash corresponding to the pcap file submission on PacketTotal.com
     """
-    if response.status_code not in [200, 301]:
+    if response.status_code not in [200, 303]:
         print('An error occurred. [{}]'.format(response.status_code), file=sys.stderr)
         print(response.text, file=sys.stderr)
     elif response.status_code == 202:
@@ -50,8 +50,8 @@ def _output_binary_response(response: requests.Response, pcap_id: str) -> None:
 def _parse_cmdline() -> argparse.Namespace:
     """
     positional arguments:
-    mode                  The mode to invoke [ analyze|search|deep_search
-                        |deep_search_results|info|analysis|download|similar ]
+    mode                 The mode to invoke [ analyze|search|search_by_pcap|ioc_search|deep_search
+                        |deep_search_results|info|analysis|download|similar| ]
 
     optional arguments:
       -h, --help            show this help message and exit
@@ -72,8 +72,8 @@ def _parse_cmdline() -> argparse.Namespace:
     )
     parser.add_argument('mode', metavar='mode', type=str,
                         help='The mode to invoke '
-                             '[ analyze|search|deep_search|deep_search_results|info|analysis|'
-                             'download|similar ]')
+                             '[ analyze|search|search_by_pcap|ioc_search|deep_search|deep_search_results|info|analysis|'
+                             'download|similar|]')
     parser.add_argument(
         '--query', dest='query', help='A search query',
         required='search' in sys.argv or 'deep_search' in sys.argv)
@@ -85,7 +85,11 @@ def _parse_cmdline() -> argparse.Namespace:
         required='deep_search_results' in sys.argv)
     parser.add_argument(
         '--path', dest='pcap_path', help='The path to the pcap file you wish to analyze.',
-        required='analyze' in sys.argv)
+        required='analyze' in sys.argv or 'search_by_pcap' in sys.argv)
+    parser.add_argument('--ioc-path', dest='ioc_path', help='The path to a newline delimited text file containing '
+                                                            'search terms.', required='ioc_search' in sys.argv)
+    parser.add_argument('--skip-warnings', dest='skip_warnings', help='Skip pcap analyze warning prompt',
+                        action='store_true')
     parser.add_argument('--name', dest='pcap_name', help='The optional name associated with the pcap analysis.',
                         default=None)
     parser.add_argument('--sources', dest='pcap_sources', help='URLs referencing the original analysis.',
@@ -96,44 +100,68 @@ def _parse_cmdline() -> argparse.Namespace:
     return args
 
 
-def get_api() -> packettotal_api.PacketTotalApi:
+def get_api() -> search_tools.SearchTools:
     """
     Gets an authenticated API instance either by prompting the user or retrieving credential from disk
     :return: Returns a validated packettotal_api.PacketTotalApi instance
     """
     try:
         with open('.pt_api_key', 'r') as f:
-            api = packettotal_api.PacketTotalApi(f.read())
+            api = search_tools.SearchTools(f.read())
             if not validate_api_key(api):
                 api = prompt_api_token()
     except FileNotFoundError:
         api = prompt_api_token()
     return api
 
 
-def prompt_api_token() -> packettotal_api.PacketTotalApi:
+def prompt_analysis_disclaimer():
+    print("""
+        WARNING: Analysis will result in the network traffic becoming public at https://packettotal.com.
+
+        ADVERTENCIA: El análisis hará que el tráfico de la red se haga público en https://packettotal.com.
+
+        WARNUNG: Die Analyse führt dazu, dass der Netzwerkverkehr unter https://packettotal.com öffentlich wird.
+
+        ПРЕДУПРЕЖДЕНИЕ. Анализ приведет к тому, что сетевой трафик станет общедоступным на https://packettotal.com.
+
+        चेतावनी: विश्लेषण का परिणाम नेटवर्क ट्रैफिक https://packettotal.com पर सार्वजनिक हो जाएगा
+
+        警告：分析将导致网络流量在https://packettotal.com上公开
+
+        警告：分析により、ネットワークトラフィックはhttps://packettotal.comで公開されます。
+
+        tahdhir: sayuadiy altahlil 'iilaa 'an tusbih harakat murur alshabakat eamat ealaa https://packettotal.com
+    """)
+
+    answer = input('Continue? [Y/n]: ')
+    if answer.lower() == 'n':
+        exit(0)
+
+
+def prompt_api_token() -> search_tools.SearchTools:
     """
     Prompts the user to enter their API key, tests the validity of the key, and if valid caches it to disk
 
-    :return: Returns a validated packettotal_api.PacketTotalApi instance
+    :return: Returns a validated search_tools.SearchTools instance
     """
     valid = False
     api = None
     while not valid:
         api_key = getpass.getpass('Enter your API key: ', stream=None)
-        api = packettotal_api.PacketTotalApi(api_key)
+        api = search_tools.SearchTools(api_key)
         if validate_api_key(api):
             with open('.pt_api_key', 'w') as f:
                 f.write(api_key)
             valid = True
     return api
 
 
-def validate_api_key(api: packettotal_api.PacketTotalApi) -> bool:
+def validate_api_key(api: search_tools.SearchTools) -> bool:
     """
     Determines whether an API key is valid
 
-    :param api: An unvalidated packettotal_api.PacketTotalApi instance
+    :param api: An unvalidated search_tools.SearchTools instance
     :return: True if api key is valid
     """
     valid = False
@@ -151,6 +179,8 @@ def main():
         return
     api = get_api()
     if args.mode == 'analyze':
+        if not args.skip_warnings:
+            prompt_analysis_disclaimer()
         if args.pcap_sources:
             pcap_sources = args.pcap_sources
         else:
@@ -161,6 +191,8 @@ def main():
         _output_text_response(api.search(args.query), args.output)
     elif args.mode == 'deep_search':
         _output_text_response(api.deep_search_create(args.query), args.output)
+    elif args.mode == 'ioc_search':
+        _output_text_response(api.search_by_iocs(open(args.ioc_path, 'r')), args.output)
     elif args.mode == 'deep_search_results':
         _output_text_response(api.deep_search_get(args.search_id), args.output)
     elif args.mode == 'info':
@@ -183,6 +215,10 @@ def main():
         _output_text_response(api.pcap_similar(args.pcap_id), args.output)
     elif args.mode == 'usage':
         _output_text_response(api.usage(), args.output)
+    elif args.mode == 'search_by_pcap':
+        if not args.skip_warnings:
+            prompt_analysis_disclaimer()
+        _output_text_response(api.search_by_pcap(open(args.pcap_path, 'rb')), args.output)
     else:
         print('Invalid mode. Valid modes are: [{}]'.format('analyze|search|deep_search|deep_search_results|'
                                                            'info|analysis|download|similar'),