From 46a89afd5048212779ef58dab7d8c707ce942ddc Mon Sep 17 00:00:00 2001 From: "Iain R. Learmonth" Date: Sun, 26 Aug 2018 14:48:18 +0100 Subject: [PATCH 1/2] Adds a helper function to get number of tests --- pathspider/base.py | 7 +++++++ pathspider/forge.py | 4 ++-- pathspider/tests/test_test_count.py | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 pathspider/tests/test_test_count.py diff --git a/pathspider/base.py b/pathspider/base.py index 3300b0e..cb2bde5 100644 --- a/pathspider/base.py +++ b/pathspider/base.py @@ -125,6 +125,13 @@ def __set_interface_addresses(self): else: self.source = ("127.0.0.1", "::1") + def _get_test_count(self): + if hasattr(self, 'packets'): + return self.packets # pylint: disable=no-member + if hasattr(self, 'configurations'): + return len(self.configurations) # pylint: disable=no-member + if hasattr(self, 'connections'): + return len(self.connections) # pylint: disable=no-member def configurator(self): raise NotImplementedError("Cannot instantiate an abstract Spider") diff --git a/pathspider/forge.py b/pathspider/forge.py index 259ad78..cb80253 100644 --- a/pathspider/forge.py +++ b/pathspider/forge.py @@ -12,8 +12,8 @@ class ForgeSpider(DesynchronizedSpider): chains = [BasicChain] packets = 0 - def __init__(self, worker_count, libtrace_uri, args): - super().__init__(worker_count, libtrace_uri, args) + def __init__(self, worker_count, libtrace_uri, args, server_mode=False): + super().__init__(worker_count, libtrace_uri, args, server_mode) self.__logger = logging.getLogger('forge') self._config_count = self.packets diff --git a/pathspider/tests/test_test_count.py b/pathspider/tests/test_test_count.py new file mode 100644 index 0000000..e0a6309 --- /dev/null +++ b/pathspider/tests/test_test_count.py @@ -0,0 +1,17 @@ +from nose.tools import assert_equals + +from pathspider.plugins.ecn import ECN +from pathspider.plugins.dscp import DSCP +from pathspider.plugins.evilbit import EvilBit + +def test_ecn_test_count(): + """ + This test checks that the ECN, DSCP and EvilBit plugins are all found + to have 2 tests each by the `_get_test_count(self)` function. These plugins + are used to have one Sync, Desync and Forge spider each. + """ + + for test_spider in [ECN, DSCP, EvilBit]: + print("Trying " + test_spider.name) + spider = test_spider(0, "", None, False) + assert_equals(spider._get_test_count(), 2) From 3ec3db0dc1e09f9bbc08c6e358bfb93bca586120 Mon Sep 17 00:00:00 2001 From: "Iain R. Learmonth" Date: Tue, 21 Aug 2018 19:12:09 +0100 Subject: [PATCH 2/2] poc for generating traceroute traffic from jobs --- pathspider/base.py | 60 +++++++++++++++++++++++++++++++++++++-- pathspider/cmd/base.py | 2 ++ pathspider/cmd/measure.py | 7 ++--- pathspider/cmd/trace.py | 58 +++++++++++++++++++++++++++++++++++++ pathspider/forge.py | 3 -- 5 files changed, 120 insertions(+), 10 deletions(-) create mode 100644 pathspider/cmd/trace.py diff --git a/pathspider/base.py b/pathspider/base.py index cb2bde5..1beb088 100644 --- a/pathspider/base.py +++ b/pathspider/base.py @@ -17,6 +17,8 @@ import queue from datetime import datetime +from scapy.all import send + from pathspider.network import ipv4_address from pathspider.network import ipv6_address from pathspider.network import ipv4_address_public @@ -139,6 +141,33 @@ def configurator(self): def worker(self, worker_number): raise NotImplementedError("Cannot instantiate an abstract Spider") + def traceroute_worker(self, worker_number): + while self.running: + try: + job = self.jobqueue.get_nowait() + # Break on shutdown sentinel + if job == SHUTDOWN_SENTINEL: + self.jobqueue.task_done() + self.__logger.debug( + "shutting down worker %d on sentinel", + worker_number) + worker_active = False + with self.active_worker_lock: + self.active_worker_count -= 1 + self.__logger.debug("%d workers still active", + self.active_worker_count) + break + + self.__logger.debug("got a job: " + repr(job)) + except queue.Empty: + time.sleep(QUEUE_SLEEP) + else: + conns = [] + for seq in range(0, self._get_test_count): + conns.append(self._trace_wrapper(job, seq)) + self.__logger.debug("trace job complete: " + repr(job)) + self.jobqueue.task_done() + def _connect_wrapper(self, job, config, connect=None): start = str(datetime.utcnow()) if connect is None: @@ -150,6 +179,33 @@ def _connect_wrapper(self, job, config, connect=None): conn['spdr_start'] = start return conn + def trace(self, job, seq, template=None): + logger = logging.getLogger('hopspider') + if template is None: + try: + template = self.forge(job, seq) + except NotImplementedError: + logger.error("This plugin has not implemented packet forging" + "which is required for standalone traceroute") + sys.exit(1) + for hop in range(1, 31): + pkt = template.copy() + pkt.ttl = hop + # TODO: A series of functions are needed to "mark" packets + # using different strategies so we can identify them + # when they come back. + send(pkt, verbose=0) + return {'sp': pkt.getlayer(1).sport} + + def _trace_wrapper(self, job, seq): + start = str(datetime.utcnow()) + conn = self.trace(job, seq) + conn['spdr_start'] = start + return conn + + def forge(self, job, seq): + raise NotImplementedError("This plugin has not implemented packet forging") + def create_observer(self): """ Create a flow observer. @@ -375,7 +431,7 @@ def _finalise_conns(self, job, jobId, conns): self.resqueue.put(conn) config += 1 - def start(self): + def start(self, worker=None): """ This function starts a PATHspider plugin by: @@ -432,7 +488,7 @@ def start(self): self.active_worker_count = self.worker_count for i in range(self.worker_count): worker_thread = threading.Thread( - args=(self.worker, i), + args=(worker or self.worker, i), target=self.exception_wrapper, name='worker_{}'.format(i), daemon=True) diff --git a/pathspider/cmd/base.py b/pathspider/cmd/base.py index f2745e9..5ed98c8 100644 --- a/pathspider/cmd/base.py +++ b/pathspider/cmd/base.py @@ -9,6 +9,7 @@ import pathspider.cmd.metadata import pathspider.cmd.observe import pathspider.cmd.test +import pathspider.cmd.trace cmds = [ pathspider.cmd.analyze, @@ -17,6 +18,7 @@ pathspider.cmd.metadata, pathspider.cmd.observe, pathspider.cmd.test, + pathspider.cmd.trace, ] def handle_args(argv): diff --git a/pathspider/cmd/measure.py b/pathspider/cmd/measure.py index 634b8f2..22d636c 100644 --- a/pathspider/cmd/measure.py +++ b/pathspider/cmd/measure.py @@ -64,10 +64,7 @@ def job_feeder_csv(inputfile, spider): spider.shutdown() logger.debug("job_feeder: stopped") - - - -def run_measurement(args): +def run_measurement(args, traceroute=None): logger = logging.getLogger("pathspider") try: @@ -84,7 +81,7 @@ def run_measurement(args): logger.info("activating spider...") - spider.start() + spider.start(worker=spider.traceroute_worker if traceroute else None) logger.debug("starting job feeder...") if args.csv_input: diff --git a/pathspider/cmd/trace.py b/pathspider/cmd/trace.py new file mode 100644 index 0000000..73d428a --- /dev/null +++ b/pathspider/cmd/trace.py @@ -0,0 +1,58 @@ + +import argparse +import logging +import json +import sys +import threading +import csv + +from straight.plugin import load + +from pathspider.base import PluggableSpider +from pathspider.base import SHUTDOWN_SENTINEL +from pathspider.cmd.measure import job_feeder_csv +from pathspider.cmd.measure import job_feeder_ndjson +from pathspider.cmd.measure import run_measurement +from pathspider.network import interface_up + +plugins = load("pathspider.plugins", subclasses=PluggableSpider) + +def run_traceroute(args): + run_measurement(args, traceroute=True) + +def register_args(subparsers): + class SubcommandHelpFormatter(argparse.RawDescriptionHelpFormatter): + def _format_action(self, action): + parts = super()._format_action(action) + if action.nargs == argparse.PARSER: + parts = "\n".join([line for line in parts.split("\n")[1:]]) + parts += "\n\nSpider safely!" + return parts + + parser = subparsers.add_parser(name='trace', + help="Perform a PATHspider traceroute", + formatter_class=SubcommandHelpFormatter) + parser.add_argument('-i', '--interface', default="eth0", + help="The interface to use for the observer. (Default: eth0)") + parser.add_argument('-w', '--workers', type=int, default=20, + help="Number of workers to use. (Default: 20)") + parser.add_argument('--input', default='/dev/stdin', metavar='INPUTFILE', + help=("A file containing a list of PATHspider jobs. " + "Defaults to standard input.")) + parser.add_argument('--csv-input', action='store_true', + help=("Indicate CSV format.")) + parser.add_argument('--output', default='/dev/stdout', metavar='OUTPUTFILE', + help=("The file to output results data to. " + "Defaults to standard output.")) + parser.add_argument('--output-flows', action='store_true', + help="Include flow results in output.") + + # Set the command entry point + parser.set_defaults(cmd=run_traceroute) + + # Add plugins + plugin_subparsers = parser.add_subparsers(title="Plugins", + description="The following plugins are available for use:", + metavar='PLUGIN', help='plugin to use') + for plugin in plugins: + plugin.register_args(plugin_subparsers) diff --git a/pathspider/forge.py b/pathspider/forge.py index cb80253..fd5913f 100644 --- a/pathspider/forge.py +++ b/pathspider/forge.py @@ -24,9 +24,6 @@ def connect(self, job, seq): send(pkt, verbose=0) return {'sp': pkt.getlayer(1).sport} - def forge(self, job, config): - raise NotImplementedError("Cannot register an abstract plugin") - @classmethod def register_args(cls, subparsers): # pylint: disable=no-member