change async to async_ (#803)

* change async to async_

* change async to async_ in tests

* change async_ to async_mode

Author: vibiu

pyspider/fetcher/tornado_fetcher.py

@@ -78,15 +78,15 @@ class Fetcher(object):
     splash_lua_source = open(os.path.join(os.path.dirname(__file__), "splash_fetcher.lua")).read()
     robot_txt_age = 60*60  # 1h
 
-    def __init__(self, inqueue, outqueue, poolsize=100, proxy=None, async=True):
+    def __init__(self, inqueue, outqueue, poolsize=100, proxy=None, async_mode=True):
         self.inqueue = inqueue
         self.outqueue = outqueue
 
         self.poolsize = poolsize
         self._running = False
         self._quit = False
         self.proxy = proxy
-        self.async = async
+        self.async = async_mode
         self.ioloop = tornado.ioloop.IOLoop()
 
         self.robots_txt_cache = {}

pyspider/run.py

@@ -228,7 +228,7 @@ def scheduler(ctx, xmlrpc, xmlrpc_host, xmlrpc_port,
 @click.pass_context
 def fetcher(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, poolsize, proxy, user_agent,
             timeout, phantomjs_endpoint, splash_endpoint, fetcher_cls,
-            async=True, get_object=False, no_input=False):
+            async_mode=True, get_object=False, no_input=False):
     """
     Run Fetcher.
     """
@@ -242,7 +242,7 @@ def fetcher(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, poolsize, proxy, user_agent,
         inqueue = g.scheduler2fetcher
         outqueue = g.fetcher2processor
     fetcher = Fetcher(inqueue=inqueue, outqueue=outqueue,
-                      poolsize=poolsize, proxy=proxy, async=async)
+                      poolsize=poolsize, proxy=proxy, async_mode=async_mode)
     fetcher.phantomjs_proxy = phantomjs_endpoint or g.phantomjs_proxy
     fetcher.splash_endpoint = splash_endpoint
     if user_agent:
@@ -362,7 +362,7 @@ def webui(ctx, host, port, cdn, scheduler_rpc, fetcher_rpc, max_rate, max_burst,
     else:
         # get fetcher instance for webui
         fetcher_config = g.config.get('fetcher', {})
-        webui_fetcher = ctx.invoke(fetcher, async=False, get_object=True, no_input=True, **fetcher_config)
+        webui_fetcher = ctx.invoke(fetcher, async_mode=False, get_object=True, no_input=True, **fetcher_config)
 
         app.config['fetch'] = lambda x: webui_fetcher.fetch(x)
 

tests/test_fetcher_processor.py

@@ -23,7 +23,7 @@ class TestFetcherProcessor(unittest.TestCase):
     @classmethod
     def setUpClass(self):
         self.projectdb = ProjectDB([os.path.join(os.path.dirname(__file__), 'data_fetcher_processor_handler.py')])
-        self.fetcher = Fetcher(None, None, async=False)
+        self.fetcher = Fetcher(None, None, async_mode=False)
         self.status_queue = Queue()
         self.newtask_queue = Queue()
         self.result_queue = Queue()

tests/test_response.py

@@ -29,7 +29,7 @@ class TestResponse(unittest.TestCase):
 
     @classmethod
     def setUpClass(self):
-        self.fetcher = Fetcher(None, None, async=False)
+        self.fetcher = Fetcher(None, None, async_mode=False)
         self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887, passthrough_errors=False)
         self.httpbin = 'http://127.0.0.1:14887'
         time.sleep(0.5)