Fix mysql return bytes as field name type (#787)

* use pip version of mysql-connector-python for testing

* fix mysql return bytes as field names type

* fix raise Unread result found error

This error raise on mysql-connector-python with C extension

* fix test

Pure version raise InterfaceError,
  but C extension version raise DatabaseError

Author: sdvcrx

.travis.yml

@@ -36,7 +36,7 @@ before_script:
     - psql -c "CREATE DATABASE pyspider_test_resultdb ENCODING 'UTF8' TEMPLATE=template0;" -U postgres
     - sleep 10
 install:
-    - pip install http://cdn.mysql.com/Downloads/Connector-Python/mysql-connector-python-2.0.4.zip#md5=3df394d89300db95163f17c843ef49df
+    - pip install mysql-connector-python
     - pip install https://github.com/marcus67/easywebdav/archive/master.zip
     - pip install --no-use-wheel lxml
     - pip install --allow-all-external -e .[all,test]

pyspider/database/basedb.py

@@ -11,6 +11,7 @@
 logger = logging.getLogger('database.basedb')
 
 from six import itervalues
+from pyspider.libs import utils
 
 
 class BaseDB:
@@ -72,7 +73,10 @@ def _select2dic(self, tablename=None, what="*", where="", where_values=[],
         logger.debug("<sql: %s>", sql_query)
 
         dbcur = self._execute(sql_query, where_values)
-        fields = [f[0] for f in dbcur.description]
+
+        # f[0] may return bytes type
+        # https://github.com/mysql/mysql-connector-python/pull/37
+        fields = [utils.text(f[0]) for f in dbcur.description]
 
         for row in dbcur:
             yield dict(zip(fields, row))

pyspider/database/mysql/mysqlbase.py

@@ -17,6 +17,8 @@ def dbcur(self):
         try:
             if self.conn.unread_result:
                 self.conn.get_rows()
+                if hasattr(self.conn, 'free_result'):
+                    self.conn.free_result()
             return self.conn.cursor()
         except (mysql.connector.OperationalError, mysql.connector.InterfaceError):
             self.conn.ping(reconnect=True)

tests/test_run.py

@@ -72,7 +72,7 @@ def test_20_cli_config(self):
         self.assertEqual(ctx.obj.debug, True)
 
         import mysql.connector
-        with self.assertRaises(mysql.connector.InterfaceError):
+        with self.assertRaises(mysql.connector.Error):
             ctx.obj.taskdb
 
         with self.assertRaises(Exception):