Merge pull request #2 from cedadev/py3-devel

Py3 devel

Author: philipkershaw

.pydevproject

@@ -1,9 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<?eclipse-pydev version="1.0"?>
-
-<pydev_project>
-<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">ndg-httpsclient-py2.7</pydev_property>
-<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
+<?eclipse-pydev version="1.0"?><pydev_project>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">ndg-https-client-py3.4</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 3.0</pydev_property>
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
 <path>/ndg_httpsclient</path>
 </pydev_pathproperty>

INSTALL

@@ -0,0 +1,8 @@
+Installation notes
+==================
+
+For install on Mac Yosemite, special header file include paths are needed for
+PyOpenSSL dependency source builds.  Install XCode and add explicit include path
+for ffi package and generic include:
+
+export CPATH=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.10.sdk/usr/include/ffi:/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.10.sdk/usr/include/

README.md

@@ -2,3 +2,92 @@ A HTTPS client implementation for httplib and urllib2 based on
 PyOpenSSL.  PyOpenSSL provides a more fully featured SSL implementation over the
 default provided with Python and importantly enables full verification of the
 SSL peer.
+
+Releases
+========
+0.4.0
+-----
+ * Made dual compatible with Python 2 / 3.
+ 
+0.3.3
+-----
+ * Fix to add in AnotherName for ``subjectAltNames`` field - added for support for CACert issued
+   certs (thanks to Gu1).
+ * Fix to HTTP Basic Auth option for ``ndg.httpsclient.utils.main``
+ * Fix to ``ServerSSLCertVerification`` so that it can pass a function-based callback instead of using ``__call__``. In newer versions of OpenSSL (>= 0.14) the latter failed because of a request for ``__name__`` attribute.
+
+0.3.2
+-----
+ * Fix to SubjectAltNames support check - should only be enabled if pyasn1 is 
+   installed.
+ * Fix to open_url: HTTP Request object was being created inside if headers is 
+   None block - now corrected to create regardless.
+ * Added http basic auth support to script. (Thanks to Willem van Engen)
+ 
+0.3.1
+-----
+ * extended utils functions to support keyword for passing additional urllib2
+   handlers.
+
+0.3.0
+-----
+ * Added ndg.httpsclient.utils.fetch_stream_from_url function and added
+   parameter for data to post in open_url and fetch_* methods.
+ * fix to ndg.httpsclient.utils module _should_use_proxy and open_url functions
+
+0.2.0
+-----
+ * added support for SSL verification with subjectAltNames using pyasn1
+ * fixed minor bug - SSL cert DN prefix matching
+
+0.1.0
+-----
+Initial release
+
+Prerequisites
+=============
+This has been developed and tested for Python 2.6 and 2.7 with pyOpenSSL 0.13 and 0.14.  
+Version 0.4.0 tested with pyOpenSSL 0.15.1 and Python 2.7 and 3.4.  Note that proxy support 
+is only available from Python 2.6.2 onwards.  pyasn1 is required for correct SSL 
+verification with subjectAltNames.
+
+Installation
+============
+Installation can be performed using easy_install or pip.
+
+Running ndg_httpclient
+======================
+A simple script for fetching data using HTTP or HTTPS GET from a specified URL.
+
+Parameter:
+
+``url``
+  The URL of the resource to be fetched
+
+Options:
+
+``-h, --help``
+  Show help message and exit.
+
+``-c FILE, --certificate=FILE``
+  Certificate file - defaults to ``$HOME/credentials.pem``
+
+``-k FILE, --private-key=FILE``
+  Private key file - defaults to the certificate file
+
+``-t DIR, --ca-certificate-dir=DIR``
+  Trusted CA certificate file directory.
+
+``-d, --debug``
+  Print debug information - this may be useful in solving problems with HTTP or 
+  HTTPS access to a server.
+    
+``-p FILE, --post-data-file=FILE``
+  POST data file
+    
+``-f FILE, --fetch=FILE``
+  Output file
+    
+``-n, --no-verify-peer``
+  Skip verification of peer certificate.
+  

ndg/httpsclient/https.py

@@ -12,9 +12,18 @@
 __revision__ = '$Id$'
 import logging
 import socket
-from httplib import HTTPS_PORT
-from httplib import HTTPConnection
-from urllib2 import AbstractHTTPHandler
+import sys
+
+if sys.version_info[0] > 2:
+    from http.client import HTTPS_PORT
+    from http.client import HTTPConnection
+
+    from urllib.request import AbstractHTTPHandler
+else:
+    from httplib import HTTPS_PORT
+    from httplib import HTTPConnection
+
+    from urllib2 import AbstractHTTPHandler
 
 
 from OpenSSL import SSL
@@ -81,7 +90,8 @@ def connect(self):
 
     def close(self):
         """Close socket and shut down SSL connection"""
-        self.sock.close()
+        if hasattr(self.sock, "close"):
+            self.sock.close()
 
 
 class HTTPSContextHandler(AbstractHTTPHandler):
@@ -106,7 +116,7 @@ def __init__(self, ssl_context, debuglevel=0):
                                 ssl_context)
             self.ssl_context = ssl_context
         else:
-            self.ssl_context = SSL.Context(SSL.SSLv23_METHOD)
+            self.ssl_context = SSL.Context(SSL.TLSv1_METHOD)
 
     def https_open(self, req):
         """Opens HTTPS request

ndg/httpsclient/ssl_context_util.py

@@ -8,7 +8,12 @@
 __license__ = "BSD - see LICENSE file in top-level directory"
 __contact__ = "[email protected]"
 __revision__ = '$Id$'
-import urlparse
+import sys
+
+if sys.version_info[0] > 2:
+    import urllib.parse as urlparse_
+else:
+    import urlparse as urlparse_
 
 from OpenSSL import SSL
 
@@ -85,7 +90,7 @@ def set_peer_verification_for_url_hostname(ssl_context, url,
     '''Convenience routine to set peer verification callback based on
     ServerSSLCertVerification class'''
     if not if_verify_enabled or (ssl_context.get_verify_mode() & SSL.VERIFY_PEER):
-        urlObj = urlparse.urlparse(url)
+        urlObj = urlparse_.urlparse(url)
         hostname = urlObj.hostname
         server_ssl_cert_verif = ServerSSLCertVerification(hostname=hostname)
         verify_callback_ = server_ssl_cert_verif.get_verify_server_cert_func()

ndg/httpsclient/ssl_peer_verification.py

@@ -14,7 +14,8 @@
     from ndg.httpsclient.subj_alt_name import SubjectAltName
     from pyasn1.codec.der import decoder as der_decoder
     SUBJ_ALT_NAME_SUPPORT = True
-except ImportError, e:
+    
+except ImportError as e:
     SUBJ_ALT_NAME_SUPPORT = False
     SUBJ_ALT_NAME_SUPPORT_MSG = (
         'SubjectAltName support is disabled - check pyasn1 package '
@@ -40,7 +41,7 @@ class ServerSSLCertVerification(object):
         'userid':                   'UID'
     }
     SUBJ_ALT_NAME_EXT_NAME = 'subjectAltName'
-    PARSER_RE_STR = '/(%s)=' % '|'.join(DN_LUT.keys() + DN_LUT.values())
+    PARSER_RE_STR = '/(%s)=' % '|'.join(list(DN_LUT.keys()) + list(DN_LUT.values()))
     PARSER_RE = re.compile(PARSER_RE_STR)
 
     __slots__ = ('__hostname', '__certDN', '__subj_alt_name_match')
@@ -156,12 +157,12 @@ def __call__(self, connection, peerCert, errorStatus, errorDepth,
             return preverifyOK
 
     def get_verify_server_cert_func(self):
-         def verify_server_cert(connection, peerCert, errorStatus, errorDepth,
-                 preverifyOK):
-              return self.__call__(connection, peerCert, errorStatus,
-                                   errorDepth, preverifyOK)
-
-         return verify_server_cert
+        def verify_server_cert(connection, peerCert, errorStatus, errorDepth,
+                preverifyOK):
+            return self.__call__(connection, peerCert, errorStatus,
+                                 errorDepth, preverifyOK)
+        
+        return verify_server_cert
 
     @classmethod
     def _get_subj_alt_name(cls, peer_cert):
@@ -195,15 +196,15 @@ def _getCertDN(self):
         return self.__certDN
 
     def _setCertDN(self, val):
-        if isinstance(val, basestring):
+        if isinstance(val, str):
             # Allow for quoted DN
             certDN = val.strip('"')
 
             dnFields = self.__class__.PARSER_RE.split(certDN)
             if len(dnFields) < 2:
                 raise TypeError('Error parsing DN string: "%s"' % certDN)
 
-            self.__certDN = zip(dnFields[1::2], dnFields[2::2])
+            self.__certDN = list(zip(dnFields[1::2], dnFields[2::2]))
             self.__certDN.sort()
 
         elif not isinstance(val, list):
@@ -226,7 +227,7 @@ def _getHostname(self):
         return self.__hostname
 
     def _setHostname(self, val):
-        if not isinstance(val, basestring):
+        if not isinstance(val, str):
             raise TypeError("Expecting string type for hostname "
                                  "attribute")
         self.__hostname = val

ndg/httpsclient/ssl_socket.py

@@ -12,7 +12,7 @@
 from datetime import datetime
 import logging
 import socket
-from cStringIO import StringIO
+from io import BytesIO
 
 from OpenSSL import SSL
 
@@ -61,22 +61,20 @@ def buf_size(self):
     @buf_size.setter
     def buf_size(self, value):
         """Buffer size for makefile method recv() operations"""
-        if not isinstance(value, (int, long)):
-            raise TypeError('Expecting int or long type for "buf_size"; '
+        if not isinstance(value, int):
+            raise TypeError('Expecting int type for "buf_size"; '
                             'got %r instead' % type(value))
         self.__buf_size = value
 
     def close(self):
         """Shutdown the SSL connection and call the close method of the
         underlying socket"""
-#        try:
-#            self.__ssl_conn.shutdown()
-#        except SSL.Error:
-#            # Make errors on shutdown non-fatal
-#            pass
-
         if self._makefile_refs < 1:        
-            self.__ssl_conn.shutdown()
+            try:
+                self.__ssl_conn.shutdown()
+            except (SSL.Error, SSL.SysCallError):
+                # Make errors on shutdown non-fatal
+                pass
         else:
             self._makefile_refs -= 1
 
@@ -236,7 +234,7 @@ def makefile(self, *args):
         _buf_size = self.buf_size
 
         i=0
-        stream = StringIO()
+        stream = BytesIO()
         startTime = datetime.utcnow()
         try:
             dat = self.__ssl_conn.recv(_buf_size)
@@ -261,17 +259,6 @@ def makefile(self, *args):
         stream.seek(0)
 
         return stream
-
-#    def makefile(self, mode='r', bufsize=-1):
-#
-#        """Make and return a file-like object that
-#        works with the SSL connection.  Just use the code
-#        from the socket module."""
-#
-#        self._makefile_refs += 1
-#        # close=True so as to decrement the reference count when done with
-#        # the file-like object.
-#        return socket._fileobject(self.socket, mode, bufsize, close=True)
 
     def getsockname(self):
         """

ndg/httpsclient/subj_alt_name.py

@@ -14,7 +14,7 @@
 try:
     from pyasn1.type import univ, constraint, char, namedtype, tag
 
-except ImportError, e:
+except ImportError as e:
     import_error_msg = ('Error importing pyasn1, subjectAltName check for SSL '
                         'peer verification will be disabled.  Import error '
                         'is: %s' % e)

ndg/httpsclient/test/pki/ca/08bd99c7.0

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDLjCCAhagAwIBAgIBATANBgkqhkiG9w0BAQUFADA3MREwDwYDVQQLDAhTZWN1
+cml0eTEUMBIGA1UEAwwLTkRHIFRlc3QgQ0ExDDAKBgNVBAoMA05ERzAeFw0xNTAx
+MjExNDMzMThaFw0yMDAxMjAxNDMzMThaMDcxETAPBgNVBAsMCFNlY3VyaXR5MRQw
+EgYDVQQDDAtOREcgVGVzdCBDQTEMMAoGA1UECgwDTkRHMIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEArq4QKUTRq45nCDR/p+OlHIIN8+ugUbiCfteazbTG
+rX8vIQ9HxSuz/xvxTw+E0KgA4YSK2SJJP4QiCjlMKYS3Rt8o361GNtnRmeo5qyBu
+GMSv73XL1uuqumggUZyrhhksckR7gyNFnKVXzZjAQPepsT0xBjs5uEAEqXJzAf+r
+24AnT3MZRh7gsyEe3sZjd75kZVwcrWhrocyKlMCR77yEr+uP4pg+dEMhDMKKxlaF
+C5RPMotOpWm/7AToHrGia34WSmcxvuOwxOkI4xEW6mxWMaVTBCXUh6Wb/0m/x8Nv
+9VvS2UBC4sCp4MqlDpySxQpT1RgrhMTEmtUOh50l4eEhdwIDAQABo0UwQzASBgNV
+HRMBAf8ECDAGAQH/AgEAMA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUkEvQjGOP
+Oj5DZEvsm96AdiiFXWgwDQYJKoZIhvcNAQEFBQADggEBAGD0kQASmNzvtYL+JUGf
+gTPyJhADl9Ai9GvZJsY/wX0IRTxRl5y08Dqlg3qyGG3GzL918cr1sVCYnLepNQES
+T0MIz50DCKGryNSc74JHPDxpYaSV6whmNH5iwh8fy6tmJwF3FWbGXD2ddc+ofJqP
+WPPJtzqxuuJ6iXQIFqD9mEn3iXVcvFuSzpdpH9paORTKB0j4gya9zctB8LP0ZXIE
+//wREc+4msnmoTn+qkFAOPBg9WnvoipfyCXPgbTagxlofVjZ7gAgYIefqhXBTQdd
+5tnYdyQQBRcUXQS2bBX03q8ftcxOjc3SvXI4MvrqofuFPwu4GnrspnC0KQYlXwEI
+7ds=
+-----END CERTIFICATE-----

ndg/httpsclient/test/pki/ca/ade0138a.0

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDLjCCAhagAwIBAgIBATANBgkqhkiG9w0BAQUFADA3MREwDwYDVQQLDAhTZWN1
+cml0eTEUMBIGA1UEAwwLTkRHIFRlc3QgQ0ExDDAKBgNVBAoMA05ERzAeFw0xNTAx
+MjExNDMzMThaFw0yMDAxMjAxNDMzMThaMDcxETAPBgNVBAsMCFNlY3VyaXR5MRQw
+EgYDVQQDDAtOREcgVGVzdCBDQTEMMAoGA1UECgwDTkRHMIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEArq4QKUTRq45nCDR/p+OlHIIN8+ugUbiCfteazbTG
+rX8vIQ9HxSuz/xvxTw+E0KgA4YSK2SJJP4QiCjlMKYS3Rt8o361GNtnRmeo5qyBu
+GMSv73XL1uuqumggUZyrhhksckR7gyNFnKVXzZjAQPepsT0xBjs5uEAEqXJzAf+r
+24AnT3MZRh7gsyEe3sZjd75kZVwcrWhrocyKlMCR77yEr+uP4pg+dEMhDMKKxlaF
+C5RPMotOpWm/7AToHrGia34WSmcxvuOwxOkI4xEW6mxWMaVTBCXUh6Wb/0m/x8Nv
+9VvS2UBC4sCp4MqlDpySxQpT1RgrhMTEmtUOh50l4eEhdwIDAQABo0UwQzASBgNV
+HRMBAf8ECDAGAQH/AgEAMA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUkEvQjGOP
+Oj5DZEvsm96AdiiFXWgwDQYJKoZIhvcNAQEFBQADggEBAGD0kQASmNzvtYL+JUGf
+gTPyJhADl9Ai9GvZJsY/wX0IRTxRl5y08Dqlg3qyGG3GzL918cr1sVCYnLepNQES
+T0MIz50DCKGryNSc74JHPDxpYaSV6whmNH5iwh8fy6tmJwF3FWbGXD2ddc+ofJqP
+WPPJtzqxuuJ6iXQIFqD9mEn3iXVcvFuSzpdpH9paORTKB0j4gya9zctB8LP0ZXIE
+//wREc+4msnmoTn+qkFAOPBg9WnvoipfyCXPgbTagxlofVjZ7gAgYIefqhXBTQdd
+5tnYdyQQBRcUXQS2bBX03q8ftcxOjc3SvXI4MvrqofuFPwu4GnrspnC0KQYlXwEI
+7ds=
+-----END CERTIFICATE-----