CVE-2013-2255

@@ -309,6 +309,22 @@
 # URL to get token from ec2 request. (string value)
 #keystone_ec2_url=http://localhost:5000/v2.0/ec2tokens
 
+# Required if EC2 server requires client certificate. (string
+# value)
+#keystone_ec2_keyfile=<None>
+
+# Client certificate key filename. Required if EC2 server
+# requires client certificate. (string value)
+#keystone_ec2_certfile=<None>
+
+# A PEM encoded certificate authority to use when verifying
+# HTTPS connections. Defaults to the system CAs. (string
+# value)
+#keystone_ec2_cafile=<None>
+
+# Disable SSL certificate verification. (boolean value)
+#keystone_ec2_insecure=false
+
 
 #
 # Options defined in keystone.openstack.common.eventlet_backdoor

@@ -20,9 +20,8 @@
 
 """
 
-from eventlet.green import httplib
 from oslo.config import cfg
-from six.moves import urllib
+import requests
 import webob.dec
 import webob.exc
 
@@ -34,6 +33,16 @@
     cfg.StrOpt('keystone_ec2_url',
                default='http://localhost:5000/v2.0/ec2tokens',
                help='URL to get token from ec2 request.'),
+    cfg.StrOpt('keystone_ec2_keyfile', help='Required if EC2 server requires '
+               'client certificate.'),
+    cfg.StrOpt('keystone_ec2_certfile', help='Client certificate key '
+               'filename. Required if EC2 server requires client '
+               'certificate.'),
+    cfg.StrOpt('keystone_ec2_cafile', help='A PEM encoded certificate '
+               'authority to use when verifying HTTPS connections. Defaults '
+               'to the system CAs.'),
+    cfg.BoolOpt('keystone_ec2_insecure', default=False, help='Disable SSL '
+                'certificate verification.'),
 ]
 
 CONF = config.CONF
@@ -71,23 +80,26 @@ def __call__(self, req):
         creds_json = jsonutils.dumps(creds)
         headers = {'Content-Type': 'application/json'}
 
-        # Disable 'has no x member' pylint error
-        # for httplib and urlparse
-        # pylint: disable-msg=E1101
-        o = urllib.parse.urlparse(CONF.keystone_ec2_url)
-        if o.scheme == 'http':
-            conn = httplib.HTTPConnection(o.netloc)
-        else:
-            conn = httplib.HTTPSConnection(o.netloc)
-        conn.request('POST', o.path, body=creds_json, headers=headers)
-        response = conn.getresponse().read()
-        conn.close()
+        verify = True
+        if CONF.keystone_ec2_insecure:
+            verify = False
+        elif CONF.keystone_ec2_cafile:
+            verify = CONF.keystone_ec2_cafile
+
+        cert = None
+        if CONF.keystone_ec2_certfile and CONF.keystone_ec2_keyfile:
+            cert = (CONF.keystone_ec2_certfile, CONF.keystone_ec2_keyfile)
+        elif CONF.keystone_ec2_certfile:
+            cert = CONF.keystone_ec2_certfile
+
+        response = requests.post(CONF.keystone_ec2_url, data=creds_json,
+                                 headers=headers, verify=verify, cert=cert)
 
         # NOTE(vish): We could save a call to keystone by
         #             having keystone return token, tenant,
         #             user, and roles from this call.
 
-        result = jsonutils.loads(response)
+        result = response.json()
         try:
             token_id = result['access']['token']['id']
         except (AttributeError, KeyError):

@@ -12,7 +12,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import httplib
+import netaddr
+import requests
 import urlparse
 
 from cinder.openstack.common import jsonutils
@@ -30,8 +31,8 @@ def __init__(self, message=None, response=None):
 
         if response:
             message = _('%(message)s\nStatus Code: %(_status)s\n'
-                        'Body: %(_body)s') % {'_status': response.status,
-                                              '_body': response.read()}
+                        'Body: %(_body)s') % {'_status': response.status_code,
+                                              '_body': response.text}
 
         super(OpenStackApiException, self).__init__(message)
 
@@ -76,7 +77,8 @@ def __init__(self, auth_user, auth_key, auth_uri):
         # default project_id
         self.project_id = 'openstack'
 
-    def request(self, url, method='GET', body=None, headers=None):
+    def request(self, url, method='GET', body=None, headers=None,
+                ssl_verify=True, stream=False):
         _headers = {'Content-Type': 'application/json'}
         _headers.update(headers or {})
 
@@ -85,14 +87,8 @@ def request(self, url, method='GET', body=None, headers=None):
         hostname = parsed_url.hostname
         scheme = parsed_url.scheme
 
-        if scheme == 'http':
-            conn = httplib.HTTPConnection(hostname,
-                                          port=port)
-        elif scheme == 'https':
-            conn = httplib.HTTPSConnection(hostname,
-                                           port=port)
-        else:
-            raise OpenStackApiException("Unknown scheme: %s" % url)
+        if netaddr.valid_ipv6(hostname):
+            hostname = "[%s]" % hostname
 
         relative_url = parsed_url.path
         if parsed_url.query:
@@ -102,8 +98,14 @@ def request(self, url, method='GET', body=None, headers=None):
         if body:
             LOG.info(_("Body: %s") % body)
 
-        conn.request(method, relative_url, body, _headers)
-        response = conn.getresponse()
+        if port:
+            _url = "%s://%s:%d%s" % (scheme, hostname, int(port), relative_url)
+        else:
+            _url = "%s://%s%s" % (scheme, hostname, relative_url)
+
+        response = requests.request(method, _url, data=body, headers=_headers,
+                                    verify=ssl_verify, stream=stream)
+
         return response
 
     def _authenticate(self):
@@ -117,18 +119,14 @@ def _authenticate(self):
         response = self.request(auth_uri,
                                 headers=headers)
 
-        http_status = response.status
+        http_status = response.status_code
         LOG.debug(_("%(auth_uri)s => code %(http_status)s"),
                   {'auth_uri': auth_uri, 'http_status': http_status})
 
         if http_status == 401:
             raise OpenStackApiAuthenticationException(response=response)
 
-        auth_headers = {}
-        for k, v in response.getheaders():
-            auth_headers[k] = v
-
-        self.auth_result = auth_headers
+        self.auth_result = response.headers
         return self.auth_result
 
     def api_request(self, relative_uri, check_response_status=None, **kwargs):
@@ -144,7 +142,7 @@ def api_request(self, relative_uri, check_response_status=None, **kwargs):
 
         response = self.request(full_uri, **kwargs)
 
-        http_status = response.status
+        http_status = response.status_code
         LOG.debug(_("%(relative_uri)s => code %(http_status)s"),
                   {'relative_uri': relative_uri, 'http_status': http_status})
 
@@ -162,7 +160,7 @@ def api_request(self, relative_uri, check_response_status=None, **kwargs):
         return response
 
     def _decode_json(self, response):
-        body = response.read()
+        body = response.text
         LOG.debug(_("Decoding JSON: %s") % (body))
         if body:
             return jsonutils.loads(body)

@@ -36,6 +36,6 @@ def _get_flags(self):
     def test_get_foxnsocks(self):
         """Simple check that fox-n-socks works."""
         response = self.api.api_request('/foxnsocks')
-        foxnsocks = response.read()
+        foxnsocks = response.text
         LOG.debug("foxnsocks: %s" % foxnsocks)
         self.assertEqual('Try to say this Mr. Knox, sir...', foxnsocks)

@@ -42,8 +42,9 @@ def test_namespace_volumes(self):
         headers = {}
         headers['Accept'] = 'application/xml'
 
-        response = self.api.api_request('/volumes', headers=headers)
-        data = response.read()
+        response = self.api.api_request('/volumes', headers=headers,
+                                        stream=True)
+        data = response.raw
         LOG.warn("data: %s" % data)
-        root = etree.XML(data)
+        root = etree.parse(data).getroot()
         self.assertEqual(root.nsmap.get(None), common.XML_NS_V1)

@@ -6,7 +6,10 @@
 # The following parameters are supported:
 #   servers               :  <host:port>[,<host:port>]*   (Error if not set)
 #   server_auth           :  <username:password>          (default: no auth)
-#   server_ssl            :  True | False                 (default: False)
+#   server_ssl            :  True | False                 (default: True)
+#   ssl_cert_directory    :  <path>                       (default: /etc/neutron/plugins/bigswitch/ssl)
+#   no_ssl_validation     :  True | False                 (default: False)
+#   ssl_sticky            :  True | False                 (default: True)
 #   sync_data             :  True | False                 (default: False)
 #   auto_sync_on_failure  :  True | False                 (default: True)
 #   server_timeout        :  <integer>                    (default: 10 seconds)
@@ -21,7 +24,20 @@ servers=localhost:8080
 # server_auth=username:password
 
 # Use SSL when connecting to the BigSwitch or Floodlight controller.
-# server_ssl=False
+# server_ssl=True
+
+# Directory which contains the ca_certs and host_certs to be used to validate
+# controller certificates.
+# ssl_cert_directory=/etc/neutron/plugins/bigswitch/ssl/
+
+# If a certificate does not exist for a controller, trust and store the first
+# certificate received for that controller and use it to validate future
+# connections to that controller.
+# ssl_sticky=True
+
+# Do not validate the controller certificates for SSL
+# Warning: This will not provide protection against man-in-the-middle attacks
+# no_ssl_validation=False
 
 # Sync data on connect
 # sync_data=False

@@ -0,0 +1,3 @@
+Certificates in this folder will be used to
+verify signatures for any controllers the plugin
+connects to.

@@ -0,0 +1,6 @@
+Certificates in this folder must match the name
+of the controller they should be used to authenticate
+with a .pem extension.
+
+For example, the certificate for the controller
+"192.168.0.1" should be named "192.168.0.1.pem".

@@ -39,9 +39,21 @@
     cfg.StrOpt('server_auth', default=None, secret=True,
                help=_("The username and password for authenticating against "
                       " the BigSwitch or Floodlight controller.")),
-    cfg.BoolOpt('server_ssl', default=False,
+    cfg.BoolOpt('server_ssl', default=True,
                 help=_("If True, Use SSL when connecting to the BigSwitch or "
                        "Floodlight controller.")),
+    cfg.BoolOpt('ssl_sticky', default=True,
+                help=_("Trust and store the first certificate received for "
+                       "each controller address and use it to validate future "
+                       "connections to that address.")),
+    cfg.BoolOpt('no_ssl_validation', default=False,
+                help=_("Disables SSL certificate validation for controllers")),
+    cfg.BoolOpt('cache_connections', default=True,
+                help=_("Re-use HTTP/HTTPS connections to the controller.")),
+    cfg.StrOpt('ssl_cert_directory',
+               default='/etc/neutron/plugins/bigswitch/ssl',
+               help=_("Directory containing ca_certs and host_certs "
+                      "certificate directories.")),
     cfg.BoolOpt('sync_data', default=False,
                 help=_("Sync data on connect")),
     cfg.BoolOpt('auto_sync_on_failure', default=True,

@@ -27,20 +27,24 @@
 The following functionality is handled by this module:
 - Translation of rest_* function calls to HTTP/HTTPS calls to the controllers
 - Automatic failover between controllers
+- SSL Certificate enforcement
 - HTTP Authentication
 
 """
 import base64
 import httplib
 import json
+import os
 import socket
+import ssl
 import time
 
 import eventlet
 from oslo.config import cfg
 
 from neutron.common import exceptions
 from neutron.common import utils
+from neutron.openstack.common import excutils
 from neutron.openstack.common import log as logging
 from neutron.plugins.bigswitch.db import consistency_db as cdb
 
@@ -85,7 +89,7 @@ class ServerProxy(object):
     """REST server proxy to a network controller."""
 
     def __init__(self, server, port, ssl, auth, neutron_id, timeout,
-                 base_uri, name, mypool):
+                 base_uri, name, mypool, combined_cert):
         self.server = server
         self.port = port
         self.ssl = ssl
@@ -99,8 +103,11 @@ def __init__(self, server, port, ssl, auth, neutron_id, timeout,
         self.capabilities = []
         # enable server to reference parent pool
         self.mypool = mypool
+        # cache connection here to avoid a SSL handshake for every connection
+        self.currentconn = None
         if auth:
             self.auth = 'Basic ' + base64.encodestring(auth).strip()
+        self.combined_cert = combined_cert
 
     def get_capabilities(self):
         try:
@@ -114,7 +121,8 @@ def get_capabilities(self):
                                                 'cap': self.capabilities})
         return self.capabilities
 
-    def rest_call(self, action, resource, data='', headers={}, timeout=None):
+    def rest_call(self, action, resource, data='', headers={}, timeout=False,
+                  reconnect=False):
         uri = self.base_uri + resource
         body = json.dumps(data)
         if not headers:
@@ -125,6 +133,10 @@ def rest_call(self, action, resource, data='', headers={}, timeout=None):
         headers['Instance-ID'] = self.neutron_id
         headers['Orchestration-Service-ID'] = ORCHESTRATION_SERVICE_ID
         headers[HASH_MATCH_HEADER] = self.mypool.consistency_hash
+        if 'keep-alive' in self.capabilities:
+            headers['Connection'] = 'keep-alive'
+        else:
+            reconnect = True
         if self.auth:
             headers['Authorization'] = self.auth
 
@@ -136,26 +148,37 @@ def rest_call(self, action, resource, data='', headers={}, timeout=None):
                   {'resource': resource, 'data': data, 'headers': headers,
                    'action': action})
 
-        conn = None
-        timeout = timeout or self.timeout
-        if self.ssl:
-            conn = httplib.HTTPSConnection(
-                self.server, self.port, timeout=timeout)
-            if conn is None:
-                LOG.error(_('ServerProxy: Could not establish HTTPS '
-                            'connection'))
-                return 0, None, None, None
-        else:
-            conn = httplib.HTTPConnection(
-                self.server, self.port, timeout=timeout)
-            if conn is None:
-                LOG.error(_('ServerProxy: Could not establish HTTP '
-                            'connection'))
-                return 0, None, None, None
+        # unspecified timeout is False because a timeout can be specified as
+        # None to indicate no timeout.
+        if timeout is False:
+            timeout = self.timeout
+
+        if timeout != self.timeout:
+            # need a new connection if timeout has changed
+            reconnect = True
+
+        if not self.currentconn or reconnect:
+            if self.currentconn:
+                self.currentconn.close()
+            if self.ssl:
+                self.currentconn = HTTPSConnectionWithValidation(
+                    self.server, self.port, timeout=timeout)
+                self.currentconn.combined_cert = self.combined_cert
+                if self.currentconn is None:
+                    LOG.error(_('ServerProxy: Could not establish HTTPS '
+                                'connection'))
+                    return 0, None, None, None
+            else:
+                self.currentconn = httplib.HTTPConnection(
+                    self.server, self.port, timeout=timeout)
+                if self.currentconn is None:
+                    LOG.error(_('ServerProxy: Could not establish HTTP '
+                                'connection'))
+                    return 0, None, None, None
 
         try:
-            conn.request(action, uri, body, headers)
-            response = conn.getresponse()
+            self.currentconn.request(action, uri, body, headers)
+            response = self.currentconn.getresponse()
             newhash = response.getheader(HASH_MATCH_HEADER)
             if newhash:
                 self._put_consistency_hash(newhash)
@@ -168,11 +191,20 @@ def rest_call(self, action, resource, data='', headers={}, timeout=None):
                     # response was not JSON, ignore the exception
                     pass
             ret = (response.status, response.reason, respstr, respdata)
+        except httplib.ImproperConnectionState:
+            # If we were using a cached connection, try again with a new one.
+            with excutils.save_and_reraise_exception() as ctxt:
+                if not reconnect:
+                    ctxt.reraise = False
+
+            if self.currentconn:
+                self.currentconn.close()
+            return self.rest_call(action, resource, data, headers,
+                                  timeout=timeout, reconnect=True)
         except (socket.timeout, socket.error) as e:
             LOG.error(_('ServerProxy: %(action)s failure, %(e)r'),
                       {'action': action, 'e': e})
             ret = 0, None, None, None
-        conn.close()
         LOG.debug(_("ServerProxy: status=%(status)d, reason=%(reason)r, "
                     "ret=%(ret)s, data=%(data)r"), {'status': ret[0],
                                                     'reason': ret[1],
@@ -187,7 +219,7 @@ def _put_consistency_hash(self, newhash):
 
 class ServerPool(object):
 
-    def __init__(self, timeout=10,
+    def __init__(self, timeout=False,
                  base_uri=BASE_URI, name='NeutronRestProxy'):
         LOG.debug(_("ServerPool: initializing"))
         # 'servers' is the list of network controller REST end-points
@@ -200,8 +232,9 @@ def __init__(self, timeout=10,
         self.base_uri = base_uri
         self.name = name
         self.timeout = cfg.CONF.RESTPROXY.server_timeout
+        self.always_reconnect = not cfg.CONF.RESTPROXY.cache_connections
         default_port = 8000
-        if timeout is not None:
+        if timeout is not False:
             self.timeout = timeout
 
         # Function to use to retrieve topology for consistency syncs.
@@ -244,8 +277,99 @@ def get_capabilities(self):
             return self.capabilities
 
     def server_proxy_for(self, server, port):
+        combined_cert = self._get_combined_cert_for_server(server, port)
         return ServerProxy(server, port, self.ssl, self.auth, self.neutron_id,
-                           self.timeout, self.base_uri, self.name, mypool=self)
+                           self.timeout, self.base_uri, self.name, mypool=self,
+                           combined_cert=combined_cert)
+
+    def _get_combined_cert_for_server(self, server, port):
+        # The ssl library requires a combined file with all trusted certs
+        # so we make one containing the trusted CAs and the corresponding
+        # host cert for this server
+        combined_cert = None
+        if self.ssl and not cfg.CONF.RESTPROXY.no_ssl_validation:
+            base_ssl = cfg.CONF.RESTPROXY.ssl_cert_directory
+            host_dir = os.path.join(base_ssl, 'host_certs')
+            ca_dir = os.path.join(base_ssl, 'ca_certs')
+            combined_dir = os.path.join(base_ssl, 'combined')
+            combined_cert = os.path.join(combined_dir, '%s.pem' % server)
+            if not os.path.exists(base_ssl):
+                raise cfg.Error(_('ssl_cert_directory [%s] does not exist. '
+                                  'Create it or disable ssl.') % base_ssl)
+            for automake in [combined_dir, ca_dir, host_dir]:
+                if not os.path.exists(automake):
+                    os.makedirs(automake)
+
+            # get all CA certs
+            certs = self._get_ca_cert_paths(ca_dir)
+
+            # check for a host specific cert
+            hcert, exists = self._get_host_cert_path(host_dir, server)
+            if exists:
+                certs.append(hcert)
+            elif cfg.CONF.RESTPROXY.ssl_sticky:
+                self._fetch_and_store_cert(server, port, hcert)
+                certs.append(hcert)
+            if not certs:
+                raise cfg.Error(_('No certificates were found to verify '
+                                  'controller %s') % (server))
+            self._combine_certs_to_file(certs, combined_cert)
+        return combined_cert
+
+    def _combine_certs_to_file(certs, cfile):
+        '''
+        Concatenates the contents of each certificate in a list of
+        certificate paths to one combined location for use with ssl
+        sockets.
+        '''
+        with open(cfile, 'w') as combined:
+            for c in certs:
+                with open(c, 'r') as cert_handle:
+                    combined.write(cert_handle.read())
+
+    def _get_host_cert_path(self, host_dir, server):
+        '''
+        returns full path and boolean indicating existence
+        '''
+        hcert = os.path.join(host_dir, '%s.pem' % server)
+        if os.path.exists(hcert):
+            return hcert, True
+        return hcert, False
+
+    def _get_ca_cert_paths(self, ca_dir):
+        certs = [os.path.join(root, name)
+                 for name in [
+                     name for (root, dirs, files) in os.walk(ca_dir)
+                     for name in files
+                 ]
+                 if name.endswith('.pem')]
+        return certs
+
+    def _fetch_and_store_cert(self, server, port, path):
+        '''
+        Grabs a certificate from a server and writes it to
+        a given path.
+        '''
+        try:
+            cert = ssl.get_server_certificate((server, port))
+        except Exception as e:
+            raise cfg.Error(_('Could not retrieve initial '
+                              'certificate from controller %(server)s. '
+                              'Error details: %(error)s'),
+                            {'server': server, 'error': e.strerror})
+
+        LOG.warning(_("Storing to certificate for host %(server)s "
+                      "at %(path)s") % {'server': server,
+                                        'path': path})
+        self._file_put_contents(path, cert)
+
+        return cert
+
+    def _file_put_contents(path, contents):
+        # Simple method to write to file.
+        # Created for easy Mocking
+        with open(path, 'w') as handle:
+            handle.write(contents)
 
     def server_failure(self, resp, ignore_codes=[]):
         """Define failure codes as required.
@@ -264,12 +388,13 @@ def action_success(self, resp):
 
     @utils.synchronized('bsn-rest-call')
     def rest_call(self, action, resource, data, headers, ignore_codes,
-                  timeout=None):
+                  timeout=False):
         good_first = sorted(self.servers, key=lambda x: x.failed)
         first_response = None
         for active_server in good_first:
             ret = active_server.rest_call(action, resource, data, headers,
-                                          timeout)
+                                          timeout,
+                                          reconnect=self.always_reconnect)
             # If inconsistent, do a full synchronization
             if ret[0] == httplib.CONFLICT:
                 if not self.get_topo_function:
@@ -309,7 +434,7 @@ def rest_call(self, action, resource, data, headers, ignore_codes,
         return first_response
 
     def rest_action(self, action, resource, data='', errstr='%s',
-                    ignore_codes=[], headers={}, timeout=None):
+                    ignore_codes=[], headers={}, timeout=False):
         """
         Wrapper for rest_call that verifies success and raises a
         RemoteRestError on failure with a provided error string
@@ -427,3 +552,26 @@ def _consistency_watchdog(self, polling_interval=60):
             # that will be handled by the rest_call.
             time.sleep(polling_interval)
             self.servers.rest_call('GET', HEALTH_PATH)
+
+
+class HTTPSConnectionWithValidation(httplib.HTTPSConnection):
+
+    # If combined_cert is None, the connection will continue without
+    # any certificate validation.
+    combined_cert = None
+
+    def connect(self):
+        sock = socket.create_connection((self.host, self.port),
+                                        self.timeout, self.source_address)
+        if self._tunnel_host:
+            self.sock = sock
+            self._tunnel()
+
+        if self.combined_cert:
+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
+                                        cert_reqs=ssl.CERT_REQUIRED,
+                                        ca_certs=self.combined_cert)
+        else:
+            self.sock = ssl.wrap_socket(sock, self.key_file,
+                                        self.cert_file,
+                                        cert_reqs=ssl.CERT_NONE)

@@ -0,0 +1,2 @@
+ca_certs directory for SSL unit tests
+No files will be generated here, but it should exist for the tests

@@ -0,0 +1,2 @@
+combined certificates directory for SSL unit tests
+No files will be created here, but it should exist for the tests

@@ -0,0 +1,2 @@
+host_certs directory for SSL unit tests
+No files will be created here, but it should exist for the tests

@@ -139,3 +139,48 @@ def request(self, action, uri, body, headers):
                 raise Exception(msg)
         super(VerifyMultiTenantFloatingIP,
               self).request(action, uri, body, headers)
+
+
+class HTTPSMockBase(HTTPConnectionMock):
+    expected_cert = ''
+    combined_cert = None
+
+    def __init__(self, host, port=None, key_file=None, cert_file=None,
+                 strict=None, timeout=None, source_address=None):
+        self.host = host
+        super(HTTPSMockBase, self).__init__(host, port, timeout)
+
+    def request(self, method, url, body=None, headers={}):
+        self.connect()
+        super(HTTPSMockBase, self).request(method, url, body, headers)
+
+
+class HTTPSNoValidation(HTTPSMockBase):
+
+    def connect(self):
+        if self.combined_cert:
+            raise Exception('combined_cert set on NoValidation')
+
+
+class HTTPSCAValidation(HTTPSMockBase):
+    expected_cert = 'DUMMYCERTIFICATEAUTHORITY'
+
+    def connect(self):
+        contents = get_cert_contents(self.combined_cert)
+        if self.expected_cert not in contents:
+            raise Exception('No dummy CA cert in cert_file')
+
+
+class HTTPSHostValidation(HTTPSMockBase):
+    expected_cert = 'DUMMYCERTFORHOST%s'
+
+    def connect(self):
+        contents = get_cert_contents(self.combined_cert)
+        expected = self.expected_cert % self.host
+        if expected not in contents:
+            raise Exception(_('No host cert for %(server)s in cert %(cert)s'),
+                            {'server': self.host, 'cert': contents})
+
+
+def get_cert_contents(path):
+    raise Exception('METHOD MUST BE MOCKED FOR TEST')

@@ -45,6 +45,12 @@ def setup_config_files(self):
                                                 'restproxy.ini.test')]
         self.addCleanup(cfg.CONF.reset)
         config.register_config()
+        # Only try SSL on SSL tests
+        cfg.CONF.set_override('server_ssl', False, 'RESTPROXY')
+        cfg.CONF.set_override('ssl_cert_directory',
+                              os.path.join(etc_path, 'ssl'), 'RESTPROXY')
+        # The mock interferes with HTTP(S) connection caching
+        cfg.CONF.set_override('cache_connections', False, 'RESTPROXY')
 
     def setup_patches(self):
         self.httpPatch = mock.patch(HTTPCON, create=True,

@@ -0,0 +1,251 @@
+# Copyright 2014 Big Switch Networks, Inc.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Kevin Benton, kevin.benton@bigswitch.com
+#
+import os
+
+import mock
+from oslo.config import cfg
+import webob.exc
+
+from neutron.openstack.common import log as logging
+from neutron.tests.unit.bigswitch import fake_server
+from neutron.tests.unit.bigswitch import test_base
+from neutron.tests.unit import test_api_v2
+from neutron.tests.unit import test_db_plugin as test_plugin
+
+LOG = logging.getLogger(__name__)
+
+SERVERMANAGER = 'neutron.plugins.bigswitch.servermanager'
+HTTPS = SERVERMANAGER + '.HTTPSConnectionWithValidation'
+CERTCOMBINER = SERVERMANAGER + '.ServerPool._combine_certs_to_file'
+FILEPUT = SERVERMANAGER + '.ServerPool._file_put_contents'
+GETCACERTS = SERVERMANAGER + '.ServerPool._get_ca_cert_paths'
+GETHOSTCERT = SERVERMANAGER + '.ServerPool._get_host_cert_path'
+FAKECERTGET = 'neutron.tests.unit.bigswitch.fake_server.get_cert_contents'
+SSLGETCERT = 'ssl.get_server_certificate'
+
+
+class test_ssl_certificate_base(test_plugin.NeutronDbPluginV2TestCase,
+                                test_base.BigSwitchTestBase):
+
+    plugin_str = ('%s.NeutronRestProxyV2' %
+                  test_base.RESTPROXY_PKG_PATH)
+    servername = None
+    cert_base = None
+
+    def _setUp(self):
+        self.servername = test_api_v2._uuid()
+        self.cert_base = cfg.CONF.RESTPROXY.ssl_cert_directory
+        self.host_cert_val = 'DUMMYCERTFORHOST%s' % self.servername
+        self.host_cert_path = os.path.join(
+            self.cert_base,
+            'host_certs',
+            '%s.pem' % self.servername
+        )
+        self.comb_cert_path = os.path.join(
+            self.cert_base,
+            'combined',
+            '%s.pem' % self.servername
+        )
+        self.ca_certs_path = os.path.join(
+            self.cert_base,
+            'ca_certs'
+        )
+        cfg.CONF.set_override('servers', ["%s:443" % self.servername],
+                              'RESTPROXY')
+        self.setup_patches()
+
+        # Mock method SSL lib uses to grab cert from server
+        self.sslgetcert_m = mock.patch(SSLGETCERT, create=True).start()
+        self.sslgetcert_m.return_value = self.host_cert_val
+
+        # Mock methods that write and read certs from the file-system
+        self.fileput_m = mock.patch(FILEPUT, create=True).start()
+        self.certcomb_m = mock.patch(CERTCOMBINER, create=True).start()
+        self.getcacerts_m = mock.patch(GETCACERTS, create=True).start()
+
+        # this is used to configure what certificate contents the fake HTTPS
+        # lib should expect to receive
+        self.fake_certget_m = mock.patch(FAKECERTGET, create=True).start()
+
+    def setUp(self):
+        super(test_ssl_certificate_base, self).setUp(self.plugin_str)
+
+
+class TestSslSticky(test_ssl_certificate_base):
+
+    def setUp(self):
+        self.setup_config_files()
+        cfg.CONF.set_override('server_ssl', True, 'RESTPROXY')
+        cfg.CONF.set_override('ssl_sticky', True, 'RESTPROXY')
+        self.httpsPatch = mock.patch(HTTPS, create=True,
+                                     new=fake_server.HTTPSHostValidation)
+        self.httpsPatch.start()
+        self._setUp()
+        # Set fake HTTPS connection's expectation
+        self.fake_certget_m.return_value = self.host_cert_val
+        # No CA certs for this test
+        self.getcacerts_m.return_value = []
+        super(TestSslSticky, self).setUp()
+
+    def test_sticky_cert(self):
+        # SSL connection should be successful and cert should be cached
+        with self.network():
+            # CA certs should have been checked for
+            self.getcacerts_m.assert_has_calls([mock.call(self.ca_certs_path)])
+            # cert should have been fetched via SSL lib
+            self.sslgetcert_m.assert_has_calls(
+                [mock.call((self.servername, 443))]
+            )
+
+            # cert should have been recorded
+            self.fileput_m.assert_has_calls([mock.call(self.host_cert_path,
+                                                       self.host_cert_val)])
+            # no ca certs, so host cert only for this combined cert
+            self.certcomb_m.assert_has_calls([mock.call([self.host_cert_path],
+                                                        self.comb_cert_path)])
+
+
+class TestSslHostCert(test_ssl_certificate_base):
+
+    def setUp(self):
+        self.setup_config_files()
+        cfg.CONF.set_override('server_ssl', True, 'RESTPROXY')
+        cfg.CONF.set_override('ssl_sticky', False, 'RESTPROXY')
+        self.httpsPatch = mock.patch(HTTPS, create=True,
+                                     new=fake_server.HTTPSHostValidation)
+        self.httpsPatch.start()
+        self._setUp()
+        # Set fake HTTPS connection's expectation
+        self.fake_certget_m.return_value = self.host_cert_val
+        # No CA certs for this test
+        self.getcacerts_m.return_value = []
+        # Pretend host cert exists
+        self.hcertpath_p = mock.patch(GETHOSTCERT,
+                                      return_value=(self.host_cert_path, True),
+                                      create=True).start()
+        super(TestSslHostCert, self).setUp()
+
+    def test_host_cert(self):
+        # SSL connection should be successful because of pre-configured cert
+        with self.network():
+            self.hcertpath_p.assert_has_calls([
+                mock.call(os.path.join(self.cert_base, 'host_certs'),
+                          self.servername)
+            ])
+            # sticky is disabled, no fetching allowed
+            self.assertFalse(self.sslgetcert_m.call_count)
+            # no ca certs, so host cert is only for this combined cert
+            self.certcomb_m.assert_has_calls([mock.call([self.host_cert_path],
+                                                        self.comb_cert_path)])
+
+
+class TestSslCaCert(test_ssl_certificate_base):
+
+    def setUp(self):
+        self.setup_config_files()
+        cfg.CONF.set_override('server_ssl', True, 'RESTPROXY')
+        cfg.CONF.set_override('ssl_sticky', False, 'RESTPROXY')
+        self.httpsPatch = mock.patch(HTTPS, create=True,
+                                     new=fake_server.HTTPSCAValidation)
+        self.httpsPatch.start()
+        self._setUp()
+
+        # pretend to have a few ca certs
+        self.getcacerts_m.return_value = ['ca1.pem', 'ca2.pem']
+
+        # Set fake HTTPS connection's expectation
+        self.fake_certget_m.return_value = 'DUMMYCERTIFICATEAUTHORITY'
+
+        super(TestSslCaCert, self).setUp()
+
+    def test_ca_cert(self):
+        # SSL connection should be successful because CA cert was present
+        # If not, attempting to create a network would raise an exception
+        with self.network():
+            # sticky is disabled, no fetching allowed
+            self.assertFalse(self.sslgetcert_m.call_count)
+            # 2 CAs and no host cert so combined should only contain both CAs
+            self.certcomb_m.assert_has_calls([mock.call(['ca1.pem', 'ca2.pem'],
+                                                        self.comb_cert_path)])
+
+
+class TestSslWrongHostCert(test_ssl_certificate_base):
+
+    def setUp(self):
+        self.setup_config_files()
+        cfg.CONF.set_override('server_ssl', True, 'RESTPROXY')
+        cfg.CONF.set_override('ssl_sticky', True, 'RESTPROXY')
+        self.httpsPatch = mock.patch(HTTPS, create=True,
+                                     new=fake_server.HTTPSHostValidation)
+        self.httpsPatch.start()
+        self._setUp()
+
+        # Set fake HTTPS connection's expectation to something wrong
+        self.fake_certget_m.return_value = 'OTHERCERT'
+
+        # No CA certs for this test
+        self.getcacerts_m.return_value = []
+
+        # Pretend host cert exists
+        self.hcertpath_p = mock.patch(GETHOSTCERT,
+                                      return_value=(self.host_cert_path, True),
+                                      create=True).start()
+        super(TestSslWrongHostCert, self).setUp()
+
+    def test_error_no_cert(self):
+        # since there will already be a host cert, sticky should not take
+        # effect and there will be an error because the host cert's contents
+        # will be incorrect
+        tid = test_api_v2._uuid()
+        data = {}
+        data['network'] = {'tenant_id': tid, 'name': 'name',
+                           'admin_state_up': True}
+        req = self.new_create_request('networks', data, 'json')
+        res = req.get_response(self.api)
+        self.assertEqual(res.status_int,
+                         webob.exc.HTTPInternalServerError.code)
+        self.hcertpath_p.assert_has_calls([
+            mock.call(os.path.join(self.cert_base, 'host_certs'),
+                      self.servername)
+        ])
+        # sticky is enabled, but a host cert already exists so it shant fetch
+        self.assertFalse(self.sslgetcert_m.call_count)
+        # no ca certs, so host cert only for this combined cert
+        self.certcomb_m.assert_has_calls([mock.call([self.host_cert_path],
+                                                    self.comb_cert_path)])
+
+
+class TestSslNoValidation(test_ssl_certificate_base):
+
+    def setUp(self):
+        self.setup_config_files()
+        cfg.CONF.set_override('server_ssl', True, 'RESTPROXY')
+        cfg.CONF.set_override('ssl_sticky', False, 'RESTPROXY')
+        cfg.CONF.set_override('no_ssl_validation', True, 'RESTPROXY')
+        self.httpsPatch = mock.patch(HTTPS, create=True,
+                                     new=fake_server.HTTPSNoValidation)
+        self.httpsPatch.start()
+        self._setUp()
+        super(TestSslNoValidation, self).setUp()
+
+    def test_validation_disabled(self):
+        # SSL connection should be successful without any certificates
+        # If not, attempting to create a network will raise an exception
+        with self.network():
+            # no sticky grabbing and no cert combining with no enforcement
+            self.assertFalse(self.sslgetcert_m.call_count)
+            self.assertFalse(self.certcomb_m.call_count)

@@ -47,7 +47,10 @@ data_files =
         etc/neutron/rootwrap.d/ryu-plugin.filters
         etc/neutron/rootwrap.d/vpnaas.filters
     etc/init.d = etc/init.d/neutron-server
-    etc/neutron/plugins/bigswitch = etc/neutron/plugins/bigswitch/restproxy.ini
+    etc/neutron/plugins/bigswitch =
+        etc/neutron/plugins/bigswitch/restproxy.ini
+        etc/neutron/plugins/bigswitch/ssl/ca_certs/README
+        etc/neutron/plugins/bigswitch/ssl/host_certs/README
     etc/neutron/plugins/brocade = etc/neutron/plugins/brocade/brocade.ini
     etc/neutron/plugins/cisco = etc/neutron/plugins/cisco/cisco_plugins.ini
     etc/neutron/plugins/hyperv = etc/neutron/plugins/hyperv/hyperv_neutron_plugin.ini

@@ -193,6 +193,10 @@ Configuration Options
 * ``certfile``: (required, if Keystone server requires client cert)
 * ``keyfile``: (required, if Keystone server requires client cert)  This can be
   the same as the certfile if the certfile includes the private key.
+* ``cafile``: (optional, defaults to use system CA bundle) the path to a PEM
+  encoded CA file/bundle that will be used to verify HTTPS connections.
+* ``insecure``: (optional, default `False`) Don't verify HTTPS connections
+  (overrides `cafile`).
 
 Caching for improved response
 -----------------------------

@@ -145,9 +145,9 @@
 """
 
 import datetime
-import httplib
 import logging
 import os
+import requests
 import stat
 import tempfile
 import time
@@ -259,6 +259,10 @@
                help='Required if Keystone server requires client certificate'),
     cfg.StrOpt('keyfile',
                help='Required if Keystone server requires client certificate'),
+    cfg.StrOpt('cafile', default=None,
+               help='A PEM encoded Certificate Authority to use when '
+                    'verifying HTTPs connections. Defaults to system CAs.'),
+    cfg.BoolOpt('insecure', default=False, help='Verify HTTPS connections.'),
     cfg.StrOpt('signing_dir',
                help='Directory used to cache files related to PKI tokens'),
     cfg.ListOpt('memcached_servers',
@@ -354,43 +358,35 @@ def __init__(self, app, conf):
                                     (True, 'true', 't', '1', 'on', 'yes', 'y'))
 
         # where to find the auth service (we use this to validate tokens)
-        self.auth_host = self._conf_get('auth_host')
-        self.auth_port = int(self._conf_get('auth_port'))
-        self.auth_protocol = self._conf_get('auth_protocol')
-        if not self._conf_get('http_handler'):
-            if self.auth_protocol == 'http':
-                self.http_client_class = httplib.HTTPConnection
-            else:
-                self.http_client_class = httplib.HTTPSConnection
-        else:
-            # Really only used for unit testing, since we need to
-            # have a fake handler set up before we issue an http
-            # request to get the list of versions supported by the
-            # server at the end of this initialization
-            self.http_client_class = self._conf_get('http_handler')
-
+        auth_host = self._conf_get('auth_host')
+        auth_port = int(self._conf_get('auth_port'))
+        auth_protocol = self._conf_get('auth_protocol')
         self.auth_admin_prefix = self._conf_get('auth_admin_prefix')
         self.auth_uri = self._conf_get('auth_uri')
+
+        if netaddr.valid_ipv6(auth_host):
+            # Note(dzyu) it is an IPv6 address, so it needs to be wrapped
+            # with '[]' to generate a valid IPv6 URL, based on
+            # http://www.ietf.org/rfc/rfc2732.txt
+            auth_host = '[%s]' % auth_host
+
+        self.request_uri = '%s://%s:%s' % (auth_protocol, auth_host, auth_port)
+
         if self.auth_uri is None:
             self.LOG.warning(
                 'Configuring auth_uri to point to the public identity '
                 'endpoint is required; clients may not be able to '
                 'authenticate against an admin endpoint')
-            host = self.auth_host
-            if netaddr.valid_ipv6(host):
-                # Note(dzyu) it is an IPv6 address, so it needs to be wrapped
-                # with '[]' to generate a valid IPv6 URL, based on
-                # http://www.ietf.org/rfc/rfc2732.txt
-                host = '[%s]' % host
+
             # FIXME(dolph): drop support for this fallback behavior as
             # documented in bug 1207517
-            self.auth_uri = '%s://%s:%s' % (self.auth_protocol,
-                                            host,
-                                            self.auth_port)
+            self.auth_uri = self.request_uri
 
         # SSL
         self.cert_file = self._conf_get('certfile')
         self.key_file = self._conf_get('keyfile')
+        self.ssl_ca_file = self._conf_get('cafile')
+        self.ssl_insecure = self._conf_get('insecure')
 
         # signing
         self.signing_dirname = self._conf_get('signing_dir')
@@ -403,7 +399,7 @@ def __init__(self, app, conf):
         val = '%s/signing_cert.pem' % self.signing_dirname
         self.signing_cert_file_name = val
         val = '%s/cacert.pem' % self.signing_dirname
-        self.ca_file_name = val
+        self.signing_ca_file_name = val
         val = '%s/revoked.pem' % self.signing_dirname
         self.revoked_file_name = val
 
@@ -505,12 +501,12 @@ def _choose_api_version(self):
     def _get_supported_versions(self):
         versions = []
         response, data = self._json_request('GET', '/')
-        if response.status == 501:
+        if response.status_code == 501:
             self.LOG.warning("Old keystone installation found...assuming v2.0")
             versions.append("v2.0")
-        elif response.status != 300:
+        elif response.status_code != 300:
             self.LOG.error('Unable to get version info from keystone: %s' %
-                           response.status)
+                           response.status_code)
             raise ServiceError('Unable to get version info from keystone')
         else:
             try:
@@ -648,17 +644,6 @@ def get_admin_token(self):
 
         return self.admin_token
 
-    def _get_http_connection(self):
-        if self.auth_protocol == 'http':
-            return self.http_client_class(self.auth_host, self.auth_port,
-                                          timeout=self.http_connect_timeout)
-        else:
-            return self.http_client_class(self.auth_host,
-                                          self.auth_port,
-                                          self.key_file,
-                                          self.cert_file,
-                                          timeout=self.http_connect_timeout)
-
     def _http_request(self, method, path, **kwargs):
         """HTTP request helper used to make unspecified content type requests.
 
@@ -668,28 +653,35 @@ def _http_request(self, method, path, **kwargs):
         :raise ServerError when unable to communicate with keystone
 
         """
-        conn = self._get_http_connection()
+        url = "%s/%s" % (self.request_uri, path.lstrip('/'))
+
+        kwargs.setdefault('timeout', self.http_connect_timeout)
+        if self.cert_file and self.key_file:
+            kwargs['cert'] = (self.cert_file, self.key_file)
+        elif self.cert_file or self.key_file:
+            self.LOG.warn('Cannot use only a cert or key file. '
+                          'Please provide both. Ignoring.')
+
+        kwargs['verify'] = self.ssl_ca_file or True
+        if self.ssl_insecure:
+            kwargs['verify'] = False
 
         RETRIES = self.http_request_max_retries
         retry = 0
         while True:
             try:
-                conn.request(method, path, **kwargs)
-                response = conn.getresponse()
-                body = response.read()
+                response = requests.request(method, url, **kwargs)
                 break
             except Exception as e:
-                if retry == RETRIES:
-                    self.LOG.error('HTTP connection exception: %s' % e)
+                if retry >= RETRIES:
+                    self.LOG.error('HTTP connection exception: %s', e)
                     raise NetworkError('Unable to communicate with keystone')
                 # NOTE(vish): sleep 0.5, 1, 2
                 self.LOG.warn('Retrying on HTTP connection exception: %s' % e)
                 time.sleep(2.0 ** retry / 2)
                 retry += 1
-            finally:
-                conn.close()
 
-        return response, body
+        return response
 
     def _json_request(self, method, path, body=None, additional_headers=None):
         """HTTP request helper used to make json requests.
@@ -714,14 +706,14 @@ def _json_request(self, method, path, body=None, additional_headers=None):
             kwargs['headers'].update(additional_headers)
 
         if body:
-            kwargs['body'] = jsonutils.dumps(body)
+            kwargs['data'] = jsonutils.dumps(body)
 
         path = self.auth_admin_prefix + path
 
-        response, body = self._http_request(method, path, **kwargs)
+        response = self._http_request(method, path, **kwargs)
 
         try:
-            data = jsonutils.loads(body)
+            data = jsonutils.loads(response.text)
         except ValueError:
             self.LOG.debug('Keystone did not return json-encoded body')
             data = {}
@@ -1090,18 +1082,18 @@ def verify_uuid_token(self, user_token, retry=True):
                 '/v2.0/tokens/%s' % safe_quote(user_token),
                 additional_headers=headers)
 
-        if response.status == 200:
+        if response.status_code == 200:
             return data
-        if response.status == 404:
+        if response.status_code == 404:
             self.LOG.warn("Authorization failed for token %s", user_token)
             raise InvalidUserToken('Token authorization failed')
-        if response.status == 401:
+        if response.status_code == 401:
             self.LOG.info(
                 'Keystone rejected admin token %s, resetting', headers)
             self.admin_token = None
         else:
             self.LOG.error('Bad response code while validating token: %s' %
-                           response.status)
+                           response.status_code)
         if retry:
             self.LOG.info('Retrying validation')
             return self._validate_user_token(user_token, False)
@@ -1135,13 +1127,14 @@ def cms_verify(self, data):
         while True:
             try:
                 output = cms.cms_verify(data, self.signing_cert_file_name,
-                                        self.ca_file_name)
+                                        self.signing_ca_file_name)
             except cms.subprocess.CalledProcessError as err:
                 if self.cert_file_missing(err.output,
                                           self.signing_cert_file_name):
                     self.fetch_signing_cert()
                     continue
-                if self.cert_file_missing(err.output, self.ca_file_name):
+                if self.cert_file_missing(err.output,
+                                          self.signing_ca_file_name):
                     self.fetch_ca_cert()
                     continue
                 self.LOG.warning('Verify error: %s' % err)
@@ -1221,14 +1214,14 @@ def fetch_revocation_list(self, retry=True):
         headers = {'X-Auth-Token': self.get_admin_token()}
         response, data = self._json_request('GET', '/v2.0/tokens/revoked',
                                             additional_headers=headers)
-        if response.status == 401:
+        if response.status_code == 401:
             if retry:
                 self.LOG.info(
                     'Keystone rejected admin token %s, resetting admin token',
                     headers)
                 self.admin_token = None
                 return self.fetch_revocation_list(retry=False)
-        if response.status != 200:
+        if response.status_code != 200:
             raise ServiceError('Unable to fetch token revocation list.')
         if 'signed' not in data:
             raise ServiceError('Revocation list improperly formatted.')
@@ -1237,7 +1230,7 @@ def fetch_revocation_list(self, retry=True):
     def fetch_signing_cert(self):
         path = self.auth_admin_prefix.rstrip('/')
         path += '/v2.0/certificates/signing'
-        response, data = self._http_request('GET', path)
+        response = self._http_request('GET', path)
 
         def write_cert_file(data):
             with open(self.signing_cert_file_name, 'w') as certfile:
@@ -1246,26 +1239,26 @@ def write_cert_file(data):
         try:
             #todo check response
             try:
-                write_cert_file(data)
+                write_cert_file(response.text)
             except IOError:
                 self.verify_signing_dir()
-                write_cert_file(data)
+                write_cert_file(response.text)
         except (AssertionError, KeyError):
             self.LOG.warn(
-                "Unexpected response from keystone service: %s", data)
+                "Unexpected response from keystone service: %s", response.text)
             raise ServiceError('invalid json response')
 
     def fetch_ca_cert(self):
         path = self.auth_admin_prefix.rstrip('/') + '/v2.0/certificates/ca'
-        response, data = self._http_request('GET', path)
+        response = self._http_request('GET', path)
 
         try:
             #todo check response
-            with open(self.ca_file_name, 'w') as certfile:
-                certfile.write(data)
+            with open(self.signing_ca_file_name, 'w') as certfile:
+                certfile.write(response.text)
         except (AssertionError, KeyError):
             self.LOG.warn(
-                "Unexpected response from keystone service: %s", data)
+                "Unexpected response from keystone service: %s", response.text)
             raise ServiceError('invalid json response')
 
 

@@ -864,7 +864,7 @@ def test_fetch_signing_ca(self):
                                body=data)
         self.middleware.fetch_ca_cert()
 
-        with open(self.middleware.ca_file_name, 'r') as f:
+        with open(self.middleware.signing_ca_file_name, 'r') as f:
             self.assertEqual(f.read(), data)
 
         self.assertEqual("/testadmin/v2.0/certificates/ca",