Add openstacksdk option to openstack_controller

datadog_checks_base/datadog_checks/base/data/agent_requirements.in

@@ -21,6 +21,7 @@ ldap3==2.5
 lxml==4.2.6
 meld3==1.0.2
 ntplib==0.3.3
+openstacksdk==0.24.0
 paramiko==2.1.5
 pg8000==1.10.1
 ply==3.10

openstack_controller/datadog_checks/openstack_controller/api.py

@@ -4,6 +4,8 @@
 import requests
 import simplejson as json
 
+from openstack import connection
+from os import environ
 from six.moves.urllib.parse import urljoin
 from datadog_checks.config import is_affirmative
 
@@ -24,10 +26,21 @@ def create(logger, proxies, instance_config):
         paginated_limit = instance_config.get('paginated_limit')
         request_timeout = instance_config.get('request_timeout')
         user = instance_config.get("user")
+        openstack_config_file_path = instance_config.get("openstack_config_file_path")
+        openstack_cloud_name = instance_config.get("openstack_cloud_name")
+
+        api = None
+
+        # If an openstack configuration is specified, an OpenstackSDKApi will be created, and the authentification
+        # will be made directly from the openstack configuration file
+        if openstack_cloud_name is None:
+            api = SimpleApi(logger, keystone_server_url, timeout=request_timeout, ssl_verify=ssl_verify,
+                            proxies=proxies, limit=paginated_limit)
+            api.connect(user)
+        else:
+            api = OpenstackSDKApi(logger)
+            api.connect(openstack_config_file_path, openstack_cloud_name)
 
-        api = SimpleApi(logger, keystone_server_url, timeout=request_timeout, ssl_verify=ssl_verify, proxies=proxies,
-                        limit=paginated_limit)
-        api.connect(user)
         return api
 
 
@@ -72,6 +85,161 @@ def get_networks(self):
         raise NotImplementedError()
 
 
+class OpenstackSDKApi(AbstractApi):
+    def __init__(self, logger):
+        super(OpenstackSDKApi, self).__init__(logger)
+
+        self.connection = None
+        self.services = {}
+        self.endpoints = {}
+        self.projects = None
+
+    def connect(self, openstack_sdk_config_file_path, openstack_sdk_cloud_name):
+        if openstack_sdk_config_file_path is not None:
+            # Set the environment variable to the path of the config file for openstacksdk to find it
+            environ["OS_CLIENT_CONFIG_FILE"] = openstack_sdk_config_file_path
+
+        self.connection = connection.Connection(cloud=openstack_sdk_cloud_name)
+        # Raise error if the connection failed
+        self.connection.authorize()
+
+    def _check_authentication(self):
+        if self.connection is None:
+            raise AuthenticationNeeded()
+
+    def _get_service(self, service_name):
+        self._check_authentication()
+
+        if service_name not in self.services:
+            self.services[service_name] = self.connection.get_service(service_name)
+        return self.services[service_name]
+
+    def _get_endpoint(self, service_name):
+        self._check_authentication()
+
+        if service_name not in self.endpoints:
+            try:
+                service_filter = {u'service_id': self._get_service(service_name)[u'id']}
+                endpoints_list = self.connection.search_endpoints(filters=service_filter)
+
+                if not endpoints_list:
+                    return None
+
+                self.endpoints[service_name] = None
+                # Get the public or the internal endpoint
+                for endpoint in endpoints_list:
+                    if endpoint[u'interface'] == u'public':
+                        self.endpoints[service_name] = endpoint
+                        return self.endpoints[service_name]
+                    elif endpoint[u'interface'] == u'internal':
+                        self.endpoints[service_name] = endpoint
+            except Exception as e:
+                self.logger.debug("Error contacting openstack endpoint with openstacksdk: %s", e)
+
+        return self.endpoints[service_name]
+
+    def get_keystone_endpoint(self):
+        keystone_endpoint = self._get_endpoint(u'keystone')
+
+        if keystone_endpoint is None:
+            raise KeystoneUnreachable()
+        return keystone_endpoint[u'links'][u'self']
+
+    def get_nova_endpoint(self):
+        nova_endpoint = self._get_endpoint(u'nova')
+
+        if nova_endpoint is None:
+            raise MissingNovaEndpoint()
+        return nova_endpoint[u'links'][u'self']
+
+    def get_neutron_endpoint(self):
+        neutron_endpoint = self._get_endpoint(u'neutron')
+
+        if neutron_endpoint is None:
+            raise MissingNeutronEndpoint()
+        return neutron_endpoint[u'links'][u'self']
+
+    def get_projects(self):
+        self._check_authentication()
+
+        if self.projects is None:
+            self.projects = self.connection.search_projects()
+
+        return self.projects
+
+    def get_project_limits(self, project_id):
+        self._check_authentication()
+
+        # Raise exception if the project is not found
+        project_limits_raw = self.connection.get_compute_limits(project_id)
+        project_limits = project_limits_raw["properties"]
+
+        # Used to convert the project_limits_raw key name
+        key_name_conversion = {
+            "max_personality": "maxPersonality",
+            "max_personality_size": "maxPersonalitySize",
+            "max_server_groups": "maxServerGroups",
+            "max_server_group_members": "maxServerGroupMembers",
+            "max_server_meta": "maxServerMeta",
+            "max_total_cores": "maxTotalCores",
+            "max_total_keypairs": "maxTotalKeypairs",
+            "max_total_instances": "maxTotalInstances",
+            "max_total_ram_size": "maxTotalRAMSize",
+            "total_cores_used": "totalCoresUsed",
+            "total_instances_used": "totalInstancesUsed",
+            "total_ram_used": "totalRAMUsed",
+            "total_server_groups_used": "totalServerGroupsUsed"
+        }
+
+        for raw_value, value in key_name_conversion.items():
+            project_limits[value] = project_limits_raw[raw_value]
+
+        return project_limits
+
+    def get_os_hypervisors_detail(self):
+        self._check_authentication()
+
+        return self.connection.list_hypervisors()
+
+    def get_os_hypervisor_uptime(self, hypervisor_id):
+        # Hypervisor uptime is not available in openstacksdk 0.24.0.
+        self.logger.warning("Hypervisor uptime is not available with this version of openstacksdk")
+        raise NotImplementedError()
+
+    def get_os_aggregates(self):
+        # Each aggregate is missing the 'uuid' attribute compared to what is returned by SimpleApi
+        self._check_authentication()
+
+        return self.connection.list_aggregates()
+
+    def get_flavors_detail(self, query_params):
+        self._check_authentication()
+
+        return self.connection.search_flavors(filters=query_params)
+
+    def get_networks(self):
+        self._check_authentication()
+
+        return self.connection.list_networks()
+
+    def get_servers_detail(self, query_params):
+        # Each server is missing some attributes compared to what is returned by SimpleApi.
+        # They are all unused for the moment.
+        # SimpleApi:
+        # https://developer.openstack.org/api-ref/compute/?expanded=list-flavors-with-details-detail,list-servers-detailed-detail#list-servers-detailed
+        # OpenstackSDKApi:
+        # https://docs.openstack.org/openstacksdk/latest/user/connection.html#openstack.connection.Connection
+
+        self._check_authentication()
+
+        return self.connection.list_servers(detailed=True, all_projects=True, filters=query_params)
+
+    def get_server_diagnostics(self, server_id):
+        # Server diagnostics is not available in openstacksdk 0.24.0. It should be available in the next release.
+        self.logger.warning("Server diagnostics is not available with this version of openstacksdk")
+        raise NotImplementedError()
+
+
 class SimpleApi(AbstractApi):
     def __init__(self, logger, keystone_endpoint, ssl_verify=False, proxies=None,
                  timeout=DEFAULT_API_REQUEST_TIMEOUT, limit=DEFAULT_PAGINATED_LIMIT):

openstack_controller/datadog_checks/openstack_controller/data/conf.yaml.example

@@ -8,17 +8,29 @@ init_config:
 
 instances:
 
-## @param name - required - string
-## Unique identifier for this instance.
-#
+  ## @param name - string - required
+  ## Unique identifier for this instance.
+  #
   - name: <INSTANCE_NAME>
 
   ## @param keystone_server_url - string - required
   ## URL of your identity server. Note that the server must support Identity API v3
   #
-    keystone_server_url: <"https://my-keystone-server.com:<port>/">
+    keystone_server_url: https://<KEYSTONE_ENDPOINT>:<PORT>/
 
-  ## @param whitelist_project_names - list of string - optional - default:[]
+  ## @param user - object - required
+  ## Password authentication is the only auth method supported
+  ## User expects username, password, and user domain id
+  ## `user` should resolve to a structure like {'password': '<PASSWORD>', 'name': '<USER_NAME>', 'domain': {'id': '<DOMAIN_ID>'}}
+  ## The check uses the Unscoped token method to collect information about all available projects to the user.
+  #
+    user:
+        password: <PASSWORD>
+        name: <USER_NAME>
+        domain:
+            id: <DOMAIN_ID>
+
+  ## @param whitelist_project_names - list of strings - optional - default: []
   ## IDs of projects to whitelist for monitoring (by default the agent collects limit metrics from all projects)
   ## Regex expressions for the project names are supported.
   ## Whitelist takes precedence over blacklist in case of overlap.
@@ -27,7 +39,7 @@ instances:
   #    - <PROJECT_NAME>
   #    - <PROJECT_PREFIX>*
 
-  ## @param blacklist_project_names - list of string - optional - default:[]
+  ## @param blacklist_project_names - list of strings - optional - default: []
   ## IDs of projects to blacklist for monitoring (by default the agent collects limit metrics from all projects)
   ## Regex expressions for the project names are supported.
   ## Whitelist takes precedence over blacklist in case of overlap.
@@ -36,92 +48,93 @@ instances:
   #    - <PROJECT_NAME>
   #    - <PROJECT_PREFIX>*
 
-  ## @param exclude_server_ids - list of string - optional - default:[]
+  ## @param exclude_server_ids - list of strings - optional - default: []
   ## IDs of servers to exclude from monitoring. (by default the agent collects metrics from all guest servers)
   ## Regex expressions for the server IDs are supported.
   #
   #  exclude_server_ids:
   #    - <SERVER_ID>
   #    - <SERVER_ID_PREFIX>*
 
-  ## @param exclude_network_ids - optional - list of string - default:[]
+  ## @param exclude_network_ids - list of strings - optional - default: []
   ## IDs of networks to exclude from monitoring (by default the agent collects metrics from networks returned by the neutron:get_networks operation)
   ## Regex expressions for the network IDs to exclude are supported.
   #
   #  exclude_network_ids:
   #    - <NETWORK_ID>
   #    - <NETWORK_ID_PREFIX>*
 
-  ## @param ssl_verify - optional - boolean - default:true
+  ## @param ssl_verify - boolean - optional - default: true
   ## Whether to enable SSL certificate verification for HTTP requests. Defaults to true, you may
   ## need to set to false when using self-signed certs.
   #
   #  ssl_verify: true
 
-  ## @param use_agent_proxy - optional - boolean - default:true
+  ## @param use_agent_proxy - boolean - optional - default: true
   ## Whether the dd-agent proxy should also be used for openstack API requests (if set).
   #
   #  use_agent_proxy: true
 
-  ## @param paginated_limit - optional - integer - default:1000
-  ## paginated_limit set the number of items some api calls should return
+  ## @param paginated_limit - integer - optional - default: 1000
+  ## paginated_limit sets the number of items some api calls should return.
   #
   #  paginated_limit: 1000
 
-  ## @param request_timeout - optional - integer - default:10
-  ## request_timeout set the timeout in second used when making api calls
+  ## @param request_timeout - integer - optional - default: 10
+  ## request_timeout sets the timeout in second used when making api calls.
   #
   #  request_timeout: 10
 
-  ## @param user - required - user_object
-  ## Password authentication is the only auth method supported
-  ## User expects username, password, and user domain id
-  ## `user` should resolve to a structure like {'password': '<PASSWORD>', 'name': '<USER_NAME>', 'domain': {'id': '<DOMAIN_ID>'}}
-  ## The check uses the Unscoped token method to collect information about all available projects to the user.
+  ## @param openstack_config_file_path - string - optional
+  ## Absolute path of the configuration file for the connection to openstack with openstacksdk.
   #
-    user:
-        password: <PASSWORD>
-        name: <USER_NAME>
-        domain:
-            id: <DOMAIN_ID>
+  #  openstack_config_file_path: <PATH_TO_YAML_FILE>
+
+  ## @param openstack_cloud_name - string - optional
+  ## Name of the cloud configuration to use.
+  ## If this parameter is specified, the connection to Openstack will be using only the configuration file specified in
+  ## openstack_config_file_path, or at the default location:
+  ## ~/.config/openstack or /etc/openstack
+  #
+  #  openstack_cloud_name: <CLOUD_NAME>
 
-  ## @param collect_hypervisor_load - optional - boolean - default:true
+  ## @param collect_hypervisor_load - boolean - optional - default: true
   ## Collects hypervisor_load.1/5/15 for each hypervisor
   ## Disabled by default to increase performance of the check.
   ## With this enabled there is an extra N requests per check run (N = # of hypervisors)
   ## If the Agent is installed on each hypervisor, this metric is available as system.load.1/5/15
   #
   #  collect_hypervisor_load: true
 
-  ## @param collect_hypervisor_metrics - optional - boolean - default:true
+  ## @param collect_hypervisor_metrics - boolean - optional - default: true
   ## Collects hypervisor metrics (including hypervisor_load).
   #
   #  collect_hypervisor_metrics: true
 
-  ## @param collect_project_metrics - optional - boolean - default:true
+  ## @param collect_project_metrics - boolean - optional - default: true
   ## The admin user defined for Datadog starts with a "default" project.
   ## If this option is disabled, project limits metrics are only collected from projects the
   ## datadog defined user has access to view.
   #
   #  collect_project_metrics: true
 
-  ## @param collect_network_metrics - optional - boolean - default:true
+  ## @param collect_network_metrics - boolean - optional - default: true
   ## Collects network metrics.
   #
   #  collect_network_metrics: true
 
-  ## @param collect_server_flavor_metrics - optional - boolean - default:true
-  ## Collect server flavor metrics
+  ## @param collect_server_flavor_metrics - boolean - optional - default: true
+  ## Collect server flavor metrics.
   #
   #  collect_server_flavor_metrics: true
 
-  ## @param collect_diagnostic_flavor_metrics - optional - boolean - default:true
+  ## @param collect_diagnostic_flavor_metrics - boolean - optional - default: true
   ## Collect server diagnostic metrics. Enabling this may result in performance decrease.
-  ## This is fine for small deployments. For large deployment, turn it off and install the agent on VMs directly
+  ## This is fine for small deployments. For large deployment, turn it off and install the agent on VMs directly.
   #
   #  collect_server_diagnostic_metrics: true
 
-  ## @param use_shortname - optional - boolean - default:false
+  ## @param use_shortname - boolean - optional - default: false
   ## In some OpenStack environments, the hostname registered to Nova is the shortname. 
   ## Enabling this enforces the check to split the hostname up to the first period when
   ## comparing against nova responses. Only affects aggregate tagging. 
@@ -135,4 +148,4 @@ instances:
   #
   #  tags:
   #    - <KEY_1>:<VALUE_1>
-  #    - <KEY_2>:<VALUE_2>
+  #    - <KEY_2>:<VALUE_2>

openstack_controller/requirements.in

@@ -0,0 +1 @@
+openstacksdk==0.24.0