Add requests wrapper to mesos_slave (#4222)

* Add requests wrapper to mesos_slave * Add standard config options * Update with default_timeout support * Update with feedback * support read/connect timeout * update __init__ * Update __init__ * fix test * update with instance * fix error * fix indent * fix endpoint in get_stats * Add timeout tests * Refactor test * fix style * Update with check mocking * Update with docs review
DataDog · Aug 21, 2019 · eda40ee · eda40ee
1 parent fab3ca0
commit eda40ee
Show file tree

Hide file tree

Showing 6 changed files with 297 additions and 59 deletions.
diff --git a/mesos_slave/datadog_checks/mesos_slave/data/conf.yaml.example b/mesos_slave/datadog_checks/mesos_slave/data/conf.yaml.example
@@ -1,9 +1,30 @@
 init_config:
+    ## @param proxy - object - optional
+    ## Set HTTP or HTTPS proxies for all instances. Use the `no_proxy` list
+    ## to specify hosts that must bypass proxies.
+    ##
+    ## The SOCKS protocol is also supported:
+    ##
+    ## socks5://user:pass@host:port
+    ##
+    ## Using the scheme `socks5` causes the DNS resolution to happen on the
+    ## client, rather than on the proxy server. This is in line with `curl`,
+    ## which uses the scheme to decide whether to do the DNS resolution on
+    ## the client or proxy. If you want to resolve the domains on the proxy
+    ## server, use `socks5h` as the scheme.
+    #
+    # proxy:
+    #   http: http://<PROXY_SERVER_FOR_HTTP>:<PORT>
+    #   https: https://<PROXY_SERVER_FOR_HTTPS>:<PORT>
+    #   no_proxy:
+    #     - <HOSTNAME_1>
+    #     - <HOSTNAME_2>
 
-  ## @param default_timeout - integer - required
-  ## Default timeout to use when connecting to Mesos master url.
-  #
-  default_timeout: 10
+    ## @param skip_proxy - boolean - optional - default: false
+    ## If set to `true`, this makes the check bypass any proxy
+    ## settings enabled and attempt to reach services directly.
+    #
+    # skip_proxy: false
 
 instances:
 
@@ -23,12 +44,6 @@ instances:
     # tasks:
     #   - <TASK_NAME>
 
-    ## @param disable_ssl_validation - boolean - optional - default: false
-    ## Instructs the check to skip the validation of the SSL certificate of the URL being tested for Mesos master.
-    ## Defaults to false, set to true if you want to disable SSL certificate validation.
-    #
-    # disable_ssl_validation: false
-
     ## @param tags - list of key:value elements - optional
     ## List of tags to attach to every metric, event and service check emitted by this integration.
     ##
@@ -37,3 +52,165 @@ instances:
     # tags:
     #   - <KEY_1>:<VALUE_1>
     #   - <KEY_2>:<VALUE_2>
+
+    ## @param username - string - optional
+    ## The username to use if services are behind basic auth.
+    #
+    # username: <USERNAME>
+
+    ## @param ntlm_domain - string - optional
+    ## If your services uses NTLM authentication, you can
+    ## specify a domain that is used in the check. For NTLM auth,
+    ## append the username to domain, not as the `username` parameter.
+    ## Example: <NTLM_DOMAIN>/<USERNAME>
+    #
+    # ntlm_domain: <DOMAIN>
+
+    ## @param password - string - optional
+    ## The password to use if services are behind basic or NTLM auth.
+    #
+    # password: <PASSWORD>
+
+    ## @param connect_timeout - integer - optional
+    ## Overrides the default connection timeout value,
+    ## and fails the check if the time to establish the (TCP) connection
+    ## exceeds the connect_timeout value (in seconds)
+    #
+    # connect_timeout: <VALUE_IN_SECOND>
+
+    ## @param read_timeout - integer - optional
+    ## Overrides the default received timeout value, and fails the check if the time to receive
+    ## the server status from the server exceeds the read_timeout value (in seconds)
+    #
+    # read_timeout: <VALUE_IN_SECOND>
+
+    ## @param proxy - object - optional
+    ## This overrides the `proxy` setting in `init_config`.
+    ##
+    ## Set HTTP or HTTPS proxies. Use the `no_proxy` list
+    ## to specify hosts that must bypass proxies.
+    ##
+    ## The SOCKS protocol is also supported:
+    ##
+    ## socks5://user:pass@host:port
+    ##
+    ## Using the scheme `socks5` causes the DNS resolution to happen on the
+    ## client, rather than on the proxy server. This is in line with `curl`,
+    ## which uses the scheme to decide whether to do the DNS resolution on
+    ## the client or proxy. If you want to resolve the domains on the proxy
+    ## server, use `socks5h` as the scheme.
+    #
+    # proxy:
+    #   http: http://<PROXY_SERVER_FOR_HTTP>:<PORT>
+    #   https: https://<PROXY_SERVER_FOR_HTTPS>:<PORT>
+    #   no_proxy:
+    #     - <HOSTNAME_1>
+    #     - <HOSTNAME_2>
+
+    ## @param skip_proxy - boolean - optional - default: false
+    ## This overrides the `skip_proxy` setting in `init_config`.
+    ##
+    ## If set to `true`, this makes the check bypass any proxy
+    ## settings enabled and attempt to reach services directly.
+    #
+    # skip_proxy: false
+
+    ## @param kerberos_auth - string - optional - default: disabled
+    ## If your service uses Kerberos authentication, you can specify the Kerberos
+    ## strategy to use between:
+    ##  * required
+    ##  * optional
+    ##  * disabled
+    ##
+    ## See https://github.com/requests/requests-kerberos#mutual-authentication
+    #
+    # kerberos_auth: disabled
+
+    ## @param kerberos_delegate - boolean - optional - default: false
+    ## Set to `true` to enable Kerberos delegation of credentials to a server that requests delegation.
+    ## See https://github.com/requests/requests-kerberos#delegation
+    #
+    # kerberos_delegate: false
+
+    ## @param kerberos_force_initiate - boolean - optional - default: false
+    ## Set to `true` to preemptively initiate the Kerberos GSS exchange and present a Kerberos ticket on the initial
+    ## request (and all subsequent requests).
+    ## See https://github.com/requests/requests-kerberos#preemptive-authentication
+    #
+    # kerberos_force_initiate: false
+
+    ## @param kerberos_hostname - string - optional
+    ## Override the hostname used for the Kerberos GSS exchange if its DNS name doesn't match its Kerberos
+    ## hostname (e.g., behind a content switch or load balancer).
+    ## See https://github.com/requests/requests-kerberos#hostname-override
+    #
+    # kerberos_hostname: null
+
+    ## @param kerberos_principal - string - optional
+    ## Set an explicit principal, to force Kerberos to look for a matching credential cache for the named user.
+    ## See https://github.com/requests/requests-kerberos#explicit-principal
+    #
+    # kerberos_principal: null
+
+    ## @param kerberos_keytab - string - optional
+    ## Set the path to your Kerberos key tab file.
+    #
+    # kerberos_keytab: <KEYTAB_FILE_PATH>
+
+    ## @param tls_verify - boolean - optional - default: true
+    ## Instructs the check to validate the TLS certificate of services.
+    #
+    # tls_verify: true
+
+    ## @param tls_ignore_warning - boolean - optional - default: false
+    ## If `tls_verify` is disabled, security warnings are logged by the check.
+    ## Disable these by setting `tls_ignore_warning` to true.
+    #
+    # tls_ignore_warning: false
+
+    ## @param tls_cert - string - optional
+    ## The path to a single file in PEM format containing a certificate as well as any
+    ## number of CA certificates needed to establish the certificate’s authenticity for
+    ## use when connecting to services. It may also contain an unencrypted private key to use.
+    #
+    # tls_cert: <CERT_PATH>
+
+    ## @param tls_private_key - string - optional
+    ## The unencrypted private key to use for `tls_cert` when connecting to services. This is
+    ## required if `tls_cert` is set and it does not already contain a private key.
+    #
+    # tls_private_key: <PRIVATE_KEY_PATH>
+
+    ## @param tls_ca_cert - string - optional
+    ## The path to a file of concatenated CA certificates in PEM format or a directory
+    ## containing several CA certificates in PEM format. If a directory, the directory
+    ## must have been processed using the c_rehash utility supplied with OpenSSL. See:
+    ## https://www.openssl.org/docs/manmaster/man3/SSL_CTX_load_verify_locations.html
+    #
+    # tls_ca_cert: <CA_CERT_PATH>
+
+    ## @param headers - list of key:value elements - optional
+    ## The headers parameter allows you to send specific headers with every request.
+    ## You can use it for explicitly specifying the host header or adding headers for
+    ## authorization purposes.
+    ##
+    ## This overrides any default headers.
+    #
+    # headers:
+    #   Host: <ALTERNATIVE_HOSTNAME>
+    #   X-Auth-Token: <AUTH_TOKEN>
+
+    ## @param timeout - integer - optional - default: 10
+    ## The timeout for connecting to services.
+    #
+    # timeout: 10
+
+    ## @param log_requests - boolean - optional - default: false
+    ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
+    #
+    # log_requests: false
+
+    ## @param persist_connections - boolean - optional - default: false
+    ## Whether or not to persist cookies and use connection pooling for increased performance.
+    #
+    # persist_connections: false
diff --git a/mesos_slave/datadog_checks/mesos_slave/mesos_slave.py b/mesos_slave/datadog_checks/mesos_slave/mesos_slave.py
@@ -12,7 +12,6 @@
 from six.moves.urllib.parse import urlparse
 
 from datadog_checks.checks import AgentCheck
-from datadog_checks.config import _is_affirmative
 from datadog_checks.errors import CheckException
 
 DEFAULT_MASTER_PORT = 5050
@@ -23,6 +22,7 @@ class MesosSlave(AgentCheck):
     MONOTONIC_COUNT = AgentCheck.monotonic_count
     SERVICE_CHECK_NAME = "mesos_slave.can_connect"
     service_check_needed = True
+    DEFAULT_TIMEOUT = 5
 
     TASK_STATUS = {
         'TASK_STARTING': AgentCheck.OK,
@@ -93,22 +93,34 @@ class MesosSlave(AgentCheck):
         'slave/valid_status_updates': ('mesos.slave.valid_status_updates', GAUGE),
     }
 
-    def __init__(self, name, init_config, agentConfig, instances=None):
-        AgentCheck.__init__(self, name, init_config, agentConfig, instances)
+    HTTP_CONFIG_REMAPPER = {'disable_ssl_validation': {'name': 'tls_verify', 'invert': True, 'default': False}}
+
+    def __init__(self, name, init_config, instances):
+        super(MesosSlave, self).__init__(name, init_config, instances)
         self.cluster_name = None
-        for instance in instances or []:
-            url = instance.get('url', '')
-            parsed_url = urlparse(url)
-            ssl_verify = not _is_affirmative(instance.get('disable_ssl_validation', False))
-            if not ssl_verify and parsed_url.scheme == 'https':
-                self.log.warning('Skipping SSL cert validation for {0} based on configuration.'.format(url))
-
-    def _get_json(self, url, timeout, verify, failure_expected=False, tags=None):
+        url = self.instance.get('url', '')
+        parsed_url = urlparse(url)
+        if self.http.options['verify'] and parsed_url.scheme == 'https':
+            self.log.warning('Skipping TLS cert validation for %s based on configuration.' % url)
+        if not ('read_timeout' in self.instance or 'connect_timeout' in self.instance):
+            # `default_timeout` config option will be removed with Agent 5
+            timeout = (
+                self.instance.get('timeout')
+                or self.instance.get('default_timeout')
+                or self.init_config.get('timeout')
+                or self.init_config.get('default_timeout')
+                or self.DEFAULT_TIMEOUT
+            )
+            self.http.options['timeout'] = (timeout, timeout)
+
+    def _get_json(self, url, failure_expected=False, tags=None):
         tags = tags + ["url:%s" % url] if tags else ["url:%s" % url]
         msg = None
         status = None
+        timeout = self.http.options['timeout']
+
         try:
-            r = requests.get(url, timeout=timeout, verify=verify)
+            r = self.http.get(url)
             if r.status_code != 200:
                 status = AgentCheck.CRITICAL
                 msg = "Got %s when hitting %s" % (r.status_code, url)
@@ -125,6 +137,7 @@ def _get_json(self, url, timeout, verify, failure_expected=False, tags=None):
         finally:
             self.log.debug('Request to url : {0}, timeout: {1}, message: {2}'.format(url, timeout, msg))
             self._send_service_check(url, r, status, failure_expected=failure_expected, tags=tags, message=msg)
+
         if r.encoding is None:
             r.encoding = 'UTF8'
 
@@ -141,11 +154,11 @@ def _send_service_check(self, url, response, status, failure_expected=False, tag
             self.service_check(self.SERVICE_CHECK_NAME, status, tags=tags, message=message)
             self.service_check_needed = False
 
-    def _get_state(self, url, timeout, verify, tags):
+    def _get_state(self, url, tags):
         try:
             # Mesos version >= 0.25
             endpoint = url + '/state'
-            master_state = self._get_json(endpoint, timeout, verify, failure_expected=True, tags=tags)
+            master_state = self._get_json(endpoint, failure_expected=True, tags=tags)
         except CheckException:
             # Mesos version < 0.25
             old_endpoint = endpoint + '.json'
@@ -154,29 +167,26 @@ def _get_state(self, url, timeout, verify, tags):
                     endpoint, old_endpoint
                 )
             )
-            master_state = self._get_json(old_endpoint, timeout, verify, tags=tags)
+            master_state = self._get_json(old_endpoint, tags=tags)
         return master_state
 
-    def _get_stats(self, url, timeout, verify, tags):
+    def _get_stats(self, url, tags):
         if self.version >= [0, 22, 0]:
             endpoint = url + '/metrics/snapshot'
         else:
             endpoint = url + '/stats.json'
-        return self._get_json(endpoint, timeout, verify, tags=tags)
+        return self._get_json(endpoint, tags=tags)
 
-    def _get_constant_attributes(self, url, timeout, master_port, verify, tags):
+    def _get_constant_attributes(self, url, master_port, tags):
         state_metrics = None
         parsed_url = urlparse(url)
         if self.cluster_name is None:
-            state_metrics = self._get_state(url, timeout, verify, tags)
+            state_metrics = self._get_state(url, tags)
             if state_metrics is not None:
                 self.version = [int(i) for i in state_metrics['version'].split('.')]
                 if 'master_hostname' in state_metrics:
                     master_state = self._get_state(
-                        '{0}://{1}:{2}'.format(parsed_url.scheme, state_metrics['master_hostname'], master_port),
-                        timeout,
-                        verify,
-                        tags,
+                        '{0}://{1}:{2}'.format(parsed_url.scheme, state_metrics['master_hostname'], master_port), tags
                     )
                     if master_state is not None:
                         self.cluster_name = master_state.get('cluster')
@@ -192,16 +202,13 @@ def check(self, instance):
         if instance_tags is None:
             instance_tags = []
         tasks = instance.get('tasks', [])
-        default_timeout = self.init_config.get('default_timeout', 5)
-        timeout = float(instance.get('timeout', default_timeout))
         master_port = instance.get("master_port", DEFAULT_MASTER_PORT)
-        ssl_verify = not _is_affirmative(instance.get('disable_ssl_validation', False))
 
-        state_metrics = self._get_constant_attributes(url, timeout, master_port, ssl_verify, instance_tags)
+        state_metrics = self._get_constant_attributes(url, master_port, instance_tags)
         tags = None
 
         if state_metrics is None:
-            state_metrics = self._get_state(url, timeout, ssl_verify, instance_tags)
+            state_metrics = self._get_state(url, instance_tags)
         if state_metrics:
             tags = ['mesos_pid:{0}'.format(state_metrics['pid']), 'mesos_node:slave']
             if self.cluster_name:
@@ -218,7 +225,7 @@ def check(self, instance):
                                 for key_name, (metric_name, metric_func) in iteritems(self.TASK_METRICS):
                                     metric_func(self, metric_name, t['resources'][key_name], tags=task_tags)
 
-        stats_metrics = self._get_stats(url, timeout, ssl_verify, instance_tags)
+        stats_metrics = self._get_stats(url, instance_tags)
         if stats_metrics:
             tags = tags if tags else instance_tags
             metrics = [

diff --git a/mesos_slave/tests/conftest.py b/mesos_slave/tests/conftest.py
@@ -10,6 +10,7 @@
 from datadog_checks.mesos_slave import MesosSlave
 
 from . import common
+from .utils import read_fixture
 
 
 @pytest.fixture(scope='session')
@@ -27,17 +28,13 @@ def instance():
 
 @pytest.fixture
 def check():
-    return MesosSlave(common.CHECK_NAME, {}, {})
+    return mock
 
 
-@pytest.fixture
-def check_mock(check):
-    check._get_stats = lambda v, x, y, z: json.loads(read_fixture('stats.json'))
-    check._get_state = lambda v, x, y, z: json.loads(read_fixture('state.json'))
-
-    return check
+def mock(init_config, instance):
+    check = MesosSlave(common.CHECK_NAME, init_config, [instance])
 
+    check._get_stats = lambda v, x: json.loads(read_fixture('stats.json'))
+    check._get_state = lambda v, x: json.loads(read_fixture('state.json'))
 
-def read_fixture(name):
-    with open(os.path.join(common.FIXTURE_DIR, name)) as f:
-        return f.read()
+    return check