[windows] new service and packaging

.pylintrc

@@ -175,7 +175,7 @@ dummy-variables-rgx=_|dummy
 
 # List of additional names supposed to be defined in builtins. Remember that
 # you should avoid to define new builtins when possible.
-additional-builtins=
+additional-builtins=WindowsError
 
 
 [CLASSES]

agent.py

@@ -37,15 +37,16 @@
 from util import (
     EC2,
     get_hostname,
-    Watchdog,
 )
 from utils.flare import Flare
 from utils.configcheck import configcheck, sd_configcheck
 from utils.jmx import jmx_command
 from utils.pidfile import PidFile
+from utils.platform import Platform
 from utils.profile import AgentProfiler
 from utils.service_discovery.config_stores import get_config_store
 from utils.service_discovery.sd_backend import get_sd_backend
+from utils.watchdog import new_watchdog
 
 # Constants
 PID_NAME = "dd-agent"
@@ -129,21 +130,22 @@ def run(self, config=None):
         # Gracefully exit on sigterm.
         signal.signal(signal.SIGTERM, self._handle_sigterm)
 
-        # A SIGUSR1 signals an exit with an autorestart
-        signal.signal(signal.SIGUSR1, self._handle_sigusr1)
+        if not Platform.is_windows():
+            # A SIGUSR1 signals an exit with an autorestart
+            signal.signal(signal.SIGUSR1, self._handle_sigusr1)
 
-        # Handle Keyboard Interrupt
-        signal.signal(signal.SIGINT, self._handle_sigterm)
+            # Handle Keyboard Interrupt
+            signal.signal(signal.SIGINT, self._handle_sigterm)
 
-        # A SIGHUP signals a configuration reload
-        signal.signal(signal.SIGHUP, self._handle_sighup)
+            # A SIGHUP signals a configuration reload
+            signal.signal(signal.SIGHUP, self._handle_sighup)
 
         # Save the agent start-up stats.
         CollectorStatus().persist()
 
         # Intialize the collector.
         if not config:
-            config = get_config(parse_args=True)
+            config = get_config(parse_args=True, can_write_conf=True)
 
         self._agentConfig = self._set_agent_config_hostname(config)
         hostname = get_hostname(self._agentConfig)
@@ -256,8 +258,8 @@ def _get_emitters(self):
     def _get_watchdog(self, check_freq):
         watchdog = None
         if self._agentConfig.get("watchdog", True):
-            watchdog = Watchdog(check_freq * WATCHDOG_MULTIPLIER,
-                                max_mem_mb=self._agentConfig.get('limit_memory_consumption', None))
+            watchdog = new_watchdog(check_freq * WATCHDOG_MULTIPLIER,
+                                    max_mem_mb=self._agentConfig.get('limit_memory_consumption', None))
             watchdog.reset()
         return watchdog
 

checks/collector.py

@@ -764,11 +764,7 @@ def _run_gohai_processes(self):
     def _run_gohai(self, options):
         output = None
         try:
-            if not Platform.is_windows():
-                command = "gohai"
-            else:
-                command = "gohai\gohai.exe"
-            output, err, _ = get_subprocess_output([command] + options, log)
+            output, err, _ = get_subprocess_output(["gohai"] + options, log)
             if err:
                 log.warning("GOHAI LOG | {0}".format(err))
         except OSError as e:

config.py

@@ -157,12 +157,15 @@ def _windows_confd_path():
     common_data = _windows_commondata_path()
     return _confd_path(os.path.join(common_data, 'Datadog'))
 
+def _windows_extra_checksd_path():
+    common_data = _windows_commondata_path()
+    return _checksd_path(os.path.join(common_data, 'Datadog'))
 
 def _windows_checksd_path():
     if hasattr(sys, 'frozen'):
         # we're frozen - from py2exe
         prog_path = os.path.dirname(sys.executable)
-        return _checksd_path(os.path.join(prog_path, '..'))
+        return _checksd_path(os.path.normpath(os.path.join(prog_path, '..', 'agent')))
     else:
         cur_path = os.path.dirname(__file__)
         return _checksd_path(cur_path)
@@ -325,7 +328,7 @@ def clean_dd_url(url):
     return url[:-1] if url.endswith('/') else url
 
 
-def get_config(parse_args=True, cfg_path=None, options=None):
+def get_config(parse_args=True, cfg_path=None, options=None, can_write_conf=False):
     if parse_args:
         options, _ = get_parsed_args()
 
@@ -349,6 +352,8 @@ def get_config(parse_args=True, cfg_path=None, options=None):
 
     if Platform.is_mac():
         agentConfig['additional_checksd'] = '/opt/datadog-agent/etc/checks.d'
+    elif Platform.is_windows():
+        agentConfig['additional_checksd'] = _windows_extra_checksd_path()
 
     # Config handling
     try:
@@ -438,10 +443,6 @@ def get_config(parse_args=True, cfg_path=None, options=None):
         # the linux directory is set by default
         if config.has_option('Main', 'additional_checksd'):
             agentConfig['additional_checksd'] = config.get('Main', 'additional_checksd')
-        elif get_os() == 'windows':
-            # default windows location
-            common_path = _windows_commondata_path()
-            agentConfig['additional_checksd'] = os.path.join(common_path, 'Datadog', 'checks.d')
 
         if config.has_option('Main', 'use_dogstatsd'):
             agentConfig['use_dogstatsd'] = config.get('Main', 'use_dogstatsd').lower() in ("yes", "true")
@@ -620,9 +621,62 @@ def get_config(parse_args=True, cfg_path=None, options=None):
     else:
         agentConfig['ssl_certificate'] = agentConfig['ca_certs']
 
+    if agentConfig['api_key'] == 'APIKEYHERE' and Platform.is_windows():
+        log.debug('Querying registry to get missing config options')
+        registry_conf = get_registry_conf(agentConfig)
+        agentConfig.update(registry_conf)
+        # If it's the collector, conf is updated
+        if can_write_conf and registry_conf:
+            log.debug('Updating conf file options: %s', registry_conf.keys())
+            update_conf_file(registry_conf, config_path)
+
     return agentConfig
 
 
+def get_registry_conf(agentConfig):
+    registry_conf = {}
+    try:
+        import _winreg
+        with _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE,
+                             "SOFTWARE\\Datadog\\Datadog Agent") as reg_key:
+            for attribute in ['api_key', 'tags', 'hostname']:
+                option = _winreg.QueryValueEx(reg_key, attribute)[0]
+                if option != '':
+                    registry_conf[attribute] = option
+    except (ImportError, WindowsError) as e:
+        log.error('Unable to get config keys from Registry: %s', e)
+
+    return registry_conf
+
+
+def update_conf_file(registry_conf, config_path):
+    buffer = ''
+    try:
+        with open(config_path, 'r') as config_file:
+            for line in config_file:
+                matched = False
+                for attribute in registry_conf:
+                    # Is it the line configuring the attribute ?
+                    if conf_match(line, attribute):
+                        buffer += '{}: {}\n'.format(attribute, registry_conf[attribute])
+                        matched = True
+                if not matched:
+                    buffer += line
+    except Exception as e:
+        log.error('An exception happened while updating the conf file. No change has been done.'
+                  ' Error: %s', e)
+    else:
+        with open(config_path, 'w') as config_file:
+            config_file.write(buffer)
+
+
+def conf_match(line, attribute):
+    attribute += ':'
+    return (line.startswith(attribute) or
+            line.startswith('#' + attribute) or
+            line.startswith('# ' + attribute))
+
+
 def get_system_stats(proc_path=None):
     systemStats = {
         'machine': platform.machine(),
@@ -1104,10 +1158,13 @@ def get_logging_config(cfg_path=None):
         'syslog_port': None,
     }
     if system_os == 'windows':
-        logging_config['windows_collector_log_file'] = os.path.join(_windows_commondata_path(), 'Datadog', 'logs', 'collector.log')
-        logging_config['windows_forwarder_log_file'] = os.path.join(_windows_commondata_path(), 'Datadog', 'logs', 'forwarder.log')
-        logging_config['windows_dogstatsd_log_file'] = os.path.join(_windows_commondata_path(), 'Datadog', 'logs', 'dogstatsd.log')
+        logging_config['collector_log_file'] = os.path.join(_windows_commondata_path(), 'Datadog', 'logs', 'collector.log')
+        logging_config['forwarder_log_file'] = os.path.join(_windows_commondata_path(), 'Datadog', 'logs', 'forwarder.log')
+        logging_config['dogstatsd_log_file'] = os.path.join(_windows_commondata_path(), 'Datadog', 'logs', 'dogstatsd.log')
         logging_config['jmxfetch_log_file'] = os.path.join(_windows_commondata_path(), 'Datadog', 'logs', 'jmxfetch.log')
+        logging_config['supervisor_log_file'] = os.path.join(_windows_commondata_path(), 'Datadog', 'logs', 'supervisor.log')
+        logging_config['service_log_file'] = os.path.join(_windows_commondata_path(), 'Datadog', 'logs', 'service.log')
+        logging_config['log_to_syslog'] = False
     else:
         logging_config['collector_log_file'] = '/var/log/datadog/collector.log'
         logging_config['forwarder_log_file'] = '/var/log/datadog/forwarder.log'

ddagent.py

@@ -55,10 +55,9 @@
     get_tornado_ioloop,
     get_uuid,
     json,
-    Watchdog,
 )
 from utils.logger import RedactedLogRecord
-
+from utils.watchdog import new_watchdog
 
 logging.LogRecord = RedactedLogRecord
 log = logging.getLogger('forwarder')
@@ -419,11 +418,9 @@ def __init__(self, port, agentConfig, watchdog=True,
         # Monitor activity
         if watchdog:
             watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER / 1000
-            self._watchdog = Watchdog(
-                watchdog_timeout,
-                max_mem_mb=agentConfig.get('limit_memory_consumption', None),
-                max_resets=WATCHDOG_HIGH_ACTIVITY_THRESHOLD
-            )
+            self._watchdog = new_watchdog(watchdog_timeout,
+                                          max_mem_mb=agentConfig.get('limit_memory_consumption', None),
+                                          max_resets=WATCHDOG_HIGH_ACTIVITY_THRESHOLD)
 
     def log_request(self, handler):
         """ Override the tornado logging method.

dogstatsd.py

@@ -7,7 +7,6 @@
 """
 A Python Statsd implementation with some datadog special sauce.
 """
-
 # set up logging before importing any other components
 from config import initialize_logging  # noqa
 initialize_logging('dogstatsd')
@@ -17,9 +16,9 @@
 set_no_proxy_settings()
 
 # stdlib
+import os
 import logging
 import optparse
-import os
 import select
 import signal
 import socket
@@ -43,9 +42,10 @@
 from config import get_config, get_version
 from daemon import AgentSupervisor, Daemon
 from util import chunks, get_hostname, get_uuid, plural
-from utils.pidfile import PidFile
 from utils.net import inet_pton
 from utils.net import IPV6_V6ONLY, IPPROTO_IPV6
+from utils.pidfile import PidFile
+from utils.watchdog import new_watchdog
 
 # urllib3 logs a bunch of stuff at the info level
 requests_log = logging.getLogger("requests.packages.urllib3")
@@ -193,8 +193,7 @@ def __init__(self, interval, metrics_aggregator, api_host, api_key=None,
 
         self.watchdog = None
         if use_watchdog:
-            from util import Watchdog
-            self.watchdog = Watchdog(WATCHDOG_TIMEOUT)
+            self.watchdog = new_watchdog(WATCHDOG_TIMEOUT)
 
         self.api_key = api_key
         self.api_host = api_host

jmxfetch.py

@@ -443,11 +443,9 @@ def _is_jmx_check(check_config, check_name, checks_list):
         return is_jmx, java_bin_path, java_options, tools_jar_path, custom_jar_paths
 
     def _get_path_to_jmxfetch(self):
-        if not Platform.is_windows():
-            return os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "checks",
-                                    "libs", JMX_FETCH_JAR_NAME))
-        return os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "..",
-                                "jmxfetch", JMX_FETCH_JAR_NAME))
+        return os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "checks",
+            "libs", JMX_FETCH_JAR_NAME))
+
 
 
 def init(config_path=None):