config: override config values using env var

Makefile

@@ -18,8 +18,11 @@ e2e:
 unit:
 	py.test -vv test
 
-check:
+flake8:
 	flake8 src test setup.py
+
+check:
+	make flake8
 	make test
 
 unit-cov:

src/iris/api.py

@@ -20,8 +20,6 @@
 from falcon import HTTP_200, HTTP_201, HTTP_204, HTTPBadRequest, HTTPNotFound, HTTPUnauthorized, HTTPForbidden, HTTPFound, API
 from falcon_cors import CORS
 from sqlalchemy.exc import IntegrityError
-from importlib import import_module
-import yaml
 import falcon.uri
 
 from collections import defaultdict
@@ -31,6 +29,7 @@
 from . import utils
 from . import cache
 from . import ui
+from .config import load_config
 from iris.sender import auditlog
 from iris.sender.quota import (get_application_quotas_query, insert_application_quota_query,
                                required_quota_keys, quota_int_keys)
@@ -502,46 +501,6 @@ def ts_to_sql_datetime(ts):
 uuid4hex = re.compile('[0-9a-f]{32}\Z', re.I)
 
 
-def process_config_hook(config):
-    ''' Examine config dict for hooks and run them if present '''
-    if 'init_config_hook' in config:
-        try:
-            module = config['init_config_hook']
-            logger.info('Bootstrapping config using %s', module)
-            getattr(import_module(module), module.split('.')[-1])(config)
-        except ImportError:
-            logger.exception('Failed loading config hook %s', module)
-
-    return config
-
-
-def load_config_file(path=None):
-    ''' Get config from path to file, defaulting to cli arg. This can easily be monkey patched. '''
-    if not path:
-        import sys
-        if len(sys.argv) <= 1:
-            print 'ERROR: missing config file.'
-            print 'usage: %s API_CONFIG_FILE' % sys.argv[0]
-            sys.exit(1)
-        path = sys.argv[1]
-
-    with open(path) as h:
-        return yaml.safe_load(h)
-
-
-def load_config(path=None):
-    '''
-      Generate configs for iris. This first calls load_config_file, which
-      will read configs from a yaml file. It will then pass that through
-      process_config_hook() which looks for a key called init_config_hook
-      which is the name of a module to call which can tweak the config further.
-
-      load_config_file() can be monkey patched, in which case this config
-      loading functionality can be customized further.
-    '''
-    return process_config_hook(load_config_file(path))
-
-
 def stream_incidents_with_context(cursor):
     for row in cursor:
         row['context'] = ujson.loads(row['context'])
@@ -3259,8 +3218,6 @@ def construct_falcon_api(debug, healthcheck_path, allowed_origins, iris_sender_a
 
 
 def get_api(config):
-    logging.basicConfig(format='[%(asctime)s] [%(process)d] [%(levelname)s] %(name)s %(message)s',
-                        level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S %z')
     db.init(config)
     spawn(update_cache_worker)
     init_plugins(config.get('plugins', {}))
@@ -3284,4 +3241,6 @@ def get_api(config):
 
 
 def get_api_app():
+    logging.basicConfig(format='[%(asctime)s] [%(process)d] [%(levelname)s] %(name)s %(message)s',
+                        level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S %z')
     return get_api(load_config())

src/iris/config.py

@@ -0,0 +1,64 @@
+# -*- coding:utf-8 -*-
+# Copyright (c) LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license.
+# See LICENSE in the project root for license information.
+
+import sys
+import os
+import logging
+from importlib import import_module
+import yaml
+
+logger = logging.getLogger(__name__)
+
+
+def load_config_file(path=None):
+    ''' Get config from path to file, defaulting to cli arg. This can easily be monkey patched. '''
+    if not path:
+        if len(sys.argv) <= 1:
+            print 'ERROR: missing config file.'
+            print 'usage: %s API_CONFIG_FILE' % sys.argv[0]
+            sys.exit(1)
+        path = sys.argv[1]
+
+    with open(path) as h:
+        return yaml.safe_load(h)
+
+
+def process_config_hook(config):
+    ''' Examine config dict for hooks and run them if present '''
+    if 'init_config_hook' in config:
+        try:
+            module = config['init_config_hook']
+            logger.info('Bootstrapping config using %s', module)
+            getattr(import_module(module), module.split('.')[-1])(config)
+        except ImportError:
+            logger.exception('Failed loading config hook %s', module)
+
+    return config
+
+
+def load_config(path=None):
+    '''
+      Generate configs for iris in the following steps:
+
+        * reads config from yaml file by calling `load_config_file()`.
+        * reads more config values from environment variables and use them to
+          override values from the config file.
+        * pass config through process_config_hook(). It looks for a key called
+          init_config_hook in the config, which is the name of a module to call
+          which can tweak the config further.
+
+      load_config_file() can be monkey patched, in which case this config
+      loading functionality can be customized further.
+    '''
+    config = load_config_file(path)
+    if not config:
+        sys.exit('Failed to load config from ' + path)
+
+    if 'IRIS_CFG_DB_HOST' in os.environ:
+        config['db']['conn']['kwargs']['host'] = os.environ['IRIS_CFG_DB_HOST']
+    if 'IRIS_CFG_DB_USER' in os.environ:
+        config['db']['conn']['kwargs']['user'] = os.environ['IRIS_CFG_DB_USER']
+    if 'IRIS_CFG_DB_PASS' in os.environ:
+        config['db']['conn']['kwargs']['password'] = os.environ['IRIS_CFG_DB_PASSWORD']
+    return process_config_hook(config)

src/iris/wrappers/gunicorn.py

@@ -1,3 +1,8 @@
 import os
-from iris.api import load_config, get_api
+import logging
+from iris.config import load_config
+from iris.api import get_api
+
+logging.basicConfig(format='[%(asctime)s] [%(process)d] [%(levelname)s] %(name)s %(message)s',
+                    level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S %z')
 application = get_api(load_config(os.environ['CONFIG']))

test/test_config.py

@@ -0,0 +1,28 @@
+# -*- coding:utf-8 -*-
+# Copyright (c) LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license.
+# See LICENSE in the project root for license information.
+
+
+def test_load_config(mocker):
+    import os
+    from tempfile import NamedTemporaryFile
+    from iris.config import load_config
+
+    mocker.patch.dict(os.environ, {'IRIS_CFG_DB_USER': 'iris_dev'})
+
+    with NamedTemporaryFile() as temp_config:
+        temp_config.write('''
+db:
+  conn:
+    kwargs:
+      scheme: mysql+pymysql
+      user: iris
+      password: iris
+      host: 127.0.0.1
+      database: iris
+      charset: utf8
+    str: "%(scheme)s://%(user)s:%(password)s@%(host)s/%(database)s?charset=%(charset)s"
+''')
+        temp_config.flush()
+        config = load_config(temp_config.name)
+        assert config['db']['conn']['kwargs']['user'] == 'iris_dev'