sender(feat): wait for api to go online on init

Makefile

@@ -33,8 +33,8 @@ e2e-cov:
 
 combined-cov:
 	rm -f .coverage*
-	make unit-cov
 	SUPPORT_COMBINED_COVERAGE=1 make e2e-cov
+	make unit-cov
 	coverage combine
 	coverage report -m
 

dev_requirements.txt

@@ -11,3 +11,4 @@ tox
 Sphinx
 sphinxcontrib-httpdomain
 sphinx_rtd_theme
+httmock

src/iris/bin/sender.py

@@ -1024,8 +1024,10 @@ def mock_gwatch_renewer():
 
 
 def init_sender(config):
+    api_host = config['sender'].get('api_host', 'http://localhost:16649')
+
     db.init(config)
-    cache.init(config)
+    cache.init(api_host, config)
     metrics.init(config, 'iris-sender', default_sender_metrics)
     api_cache.cache_priorities()
     api_cache.cache_applications()
@@ -1052,7 +1054,6 @@ def init_sender(config):
 
 
 def main():
-
     global config
     config = load_config()
 

src/iris/sender/cache.py

@@ -42,8 +42,6 @@ def __init__(self, base, version=0):
     def get(self, path, *args, **kwargs):
         return super(IrisClient, self).get(self.url + path, *args, **kwargs)
 
-    def post(self, path, *args, **kwargs):
-        return super(IrisClient, self).post(self.url + path, *args, **kwargs)
 
 
 class Cache():
@@ -403,11 +401,32 @@ def purge():
     incidents.purge()
 
 
-def init(config):
+def init(api_host, config):
     global targets_for_role, target_names, target_reprioritization, plan_notifications, targets
     global roles, incidents, templates, plans, iris_client
 
-    iris_client = IrisClient(config['sender'].get('api_host', 'http://localhost:16649'), 0)
+    iris_client = IrisClient(api_host, 0)
+
+    # make sure API is online
+    max_trey = 36
+    api_chk_cnt = 0
+    while api_chk_cnt < max_trey:
+        try:
+            re = iris_client.get('target_roles')
+            if re.status_code == 200:
+                break
+        except:
+            pass
+        api_chk_cnt += 1
+        logger.warning(
+            'Not able to connect to Iris API %s, retry in 5 seconds.',
+            iris_client.url)
+        sleep(5)
+
+    if api_chk_cnt >= max_trey:
+        import sys
+        sys.exit('FATAL: Not able to connect to Iris API: %s' % iris_client.url)
+
     plans = Plans(db.engine)
     templates = Templates(db.engine)
     incidents = Cache(db.engine,

test/test_sender.py

@@ -8,6 +8,12 @@
 import time
 from iris.bin.sender import init_sender
 import msgpack
+from httmock import all_requests, HTTMock
+
+
+@all_requests
+def mock_api_response(url, request):
+    return {'status_code': 200}
 
 
 def test_configure(mocker):
@@ -16,36 +22,37 @@ def test_configure(mocker):
     mocker.patch('iris.bin.sender.api_cache.cache_priorities')
     mocker.patch('iris.bin.sender.api_cache.cache_applications')
     mocker.patch('iris.bin.sender.api_cache.cache_modes')
-    init_sender({
-        'db': {
-            'conn': {
-                'kwargs': {
-                    'scheme': 'mysql+pymysql',
-                    'user': 'foo',
-                    'password': 'bar',
-                    'host': '127.0.0.1',
-                    'database': 'iris',
-                    'charset': 'utf8',
+    with HTTMock(mock_api_response):
+        init_sender({
+            'db': {
+                'conn': {
+                    'kwargs': {
+                        'scheme': 'mysql+pymysql',
+                        'user': 'foo',
+                        'password': 'bar',
+                        'host': '127.0.0.1',
+                        'database': 'iris',
+                        'charset': 'utf8',
+                    },
+                    'str': '%(scheme)s://%(user)s:%(password)s@%(host)s/%(database)s?charset=%(charset)s'
                 },
-                'str': '%(scheme)s://%(user)s:%(password)s@%(host)s/%(database)s?charset=%(charset)s'
+                'kwargs': {
+                    'pool_recycle': 3600,
+                    'echo': True,
+                    'pool_size': 100,
+                    'max_overflow': 100,
+                    'pool_timeout': 60,
+                }
             },
-            'kwargs': {
-                'pool_recycle': 3600,
-                'echo': True,
-                'pool_size': 100,
-                'max_overflow': 100,
-                'pool_timeout': 60,
-            }
-        },
-        'sender': {
-            'debug': True,
-        },
-        'oncall': 'http://localhost:8002',
-        'role_lookup': 'dummy',
-        'metrics': 'dummy',
-        'skipsend': True,
-        'skipgmailwatch': True,
-    })
+            'sender': {
+                'debug': True,
+            },
+            'oncall': 'http://localhost:8002',
+            'role_lookup': 'dummy',
+            'metrics': 'dummy',
+            'skipsend': True,
+            'skipgmailwatch': True,
+        })
 
 
 fake_message = {