refactor: Separate configs based on the backends/proxies (#279)

* fixes pip install and configs Signed-off-by: verdan <[email protected]> * cleanup Signed-off-by: verdan <[email protected]> * test fix Signed-off-by: verdan <[email protected]> * removes the requirements changes from branch Signed-off-by: verdan <[email protected]>
amundsen-io · Mar 12, 2021 · fc6c0f8 · fc6c0f8
1 parent ed93cab
commit fc6c0f8
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 41 deletions.
diff --git a/metadata_service/config.py b/metadata_service/config.py
@@ -54,13 +54,6 @@ class Config:
 
     IS_STATSD_ON = False
 
-    # Used to differentiate tables with other entities in Atlas. For more details:
-    # https://github.com/amundsen-io/amundsenmetadatalibrary/blob/master/docs/proxy/atlas_proxy.md
-    ATLAS_TABLE_ENTITY = 'Table'
-
-    # The relationalAttribute name of Atlas Entity that identifies the database entity.
-    ATLAS_DB_ATTRIBUTE = 'db'
-
     # Configurable dictionary to influence format of column statistics displayed in UI
     STATISTICS_FORMAT_SPEC: Dict[str, Dict] = {}
 
@@ -82,17 +75,19 @@ class Config:
     # List of regexes which will exclude certain parameters from appearing as Programmatic Descriptions
     PROGRAMMATIC_DESCRIPTIONS_EXCLUDE_FILTERS = []  # type: list
 
-    # List of accepted date formats for AtlasProxy Watermarks. With this we allow more than one datetime partition
-    # format to be used in tables
-    WATERMARK_DATE_FORMATS = ['%Y%m%d']
-
     # Custom kwargs that will be passed to proxy client. Can be used to fine-tune parameters like timeout
     # or num of retries
     PROXY_CLIENT_KWARGS: Dict = dict()
 
+    SWAGGER_TEMPLATE_PATH = os.path.join('api', 'swagger_doc', 'template.yml')
+    SWAGGER = {
+        'openapi': '3.0.2',
+        'title': 'Metadata Service',
+        'uiversion': 3
+    }
 
-# NB: If you're using the gremlin proxy, the appropriate GremlinConfig must be added to any other configs
-class LocalConfig(LocalGremlinConfig, Config):
+
+class LocalConfig(Config):
     DEBUG = True
     TESTING = False
     LOG_LEVEL = 'DEBUG'
@@ -104,31 +99,33 @@ class LocalConfig(LocalGremlinConfig, Config):
     PROXY_ENCRYPTED = bool(distutils.util.strtobool(os.environ.get(PROXY_ENCRYPTED, 'True')))
     PROXY_VALIDATE_SSL = bool(distutils.util.strtobool(os.environ.get(PROXY_VALIDATE_SSL, 'False')))
 
-    JANUS_GRAPH_URL = None
-
     IS_STATSD_ON = bool(distutils.util.strtobool(os.environ.get(IS_STATSD_ON, 'False')))
 
     SWAGGER_ENABLED = True
-    SWAGGER_TEMPLATE_PATH = os.path.join('api', 'swagger_doc', 'template.yml')
-    SWAGGER = {
-        'openapi': '3.0.2',
-        'title': 'Metadata Service',
-        'uiversion': 3
-    }
 
 
-class NeptuneConfig(LocalGremlinConfig, Config):
+class AtlasConfig(LocalConfig):
+    PROXY_HOST = os.environ.get('PROXY_HOST', 'localhost')
+    PROXY_PORT = os.environ.get('PROXY_PORT', '21000')
+    PROXY_CLIENT = PROXY_CLIENTS['ATLAS']
+
+    # List of accepted date formats for AtlasProxy Watermarks. With this we allow more than one datetime partition
+    # format to be used in tables
+    WATERMARK_DATE_FORMATS = ['%Y%m%d']
+
+
+class GremlinConfig(LocalGremlinConfig, LocalConfig):
+    JANUS_GRAPH_URL = None
+
+
+class NeptuneConfig(LocalGremlinConfig, LocalConfig):
     DEBUG = False
-    TESTING = False
     LOG_LEVEL = 'INFO'
-    LOCAL_HOST = '0.0.0.0'
 
-    # FORMAT: wss://<NEPTUNE_URL>:<NEPTUNE_PORT>/gremlin
-    PROXY_HOST = os.environ.get('PROXY_HOST')
-    PROXY_PORT = None
+    # PROXY_HOST FORMAT: wss://<NEPTUNE_URL>:<NEPTUNE_PORT>/gremlin
+    PROXY_PORT = None   # type: ignore
+
     PROXY_CLIENT = PROXY_CLIENTS['NEPTUNE']
-    PROXY_ENCRYPTED = bool(distutils.util.strtobool(os.environ.get(PROXY_ENCRYPTED, 'True')))
-    PROXY_VALIDATE_SSL = bool(distutils.util.strtobool(os.environ.get(PROXY_VALIDATE_SSL, 'False')))
     PROXY_PASSWORD = boto3.session.Session(region_name=os.environ.get('AWS_REGION', 'us-east-1'))
 
     PROXY_CLIENT_KWARGS = {
@@ -137,13 +134,3 @@ class NeptuneConfig(LocalGremlinConfig, Config):
     }
 
     JANUS_GRAPH_URL = None
-
-    IS_STATSD_ON = bool(distutils.util.strtobool(os.environ.get(IS_STATSD_ON, 'False')))
-
-    SWAGGER_ENABLED = True
-    SWAGGER_TEMPLATE_PATH = os.path.join('api', 'swagger_doc', 'template.yml')
-    SWAGGER = {
-        'openapi': '3.0.2',
-        'title': 'Metadata Service',
-        'uiversion': 3
-    }
diff --git a/metadata_service/proxy/atlas_proxy.py b/metadata_service/proxy/atlas_proxy.py
@@ -51,8 +51,8 @@ class AtlasProxy(BaseProxy):
     Atlas Proxy client for the amundsen metadata
     {ATLAS_API_DOCS} = https://atlas.apache.org/api/v2/
     """
-    TABLE_ENTITY = app.config['ATLAS_TABLE_ENTITY']
-    DB_ATTRIBUTE = app.config['ATLAS_DB_ATTRIBUTE']
+    TABLE_ENTITY = 'Table'
+    DB_ATTRIBUTE = 'db'
     STATISTICS_FORMAT_SPEC = app.config['STATISTICS_FORMAT_SPEC']
     BOOKMARK_TYPE = 'Bookmark'
     USER_TYPE = 'User'