🎉 Initial commit.

amundsen-io · Jul 13, 2020 · 40cafc9 · 40cafc9
1 parent 5a8c117
commit 40cafc9
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 5 deletions.
diff --git a/metadata_service/config.py b/metadata_service/config.py
@@ -67,6 +67,9 @@ class Config:
     # Number of minimum reader count to qualify for popular table
     POPULAR_TABLE_MINIMUM_READER_COUNT = 10  # type: int
 
+    # List of regexes which will exclude certain parameters from appearing as Programmatic Descriptions
+    PROGRAMMATIC_DESCRIPTIONS_EXCLUDE_FILTERS = []  # type: list
+
 
 class LocalConfig(Config):
     DEBUG = True

diff --git a/metadata_service/proxy/atlas_proxy.py b/metadata_service/proxy/atlas_proxy.py
@@ -4,7 +4,7 @@
 from typing import Any, Dict, List, Union, Optional
 
 from amundsen_common.models.popular_table import PopularTable
-from amundsen_common.models.table import Column, Statistics, Table, Tag, User, Reader
+from amundsen_common.models.table import Column, Statistics, Table, Tag, User, Reader, ProgrammaticDescription
 from amundsen_common.models.user import User as UserEntity
 from amundsen_common.models.dashboard import DashboardSummary
 from atlasclient.client import Atlas
@@ -363,6 +363,8 @@ def get_table(self, *, table_uri: str) -> Table:
         try:
             attrs = table_details[self.ATTRS_KEY]
 
+            programmatic_descriptions = self._get_programmatic_descriptions(attrs.get('parameters'))
+
             table_qn = parse_table_qualified_name(
                 qualified_name=attrs.get(self.QN_KEY)
             )
@@ -389,7 +391,8 @@ def get_table(self, *, table_uri: str) -> Table:
                 owners=[User(email=attrs.get('owner'))],
                 columns=columns,
                 table_readers=self._get_readers(attrs.get(self.QN_KEY)),
-                last_updated_timestamp=self._parse_date(table_details.get('updateTime')))
+                last_updated_timestamp=self._parse_date(table_details.get('updateTime')),
+                programmatic_descriptions=programmatic_descriptions)
 
             return table
         except KeyError as ex:
@@ -736,6 +739,31 @@ def _get_readers(self, qualified_name: str, top: Optional[int] = 15) -> List[Rea
 
         return results
 
+    def _get_programmatic_descriptions(self, parameters: dict) -> List[ProgrammaticDescription]:
+        _programmatic_descriptions = {}
+        programmatic_descriptions: List[ProgrammaticDescription] = []
+
+        for source, text in parameters.items():
+            use_parameter = True
+
+            for f in app.config['PROGRAMMATIC_DESCRIPTIONS_EXCLUDE_FILTERS']:
+                pattern = re.compile(f)
+
+                if pattern.match(source):
+                    use_parameter = False
+                    break
+
+            if use_parameter:
+                source = re.sub("([a-z])([A-Z])", "\g<1> \g<2>", source).lower()
+                _programmatic_descriptions[source] = text
+
+        _programmatic_descriptions = dict(sorted(_programmatic_descriptions.items()))
+
+        for source, text in _programmatic_descriptions.items():
+            programmatic_descriptions.append(ProgrammaticDescription(source=source, text=text))
+
+        return programmatic_descriptions
+
     def get_dashboard(self,
                       dashboard_uri: str,
                       ) -> DashboardDetailEntity:

diff --git a/tests/unit/proxy/fixtures/atlas_test_data.py b/tests/unit/proxy/fixtures/atlas_test_data.py
@@ -102,7 +102,11 @@ class Data:
             'owner': '[email protected]',
             'db': db_entity,
             'popularityScore': 100,
-            'partitions': list()
+            'partitions': list(),
+            'parameters': {
+                'testParameterKey': 'testParameterValue',
+                'spark.sql.param': 1
+            }
         },
         'relationshipAttributes': {
             'db': db_entity,

diff --git a/tests/unit/proxy/test_atlas_proxy.py b/tests/unit/proxy/test_atlas_proxy.py
@@ -3,7 +3,7 @@
 from typing import Any, Dict, Optional, cast, List
 
 from amundsen_common.models.popular_table import PopularTable
-from amundsen_common.models.table import Column, Statistics, Table, Tag, User, Reader
+from amundsen_common.models.table import Column, Statistics, Table, Tag, User, Reader, ProgrammaticDescription
 from atlasclient.exceptions import BadRequest
 from mock import MagicMock, patch
 from tests.unit.proxy.fixtures.atlas_test_data import Data, DottedDict
@@ -18,6 +18,7 @@
 class TestAtlasProxy(unittest.TestCase, Data):
     def setUp(self) -> None:
         self.app = create_app(config_module_class='metadata_service.config.LocalConfig')
+        self.app.config['PROGRAMMATIC_DESCRIPTIONS_EXCLUDE_FILTERS'] = ['spark.*']
         self.app_context = self.app.app_context()
         self.app_context.push()
 
@@ -126,7 +127,9 @@ def _get_table(self, custom_stats_format: bool = False) -> None:
                          description=ent_attrs['description'],
                          owners=[User(email=ent_attrs['owner'])],
                          last_updated_timestamp=int(str(self.entity1['updateTime'])[:10]),
-                         columns=[exp_col] * self.active_columns)
+                         columns=[exp_col] * self.active_columns,
+                         programmatic_descriptions=[ProgrammaticDescription(source='test parameter key',
+                                                                            text='testParameterValue')])
 
         self.assertEqual(str(expected), str(response))