metadataservice: add neptune proxy

add a bunch of test only proxy endpoints to support roundtrip tests.
pin specific common version to reduce surprises.

Signed-off-by: Joshua Hoskins <[email protected]>

README.md

@@ -108,3 +108,9 @@ We have Swagger documentation setup with OpenApi 3.0.2. This documentation is ge
 
 ## Code structure
 Please visit [Code Structure](docs/structure.md) to read how different modules are structured in Amundsen Metadata service.
+
+## Roundtrip tests
+Roundtrip tests are a new feature - by implementing the abstract_proxy_tests and some test setup endpoints in the base_proxy, you can validate your proxy code against the actual data store. These tests do not run by default, but can be run by passing the `--roundtrip` argument.
+```bash
+$ python -m pytest --roundtrip .
+```

metadata_service/config.py

@@ -5,6 +5,9 @@
 import os
 from typing import List, Dict, Optional, Set  # noqa: F401
 from metadata_service.entity.badge import Badge
+from amundsen_gremlin.config import (
+    LocalGremlinConfig
+)
 
 # PROXY configuration keys
 PROXY_HOST = 'PROXY_HOST'
@@ -77,7 +80,8 @@ class Config:
     PROGRAMMATIC_DESCRIPTIONS_EXCLUDE_FILTERS = []  # type: list
 
 
-class LocalConfig(Config):
+# NB: If you're using the gremlin proxy, the appropriate GremlinConfig must be added to any other configs
+class LocalConfig(LocalGremlinConfig, Config):
     DEBUG = True
     TESTING = False
     LOG_LEVEL = 'DEBUG'

metadata_service/proxy/atlas_proxy.py

@@ -8,7 +8,7 @@
 
 from amundsen_common.models.dashboard import DashboardSummary
 from amundsen_common.models.popular_table import PopularTable
-from amundsen_common.models.table import Column, Statistics, Table, Tag, User, Reader, \
+from amundsen_common.models.table import Application, Column, Stat, Table, Tag, User, Reader, \
     ProgrammaticDescription, ResourceReport
 from amundsen_common.models.user import User as UserEntity
 from atlasclient.client import Atlas
@@ -308,7 +308,7 @@ def _serialize_columns(self, *, entity: EntityUniqueAttribute) -> \
             Union[List[Column], List]:
         """
         Helper function to fetch the columns from entity and serialize them
-        using Column and Statistics model.
+        using Column and Stat model.
         :param entity: EntityUniqueAttribute object,
         along with relationshipAttributes
         :return: A list of Column objects, if there are any columns available,
@@ -348,7 +348,7 @@ def _serialize_columns(self, *, entity: EntityUniqueAttribute) -> \
                     end_epoch = stats_attrs.get('end_epoch')
 
                     statistics.append(
-                        Statistics(
+                        Stat(
                             stat_type=stat_type,
                             stat_val=stat_val,
                             start_epoch=start_epoch,
@@ -997,3 +997,34 @@ def get_resources_using_table(self, *,
                                   id: str,
                                   resource_type: ResourceType) -> Dict[str, List[DashboardSummary]]:
         return {}
+
+    def put_user(self, *, data: User) -> None:
+        pass
+
+    def post_users(self, *, data: List[User]) -> None:
+        pass
+
+    def put_app(self, *, data: Application) -> None:
+        pass
+
+    def post_apps(self, *, data: List[Application]) -> None:
+        pass
+
+    def put_table(self, *, table: Table) -> None:
+        pass
+
+    def post_tables(self, *, tables: List[Table]) -> None:
+        pass
+
+    def put_column(self, *, table_uri: str, column: Column) -> None:
+        pass
+
+    def put_programmatic_table_description(self, *, table_uri: str, description: ProgrammaticDescription) -> None:
+        pass
+
+    def put_programmatic_column_description(self, *, table_uri: str, column_name: str,
+                                            description: Description) -> None:
+        pass
+
+    def add_read_count(self, *, table_uri: str, user_id: str, read_count: int) -> None:
+        pass

metadata_service/proxy/base_proxy.py

@@ -5,8 +5,8 @@
 from typing import Any, Dict, List, Union
 
 from amundsen_common.models.popular_table import PopularTable
-from amundsen_common.models.table import Table
-from amundsen_common.models.user import User as UserEntity
+from amundsen_common.models.table import Table, Application, Column, ProgrammaticDescription
+from amundsen_common.models.user import User
 from amundsen_common.models.dashboard import DashboardSummary
 
 from metadata_service.entity.dashboard_detail import DashboardDetail as DashboardDetailEntity
@@ -22,11 +22,11 @@ class BaseProxy(metaclass=ABCMeta):
     """
 
     @abstractmethod
-    def get_user(self, *, id: str) -> Union[UserEntity, None]:
+    def get_user(self, *, id: str) -> Union[User, None]:
         pass
 
     @abstractmethod
-    def get_users(self) -> List[UserEntity]:
+    def get_users(self) -> List[User]:
         pass
 
     @abstractmethod
@@ -151,3 +151,42 @@ def get_resources_using_table(self, *,
                                   id: str,
                                   resource_type: ResourceType) -> Dict[str, List[DashboardSummary]]:
         pass
+
+    # NB: the following methods exist solely for testing in the roundtrip tests.
+    # These may not be suitably performant for handling large batches
+
+    @abstractmethod
+    def put_user(self, *, data: User) -> None:
+        pass
+
+    @abstractmethod
+    def post_users(self, *, data: List[User]) -> None:
+        pass
+
+    @abstractmethod
+    def put_app(self, *, data: Application) -> None:
+        pass
+
+    @abstractmethod
+    def post_apps(self, *, data: List[Application]) -> None:
+        pass
+
+    @abstractmethod
+    def put_table(self, *, table: Table) -> None:
+        pass
+
+    @abstractmethod
+    def post_tables(self, *, tables: List[Table]) -> None:
+        pass
+
+    @abstractmethod
+    def put_column(self, *, table_uri: str, column: Column) -> None:
+        pass
+
+    @abstractmethod
+    def put_programmatic_table_description(self, *, table_uri: str, description: ProgrammaticDescription) -> None:
+        pass
+
+    @abstractmethod
+    def add_read_count(self, *, table_uri: str, user_id: str, read_count: int) -> None:
+        pass