Mode dashboard owner (#200)

* Mode dashboard owner

* Not having dashboard_owner to create User node

* Update

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_owner_extractor.py

@@ -0,0 +1,76 @@
+import logging
+
+from pyhocon import ConfigTree, ConfigFactory  # noqa: F401
+from typing import Any  # noqa: F401
+
+from databuilder.extractor.base_extractor import Extractor
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.extractor.restapi.rest_api_extractor import MODEL_CLASS
+from databuilder.rest_api.rest_api_failure_handlers import HttpFailureSkipOnStatus
+from databuilder.rest_api.rest_api_query import RestApiQuery
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ModeDashboardOwnerExtractor(Extractor):
+    """
+    An Extractor that extracts Dashboard owner.
+
+    """
+
+    def init(self, conf):
+        # type: (ConfigTree) -> None
+        self._conf = conf
+
+        restapi_query = self._build_restapi_query()
+        self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
+            restapi_query=restapi_query,
+            conf=self._conf.with_fallback(
+                ConfigFactory.from_dict(
+                    {MODEL_CLASS: 'databuilder.models.dashboard.dashboard_owner.DashboardOwner', }
+                )
+            )
+        )
+
+    def extract(self):
+        # type: () -> Any
+
+        return self._extractor.extract()
+
+    def get_scope(self):
+        # type: () -> str
+        return 'extractor.mode_dashboard_owner'
+
+    def _build_restapi_query(self):
+        """
+        Build REST API Query. To get Mode Dashboard owner, it needs to call three APIs (spaces API, reports
+        API, and user API) joining together.
+        :return: A RestApiQuery that provides Mode Dashboard owner
+        """
+        # type: () -> RestApiQuery
+
+        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
+        report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
+
+        # https://mode.com/developer/api-reference/management/users/
+        creator_url_template = 'https://app.mode.com{creator_resource_path}'
+
+        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
+        params = ModeDashboardUtils.get_auth_params(conf=self._conf)
+
+        # Reports
+        json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.creator.href)'
+        field_names = ['dashboard_id', 'creator_resource_path']
+        creator_resource_path_query = RestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params,
+                                                   json_path=json_path, field_names=field_names, skip_no_result=True,
+                                                   json_path_contains_or=True)
+
+        json_path = 'email'
+        field_names = ['email']
+        failure_handler = HttpFailureSkipOnStatus(status_codes_to_skip={404})
+        owner_email_query = RestApiQuery(query_to_join=creator_resource_path_query, url=creator_url_template,
+                                         params=params,
+                                         json_path=json_path, field_names=field_names, skip_no_result=True,
+                                         can_skip_failure=failure_handler.can_skip_failure)
+
+        return owner_email_query

databuilder/models/dashboard/dashboard_owner.py

@@ -0,0 +1,76 @@
+import logging
+
+from typing import Optional, Dict, Any, Union, Iterator  # noqa: F401
+
+from databuilder.models.dashboard_metadata import DashboardMetadata
+from databuilder.models.neo4j_csv_serde import (
+    Neo4jCsvSerializable, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
+    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
+from databuilder.models.owner_constants import OWNER_OF_OBJECT_RELATION_TYPE, OWNER_RELATION_TYPE
+from databuilder.models.user import User
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+class DashboardOwner(Neo4jCsvSerializable):
+    """
+    A model that encapsulate Dashboard's owner.
+    Note that it does not create new user as it has insufficient information about user but it builds relation
+    between User and Dashboard
+    """
+
+    DASHBOARD_EXECUTION_RELATION_TYPE = 'LAST_EXECUTED'
+    EXECUTION_DASHBOARD_RELATION_TYPE = 'LAST_EXECUTION_OF'
+
+    def __init__(self,
+                 dashboard_group_id,  # type: str
+                 dashboard_id,  # type: str
+                 email,  # type: str
+                 product='',  # type: Optional[str]
+                 cluster='gold',  # type: str
+                 **kwargs
+                 ):
+        self._dashboard_group_id = dashboard_group_id
+        self._dashboard_id = dashboard_id
+        self._email = email
+        self._product = product
+        self._cluster = cluster
+
+        self._relation_iterator = self._create_relation_iterator()
+
+    def create_next_node(self):
+        # type: () -> Union[Dict[str, Any], None]
+        return None
+
+    def create_next_relation(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._relation_iterator)
+        except StopIteration:
+            return None
+
+    def _create_relation_iterator(self):
+        # type: () -> Iterator[[Dict[str, Any]]]
+        yield {
+            RELATION_START_LABEL: DashboardMetadata.DASHBOARD_NODE_LABEL,
+            RELATION_END_LABEL: User.USER_NODE_LABEL,
+            RELATION_START_KEY: DashboardMetadata.DASHBOARD_KEY_FORMAT.format(
+                product=self._product,
+                cluster=self._cluster,
+                dashboard_group=self._dashboard_group_id,
+                dashboard_name=self._dashboard_id
+            ),
+            RELATION_END_KEY: User.get_user_model_key(email=self._email),
+            RELATION_TYPE: OWNER_RELATION_TYPE,
+            RELATION_REVERSE_TYPE: OWNER_OF_OBJECT_RELATION_TYPE
+        }
+
+    def __repr__(self):
+        return 'DashboardOwner({!r}, {!r}, {!r}, {!r}, {!r})'.format(
+            self._dashboard_group_id,
+            self._dashboard_id,
+            self._email,
+            self._product,
+            self._cluster
+        )

databuilder/models/owner_constants.py

@@ -0,0 +1,3 @@
+
+OWNER_RELATION_TYPE = 'OWNER'
+OWNER_OF_OBJECT_RELATION_TYPE = 'OWNER_OF'

databuilder/models/table_owner.py

@@ -3,7 +3,7 @@
 from databuilder.models.neo4j_csv_serde import Neo4jCsvSerializable, NODE_KEY, \
     NODE_LABEL, RELATION_START_KEY, RELATION_START_LABEL, RELATION_END_KEY, \
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE
-
+from databuilder.models.owner_constants import OWNER_RELATION_TYPE, OWNER_OF_OBJECT_RELATION_TYPE
 from databuilder.models.user import User
 
 
@@ -12,8 +12,8 @@ class TableOwner(Neo4jCsvSerializable):
     """
     Hive table owner model.
     """
-    OWNER_TABLE_RELATION_TYPE = 'OWNER_OF'
-    TABLE_OWNER_RELATION_TYPE = 'OWNER'
+    OWNER_TABLE_RELATION_TYPE = OWNER_OF_OBJECT_RELATION_TYPE
+    TABLE_OWNER_RELATION_TYPE = OWNER_RELATION_TYPE
 
     def __init__(self,
                  db_name,  # type: str

databuilder/rest_api/rest_api_failure_handlers.py

@@ -0,0 +1,36 @@
+import abc
+
+import six
+from requests.exceptions import HTTPError
+from typing import Iterable, Union, List, Dict, Any, Optional  # noqa: F401
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseFailureHandler(object):
+
+    @abc.abstractmethod
+    def can_skip_failure(self,
+                         exception,  # type: Exception
+                         ):
+        # type: (...) -> bool
+        pass
+
+
+class HttpFailureSkipOnStatus(BaseFailureHandler):
+
+    def __init__(self,
+                 status_codes_to_skip,  # type: Iterable[int]
+                 ):
+        # type: (...) -> None
+        self._status_codes_to_skip = {v for v in status_codes_to_skip}
+
+    def can_skip_failure(self,
+                         exception,  # type: Exception
+                         ):
+        # type: (...) -> bool
+
+        if (isinstance(exception, HTTPError) or hasattr(exception, 'response')) \
+                and exception.response.status_code in self._status_codes_to_skip:
+            return True
+
+        return False

databuilder/rest_api/rest_api_query.py

@@ -4,7 +4,7 @@
 import requests
 from jsonpath_rw import parse
 from retrying import retry
-from typing import List, Dict, Any, Union  # noqa: F401
+from typing import List, Dict, Any, Union, Iterator, Callable  # noqa: F401
 
 from databuilder.rest_api.base_rest_api_query import BaseRestApiQuery
 
@@ -56,6 +56,7 @@ def __init__(self,
                  fail_no_result=False,  # type: bool
                  skip_no_result=False,  # type: bool
                  json_path_contains_or=False,  # type: bool
+                 can_skip_failure=None,  # type: Callable
                  ):
         # type: (...) -> None
         """
@@ -107,6 +108,8 @@ def __init__(self,
 
             ["1", "2", "baz", "box"]
 
+        :param can_skip_failure A function that can determine if it can skip the failure. See BaseFailureHandler for
+        the function interface
 
         """
         self._inner_rest_api_query = query_to_join
@@ -121,9 +124,10 @@ def __init__(self,
         self._skip_no_result = skip_no_result
         self._field_names = field_names
         self._json_path_contains_or = json_path_contains_or
+        self._can_skip_failure = can_skip_failure
         self._more_pages = False
 
-    def execute(self):
+    def execute(self):  # noqa: C901
         # type: () -> Iterator[Dict[str, Any]]
         self._authenticate()
 
@@ -134,7 +138,13 @@ def execute(self):
                 first_try = False
 
                 url = self._preprocess_url(record=record_dict)
-                response = self._send_request(url=url)
+
+                try:
+                    response = self._send_request(url=url)
+                except Exception as e:
+                    if self._can_skip_failure and self._can_skip_failure(exception=e):
+                        continue
+                    raise e
 
                 response_json = response.json()  # type: Union[List[Any], Dict[str, Any]]
 
@@ -193,7 +203,7 @@ def _send_request(self,
         return response
 
     @classmethod
-    def _compute_sub_records(self,
+    def _compute_sub_records(cls,
                              result_list,  # type: List
                              field_names,  # type: List[str]
                              json_path_contains_or=False,  # type: bool

tests/unit/models/dashboard/test_dashboard_owner.py

@@ -0,0 +1,28 @@
+import unittest
+
+from databuilder.models.dashboard.dashboard_owner import DashboardOwner
+from databuilder.models.neo4j_csv_serde import RELATION_START_KEY, RELATION_START_LABEL, RELATION_END_KEY, \
+    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE
+
+
+class TestDashboardOwner(unittest.TestCase):
+
+    def test_dashboard_owner_nodes(self):
+        # type: () -> None
+        dashboard_owner = DashboardOwner(email='[email protected]', cluster='cluster_id', product='product_id',
+                                         dashboard_id='dashboard_id', dashboard_group_id='dashboard_group_id')
+
+        actual = dashboard_owner.create_next_node()
+        self.assertIsNone(actual)
+
+    def test_dashboard_owner_relations(self):
+        # type: () -> None
+        dashboard_owner = DashboardOwner(email='[email protected]', cluster='cluster_id', product='product_id',
+                                         dashboard_id='dashboard_id', dashboard_group_id='dashboard_group_id')
+
+        actual = dashboard_owner.create_next_relation()
+        expected = {RELATION_END_KEY: '[email protected]', RELATION_START_LABEL: 'Dashboard', RELATION_END_LABEL: 'User',
+                    RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
+                    RELATION_TYPE: 'OWNER',
+                    RELATION_REVERSE_TYPE: 'OWNER_OF'}
+        self.assertDictEqual(actual, expected)

tests/unit/rest_api/test_rest_api_failure_handlers.py

@@ -0,0 +1,22 @@
+import unittest
+
+from databuilder.rest_api.rest_api_failure_handlers import HttpFailureSkipOnStatus
+from mock import MagicMock
+
+
+class TestHttpFailureSkipOnStatus(unittest.TestCase):
+
+    def testSkip(self):
+        # typ: (...) -> None
+
+        failure_handler = HttpFailureSkipOnStatus([404, 400])
+
+        exception = MagicMock()
+        exception.response.status_code = 404
+        self.assertTrue(failure_handler.can_skip_failure(exception=exception))
+
+        exception.response.status_code = 400
+        self.assertTrue(failure_handler.can_skip_failure(exception=exception))
+
+        exception.response.status_code = 500
+        self.assertFalse(failure_handler.can_skip_failure(exception=exception))