Mode Dashboard execution timestamp and state (#197)

* Initial commit

* Update

* Update

* Update

* flake8

* Update docstring

* Update

* Update

* Add mode_dashboard_constants and mode_dashboard_utils

* Add more to Util

* Added back model class to ModeDashboardExtractor

* flake8

* Address PR comments

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_constants.py

@@ -0,0 +1,3 @@
+ORGANIZATION = 'organization'
+MODE_ACCESS_TOKEN = 'mode_user_token'
+MODE_PASSWORD_TOKEN = 'mode_password_token'

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_executions_extractor.py

@@ -0,0 +1,91 @@
+import logging
+
+from pyhocon import ConfigTree, ConfigFactory  # noqa: F401
+from typing import Any  # noqa: F401
+
+from databuilder import Scoped
+from databuilder.extractor.base_extractor import Extractor
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.transformer.base_transformer import ChainedTransformer
+from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
+from databuilder.transformer.timestamp_string_to_epoch import TimestampStringToEpoch, FIELD_NAME
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ModeDashboardExecutionsExtractor(Extractor):
+    """
+    A Extractor that extracts run (execution) status and timestamp.
+
+    """
+
+    def init(self, conf):
+        # type: (ConfigTree) -> None
+        self._conf = conf
+
+        restapi_query = self._build_restapi_query()
+        self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
+            restapi_query=restapi_query,
+            conf=self._conf
+        )
+
+        # Payload from RestApiQuery has timestamp which is ISO8601. Here we are using TimestampStringToEpoch to
+        # transform into epoch and then using DictToModel to convert Dictionary to Model
+        transformers = []
+        timestamp_str_to_epoch_transformer = TimestampStringToEpoch()
+        timestamp_str_to_epoch_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, timestamp_str_to_epoch_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict({FIELD_NAME: 'execution_timestamp', })))
+
+        transformers.append(timestamp_str_to_epoch_transformer)
+
+        dict_to_model_transformer = DictToModel()
+        dict_to_model_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, dict_to_model_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict(
+                    {MODEL_CLASS: 'databuilder.models.dashboard.dashboard_execution.DashboardExecution'})))
+        transformers.append(dict_to_model_transformer)
+
+        self._transformer = ChainedTransformer(transformers=transformers)
+
+    def extract(self):
+        # type: () -> Any
+        record = self._extractor.extract()
+        if not record:
+            return None
+
+        return self._transformer.transform(record=record)
+
+    def get_scope(self):
+        # type: () -> str
+        return 'extractor.mode_dashboard_execution'
+
+    def _build_restapi_query(self):
+        """
+        Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports
+        API, and run API) joining together.
+        :return: A RestApiQuery that provides Mode Dashboard execution (run)
+        """
+        # type: () -> RestApiQuery
+
+        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
+        params = ModeDashboardUtils.get_auth_params(conf=self._conf)
+
+        # Reports
+        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
+        url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
+        json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.last_run.href)'
+        field_names = ['dashboard_id', 'last_run_resource_path']
+        last_run_resource_path_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
+                                                    json_path=json_path, field_names=field_names, skip_no_result=True,
+                                                    json_path_contains_or=True)
+
+        # https://mode.com/developer/api-reference/analytics/report-runs/#getReportRun
+        url = 'https://app.mode.com{last_run_resource_path}'
+        json_path = '[state,completed_at]'
+        field_names = ['execution_state', 'execution_timestamp']
+        last_run_state_query = RestApiQuery(query_to_join=last_run_resource_path_query, url=url, params=params,
+                                            json_path=json_path, field_names=field_names, skip_no_result=True)
+
+        return last_run_state_query

databuilder/extractor/dashboard/mode_dashboard_extractor.py → databuilder/extractor/dashboard/mode_analytics/mode_dashboard_extractor.py

@@ -1,14 +1,11 @@
 import logging
 
 from pyhocon import ConfigTree, ConfigFactory  # noqa: F401
-from requests.auth import HTTPBasicAuth
 from typing import Any  # noqa: F401
 
-from databuilder import Scoped
 from databuilder.extractor.base_extractor import Extractor
-from databuilder.extractor.restapi.rest_api_extractor import RestAPIExtractor, REST_API_QUERY, MODEL_CLASS, \
-    STATIC_RECORD_DICT
-from databuilder.rest_api.base_rest_api_query import RestApiQuerySeed
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.extractor.restapi.rest_api_extractor import MODEL_CLASS
 from databuilder.rest_api.rest_api_query import RestApiQuery
 
 # CONFIG KEYS
@@ -39,19 +36,15 @@ def init(self, conf):
         self._conf = conf
 
         restapi_query = self._build_restapi_query()
-        self._extractor = RestAPIExtractor()
-        rest_api_extractor_conf = Scoped.get_scoped_conf(conf, self._extractor.get_scope()).with_fallback(
-            ConfigFactory.from_dict(
-                {
-                    REST_API_QUERY: restapi_query,
-                    MODEL_CLASS: 'databuilder.models.dashboard_metadata.DashboardMetadata',
-                    STATIC_RECORD_DICT: {'product': 'mode'}
-                }
+        self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
+            restapi_query=restapi_query,
+            conf=self._conf.with_fallback(
+                ConfigFactory.from_dict(
+                    {MODEL_CLASS: 'databuilder.models.dashboard_metadata.DashboardMetadata', }
+                )
             )
         )
 
-        self._extractor.init(conf=rest_api_extractor_conf)
-
     def extract(self):
         # type: () -> Any
 
@@ -70,23 +63,11 @@ def _build_restapi_query(self):
         """
         # type: () -> RestApiQuery
 
-        spaces_url_template = 'https://app.mode.com/api/{organization}/spaces?filter=all'
+        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
         reports_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
 
-        # Seed query record for next query api to join with
-        seed_record = [{'organization': self._conf.get_string(ORGANIZATION)}]
-        seed_query = RestApiQuerySeed(seed_record=seed_record)
-
-        params = {'auth': HTTPBasicAuth(self._conf.get_string(MODE_ACCESS_TOKEN),
-                                        self._conf.get_string(MODE_PASSWORD_TOKEN))}
-
-        # Spaces
-        # JSONPATH expression. it goes into array which is located in _embedded.spaces and then extracts token, name,
-        # and description
-        json_path = '_embedded.spaces[*].[token,name,description]'
-        field_names = ['dashboard_group_id', 'dashboard_group', 'dashboard_group_description']
-        spaces_query = RestApiQuery(query_to_join=seed_query, url=spaces_url_template, params=params,
-                                    json_path=json_path, field_names=field_names)
+        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
+        params = ModeDashboardUtils.get_auth_params(conf=self._conf)
 
         # Reports
         # JSONPATH expression. it goes into array which is located in _embedded.reports and then extracts token, name,

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_utils.py

@@ -0,0 +1,74 @@
+from pyhocon import ConfigTree, ConfigFactory  # noqa: F401
+from requests.auth import HTTPBasicAuth
+
+from databuilder import Scoped
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_constants import ORGANIZATION, MODE_ACCESS_TOKEN, \
+    MODE_PASSWORD_TOKEN
+from databuilder.extractor.restapi.rest_api_extractor import RestAPIExtractor, REST_API_QUERY, STATIC_RECORD_DICT
+from databuilder.rest_api.base_rest_api_query import BaseRestApiQuery  # noqa: F401
+from databuilder.rest_api.base_rest_api_query import RestApiQuerySeed
+from databuilder.rest_api.rest_api_query import RestApiQuery  # noqa: F401
+
+
+class ModeDashboardUtils(object):
+
+    @staticmethod
+    def get_spaces_query_api(conf,  # type: ConfigTree
+                             ):
+        """
+        Provides RestApiQuerySeed where it will provides iterator of dictionaries as records where dictionary keys are
+         organization, dashboard_group_id, dashboard_group and dashboard_group_description
+        :param conf:
+        :return:
+        """
+        # type: (...) -> BaseRestApiQuery
+
+        # https://mode.com/developer/api-reference/management/spaces/#listSpaces
+        spaces_url_template = 'https://app.mode.com/api/{organization}/spaces?filter=all'
+
+        # Seed query record for next query api to join with
+        seed_record = [{'organization': conf.get_string(ORGANIZATION)}]
+        seed_query = RestApiQuerySeed(seed_record=seed_record)
+
+        # Spaces
+        params = {'auth': HTTPBasicAuth(conf.get_string(MODE_ACCESS_TOKEN),
+                                        conf.get_string(MODE_PASSWORD_TOKEN))}
+
+        json_path = '_embedded.spaces[*].[token,name,description]'
+        field_names = ['dashboard_group_id', 'dashboard_group', 'dashboard_group_description']
+        spaces_query = RestApiQuery(query_to_join=seed_query, url=spaces_url_template, params=params,
+                                    json_path=json_path, field_names=field_names)
+
+        return spaces_query
+
+    @staticmethod
+    def get_auth_params(conf,  # type: ConfigTree
+                        ):
+        params = {'auth': HTTPBasicAuth(conf.get_string(MODE_ACCESS_TOKEN),
+                                        conf.get_string(MODE_PASSWORD_TOKEN)
+                                        )
+                  }
+        return params
+
+    @staticmethod
+    def create_mode_rest_api_extractor(restapi_query,  # type: BaseRestApiQuery
+                                       conf,  # type: ConfigTree
+                                       ):
+        """
+        Creates RestAPIExtractor. Note that RestAPIExtractor is already initialized
+        :param restapi_query:
+        :param conf:
+        :return: RestAPIExtractor. Note that RestAPIExtractor is already initialized
+        """
+        extractor = RestAPIExtractor()
+        rest_api_extractor_conf = \
+            Scoped.get_scoped_conf(conf, extractor.get_scope())\
+                  .with_fallback(conf)\
+                  .with_fallback(ConfigFactory.from_dict({REST_API_QUERY: restapi_query,
+                                                          STATIC_RECORD_DICT: {'product': 'mode'}
+                                                          }
+                                                         )
+                                 )
+
+        extractor.init(conf=rest_api_extractor_conf)
+        return extractor

databuilder/models/dashboard/dashboard_execution.py

@@ -0,0 +1,100 @@
+import logging
+
+from typing import Optional, Dict, Any, Union, Iterator  # noqa: F401
+
+from databuilder.models.dashboard_metadata import DashboardMetadata
+from databuilder.models.neo4j_csv_serde import (
+    Neo4jCsvSerializable, NODE_LABEL, NODE_KEY, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
+    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
+
+LOGGER = logging.getLogger(__name__)
+
+
+class DashboardExecution(Neo4jCsvSerializable):
+    """
+    A model that encapsulate Dashboard's execution timestamp in epoch and execution state
+    """
+    DASHBOARD_EXECUTION_LABEL = 'Execution'
+    DASHBOARD_EXECUTION_KEY_FORMAT = '{product}_dashboard://{cluster}.{dashboard_group_id}/' \
+                                     '{dashboard_id}/execution/{execution_id}'
+    DASHBOARD_EXECUTION_RELATION_TYPE = 'LAST_EXECUTED'
+    EXECUTION_DASHBOARD_RELATION_TYPE = 'LAST_EXECUTION_OF'
+
+    def __init__(self,
+                 dashboard_group_id,  # type: Optional[str]
+                 dashboard_id,  # type: Optional[str]
+                 execution_timestamp,  # type: int
+                 execution_state,  # type: str
+                 execution_id='_last_execution',  # type: str
+                 product='',  # type: Optional[str]
+                 cluster='gold',  # type: str
+                 **kwargs
+                 ):
+        self._dashboard_group_id = dashboard_group_id
+        self._dashboard_id = dashboard_id
+        self._execution_timestamp = execution_timestamp
+        self._execution_state = execution_state
+        self._execution_id = execution_id
+        self._product = product
+        self._cluster = cluster
+        self._node_iterator = self._create_node_iterator()
+        self._relation_iterator = self._create_relation_iterator()
+
+    def create_next_node(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._node_iterator)
+        except StopIteration:
+            return None
+
+    def _create_node_iterator(self):  # noqa: C901
+        # type: () -> Iterator[[Dict[str, Any]]]
+        yield {
+            NODE_LABEL: DashboardExecution.DASHBOARD_EXECUTION_LABEL,
+            NODE_KEY: self._get_last_execution_node_key(),
+            'time_stamp': self._execution_timestamp,
+            'state': self._execution_state
+        }
+
+    def create_next_relation(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._relation_iterator)
+        except StopIteration:
+            return None
+
+    def _create_relation_iterator(self):
+        # type: () -> Iterator[[Dict[str, Any]]]
+        yield {
+            RELATION_START_LABEL: DashboardMetadata.DASHBOARD_NODE_LABEL,
+            RELATION_END_LABEL: DashboardExecution.DASHBOARD_EXECUTION_LABEL,
+            RELATION_START_KEY: DashboardMetadata.DASHBOARD_KEY_FORMAT.format(
+                product=self._product,
+                cluster=self._cluster,
+                dashboard_group=self._dashboard_group_id,
+                dashboard_name=self._dashboard_id
+            ),
+            RELATION_END_KEY: self._get_last_execution_node_key(),
+            RELATION_TYPE: DashboardExecution.DASHBOARD_EXECUTION_RELATION_TYPE,
+            RELATION_REVERSE_TYPE: DashboardExecution.EXECUTION_DASHBOARD_RELATION_TYPE
+        }
+
+    def _get_last_execution_node_key(self):
+        return DashboardExecution.DASHBOARD_EXECUTION_KEY_FORMAT.format(
+            product=self._product,
+            cluster=self._cluster,
+            dashboard_group_id=self._dashboard_group_id,
+            dashboard_id=self._dashboard_id,
+            execution_id=self._execution_id
+        )
+
+    def __repr__(self):
+        return 'DashboardExecution({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})'.format(
+            self._dashboard_group_id,
+            self._dashboard_id,
+            self._execution_timestamp,
+            self._execution_state,
+            self._execution_id,
+            self._product,
+            self._cluster
+        )