#[UIC-2021]: replicate api enhancements (#239)

* feat(replicate-api): added support to handle optional req params and refactor code added support to handle optional req params and refactor code UIC-2037 * feat(replicate-api): added slug support in url added slug support in url UIC-2037 * feat(replicate-api): added UT added UT UIC-2037 * feat(replicate-api): updated UT updated UT UIC-2037 * feat(replicate-api): updated user password for UT updated user password for UT UIC-2037 * feat(replicate-api): updated UT updated UT UIC-2037 * feat(replicate-api): updated UT updated UT UIC-2037 * feat(replicate-api): updated UT updated UT UIC-2037 * feat(replicate-api): updated UT updated UT UIC-2037
ThalesGroup · Aug 30, 2019 · 3680648 · 3680648
1 parent bd391d0
commit 3680648
Show file tree

Hide file tree

Showing 4 changed files with 183 additions and 73 deletions.
diff --git a/superset/utils/dashboard_import_export.py b/superset/utils/dashboard_import_export.py
@@ -29,9 +29,10 @@ def import_dashboard_json(session, data, import_time=None):
     current_tt = int(time.time())
     import_time = current_tt if import_time is None else import_time
     dashboard = data['dashboards'][0]
-    Dashboard.import_obj(
+    dash_id = Dashboard.import_obj(
             dashboard, import_time=import_time)
     session.commit()
+    return dash_id
 
 def import_dashboards(session, data_stream, import_time=None):
     """Imports dashboards from a stream to databases"""

diff --git a/superset/views/add_to_dashboard.py b/superset/views/add_to_dashboard.py
@@ -3,11 +3,12 @@
 
 from flask import g 
 
-from superset import db 
+from superset import app, db, security_manager
 from superset.connectors.sqla.models import SqlaTable
 import superset.models.core as models
 from superset.utils import core as utils
 from superset.views.default_slice_metadata import update_slice_metadata
+from superset.utils import dashboard_import_export
 
 def get_table_columns(columns):
     table_columns = []
@@ -19,7 +20,7 @@ def get_table_columns(columns):
         table_columns.append(table_column)
     return table_columns
 
-def create_table(args):
+def create_table(args,fetch_metadata = False):
     database_id = int(args.get('database_id'))
     table_name =  args.get('table_name')
     schema =  args.get('schema')
@@ -45,14 +46,23 @@ def create_table(args):
         )
         db.session.add(table_model)
         db.session.commit()
+        fetch_table_metadata(table_model, fetch_metadata)
         logging.info('table is created with id = '+str(table_model.id)+' and linked with database id = '+str(database_id))
         return table_model
     else:
         # pick first one
         logging.info('reused table with id = '+str(table_models[0].id))
+        fetch_table_metadata(table_models[0], fetch_metadata)
         return table_models[0]
 
 
+def fetch_table_metadata(table_model,fetch_metadata = False):
+    if fetch_metadata:
+        table_model.fetch_metadata()
+        security_manager.merge_perm('datasource_access', table_model.get_perm())
+        if table_model.schema:
+            security_manager.merge_perm('schema_access', table_model.schema_perm)
+
 
 def add_slice_to_dashboard(request,args, datasource_type=None, datasource_id=None):
     form_data = json.loads(args.get('form_data'))
@@ -91,19 +101,24 @@ def add_slice_to_dashboard(request,args, datasource_type=None, datasource_id=Non
 
     dash.slices.append(slc)
     db.session.commit()
-    logging.info(
-    'Slice [{}] was added to dashboard [{}]'.format(
-        slc.slice_name,
-        dash.dashboard_title),
-    'info')
+    logging.info('Slice ['+ slc.slice_name +'] was added to dashboard id [ '+str(args.get('save_to_dashboard_id'))+' ]')
 
     return {
         'form_data': slc.form_data,
         'slice': slc.data,
     }
 
 
-def create_database(database_name,sqlalchemy_uri,extra,impersonate_user):
+def create_database(form):
+    # create database  connection
+    database_name = form.get('database_name')
+    if database_name is None:
+        return None
+
+    sqlalchemy_uri = form.get('sqlalchemy_uri')
+    extra = form.get('extra')
+    impersonate_user = eval(form.get('impersonate_user'))
+
     db_models = (
                 db.session.query(models.Database)
                 .filter_by(database_name=database_name)
@@ -128,11 +143,7 @@ def create_database(database_name,sqlalchemy_uri,extra,impersonate_user):
 
 def add_to_dashboard(request):
     # create database  connection
-    database_name = request.form.get('database_name')
-    sqlalchemy_uri = request.form.get('sqlalchemy_uri')
-    extra = request.form.get('extra')
-    impersonate_user = eval(request.form.get('impersonate_user'))
-    database_id = create_database(database_name,sqlalchemy_uri,extra,impersonate_user)    
+    database_id = create_database(request.form)    
 
     # create dashboard
     dash_model = models.Dashboard(
@@ -174,4 +185,73 @@ def add_to_dashboard(request):
         'datasource_name' : table_model.name
         }
 
-        add_slice_to_dashboard(request,slice_param_data)
+        add_slice_to_dashboard(request,slice_param_data)
+
+    return get_dashboard_response(request.url_root,dashboard_id,dash_model.slug)    
+
+
+def update_slices_in_dashboard(dashboards, parameters):
+    dashboard = dashboards['dashboards'][0]
+    title = parameters['dashboard_title']
+    if title:
+        dashboard.dashboard_title = title
+    # assuming sinle dashboard per JSON
+    for slice_item in dashboard.slices:
+        datasource = None
+        if slice_item.datasource_name in parameters:
+            datasource = parameters[slice_item.datasource_name]
+        if datasource is not None:
+            slice_item.alter_params(
+                        datasource_name = datasource.name,
+                        schema = datasource.name,
+                        database_name = datasource.database_name,
+                    )
+    return dashboards
+
+def get_dashboard_response(url_root,dash_id,slug = None):
+    id = str(dash_id)
+    if slug is not None:
+        id = slug 
+    return json.dumps({'dashboard_url' : url_root.rstrip('/')+ app.config.get('APPLICATION_PREFIX') + "/superset/dashboard/" + id + "/"})
+
+def replicate_dashboard(request):
+    database_id = create_database(request.form)
+
+    dashboard_title = request.form.get('dashboard_title')
+    template_parameters = {'dashboard_title': dashboard_title}
+    tables_param = {}
+    dashboard_data_param = request.form.get('template')
+    dashboard_data = dashboard_import_export.dashboard_json(dashboard_data_param)
+
+    if database_id is not None:
+        # get tables from request
+        tables = request.form.get('tables')
+        if tables is not None:
+            tables_param = json.loads(r''+request.form.get('tables'))
+
+        # update tables_param with datasource_name of  slices, not defined in  tables
+        for slice_item in dashboard_data['dashboards'][0].slices:
+            if slice_item.datasource_name not in tables_param:
+                tables_param[slice_item.datasource_name] = slice_item.datasource_name
+
+        for table_placeholder in tables_param:
+            schema_and_table_name = tables_param[table_placeholder].split('.')
+            table_name = schema_and_table_name[0]
+            schema = None
+            if len(schema_and_table_name) >= 2:
+                table_name = schema_and_table_name[1]
+                schema = schema_and_table_name[0]
+
+            params = {
+                'database_id': database_id ,
+                'table_name': table_name,
+                'schema': schema,
+                'columns': json.dumps([])
+            }
+            # create table for slice
+            table = create_table(params,True)
+            template_parameters[table_placeholder] = table
+
+    update_slices_in_dashboard(dashboard_data,template_parameters)
+    dash_id = dashboard_import_export.import_dashboard_json(db.session, dashboard_data)
+    return get_dashboard_response(request.url_root,dash_id)
diff --git a/superset/views/core.py b/superset/views/core.py
@@ -77,7 +77,7 @@
     SupersetFilter, SupersetModelView, YamlExportMixin,
 )
 from .utils import bootstrap_user_data
-from .add_to_dashboard import add_to_dashboard, create_database, create_table
+from .add_to_dashboard import add_to_dashboard, replicate_dashboard
 
 
 config = app.config
@@ -830,69 +830,20 @@ def execute_rest_action(self):
     @expose('/add_to_dashboard', methods=['POST'])
     def addtodashboard(self):
         try:
-            add_to_dashboard(request)
-            return json_success('OK')
+            response = add_to_dashboard(request)
+            return json_success(response)
         except Exception as e:
             logging.exception(e)
             return json_error_response(e) 
 
-    def update_dashboard(self, dashboards, parameters):
-        dashboard = dashboards['dashboards'][0]
-        title = parameters['dashboard_title']
-        if title:
-            dashboard.dashboard_title = title
-        # assuming sinle dashboard per JSON
-        for slice_item in dashboard.slices:
-            datasource = parameters[slice_item.datasource_name]
-            if datasource is not None:
-                slice_item.alter_params(
-                            datasource_name = datasource.name,
-                            schema = datasource.name,
-                            database_name = datasource.database_name,
-                        )
-        return dashboards
-
-
     @expose('/replicate_dashboard', methods=['POST'])
     def replicate_dashboard(self):
-        database_name = request.form.get('database_name')
-        sqlalchemy_uri = request.form.get('sqlalchemy_uri')
-        extra = request.form.get('extra')
-        impersonate_user = eval(request.form.get('impersonate_user'))
-        if database_name is not None:
-            database_id = create_database(database_name,sqlalchemy_uri,extra,impersonate_user)
-
-        dashboard_title = request.form.get('dashboard_title')
-        tables_param = json.loads(r''+request.form.get('tables'))
-        template_parameters = {'dashboard_title': dashboard_title}
-
-        if database_id is not None:
-            for table_placeholder in tables_param:
-                schema_and_table_name = tables_param[table_placeholder].split('.')
-                params = {
-                    'database_id':database_id ,
-                    'table_name': schema_and_table_name[1],
-                    'schema': schema_and_table_name[0],
-                    'columns': json.dumps([])
-                }
-                # create table for slice
-                table = create_table(params)
-                table.fetch_metadata()
-                security_manager.merge_perm('datasource_access', table.get_perm())
-                if table.schema:
-                    security_manager.merge_perm('schema_access', table.schema_perm)
-                template_parameters[table_placeholder] = table
-
-                logging.info('table is created with id = '+str(table.id)+' and linked with database id = '+str(database_id)) 
-
-        dashboard_data_param = request.form.get('template')
-        dashboard_data = dashboard_import_export.dashboard_json(dashboard_data_param)
-
-        self.update_dashboard(dashboard_data,template_parameters)
-
-        dashboard_import_export.import_dashboard_json(db.session, dashboard_data)
-        return redirect(url_for('DashboardAddView.list'))
-
+        try:
+            response = replicate_dashboard(request)
+            return json_success(response)
+        except Exception as e:
+            logging.exception(e)
+            return json_error_response(e) 
 
     @has_access_api
     @expose('/datasources/')

diff --git a/tests/add_to_dashboard_tests.py b/tests/add_to_dashboard_tests.py
@@ -0,0 +1,78 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Unit tests for Superset"""
+import json
+import unittest
+
+from flask import escape
+from sqlalchemy import func
+
+from superset import db, security_manager
+from superset.connectors.sqla.models import SqlaTable
+from superset.models import core as models
+from .base_tests import SupersetTestCase
+
+class AddToDashboardTests(SupersetTestCase):
+
+    def __init__(self, *args, **kwargs):
+        super(AddToDashboardTests, self).__init__(*args, **kwargs)
+
+    @classmethod
+    def setUpClass(cls):
+        pass
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def test_add_to_dashboard(self):
+        self.login(username='admin',password='Ym8Hg1+u3VmyM8mRul3xnWuvh2xalT/soSM3z5fTosQ=')
+        url = '/superset/add_to_dashboard'
+        self.client.post(url, data=dict(
+            database_name="test_add_to_dashboard",
+            sqlalchemy_uri="hive://[email protected]:10000/",
+            impersonate_user=False,
+            extra= json.dumps({
+                "metadata_params": {},
+                "engine_params": {},
+                "metadata_cache_timeout": {},
+                "schemas_allowed_for_csv_upload": []
+            }),
+            dashboard_title="test_add_to_dashboard",
+            slug="test_add_to_dashboard",
+            slices=json.dumps([
+                        {
+                            "table_name":"control_plane_table",
+                            "schema":"demo",
+                            "slice_name":"bardemo",
+                            "viz_type":"dist_bar",
+                            "metrics":[{"column":{"column_name":"c_call_duration"}}],
+                            "groupby":["c_to_msisdn"]}
+            ])
+
+        ))
+
+        dash_added = db.session.query(models.Dashboard).filter_by(
+            slug='test_add_to_dashboard').first()
+        slice_added = db.session.query(models.Slice).filter_by(
+            slice_name='bardemo').first()
+        assert slice_added in dash_added.slices
+
+if __name__ == '__main__':
+    unittest.main()