Reduce data loaded before loading tests (apache#6298)

* reduce data loaded before loading tests

* make cypress only load needed tests

superset/assets/cypress_build.sh

@@ -4,7 +4,7 @@ set -e
 cd "$(dirname "$0")"
 
 #run all the python steps in a background process
-(time /home/travis/build/apache/incubator-superset/superset/bin/superset db upgrade; time /home/travis/build/apache/incubator-superset/superset/bin/superset load_test_users; /home/travis/build/apache/incubator-superset/superset/bin/superset load_examples; time /home/travis/build/apache/incubator-superset/superset/bin/superset init; echo "[completed python build steps]"; flask run -p 8081 --with-threads --reload --debugger) &
+(time /home/travis/build/apache/incubator-superset/superset/bin/superset db upgrade; time /home/travis/build/apache/incubator-superset/superset/bin/superset load_test_users; /home/travis/build/apache/incubator-superset/superset/bin/superset load_examples --load-test-data; time /home/travis/build/apache/incubator-superset/superset/bin/superset init; echo "[completed python build steps]"; flask run -p 8081 --with-threads --reload --debugger) &
 
 #block on the longer running javascript process
 (time yarn install --frozen-lockfile; time npm run build; echo "[completed js build steps]")

superset/cli.py

@@ -154,42 +154,42 @@ def load_examples_run(load_test_data):
     print('Loading [Birth names]')
     data.load_birth_names()
 
-    print('Loading [Random time series data]')
-    data.load_random_time_series_data()
+    print('Loading [Unicode test data]')
+    data.load_unicode_test_data()
 
-    print('Loading [Random long/lat data]')
-    data.load_long_lat_data()
+    if not load_test_data:
+        print('Loading [Random time series data]')
+        data.load_random_time_series_data()
 
-    print('Loading [Country Map data]')
-    data.load_country_map_data()
+        print('Loading [Random long/lat data]')
+        data.load_long_lat_data()
 
-    print('Loading [Multiformat time series]')
-    data.load_multiformat_time_series()
+        print('Loading [Country Map data]')
+        data.load_country_map_data()
 
-    print('Loading [Paris GeoJson]')
-    data.load_paris_iris_geojson()
+        print('Loading [Multiformat time series]')
+        data.load_multiformat_time_series()
 
-    print('Loading [San Francisco population polygons]')
-    data.load_sf_population_polygons()
+        print('Loading [Paris GeoJson]')
+        data.load_paris_iris_geojson()
 
-    print('Loading [Flights data]')
-    data.load_flights()
+        print('Loading [San Francisco population polygons]')
+        data.load_sf_population_polygons()
 
-    print('Loading [BART lines]')
-    data.load_bart_lines()
+        print('Loading [Flights data]')
+        data.load_flights()
 
-    print('Loading [Multi Line]')
-    data.load_multi_line()
+        print('Loading [BART lines]')
+        data.load_bart_lines()
 
-    print('Loading [Misc Charts] dashboard')
-    data.load_misc_dashboard()
+        print('Loading [Multi Line]')
+        data.load_multi_line()
 
-    if load_test_data:
-        print('Loading [Unicode test data]')
-        data.load_unicode_test_data()
+        print('Loading [Misc Charts] dashboard')
+        data.load_misc_dashboard()
 
-    print('Loading DECK.gl demo')
-    data.load_deck_dash()
+        print('Loading DECK.gl demo')
+        data.load_deck_dash()
 
 
 @app.cli.command()

tests/access_tests.py

@@ -165,7 +165,7 @@ def test_override_role_permissions_drops_absent_perms(self):
         override_me = security_manager.find_role('override_me')
         override_me.permissions.append(
             security_manager.find_permission_view_menu(
-                view_menu_name=self.get_table_by_name('long_lat').perm,
+                view_menu_name=self.get_table_by_name('energy_usage').perm,
                 permission_name='datasource_access'),
         )
         db.session.flush()
@@ -251,9 +251,9 @@ def test_clean_requests_after_db_grant(self):
 
         gamma_user = security_manager.find_user(username='gamma')
         access_request1 = create_access_request(
-            session, 'table', 'long_lat', TEST_ROLE_1, 'gamma')
+            session, 'table', 'energy_usage', TEST_ROLE_1, 'gamma')
         create_access_request(
-            session, 'table', 'long_lat', TEST_ROLE_2, 'gamma2')
+            session, 'table', 'energy_usage', TEST_ROLE_2, 'gamma2')
         ds_1_id = access_request1.datasource_id
         # gamma gets granted database access
         database = session.query(models.Database).first()
@@ -351,9 +351,9 @@ def test_approve(self, mock_send_mime):
             # Case 2. Extend the role to have access to the table
 
             access_request2 = create_access_request(
-                session, 'table', 'long_lat', TEST_ROLE_NAME, 'gamma')
+                session, 'table', 'energy_usage', TEST_ROLE_NAME, 'gamma')
             ds_2_id = access_request2.datasource_id
-            long_lat_perm = access_request2.datasource.perm
+            energy_usage_perm = access_request2.datasource.perm
 
             self.client.get(EXTEND_ROLE_REQUEST.format(
                 'table', access_request2.datasource_id, 'gamma', TEST_ROLE_NAME))
@@ -369,13 +369,13 @@ def test_approve(self, mock_send_mime):
                 '[Superset] Access to the datasource {} was granted'.format(
                     self.get_table(ds_2_id).full_name), call_args[2]['Subject'])
             self.assertIn(TEST_ROLE_NAME, call_args[2].as_string())
-            self.assertIn('long_lat', call_args[2].as_string())
+            self.assertIn('energy_usage', call_args[2].as_string())
 
             # request was removed
             self.assertFalse(access_requests)
-            # table_role was extended to grant access to the long_lat table/
+            # table_role was extended to grant access to the energy_usage table/
             perm_view = security_manager.find_permission_view_menu(
-                'datasource_access', long_lat_perm)
+                'datasource_access', energy_usage_perm)
             TEST_ROLE = security_manager.find_role(TEST_ROLE_NAME)
             self.assertIn(perm_view, TEST_ROLE.permissions)
 

tests/dashboard_tests.py

@@ -203,7 +203,7 @@ def test_add_slices(self, username='admin'):
         dash = db.session.query(models.Dashboard).filter_by(
             slug='births').first()
         new_slice = db.session.query(models.Slice).filter_by(
-            slice_name='Mapbox Long/Lat').first()
+            slice_name='Energy Force Layout').first()
         existing_slice = db.session.query(models.Slice).filter_by(
             slice_name='Name Cloud').first()
         data = {
@@ -217,15 +217,15 @@ def test_add_slices(self, username='admin'):
         dash = db.session.query(models.Dashboard).filter_by(
             slug='births').first()
         new_slice = db.session.query(models.Slice).filter_by(
-            slice_name='Mapbox Long/Lat').first()
+            slice_name='Energy Force Layout').first()
         assert new_slice in dash.slices
         assert len(set(dash.slices)) == len(dash.slices)
 
         # cleaning up
         dash = db.session.query(models.Dashboard).filter_by(
             slug='births').first()
         dash.slices = [
-            o for o in dash.slices if o.slice_name != 'Mapbox Long/Lat']
+            o for o in dash.slices if o.slice_name != 'Energy Force Layout']
         db.session.commit()
 
     def test_remove_slices(self, username='admin'):

tests/load_examples_test.py

@@ -17,35 +17,5 @@ def test_load_world_bank_health_n_pop(self):
     def test_load_birth_names(self):
         data.load_birth_names()
 
-    def test_load_random_time_series_data(self):
-        data.load_random_time_series_data()
-
-    def test_load_country_map_data(self):
-        data.load_country_map_data()
-
-    def test_load_multiformat_time_series_data(self):
-        data.load_multiformat_time_series()
-
-    def test_load_paris_iris_geojson(self):
-        data.load_paris_iris_geojson()
-
-    def test_load_bart_lines(self):
-        data.load_bart_lines()
-
-    def test_load_multi_line(self):
-        data.load_multi_line()
-
-    def test_load_misc_dashboard(self):
-        data.load_misc_dashboard()
-
-    def test_load_unicode_test_data(self):
-        data.load_unicode_test_data()
-
-    def test_load_deck_dash(self):
-        data.load_long_lat_data()
-        data.load_flights()
-        data.load_sf_population_polygons()
-        data.load_deck_dash()
-
     def test_load_test_users_run(self):
         load_test_users_run()

tests/model_tests.py

@@ -73,7 +73,7 @@ def test_database_impersonate_user(self):
 
     def test_select_star(self):
         main_db = get_main_database(db.session)
-        table_name = 'bart_lines'
+        table_name = 'energy_usage'
         sql = main_db.select_star(
             table_name, show_cols=False, latest_partition=False)
         expected = textwrap.dedent("""\
@@ -85,11 +85,10 @@ def test_select_star(self):
         sql = main_db.select_star(
             table_name, show_cols=True, latest_partition=False)
         expected = textwrap.dedent("""\
-        SELECT color,
-               name,
-               path_json,
-               polyline
-        FROM bart_lines
+        SELECT source,
+               target,
+               value
+        FROM energy_usage
         LIMIT 100""".format(**locals()))
         assert sql.startswith(expected)