From 4ec08103bdb4e6218156c8bfc12d553ca9710320 Mon Sep 17 00:00:00 2001 From: Rob DiCiuccio Date: Tue, 19 Jan 2021 15:27:58 -0800 Subject: [PATCH 1/3] Skip loading example data from configs in CI --- superset/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/superset/cli.py b/superset/cli.py index 557cc9d0efd4a..63ac216572a04 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -130,6 +130,9 @@ def load_examples_run( print("Loading [Birth names]") examples.load_birth_names(only_metadata, force) + print("Loading [Tabbed dashboard]") + examples.load_tabbed_dashboard(only_metadata) + if not load_test_data: print("Loading [Random time series data]") examples.load_random_time_series_data(only_metadata, force) @@ -164,11 +167,8 @@ def load_examples_run( print("Loading DECK.gl demo") examples.load_deck_dash() - print("Loading [Tabbed dashboard]") - examples.load_tabbed_dashboard(only_metadata) - - # load examples that are stored as YAML config files - examples.load_from_configs(force) + # load examples that are stored as YAML config files + examples.load_from_configs(force) @with_appcontext From ca8ae6c2b87d6dc28af202357e5238e18e6e02e3 Mon Sep 17 00:00:00 2001 From: Rob DiCiuccio Date: Tue, 19 Jan 2021 17:54:01 -0800 Subject: [PATCH 2/3] Use .test suffix for example data required by e2e tests --- superset/cli.py | 4 ++-- .../{Unicode_Cloud.yaml => Unicode_Cloud.test.yaml} | 0 .../{Unicode_Test.yaml => Unicode_Test.test.yaml} | 0 .../{unicode_test.yaml => unicode_test.test.yaml} | 0 superset/examples/utils.py | 10 +++++++--- 5 files changed, 9 insertions(+), 5 deletions(-) rename superset/examples/configs/charts/{Unicode_Cloud.yaml => Unicode_Cloud.test.yaml} (100%) rename superset/examples/configs/dashboards/{Unicode_Test.yaml => Unicode_Test.test.yaml} (100%) rename superset/examples/configs/datasets/examples/{unicode_test.yaml => unicode_test.test.yaml} (100%) diff --git a/superset/cli.py b/superset/cli.py index 63ac216572a04..e1b46ee3dea97 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -167,8 +167,8 @@ def load_examples_run( print("Loading DECK.gl demo") examples.load_deck_dash() - # load examples that are stored as YAML config files - examples.load_from_configs(force) + # load examples that are stored as YAML config files + examples.load_from_configs(force, load_test_data) @with_appcontext diff --git a/superset/examples/configs/charts/Unicode_Cloud.yaml b/superset/examples/configs/charts/Unicode_Cloud.test.yaml similarity index 100% rename from superset/examples/configs/charts/Unicode_Cloud.yaml rename to superset/examples/configs/charts/Unicode_Cloud.test.yaml diff --git a/superset/examples/configs/dashboards/Unicode_Test.yaml b/superset/examples/configs/dashboards/Unicode_Test.test.yaml similarity index 100% rename from superset/examples/configs/dashboards/Unicode_Test.yaml rename to superset/examples/configs/dashboards/Unicode_Test.test.yaml diff --git a/superset/examples/configs/datasets/examples/unicode_test.yaml b/superset/examples/configs/datasets/examples/unicode_test.test.yaml similarity index 100% rename from superset/examples/configs/datasets/examples/unicode_test.yaml rename to superset/examples/configs/datasets/examples/unicode_test.test.yaml diff --git a/superset/examples/utils.py b/superset/examples/utils.py index 951b741b28d69..090e54104220d 100644 --- a/superset/examples/utils.py +++ b/superset/examples/utils.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import re from pathlib import Path from typing import Any, Dict @@ -24,13 +25,13 @@ YAML_EXTENSIONS = {".yaml", ".yml"} -def load_from_configs(force_data: bool = False) -> None: - contents = load_contents() +def load_from_configs(force_data: bool = False, load_test_data: bool = False) -> None: + contents = load_contents(load_test_data) command = ImportExamplesCommand(contents, overwrite=True, force_data=force_data) command.run() -def load_contents() -> Dict[str, Any]: +def load_contents(load_test_data: bool = False) -> Dict[str, Any]: """Traverse configs directory and load contents""" root = Path("examples/configs") resource_names = resource_listdir("superset", str(root)) @@ -39,6 +40,7 @@ def load_contents() -> Dict[str, Any]: contents: Dict[Path, str] = {} while queue: path_name = queue.pop() + test_re = re.compile("\.test\.|metadata\.yaml$") if resource_isdir("superset", str(path_name)): queue.extend( @@ -46,6 +48,8 @@ def load_contents() -> Dict[str, Any]: for child_name in resource_listdir("superset", str(path_name)) ) elif path_name.suffix.lower() in YAML_EXTENSIONS: + if load_test_data and test_re.search(str(path_name)) is None: + continue contents[path_name] = ( resource_stream("superset", str(path_name)).read().decode("utf-8") ) From 2d0bab4e452032c963f11e10407d2470aaa275c3 Mon Sep 17 00:00:00 2001 From: Rob DiCiuccio Date: Tue, 19 Jan 2021 19:09:11 -0800 Subject: [PATCH 3/3] Address linting error with regex --- superset/examples/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/examples/utils.py b/superset/examples/utils.py index 090e54104220d..66ca811df2d35 100644 --- a/superset/examples/utils.py +++ b/superset/examples/utils.py @@ -40,7 +40,7 @@ def load_contents(load_test_data: bool = False) -> Dict[str, Any]: contents: Dict[Path, str] = {} while queue: path_name = queue.pop() - test_re = re.compile("\.test\.|metadata\.yaml$") + test_re = re.compile(r"\.test\.|metadata\.yaml$") if resource_isdir("superset", str(path_name)): queue.extend(