From d04232f583d880f04be936785e2060049db88c14 Mon Sep 17 00:00:00 2001 From: nvuillam Date: Sun, 5 Dec 2021 13:00:57 +0100 Subject: [PATCH 1/4] Improve runtime perfs when FLAVOR_SUGGESTIONS: false --- .vscode/launch.json | 15 +++++++++++++++ CHANGELOG.md | 1 + megalinter/MegaLinter.py | 13 ++++++++++++- megalinter/flavor_factory.py | 6 ++++++ megalinter/linter_factory.py | 13 ++++++++++++- .../tests/test_megalinter/helpers/utilstest.py | 2 ++ .../tests/test_megalinter/mega_linter_1_test.py | 7 ++++++- 7 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000000..f87322650bb --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python : CurrentFile", + "type": "python", + "request": "test", + "program": "${file}", + "console": "integratedTerminal" + } + ] +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b81a1e8d32..749c24308a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Note: Can be used with `megalinter/megalinter@beta` in your GitHub Action mega-l - Docker run -- clean-up containers when exits (#1033) - Add missing Bandit config file and rules path options (#679) - Fix getting linter version of npm plugin. (#845) +- Improve runtime performances when using a flavor and defining `FLAVORS_SUGGESTION: false` - Linters - New linter `phplint` to speed-up linting of php files (#1031) diff --git a/megalinter/MegaLinter.py b/megalinter/MegaLinter.py index 8f49a83d754..5110b7c5c68 100644 --- a/megalinter/MegaLinter.py +++ b/megalinter/MegaLinter.py @@ -40,6 +40,7 @@ def __init__(self, params=None): self.workspace = self.get_workspace() config.init_config(self.workspace) # Initialize runtime config self.github_workspace = config.get("GITHUB_WORKSPACE", self.workspace) + self.megalinter_flavor = config.get("MEGALINTER_FLAVOR", "all") self.report_folder = config.get( "REPORT_OUTPUT_FOLDER", config.get("OUTPUT_FOLDER", self.github_workspace + os.path.sep + "report"), @@ -350,7 +351,17 @@ def load_linters(self): } # Build linters from descriptor files - all_linters = linter_factory.list_all_linters(linter_init_params) + # if flavor selected and no flavor suggestion, ignore linters that are not in current flavor) + if ( + self.megalinter_flavor != "all" + and config.get("FLAVOR_SUGGESTIONS", "true") != "true" + ): + all_linters = linter_factory.list_flavor_linters( + linter_init_params, self.megalinter_flavor + ) + else: + all_linters = linter_factory.list_all_linters(linter_init_params) + skipped_linters = [] for linter in all_linters: linter.master = self diff --git a/megalinter/flavor_factory.py b/megalinter/flavor_factory.py index b832afe1e03..d376d1184d2 100644 --- a/megalinter/flavor_factory.py +++ b/megalinter/flavor_factory.py @@ -32,6 +32,12 @@ def get_all_flavors(): return all_flavors +def list_flavor_linters(flavor_id): + all_flavors = get_all_flavors() + flavor_definition = all_flavors[flavor_id] + return flavor_definition['linters'] + + def list_megalinter_flavors(): flavors = { "all": {"label": "MegaLinter for any type of project"}, diff --git a/megalinter/linter_factory.py b/megalinter/linter_factory.py index 0505f09a65d..dc0af6b7e14 100644 --- a/megalinter/linter_factory.py +++ b/megalinter/linter_factory.py @@ -3,7 +3,7 @@ import os import yaml -from megalinter import Linter +from megalinter import Linter,flavor_factory # Returns directory where all .yml language descriptors are defined @@ -33,6 +33,17 @@ def list_all_linters(linters_init_params=None): linters += descriptor_linters return linters +# List flavor linters +def list_flavor_linters(linters_init_params=None,flavor_id="all"): + all_linters = list_all_linters(linters_init_params=None) + flavor_linter_ids = flavor_factory.list_flavor_linters(flavor_id) + linters = [] + for linter in all_linters: + if linter.name in flavor_linter_ids: + linters += [linter] + else: + del linter + return linters # List all descriptor files (one by language) def list_descriptor_files(): diff --git a/megalinter/tests/test_megalinter/helpers/utilstest.py b/megalinter/tests/test_megalinter/helpers/utilstest.py index 2ce5ea82a48..d4c4adcfa4d 100644 --- a/megalinter/tests/test_megalinter/helpers/utilstest.py +++ b/megalinter/tests/test_megalinter/helpers/utilstest.py @@ -40,6 +40,8 @@ def linter_test_setup(params=None): "IGNORE_GITIGNORED_FILES", "IGNORE_GENERATED_FILES", "SHOW_ELAPSED_TIME", + "MEGALINTER_FLAVOR", + "FLAVOR_SUGGESTIONS" ]: if key in os.environ: del os.environ[key] diff --git a/megalinter/tests/test_megalinter/mega_linter_1_test.py b/megalinter/tests/test_megalinter/mega_linter_1_test.py index ea09dc091f1..722dd2cac7a 100644 --- a/megalinter/tests/test_megalinter/mega_linter_1_test.py +++ b/megalinter/tests/test_megalinter/mega_linter_1_test.py @@ -363,7 +363,12 @@ def test_override_cli_executable(self): def test_print_all_files_false(self): mega_linter, output = utilstest.call_mega_linter( - {"ENABLE_LINTERS": "JAVASCRIPT_ES", "PRINT_ALL_FILES": "false"} + { + "ENABLE_LINTERS": "JAVASCRIPT_ES", + "PRINT_ALL_FILES": "false", + "MEGALINTER_FLAVOR": "javascript", + "FLAVOR_SUGGESTIONS": "false", + } ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" From 33898fc87ec4f5767903a81f404ced2bff1bb79f Mon Sep 17 00:00:00 2001 From: nvuillam Date: Sun, 5 Dec 2021 12:05:08 +0000 Subject: [PATCH 2/4] [MegaLinter] Apply linters fixes --- megalinter/flavor_factory.py | 2 +- megalinter/linter_factory.py | 6 ++++-- megalinter/tests/test_megalinter/helpers/utilstest.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/megalinter/flavor_factory.py b/megalinter/flavor_factory.py index d376d1184d2..78551dbe997 100644 --- a/megalinter/flavor_factory.py +++ b/megalinter/flavor_factory.py @@ -35,7 +35,7 @@ def get_all_flavors(): def list_flavor_linters(flavor_id): all_flavors = get_all_flavors() flavor_definition = all_flavors[flavor_id] - return flavor_definition['linters'] + return flavor_definition["linters"] def list_megalinter_flavors(): diff --git a/megalinter/linter_factory.py b/megalinter/linter_factory.py index dc0af6b7e14..369641847fa 100644 --- a/megalinter/linter_factory.py +++ b/megalinter/linter_factory.py @@ -3,7 +3,7 @@ import os import yaml -from megalinter import Linter,flavor_factory +from megalinter import Linter, flavor_factory # Returns directory where all .yml language descriptors are defined @@ -33,8 +33,9 @@ def list_all_linters(linters_init_params=None): linters += descriptor_linters return linters + # List flavor linters -def list_flavor_linters(linters_init_params=None,flavor_id="all"): +def list_flavor_linters(linters_init_params=None, flavor_id="all"): all_linters = list_all_linters(linters_init_params=None) flavor_linter_ids = flavor_factory.list_flavor_linters(flavor_id) linters = [] @@ -45,6 +46,7 @@ def list_flavor_linters(linters_init_params=None,flavor_id="all"): del linter return linters + # List all descriptor files (one by language) def list_descriptor_files(): descriptors_dir = get_descriptor_dir() diff --git a/megalinter/tests/test_megalinter/helpers/utilstest.py b/megalinter/tests/test_megalinter/helpers/utilstest.py index d4c4adcfa4d..83390f9cc5f 100644 --- a/megalinter/tests/test_megalinter/helpers/utilstest.py +++ b/megalinter/tests/test_megalinter/helpers/utilstest.py @@ -41,7 +41,7 @@ def linter_test_setup(params=None): "IGNORE_GENERATED_FILES", "SHOW_ELAPSED_TIME", "MEGALINTER_FLAVOR", - "FLAVOR_SUGGESTIONS" + "FLAVOR_SUGGESTIONS", ]: if key in os.environ: del os.environ[key] From f4faa901e958490b5e32330a8afe09dc8ba8a05e Mon Sep 17 00:00:00 2001 From: nvuillam Date: Sun, 5 Dec 2021 19:55:40 +0100 Subject: [PATCH 3/4] Fix how to get MEGALINTER_FLAVOR --- megalinter/flavor_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/megalinter/flavor_factory.py b/megalinter/flavor_factory.py index 78551dbe997..2976adb2e21 100644 --- a/megalinter/flavor_factory.py +++ b/megalinter/flavor_factory.py @@ -65,7 +65,7 @@ def list_megalinter_flavors(): def get_image_flavor(): - return os.environ.get("MEGALINTER_FLAVOR", "all") + return config.get("MEGALINTER_FLAVOR", "all") # Compare linters active for the current repo, and linters available in the current MegaLinter image flavor From 49a6fb8d893cf6022fcd9ff1622cdaba23c3442e Mon Sep 17 00:00:00 2001 From: nvuillam Date: Sun, 5 Dec 2021 20:58:01 +0100 Subject: [PATCH 4/4] Fix ignore flavor suggestion test class --- megalinter/linter_factory.py | 2 +- megalinter/tests/test_megalinter/mega_linter_1_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/megalinter/linter_factory.py b/megalinter/linter_factory.py index 369641847fa..8217104a0d8 100644 --- a/megalinter/linter_factory.py +++ b/megalinter/linter_factory.py @@ -36,7 +36,7 @@ def list_all_linters(linters_init_params=None): # List flavor linters def list_flavor_linters(linters_init_params=None, flavor_id="all"): - all_linters = list_all_linters(linters_init_params=None) + all_linters = list_all_linters(linters_init_params) flavor_linter_ids = flavor_factory.list_flavor_linters(flavor_id) linters = [] for linter in all_linters: diff --git a/megalinter/tests/test_megalinter/mega_linter_1_test.py b/megalinter/tests/test_megalinter/mega_linter_1_test.py index 722dd2cac7a..451eb75b95c 100644 --- a/megalinter/tests/test_megalinter/mega_linter_1_test.py +++ b/megalinter/tests/test_megalinter/mega_linter_1_test.py @@ -361,7 +361,7 @@ def test_override_cli_executable(self): "PHP_BUILTIN should have been processed with cli_executable = /usr/bin/php8", ) - def test_print_all_files_false(self): + def test_print_all_files_false_and_no_flavor_suggestion(self): mega_linter, output = utilstest.call_mega_linter( { "ENABLE_LINTERS": "JAVASCRIPT_ES",