diff --git a/.automation/build.py b/.automation/build.py index f32daccbd37..f1dd72741fe 100644 --- a/.automation/build.py +++ b/.automation/build.py @@ -25,7 +25,7 @@ import yaml from bs4 import BeautifulSoup from giturlparse import parse -from megalinter import utils +from megalinter import config, utils from megalinter.constants import ( DEFAULT_DOCKERFILE_APK_PACKAGES, DEFAULT_RELEASE, @@ -47,6 +47,7 @@ UPDATE_CHANGELOG = "--changelog" in sys.argv IS_LATEST = "--latest" in sys.argv DELETE_DOCKERFILES = "--delete-dockerfiles" in sys.argv +DELETE_TEST_CLASSES = "--delete-test-classes" in sys.argv # Release args management if RELEASE is True: @@ -153,7 +154,7 @@ def generate_flavor(flavor, flavor_info): descriptor_and_linters += [descriptor] flavor_descriptors += [descriptor["descriptor_id"]] # Get install instructions at linter level - linters = megalinter.linter_factory.list_all_linters() + linters = megalinter.linter_factory.list_all_linters(({"request_id": "build"})) requires_docker = False for linter in linters: if match_flavor(vars(linter), flavor, flavor_info) is True: @@ -596,7 +597,7 @@ def generate_linter_dockerfiles(): if "install" in descriptor: descriptor_items += [descriptor] descriptor_linters = megalinter.linter_factory.build_descriptor_linters( - descriptor_file, None + descriptor_file, {"request_id": "build"} ) # Browse descriptor linters for linter in descriptor_linters: @@ -686,11 +687,12 @@ def generate_linter_dockerfiles(): def generate_linter_test_classes(): test_linters_root = f"{REPO_HOME}/megalinter/tests/test_megalinter/linters" - # Remove all the contents of test_linters_root beforehand so that the result is deterministic - shutil.rmtree(os.path.realpath(test_linters_root)) - os.makedirs(os.path.realpath(test_linters_root)) + if DELETE_TEST_CLASSES is True: + # Remove all the contents of test_linters_root beforehand so that the result is deterministic + shutil.rmtree(os.path.realpath(test_linters_root)) + os.makedirs(os.path.realpath(test_linters_root)) - linters = megalinter.linter_factory.list_all_linters() + linters = megalinter.linter_factory.list_all_linters(({"request_id": "build"})) for linter in linters: if linter.name is not None: linter_name = linter.name @@ -737,7 +739,7 @@ def list_descriptors_for_build(): descriptor = megalinter.linter_factory.build_descriptor_info(descriptor_file) descriptors += [descriptor] descriptor_linters = megalinter.linter_factory.build_descriptor_linters( - descriptor_file + descriptor_file, {"request_id": "build"} ) linters_by_type[descriptor_linters[0].descriptor_type] += descriptor_linters DESCRIPTORS_FOR_BUILD_CACHE = descriptors, linters_by_type @@ -2468,7 +2470,7 @@ def generate_json_schema_enums(): outfile.write("\n") # Update list of descriptors and linters in configuration schema descriptors, _linters_by_type = list_descriptors_for_build() - linters = megalinter.linter_factory.list_all_linters() + linters = megalinter.linter_factory.list_all_linters({"request_id": "build"}) with open(CONFIG_JSON_SCHEMA, "r", encoding="utf-8") as json_file: json_schema = json.load(json_file) json_schema["definitions"]["enum_descriptor_keys"]["enum"] = [ @@ -2489,7 +2491,7 @@ def generate_json_schema_enums(): # Collect linters info from linter url, later used to build link preview card within linter documentation def collect_linter_previews(): - linters = megalinter.linter_factory.list_all_linters() + linters = megalinter.linter_factory.list_all_linters({"request_id": "build"}) # Read file with open(LINKS_PREVIEW_FILE, "r", encoding="utf-8") as json_file: data = json.load(json_file) @@ -2498,7 +2500,7 @@ def collect_linter_previews(): for linter in linters: if ( linter.linter_name not in data - or megalinter.config.get("REFRESH_LINTER_PREVIEWS", "false") == "true" + or megalinter.config.get(None, "REFRESH_LINTER_PREVIEWS", "false") == "true" ): logging.info( f"Collecting link preview info for {linter.linter_name} at {linter.linter_url}" @@ -2527,7 +2529,7 @@ def collect_linter_previews(): def generate_documentation_all_linters(): - linters_raw = megalinter.linter_factory.list_all_linters() + linters_raw = megalinter.linter_factory.list_all_linters(({"request_id": "build"})) linters = [] with open(VERSIONS_FILE, "r", encoding="utf-8") as json_file: linter_versions = json.load(json_file) @@ -3103,7 +3105,7 @@ def update_workflow_linters(file_path, linters): format="%(asctime)s [%(levelname)s] %(message)s", handlers=[logging.StreamHandler(sys.stdout)], ) - + config.init_config("build") # noinspection PyTypeChecker collect_linter_previews() generate_json_schema_enums() diff --git a/.automation/generated/linter-versions.json b/.automation/generated/linter-versions.json index 83d79cb7415..76edac5150c 100644 --- a/.automation/generated/linter-versions.json +++ b/.automation/generated/linter-versions.json @@ -94,7 +94,7 @@ "standard": "17.0.0", "stylelint": "15.6.0", "swiftlint": "0.51.0", - "syft": "0.79.0", + "syft": "0.76.1", "tekton-lint": "0.6.0", "terraform-fmt": "1.4.6", "terragrunt": "0.45.6", diff --git a/.automation/test/sample_project/.mega-linter.yml b/.automation/test/sample_project/.mega-linter.yml new file mode 100644 index 00000000000..568086a45ce --- /dev/null +++ b/.automation/test/sample_project/.mega-linter.yml @@ -0,0 +1,3 @@ +DISABLE: + - REPOSITORY + - SPELL \ No newline at end of file diff --git a/.automation/test/sample_project/groovy_good_01.groovy b/.automation/test/sample_project/groovy_good_01.groovy deleted file mode 100644 index d0375dab32e..00000000000 --- a/.automation/test/sample_project/groovy_good_01.groovy +++ /dev/null @@ -1,6 +0,0 @@ -class Example { - static void main(String[] args) { - File file = new File("E:/Example.txt") - println "The file ${file.absolutePath} has ${file.length()} bytes" - } -} diff --git a/.github/linters/.cspell.json b/.github/linters/.cspell.json index 461bb5cef71..4901621a0f1 100644 --- a/.github/linters/.cspell.json +++ b/.github/linters/.cspell.json @@ -535,6 +535,7 @@ "contextlib", "contextmanager", "copypaste", + "copytree", "coreutils", "countdef", "coursier", @@ -576,6 +577,7 @@ "disableassertions", "disablesystemassertions", "displaymath", + "distutils", "djlint", "dockerfilelint", "dockerfilelintrc", @@ -1217,6 +1219,7 @@ "shpca", "shppa", "shpss", + "shutil", "simplexml", "slshape", "smallskip", diff --git a/.mega-linter.yml b/.mega-linter.yml index 69c855299fe..c4d67bc1def 100644 --- a/.mega-linter.yml +++ b/.mega-linter.yml @@ -41,6 +41,7 @@ REPOSITORY_TRIVY_ARGUMENTS: - "--skip-dirs" - ".automation/test" SHOW_ELAPSED_TIME: true +FLAVOR_SUGGESTIONS: false EMAIL_REPORTER: false FILEIO_REPORTER: true JSON_REPORTER: true diff --git a/CHANGELOG.md b/CHANGELOG.md index b20734782fb..ec4c66aa60b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l - Core - Use relative file paths to call linters ([#1875](https://github.com/oxsecurity/megalinter/issues/1875)) + - Refactor internal configuration management to scope config to a request identifier + - New configuration variable SECURED_ENV_VARIABLES to hide some env vars from environment used when calling linters + - Replace deprecated distutils.copy_tree by shutil.copytree - Add support for idea plugins autoinstall - Upgrade base Docker image to python:3.11.3-alpine3.17 - Fix issue preventing plugins to work with flavors diff --git a/README.md b/README.md index f7f45415e3d..86bd3c69049 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ _Github PR reporter_ - [Linter specific variables](#linter-specific-variables) - [Pre-commands](#pre-commands) - [Post-commands](#post-commands) + - [Environment variables security](#environment-variables-security) - [CLI lint mode](#cli-lint-mode) - [Reporters](#reporters) - [Flavors](#flavors) @@ -782,50 +783,51 @@ Configuration is assisted with autocompletion and validation in most commonly us ### Common variables -| **ENV VAR** | **Default Value** | **Notes** | -|------------------------------------------------------------|------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **ADDITIONAL_EXCLUDED_DIRECTORIES** | \[\] | List of additional excluded directory basenames. they're excluded at any nested level. | -| [**APPLY_FIXES**](#apply-fixes) | `none` | Activates formatting and autofix [(more info)](#apply-fixes) | -| **CLEAR_REPORT_FOLDER** | `false` | Flag to clear files from report folder (usually megalinter-reports) before starting the linting process | -| **DEFAULT_BRANCH** | `HEAD` | Deprecated: The name of the repository's default branch. | -| **DEFAULT_WORKSPACE** | `/tmp/lint` | The location containing files to lint if you are running locally. | -| **DISABLE_ERRORS** | `false` | Flag to have the linter complete with exit code 0 even if errors were detected. | -| [**DISABLE**](#activation-and-deactivation) | | List of disabled descriptors keys [(more info)](#activation-and-deactivation) | -| [**DISABLE_LINTERS**](#activation-and-deactivation) | | List of disabled linters keys [(more info)](#activation-and-deactivation) | -| [**DISABLE_ERRORS_LINTERS**](#activation-and-deactivation) | | List of enabled but not blocking linters keys [(more info)](#activation-and-deactivation) | -| [**ENABLE**](#activation-and-deactivation) | | List of enabled descriptors keys [(more info)](#activation-and-deactivation) | -| [**ENABLE_LINTERS**](#activation-and-deactivation) | | List of enabled linters keys [(more info)](#activation-and-deactivation) | -| **EXCLUDED_DIRECTORIES** | \[…many values…\] | List of excluded directory basenames. they're excluded at any nested level. | -| **EXTENDS** | | Base `mega-linter.yml` config file(s) to extend local configuration from. Can be a single URL or a list of `.mega-linter.yml` config files URLs. Later files take precedence. | -| **FAIL_IF_MISSING_LINTER_IN_FLAVOR** | `false` | If set to `true`, MegaLinter fails if a linter is missing in the selected flavor | -| **FAIL_IF_UPDATED_SOURCES** | `false` | If set to `true`, MegaLinter fails if a linter or formatter has autofixed sources, even if there are no errors | -| [**FILTER_REGEX_EXCLUDE**](#filter-linted-files) | `none` | Regular expression defining which files will be excluded from linting [(more info)](#filter-linted-files) .ex: `.*src/test.*`) | -| [**FILTER_REGEX_INCLUDE**](#filter-linted-files) | `all` | Regular expression defining which files will be processed by linters [(more info)](#filter-linted-files) .ex: `.*src/.*`) | -| **FLAVOR_SUGGESTIONS** | `true` | Provides suggestions about different MegaLinter flavors to use to improve runtime performances | -| **FORMATTERS_DISABLE_ERRORS** | `true` | Formatter errors will be reported as errors (and not warnings) if this variable is set to `false` | -| **GIT_AUTHORIZATION_BEARER** | | If set, calls git with **`Authorization: Bearer`+value** | -| **GITHUB_WORKSPACE** | | Base directory for `REPORT_OUTPUT_FOLDER`, for user-defined linter rules location, for location of linted files if `DEFAULT_WORKSPACE` isn't set | -| **IGNORE_GENERATED_FILES** | `false` | If set to `true`, MegaLinter will skip files containing `@generated` marker but without `@not-generated` marker (more info at [https://generated.at](https://generated.at/)) | -| **IGNORE_GITIGNORED_FILES** | `true` | If set to `true`, MegaLinter will skip files ignored by git using `.gitignore` file | -| **JAVASCRIPT_DEFAULT_STYLE** | `standard` | Javascript default style to check/apply. `standard`,`prettier` | -| **LINTER_RULES_PATH** | `.github/linters` | Directory for all linter configuration rules.
Can be a local folder or a remote URL (ex: `https://raw.githubusercontent.com/some_org/some_repo/mega-linter-rules` ) | -| **LOG_FILE** | `mega-linter.log` | The file name for outputting logs. All output is sent to the log file regardless of `LOG_LEVEL`. Use `none` to not generate this file. | -| **LOG_LEVEL** | `INFO` | How much output the script will generate to the console. One of `INFO`, `DEBUG`, `WARNING` or `ERROR`. | -| **MARKDOWN_DEFAULT_STYLE** | `markdownlint` | Markdown default style to check/apply. `markdownlint`,`remark-lint` | -| **MEGALINTER_CONFIG** | `.mega-linter.yml` | Name of MegaLinter configuration file. Can be defined remotely, in that case set this environment variable with the remote URL of `.mega-linter.yml` config file | -| **MEGALINTER_FILES_TO_LINT** | \[\] | Comma-separated list of files to analyze. Using this variable will bypass other file listing methods | -| **PARALLEL** | `true` | Process linters in parallel to improve overall MegaLinter performance. If true, linters of same language or formats are grouped in the same parallel process to avoid lock issues if fixing the same files | -| [**PLUGINS**](#plugins) | \[\] | List of plugin urls to install and run during MegaLinter run | -| [**POST_COMMANDS**](#post-commands) | \[\] | Custom bash commands to run after linters | -| [**PRE_COMMANDS**](#pre-commands) | \[\] | Custom bash commands to run before linters | -| **PRINT_ALPACA** | `true` | Enable printing alpaca image to console | -| **PRINT_ALL_FILES** | `false` | Display all files analyzed by the linter instead of only the number | -| **REPORT_OUTPUT_FOLDER** | `${GITHUB_WORKSPACE}/megalinter-reports` | Directory for generating report files. Set to `none` to not generate reports | -| **SHOW_ELAPSED_TIME** | `false` | Displays elapsed time in reports | -| **SHOW_SKIPPED_LINTERS** | `true` | Displays all disabled linters mega-linter could have run | -| **SKIP_CLI_LINT_MODES** | \[\] | Comma-separated list of cli_lint_modes. To use if you want to skip linters with some CLI lint modes (ex: `file,project`). Available values: `file`,`cli_lint_mode`,`project`. | -| **TYPESCRIPT_DEFAULT_STYLE** | `standard` | Typescript default style to check/apply. `standard`,`prettier` | -| **VALIDATE_ALL_CODEBASE** | `true` | Will parse the entire repository and find all files to validate across all types. **NOTE:** When set to `false`, only **new** or **edited** files will be parsed for validation. | +| **ENV VAR** | **Default Value** | **Notes** | +|--------------------------------------------------------------|-----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **ADDITIONAL_EXCLUDED_DIRECTORIES** | \[\] | List of additional excluded directory basenames. they're excluded at any nested level. | +| [**APPLY_FIXES**](#apply-fixes) | `none` | Activates formatting and autofix [(more info)](#apply-fixes) | +| **CLEAR_REPORT_FOLDER** | `false` | Flag to clear files from report folder (usually megalinter-reports) before starting the linting process | +| **DEFAULT_BRANCH** | `HEAD` | Deprecated: The name of the repository's default branch. | +| **DEFAULT_WORKSPACE** | `/tmp/lint` | The location containing files to lint if you are running locally. | +| **DISABLE_ERRORS** | `false` | Flag to have the linter complete with exit code 0 even if errors were detected. | +| [**DISABLE**](#activation-and-deactivation) | | List of disabled descriptors keys [(more info)](#activation-and-deactivation) | +| [**DISABLE_LINTERS**](#activation-and-deactivation) | | List of disabled linters keys [(more info)](#activation-and-deactivation) | +| [**DISABLE_ERRORS_LINTERS**](#activation-and-deactivation) | | List of enabled but not blocking linters keys [(more info)](#activation-and-deactivation) | +| [**ENABLE**](#activation-and-deactivation) | | List of enabled descriptors keys [(more info)](#activation-and-deactivation) | +| [**ENABLE_LINTERS**](#activation-and-deactivation) | | List of enabled linters keys [(more info)](#activation-and-deactivation) | +| **EXCLUDED_DIRECTORIES** | \[…many values…\] | List of excluded directory basenames. they're excluded at any nested level. | +| **EXTENDS** | | Base `mega-linter.yml` config file(s) to extend local configuration from. Can be a single URL or a list of `.mega-linter.yml` config files URLs. Later files take precedence. | +| **FAIL_IF_MISSING_LINTER_IN_FLAVOR** | `false` | If set to `true`, MegaLinter fails if a linter is missing in the selected flavor | +| **FAIL_IF_UPDATED_SOURCES** | `false` | If set to `true`, MegaLinter fails if a linter or formatter has autofixed sources, even if there are no errors | +| [**FILTER_REGEX_EXCLUDE**](#filter-linted-files) | `none` | Regular expression defining which files will be excluded from linting [(more info)](#filter-linted-files) .ex: `.*src/test.*`) | +| [**FILTER_REGEX_INCLUDE**](#filter-linted-files) | `all` | Regular expression defining which files will be processed by linters [(more info)](#filter-linted-files) .ex: `.*src/.*`) | +| **FLAVOR_SUGGESTIONS** | `true` | Provides suggestions about different MegaLinter flavors to use to improve runtime performances | +| **FORMATTERS_DISABLE_ERRORS** | `true` | Formatter errors will be reported as errors (and not warnings) if this variable is set to `false` | +| **GIT_AUTHORIZATION_BEARER** | | If set, calls git with **`Authorization: Bearer`+value** | +| **GITHUB_WORKSPACE** | | Base directory for `REPORT_OUTPUT_FOLDER`, for user-defined linter rules location, for location of linted files if `DEFAULT_WORKSPACE` isn't set | +| **IGNORE_GENERATED_FILES** | `false` | If set to `true`, MegaLinter will skip files containing `@generated` marker but without `@not-generated` marker (more info at [https://generated.at](https://generated.at/)) | +| **IGNORE_GITIGNORED_FILES** | `true` | If set to `true`, MegaLinter will skip files ignored by git using `.gitignore` file | +| **JAVASCRIPT_DEFAULT_STYLE** | `standard` | Javascript default style to check/apply. `standard`,`prettier` | +| **LINTER_RULES_PATH** | `.github/linters` | Directory for all linter configuration rules.
Can be a local folder or a remote URL (ex: `https://raw.githubusercontent.com/some_org/some_repo/mega-linter-rules` ) | +| **LOG_FILE** | `mega-linter.log` | The file name for outputting logs. All output is sent to the log file regardless of `LOG_LEVEL`. Use `none` to not generate this file. | +| **LOG_LEVEL** | `INFO` | How much output the script will generate to the console. One of `INFO`, `DEBUG`, `WARNING` or `ERROR`. | +| **MARKDOWN_DEFAULT_STYLE** | `markdownlint` | Markdown default style to check/apply. `markdownlint`,`remark-lint` | +| **MEGALINTER_CONFIG** | `.mega-linter.yml` | Name of MegaLinter configuration file. Can be defined remotely, in that case set this environment variable with the remote URL of `.mega-linter.yml` config file | +| **MEGALINTER_FILES_TO_LINT** | \[\] | Comma-separated list of files to analyze. Using this variable will bypass other file listing methods | +| **PARALLEL** | `true` | Process linters in parallel to improve overall MegaLinter performance. If true, linters of same language or formats are grouped in the same parallel process to avoid lock issues if fixing the same files | +| [**PLUGINS**](#plugins) | \[\] | List of plugin urls to install and run during MegaLinter run | +| [**POST_COMMANDS**](#post-commands) | \[\] | Custom bash commands to run after linters | +| [**PRE_COMMANDS**](#pre-commands) | \[\] | Custom bash commands to run before linters | +| **PRINT_ALPACA** | `true` | Enable printing alpaca image to console | +| **PRINT_ALL_FILES** | `false` | Display all files analyzed by the linter instead of only the number | +| **REPORT_OUTPUT_FOLDER** | `${GITHUB_WORKSPACE}/megalinter-reports` | Directory for generating report files. Set to `none` to not generate reports | +| [**SECURED_ENV_VARIABLES**](#environment-variables-security) | MegaLinter & CI platforms sensitive variables | List of secured environment variables to hide when calling linters. Default list is GITHUB_TOKEN,PAT,SYSTEM_ACCESSTOKEN,CI_JOB_TOKEN,GITLAB_ACCESS_TOKEN_MEGALINTER,GITLAB_CUSTOM_CERTIFICATE,WEBHOOK_REPORTER_BEARER_TOKEN. If you override it, add them ! | +| **SHOW_ELAPSED_TIME** | `false` | Displays elapsed time in reports | +| **SHOW_SKIPPED_LINTERS** | `true` | Displays all disabled linters mega-linter could have run | +| **SKIP_CLI_LINT_MODES** | \[\] | Comma-separated list of cli_lint_modes. To use if you want to skip linters with some CLI lint modes (ex: `file,project`). Available values: `file`,`cli_lint_mode`,`project`. | +| **TYPESCRIPT_DEFAULT_STYLE** | `standard` | Typescript default style to check/apply. `standard`,`prettier` | +| **VALIDATE_ALL_CODEBASE** | `true` | Will parse the entire repository and find all files to validate across all types. **NOTE:** When set to `false`, only **new** or **edited** files will be parsed for validation. | ### Activation and deactivation @@ -932,6 +934,36 @@ POST_COMMANDS: cwd: "workspace" # Will be run at the root of the workspace (usually your repository root) ``` +### Environment variables security + +MegaLinter runs on a docker image and calls the linters via command line to gather their results. + +If you run it from your **CI/CD pipelines**, the docker image may have **access to your environment variables, that can contain secrets** defined in CI/CD variables. + +As it can be complicated to **trust** the authors of all the open-source linters, **MegaLinter removes variables from the environment used to call linters**. + +Thanks to this feature, you only need to [**trust MegaLinter and its internal python dependencies**](https://github.com/oxsecurity/megalinter/blob/main/megalinter/setup.py), but there is **no need to trust all the linters that are used** ! + +You can override the list of hidden variables using configuration property **SECURED_ENV_VARIABLES** in .mega-linter.yml or in an environment variable. + +Example in `.mega-linter.yml`, with default values: + +```yaml +SECURED_ENV_VARIABLES: + - GITHUB_TOKEN + - PAT + - SYSTEM_ACCESSTOKEN + - CI_JOB_TOKEN + - GITLAB_ACCESS_TOKEN_MEGALINTER + - GITLAB_CUSTOM_CERTIFICATE + - WEBHOOK_REPORTER_BEARER_TOKEN +``` + +Notes: + +- If you define SECURED_ENV_VARIABLES, it replaces the default list, so append to the default list instead of just adding the new variables that you want to secure ! +- Environment variables are secured for each command line called (linters, plugins, sarif formatter...) except for [PRE_COMMANDS](#pre-commands) , as you might need secured values within their code. + ### CLI lint mode Each linter has a lint mode by default, visible in its MegaLinter documentation ([example](https://megalinter.io/latest/descriptors/repository_trivy/#how-the-linting-is-performed)): diff --git a/linters/bicep_bicep_linter/Dockerfile b/linters/bicep_bicep_linter/Dockerfile index 38ba96d5020..4f5401d72ce 100644 --- a/linters/bicep_bicep_linter/Dockerfile +++ b/linters/bicep_bicep_linter/Dockerfile @@ -50,6 +50,7 @@ RUN apk add --update --no-cache \ make \ musl-dev \ openssh \ + icu-libs \ && git config --global core.autocrlf true #APK__END diff --git a/megalinter/Linter.py b/megalinter/Linter.py index 6f3f199eed7..c62d34904e4 100644 --- a/megalinter/Linter.py +++ b/megalinter/Linter.py @@ -45,6 +45,7 @@ def __init__(self, params=None, linter_config=None): self.linter_help_cache = None self.processing_order = 0 self.master = None + self.request_id = None # Definition fields & default values: can be overridden at custom linter class level or in YML descriptors # Ex: JAVASCRIPT self.descriptor_id = ( @@ -138,31 +139,47 @@ def __init__(self, params=None, linter_config=None): self.report_folder = "" self.reporters = [] + # Initialize parameters + default_params = { + "default_linter_activation": False, + "enable_descriptors": [], + "enable_linters": [], + "disable_descriptors": [], + "disable_linters": [], + "disable_errors_linters": [], + "post_linter_status": True, + } + if params is None: + params = default_params + else: + params = {**default_params, **params} + # Initialize with configuration data for key, value in linter_config.items(): self.__setattr__(key, value) + if "request_id" in params: + self.request_id = params["request_id"] + elif self.master is not None: + self.request_id: str = self.master.request_id + elif "master" in params: + self.request_id: str = params["master"].request_id + else: + raise Exception("Missing megalinter request_id") - # Initialize parameters - if params is None: - params = { - "default_linter_activation": False, - "enable_descriptors": [], - "enable_linters": [], - "disable_descriptors": [], - "disable_linters": [], - "disable_errors_linters": [], - "post_linter_status": True, - } - - self.is_active = params["default_linter_activation"] + self.is_active = ( + False + if "default_linter_activation" not in params + else params["default_linter_activation"] + ) # Disable errors self.disable_errors_if_less_than = None self.disable_errors = ( True if self.is_formatter is True - and not config.get("FORMATTERS_DISABLE_ERRORS", "true") == "false" + and not config.get(self.request_id, "FORMATTERS_DISABLE_ERRORS", "true") + == "false" else True - if config.get("DISABLE_ERRORS", "false") == "true" + if config.get(self.request_id, "DISABLE_ERRORS", "false") == "true" else False ) # Name @@ -178,15 +195,19 @@ def __init__(self, params=None, linter_config=None): ) if self.output_sarif is True: # Disable SARIF if linter not in specified linter list - sarif_enabled_linters = config.get_list("SARIF_REPORTER_LINTERS", None) + sarif_enabled_linters = config.get_list( + self.request_id, "SARIF_REPORTER_LINTERS", None + ) if ( sarif_enabled_linters is not None and self.name not in sarif_enabled_linters ): self.output_sarif = False # Override default executable - if config.exists(self.name + "_CLI_EXECUTABLE"): - self.cli_executable = config.get(self.name + "_CLI_EXECUTABLE") + if config.exists(self.request_id, self.name + "_CLI_EXECUTABLE"): + self.cli_executable = config.get( + self.request_id, self.name + "_CLI_EXECUTABLE" + ) if self.cli_executable is None: self.cli_executable = self.linter_name if self.cli_executable_fix is None: @@ -200,10 +221,10 @@ def __init__(self, params=None, linter_config=None): # Apply linter customization via config settings: self.file_extensions = config.get_list( - self.name + "_FILE_EXTENSIONS", self.file_extensions + self.request_id, self.name + "_FILE_EXTENSIONS", self.file_extensions ) self.file_names_regex = config.get_list( - self.name + "_FILE_NAMES_REGEX", self.file_names_regex + self.request_id, self.name + "_FILE_NAMES_REGEX", self.file_names_regex ) self.manage_activation(params) @@ -241,7 +262,7 @@ def __init__(self, params=None, linter_config=None): # because there are no other linters if ( self.lint_all_other_linters_files is True - and config.get("SINGLE_LINTER", "") != "" + and config.get(self.request_id, "SINGLE_LINTER", "") != "" ): self.lint_all_other_linters_files = False @@ -287,7 +308,9 @@ def __init__(self, params=None, linter_config=None): # Manage sub-directory filter if defined if self.files_sub_directory is not None: self.files_sub_directory = config.get( - f"{self.descriptor_id}_DIRECTORY", self.files_sub_directory + self.request_id, + f"{self.descriptor_id}_DIRECTORY", + self.files_sub_directory, ) if not os.path.isdir( self.workspace + os.path.sep + self.files_sub_directory @@ -370,23 +393,23 @@ def manage_activation(self, params): elif self.descriptor_id in params["enable_descriptors"]: self.is_active = True elif ( - config.exists("VALIDATE_" + self.name) - and config.get("VALIDATE_" + self.name) == "false" + config.exists(self.request_id, "VALIDATE_" + self.name) + and config.get(self.request_id, "VALIDATE_" + self.name) == "false" ): self.is_active = False elif ( - config.exists("VALIDATE_" + self.descriptor_id) - and config.get("VALIDATE_" + self.descriptor_id) == "false" + config.exists(self.request_id, "VALIDATE_" + self.descriptor_id) + and config.get(self.request_id, "VALIDATE_" + self.descriptor_id) == "false" ): self.is_active = False elif ( - config.exists("VALIDATE_" + self.name) - and config.get("VALIDATE_" + self.name) == "true" + config.exists(self.request_id, "VALIDATE_" + self.name) + and config.get(self.request_id, "VALIDATE_" + self.name) == "true" ): self.is_active = True elif ( - config.exists("VALIDATE_" + self.descriptor_id) - and config.get("VALIDATE_" + self.descriptor_id) == "true" + config.exists(self.request_id, "VALIDATE_" + self.descriptor_id) + and config.get(self.request_id, "VALIDATE_" + self.descriptor_id) == "true" ): self.is_active = True # check activation rules @@ -397,25 +420,41 @@ def manage_activation(self, params): def load_config_vars(self, params): # Configuration file name: try first NAME + _FILE_NAME, then LANGUAGE + _FILE_NAME # _CONFIG_FILE = _FILE_NAME (config renaming but keeping config ascending compatibility) - if config.exists(self.name + "_CONFIG_FILE"): - self.config_file_name = config.get(self.name + "_CONFIG_FILE") - elif config.exists(self.descriptor_id + "_CONFIG_FILE"): - self.config_file_name = config.get(self.descriptor_id + "_CONFIG_FILE") - elif config.exists(self.name + "_FILE_NAME"): - self.config_file_name = config.get(self.name + "_FILE_NAME") - elif config.exists(self.descriptor_id + "_FILE_NAME"): - self.config_file_name = config.get(self.descriptor_id + "_FILE_NAME") + if config.exists(self.request_id, self.name + "_CONFIG_FILE"): + self.config_file_name = config.get( + self.request_id, self.name + "_CONFIG_FILE" + ) + elif config.exists(self.request_id, self.descriptor_id + "_CONFIG_FILE"): + self.config_file_name = config.get( + self.request_id, self.descriptor_id + "_CONFIG_FILE" + ) + elif config.exists(self.request_id, self.name + "_FILE_NAME"): + self.config_file_name = config.get( + self.request_id, self.name + "_FILE_NAME" + ) + elif config.exists(self.request_id, self.descriptor_id + "_FILE_NAME"): + self.config_file_name = config.get( + self.request_id, self.descriptor_id + "_FILE_NAME" + ) # Ignore file name: try first NAME + _FILE_NAME, then LANGUAGE + _FILE_NAME if self.cli_lint_ignore_arg_name is not None: - if config.exists(self.name + "_IGNORE_FILE"): - self.ignore_file_name = config.get(self.name + "_IGNORE_FILE") - elif config.exists(self.descriptor_id + "_IGNORE_FILE"): - self.ignore_file_name = config.get(self.descriptor_id + "_IGNORE_FILE") + if config.exists(self.request_id, self.name + "_IGNORE_FILE"): + self.ignore_file_name = config.get( + self.request_id, self.name + "_IGNORE_FILE" + ) + elif config.exists(self.request_id, self.descriptor_id + "_IGNORE_FILE"): + self.ignore_file_name = config.get( + self.request_id, self.descriptor_id + "_IGNORE_FILE" + ) # Linter rules path: try first NAME + _RULE_PATH, then LANGUAGE + _RULE_PATH - if config.exists(self.name + "_RULES_PATH"): - self.linter_rules_path = config.get(self.name + "_RULES_PATH") - elif config.exists(self.descriptor_id + "_RULES_PATH"): - self.linter_rules_path = config.get(self.descriptor_id + "_RULES_PATH") + if config.exists(self.request_id, self.name + "_RULES_PATH"): + self.linter_rules_path = config.get( + self.request_id, self.name + "_RULES_PATH" + ) + elif config.exists(self.request_id, self.descriptor_id + "_RULES_PATH"): + self.linter_rules_path = config.get( + self.request_id, self.descriptor_id + "_RULES_PATH" + ) # Linter config file: # 0: LINTER_DEFAULT set in user config: let the linter find it, don't reference it in cli arguments # 1: http rules path: fetch remove file and copy it locally (then delete it after linting) @@ -568,9 +607,11 @@ def load_config_vars(self, params): ).replace(self.TEMPLATES_DIR, "") # User override of cli_lint_mode - if config.exists(self.name + "_CLI_LINT_MODE"): + if config.exists(self.request_id, self.name + "_CLI_LINT_MODE"): cli_lint_mode_descriptor = self.cli_lint_mode - cli_lint_mode_config = config.get(self.name + "_CLI_LINT_MODE") + cli_lint_mode_config = config.get( + self.request_id, self.name + "_CLI_LINT_MODE" + ) if cli_lint_mode_descriptor == "project": raise KeyError( f"You can not override {self.name} cli_lint_mode as it can " @@ -587,52 +628,72 @@ def load_config_vars(self, params): self.cli_lint_mode = cli_lint_mode_config # Include regex :try first NAME + _FILTER_REGEX_INCLUDE, then LANGUAGE + _FILTER_REGEX_INCLUDE - if config.exists(self.name + "_FILTER_REGEX_INCLUDE"): - self.filter_regex_include = config.get(self.name + "_FILTER_REGEX_INCLUDE") - elif config.exists(self.descriptor_id + "_FILTER_REGEX_INCLUDE"): + if config.exists(self.request_id, self.name + "_FILTER_REGEX_INCLUDE"): + self.filter_regex_include = config.get( + self.request_id, self.name + "_FILTER_REGEX_INCLUDE" + ) + elif config.exists( + self.request_id, self.descriptor_id + "_FILTER_REGEX_INCLUDE" + ): self.filter_regex_include = config.get( - self.descriptor_id + "_FILTER_REGEX_INCLUDE" + self.request_id, self.descriptor_id + "_FILTER_REGEX_INCLUDE" ) # User arguments from config - if config.get(self.name + "_ARGUMENTS", "") != "": - self.cli_lint_user_args = config.get_list_args(self.name + "_ARGUMENTS") + if config.get(self.request_id, self.name + "_ARGUMENTS", "") != "": + self.cli_lint_user_args = config.get_list_args( + self.request_id, self.name + "_ARGUMENTS" + ) # Get PRE_COMMANDS overridden by user - if config.get(self.name + "_PRE_COMMANDS", "") != "": - self.pre_commands = config.get_list(self.name + "_PRE_COMMANDS") + if config.get(self.request_id, self.name + "_PRE_COMMANDS", "") != "": + self.pre_commands = config.get_list( + self.request_id, self.name + "_PRE_COMMANDS" + ) # Get POST_COMMANDS overridden by user - if config.get(self.name + "_POST_COMMANDS", "") != "": - self.post_commands = config.get_list(self.name + "_POST_COMMANDS") + if config.get(self.request_id, self.name + "_POST_COMMANDS", "") != "": + self.post_commands = config.get_list( + self.request_id, self.name + "_POST_COMMANDS" + ) # Disable errors for this linter NAME + _DISABLE_ERRORS, then LANGUAGE + _DISABLE_ERRORS - if config.get(self.name + "_DISABLE_ERRORS_IF_LESS_THAN"): + if config.get(self.request_id, self.name + "_DISABLE_ERRORS_IF_LESS_THAN"): self.disable_errors_if_less_than = int( - config.get(self.name + "_DISABLE_ERRORS_IF_LESS_THAN") + config.get(self.request_id, self.name + "_DISABLE_ERRORS_IF_LESS_THAN") ) if self.disable_errors_if_less_than is not None: self.disable_errors = False elif self.name in params["disable_errors_linters"]: self.disable_errors = True - elif config.get(self.name + "_DISABLE_ERRORS", "") == "false": + elif config.get(self.request_id, self.name + "_DISABLE_ERRORS", "") == "false": self.disable_errors = False - elif config.get(self.name + "_DISABLE_ERRORS", "") == "true": + elif config.get(self.request_id, self.name + "_DISABLE_ERRORS", "") == "true": self.disable_errors = True - elif config.get(self.descriptor_id + "_DISABLE_ERRORS", "") == "false": + elif ( + config.get(self.request_id, self.descriptor_id + "_DISABLE_ERRORS", "") + == "false" + ): self.disable_errors = False - elif config.get(self.descriptor_id + "_DISABLE_ERRORS", "") == "true": + elif ( + config.get(self.request_id, self.descriptor_id + "_DISABLE_ERRORS", "") + == "true" + ): self.disable_errors = True # Exclude regex: try first NAME + _FILTER_REGEX_EXCLUDE, then LANGUAGE + _FILTER_REGEX_EXCLUDE - if config.exists(self.name + "_FILTER_REGEX_EXCLUDE"): - self.filter_regex_exclude = config.get(self.name + "_FILTER_REGEX_EXCLUDE") - elif config.exists(self.descriptor_id + "_FILTER_REGEX_EXCLUDE"): + if config.exists(self.request_id, self.name + "_FILTER_REGEX_EXCLUDE"): self.filter_regex_exclude = config.get( - self.descriptor_id + "_FILTER_REGEX_EXCLUDE" + self.request_id, self.name + "_FILTER_REGEX_EXCLUDE" + ) + elif config.exists( + self.request_id, self.descriptor_id + "_FILTER_REGEX_EXCLUDE" + ): + self.filter_regex_exclude = config.get( + self.request_id, self.descriptor_id + "_FILTER_REGEX_EXCLUDE" ) # Override default docker image version - if config.exists(self.name + "_DOCKER_IMAGE_VERSION"): + if config.exists(self.request_id, self.name + "_DOCKER_IMAGE_VERSION"): self.cli_docker_image_version = config.get( - self.name + "_DOCKER_IMAGE_VERSION" + self.request_id, self.name + "_DOCKER_IMAGE_VERSION" ) # Processes the linter @@ -821,7 +882,10 @@ def process_linter(self, file=None): def execute_lint_command(self, command): cwd = os.path.abspath(self.workspace) logging.debug(f"[{self.linter_name}] CWD: {cwd}") - subprocess_env = {**os.environ, "FORCE_COLOR": "0"} + subprocess_env = { + **config.build_env(self.request_id), + "FORCE_COLOR": "0", + } if type(command) == str: # Call linter with a sub-process process = subprocess.run( @@ -917,7 +981,9 @@ def manage_sarif_output(self, return_stdout): # Convert SARIF into human readable text for Console & Text reporters if sarif_confirmed is True and self.master.sarif_to_human is True: with open(self.sarif_output_file, "r", encoding="utf-8") as file: - self.stdout_human = utils_reporter.convert_sarif_to_human(file.read()) + self.stdout_human = utils_reporter.convert_sarif_to_human( + file.read(), self.request_id + ) # Returns linter version (can be overridden in special cases, like version has special format) def get_linter_version(self): @@ -942,9 +1008,17 @@ def get_linter_version_output(self): command = self.build_version_command() logging.debug("Linter version command: " + str(command)) cwd = os.getcwd() if command[0] != "npm" else "~/" + subprocess_env = { + **config.build_env(self.request_id), + "FORCE_COLOR": "0", + } try: process = subprocess.run( - command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=cwd, + env=subprocess_env, ) return_code = process.returncode output = utils.decode_utf8(process.stdout) @@ -982,8 +1056,15 @@ def get_linter_help(self): if cli_absolute is not None: command[0] = cli_absolute logging.debug("Linter help command: " + str(command)) + subprocess_env = { + **config.build_env(self.request_id), + "FORCE_COLOR": "0", + } process = subprocess.run( - command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=subprocess_env, ) return_code = process.returncode output += utils.decode_utf8(process.stdout) @@ -1014,7 +1095,7 @@ def manage_docker_command(self, command): return command docker_command = ["docker", "run", "--rm"] if hasattr(self, "workspace"): - volume_root = config.get("MEGALINTER_VOLUME_ROOT", "") + volume_root = config.get(self.request_id, "MEGALINTER_VOLUME_ROOT", "") if volume_root != "": workspace_value = ( volume_root diff --git a/megalinter/MegaLinter.py b/megalinter/MegaLinter.py index 8ce3cffcaab..fe1beaffcc2 100644 --- a/megalinter/MegaLinter.py +++ b/megalinter/MegaLinter.py @@ -10,6 +10,8 @@ import os import shutil import sys +from shutil import copytree +from uuid import uuid1 import chalk as c import git @@ -28,7 +30,7 @@ ML_DOC_URL, ) from megalinter.utils_reporter import log_section_end, log_section_start -from multiprocessing_logging import install_mp_handler +from multiprocessing_logging import install_mp_handler, uninstall_mp_handler # Function to run linters using multiprocessing pool @@ -53,11 +55,18 @@ def __init__(self, params=None): if "request_id" in params: self.request_id = params["request_id"] + else: + self.request_id = str(uuid1()) # Initialization for lint request cases self.workspace = self.get_workspace(params) - config.init_config(self.workspace, params) # Initialize runtime config - self.github_workspace = config.get("GITHUB_WORKSPACE", self.workspace) + # Do not send secrets to linter executables + config.init_config( + self.request_id, self.workspace, params + ) # Initialize runtime config + self.github_workspace = config.get( + self.request_id, "GITHUB_WORKSPACE", self.workspace + ) self.megalinter_flavor = flavor_factory.get_image_flavor() self.initialize_output() self.initialize_logger() @@ -86,19 +95,23 @@ def __init__(self, params=None): self.output_sarif = False # Get enable / disable vars - self.enable_descriptors = config.get_list("ENABLE", []) - self.enable_linters = config.get_list("ENABLE_LINTERS", []) - self.disable_descriptors = config.get_list("DISABLE", []) - self.disable_linters = config.get_list("DISABLE_LINTERS", []) - self.disable_errors_linters = config.get_list("DISABLE_ERRORS_LINTERS", []) + self.enable_descriptors = config.get_list(self.request_id, "ENABLE", []) + self.enable_linters = config.get_list(self.request_id, "ENABLE_LINTERS", []) + self.disable_descriptors = config.get_list(self.request_id, "DISABLE", []) + self.disable_linters = config.get_list(self.request_id, "DISABLE_LINTERS", []) + self.disable_errors_linters = config.get_list( + self.request_id, "DISABLE_ERRORS_LINTERS", [] + ) self.manage_default_linter_activation() - self.apply_fixes = config.get_list("APPLY_FIXES", "none") + self.apply_fixes = config.get_list(self.request_id, "APPLY_FIXES", "none") self.show_elapsed_time = ( - config.get("SHOW_ELAPSED_TIME", "false") == "true" - or config.get("LOG_LEVEL", "DEBUG") == "DEBUG" + config.get(self.request_id, "SHOW_ELAPSED_TIME", "false") == "true" + or config.get(self.request_id, "LOG_LEVEL", "DEBUG") == "DEBUG" ) # In case SARIF is active, convert results into human readable text for logs - self.sarif_to_human = config.get("SARIF_TO_HUMAN", "true") == "true" + self.sarif_to_human = ( + config.get(self.request_id, "SARIF_TO_HUMAN", "true") == "true" + ) # Load optional configuration self.load_config_vars() # Runtime properties @@ -111,11 +124,31 @@ def __init__(self, params=None): self.has_git_extraheader = False self.has_updated_sources = 0 self.fail_if_updated_sources = ( - config.get("FAIL_IF_UPDATED_SOURCES", "false") == "true" + config.get(self.request_id, "FAIL_IF_UPDATED_SOURCES", "false") == "true" ) self.flavor_suggestions = None + # Initialize plugins - plugin_factory.initialize_plugins() + plugin_factory.initialize_plugins(self.request_id) + + # Copy node_modules in current folder if necessary + if ( + os.path.isdir("/node_deps") + and len(os.listdir("/node_deps")) > 0 + and pre_post_factory.has_npm_or_yarn_commands(self.request_id) + ): + workspace_node_modules = os.path.join(self.workspace, "node_modules") + copytree("/node_deps", workspace_node_modules) + # Update PATH & NODE_PATH so node_modules of the currently analyzed workspace is used + config.set( + self.request_id, + "PATH", + config.get(self.request_id, "PATH").replace( + "/node-deps/node_modules/.bin:", workspace_node_modules + ), + ) + config.set(self.request_id, "NODE_PATH", workspace_node_modules) + # Run user-defined commands self.pre_commands_results = pre_post_factory.run_pre_commands(self) self.post_commands_results = [] @@ -158,12 +191,20 @@ def run(self): reporter.initialize() # Display warning if selected flavors doesn't match all linters - if flavor_factory.check_active_linters_match_flavor(active_linters) is False: + if ( + flavor_factory.check_active_linters_match_flavor( + active_linters, self.request_id + ) + is False + ): active_linters = [ linter for linter in active_linters if linter.is_active is True ] - if config.get("PARALLEL", "true") == "true" and len(active_linters) > 1: + if ( + config.get(self.request_id, "PARALLEL", "true") == "true" + and len(active_linters) > 1 + ): self.process_linters_parallel(active_linters, linters_do_fixes) else: self.process_linters_serial(active_linters) @@ -195,7 +236,7 @@ def run(self): # - VALIDATE_ALL_CODE_BASE is false, or diff failed (we don't have all the files to calculate the suggestion) if ( self.validate_all_code_base is True - and config.get("FLAVOR_SUGGESTIONS", "true") == "true" + and config.get(self.request_id, "FLAVOR_SUGGESTIONS", "true") == "true" ): self.flavor_suggestions = flavor_factory.get_megalinter_flavor_suggestions( active_linters @@ -251,8 +292,10 @@ def process_linters_parallel(self, active_linters, linters_do_fixes): linter_groups += [[linter]] linter_groups = linter_factory.sort_linters_groups_by_speed(linter_groups) # Execute linters in asynchronous pool to improve overall performances + process_number = mp.cpu_count() + logging.info(f"Processing linters on [{str(process_number)}] parallel cores…") install_mp_handler() - pool = mp.Pool(mp.cpu_count()) + pool = mp.Pool(process_number) pool_results = [] # Add linter groups to pool for linter_group in linter_groups: @@ -273,13 +316,20 @@ def process_linters_parallel(self, active_linters, linters_do_fixes): if self.linters[i].name == updated_linter.name: self.linters[i] = updated_linter break + uninstall_mp_handler() # noinspection PyMethodMayBeStatic def get_workspace(self, params): if "workspace" in params: self.arg_input = params["workspace"] - default_workspace = config.get("DEFAULT_WORKSPACE", "") - github_workspace = config.get("GITHUB_WORKSPACE", "") + if config.is_initialized_for(self.request_id): + # Use stored config vars + default_workspace = config.get(self.request_id, "DEFAULT_WORKSPACE", "") + github_workspace = config.get(self.request_id, "GITHUB_WORKSPACE", "") + else: + # Use ENV vars + default_workspace = config.get(None, "DEFAULT_WORKSPACE", "") + github_workspace = config.get(None, "GITHUB_WORKSPACE", "") # Use CLI input argument if self.arg_input is not None: if os.path.isdir(self.arg_input): @@ -393,8 +443,8 @@ def load_cli_vars(self): # Manage configuration variables def load_config_vars(self): # Linter rules root path - if config.exists("LINTER_RULES_PATH"): - linter_rules_path_val = config.get("LINTER_RULES_PATH") + if config.exists(self.request_id, "LINTER_RULES_PATH"): + linter_rules_path_val = config.get(self.request_id, "LINTER_RULES_PATH") if linter_rules_path_val.startswith("http"): self.linter_rules_path = linter_rules_path_val else: @@ -402,29 +452,33 @@ def load_config_vars(self): self.github_workspace + os.path.sep + linter_rules_path_val ) # Filtering regex (inclusion) - if config.exists("FILTER_REGEX_INCLUDE"): - self.filter_regex_include = config.get("FILTER_REGEX_INCLUDE") + if config.exists(self.request_id, "FILTER_REGEX_INCLUDE"): + self.filter_regex_include = config.get( + self.request_id, "FILTER_REGEX_INCLUDE" + ) # Filtering regex (exclusion) - if config.exists("FILTER_REGEX_EXCLUDE"): - self.filter_regex_exclude = config.get("FILTER_REGEX_EXCLUDE") + if config.exists(self.request_id, "FILTER_REGEX_EXCLUDE"): + self.filter_regex_exclude = config.get( + self.request_id, "FILTER_REGEX_EXCLUDE" + ) # Disable all fields validation if VALIDATE_ALL_CODEBASE is 'false' if ( - config.exists("VALIDATE_ALL_CODEBASE") - and config.get("VALIDATE_ALL_CODEBASE") == "false" + config.exists(self.request_id, "VALIDATE_ALL_CODEBASE") + and config.get(self.request_id, "VALIDATE_ALL_CODEBASE") == "false" ): self.validate_all_code_base = False # Manage IGNORE_GITIGNORED_FILES - if config.exists("IGNORE_GITIGNORED_FILES"): + if config.exists(self.request_id, "IGNORE_GITIGNORED_FILES"): self.ignore_gitignore_files = ( - config.get("IGNORE_GITIGNORED_FILES", "true") == "true" + config.get(self.request_id, "IGNORE_GITIGNORED_FILES", "true") == "true" ) # Manage IGNORE_GENERATED_FILES - if config.exists("IGNORE_GENERATED_FILES"): + if config.exists(self.request_id, "IGNORE_GENERATED_FILES"): self.ignore_generated_files = ( - config.get("IGNORE_GENERATED_FILES", "false") == "true" + config.get(self.request_id, "IGNORE_GENERATED_FILES", "false") == "true" ) # Manage SARIF output - if config.get("SARIF_REPORTER", "") == "true": + if config.get(self.request_id, "SARIF_REPORTER", "") == "true": self.output_sarif = True # Calculate default linter activation according to env variables @@ -433,9 +487,9 @@ def manage_default_linter_activation(self): if len(self.enable_descriptors) > 0 or len(self.enable_linters) > 0: self.default_linter_activation = False # V3 legacy variables - for env_var in config.get(): + for env_var in config.get(self.request_id): if env_var.startswith("VALIDATE_") and env_var != "VALIDATE_ALL_CODEBASE": - if config.get(env_var) == "true": + if config.get(self.request_id, env_var) == "true": self.default_linter_activation = False # Load and initialize all linters @@ -463,14 +517,14 @@ def load_linters(self): # if flavor selected and no flavor suggestion, ignore linters that aren't in current flavor) if self.megalinter_flavor == "none": # Single linter docker image - unique_linter = config.get("SINGLE_LINTER") + unique_linter = config.get(self.request_id, "SINGLE_LINTER") all_linters = linter_factory.list_linters_by_name( linter_init_params, [unique_linter] ) elif ( # Flavored MegaLinter self.megalinter_flavor != "all" - and config.get("FLAVOR_SUGGESTIONS", "true") != "true" + and config.get(self.request_id, "FLAVOR_SUGGESTIONS", "true") != "true" ): all_linters = linter_factory.list_flavor_linters( linter_init_params, self.megalinter_flavor @@ -481,7 +535,9 @@ def load_linters(self): skipped_linters = [] # Remove inactive, disabled or skipped linters - skip_cli_lint_modes = config.get_list("SKIP_CLI_LINT_MODES", []) + skip_cli_lint_modes = config.get_list( + self.request_id, "SKIP_CLI_LINT_MODES", [] + ) for linter in all_linters: linter.master = self if ( @@ -503,7 +559,9 @@ def load_linters(self): continue self.linters += [linter] # Display skipped linters in log - show_skipped_linters = config.get("SHOW_SKIPPED_LINTERS", "true") == "true" + show_skipped_linters = ( + config.get(self.request_id, "SHOW_SKIPPED_LINTERS", "true") == "true" + ) if len(skipped_linters) > 0 and show_skipped_linters: skipped_linters.sort() logging.info("Skipped linters: " + ", ".join(skipped_linters)) @@ -534,7 +592,7 @@ def compute_file_extensions(self): # Collect list of files matching extensions and regex def collect_files(self): # Collect not filtered list of files - files_to_lint = config.get_list("MEGALINTER_FILES_TO_LINT", []) + files_to_lint = config.get_list(self.request_id, "MEGALINTER_FILES_TO_LINT", []) if len(files_to_lint) > 0: # Files sent as input parameter all_files = list() @@ -647,14 +705,14 @@ def list_files_git_diff(self): ) repo = git.Repo(os.path.realpath(self.github_workspace)) # Add auth header if necessary - if config.get("GIT_AUTHORIZATION_BEARER", "") != "": + if config.get(self.request_id, "GIT_AUTHORIZATION_BEARER", "") != "": auth_bearer = "Authorization: Bearer " + config.get( - "GIT_AUTHORIZATION_BEARER" + self.request_id, "GIT_AUTHORIZATION_BEARER" ) repo.config_writer().set_value("http", "extraheader", auth_bearer).release() self.has_git_extraheader = True # Fetch base branch content - default_branch = config.get("DEFAULT_BRANCH", "HEAD") + default_branch = config.get(self.request_id, "DEFAULT_BRANCH", "HEAD") default_branch_remote = f"origin/{default_branch}" if default_branch_remote not in [ref.name for ref in repo.refs]: remote_ref = ( @@ -684,7 +742,7 @@ def list_files_all(self): ] if logging.getLogger().isEnabledFor(logging.DEBUG): logging.debug("Root dir content:" + utils.format_bullet_list(all_files)) - excluded_directories = utils.get_excluded_directories() + excluded_directories = utils.get_excluded_directories(self.request_id) for dirpath, dirnames, filenames in os.walk(self.workspace, topdown=True): dirnames[:] = [d for d in dirnames if d not in excluded_directories] all_files += [ @@ -714,8 +772,10 @@ def list_git_ignored_files(self): def initialize_output(self): self.report_folder = config.get( + self.request_id, "REPORT_OUTPUT_FOLDER", config.get( + self.request_id, "OUTPUT_FOLDER", self.github_workspace + os.path.sep + DEFAULT_REPORT_FOLDER_NAME, ), @@ -727,11 +787,15 @@ def initialize_output(self): # --output /logs/megalinter/myoutputfile.sarif self.report_folder = os.path.dirname(self.arg_output) config.set( - "SARIF_REPORTER_FILE_NAME", os.path.basename(self.arg_output) + self.request_id, + "SARIF_REPORTER_FILE_NAME", + os.path.basename(self.arg_output), ) else: # --output myoutputfile.sarif - config.set("SARIF_REPORTER_FILE_NAME", self.arg_output) + config.set( + self.request_id, "SARIF_REPORTER_FILE_NAME", self.arg_output + ) elif os.path.isdir(self.arg_output): # --output /logs/megalinter self.report_folder = self.arg_output @@ -741,7 +805,7 @@ def initialize_output(self): # Initialize output dir os.makedirs(self.report_folder, exist_ok=True) # Clear report folder if requested - if config.get("CLEAR_REPORT_FOLDER", "false") == "true": + if config.get(self.request_id, "CLEAR_REPORT_FOLDER", "false") == "true": logging.info( f"CLEAR_REPORT_FOLDER found: empty folder {self.report_folder}" ) @@ -749,7 +813,7 @@ def initialize_output(self): os.makedirs(self.report_folder, exist_ok=True) def initialize_logger(self): - logging_level_key = config.get("LOG_LEVEL", "INFO").upper() + logging_level_key = config.get(self.request_id, "LOG_LEVEL", "INFO").upper() logging_level_list = { "INFO": logging.INFO, "DEBUG": logging.DEBUG, @@ -765,9 +829,9 @@ def initialize_logger(self): else logging.INFO ) - if config.get("LOG_FILE", "") == "none" or not utils.can_write_report_files( - self - ): + if config.get( + self.request_id, "LOG_FILE", "" + ) == "none" or not utils.can_write_report_files(self): # Don't log console output in a file logging.basicConfig( force=True, @@ -781,7 +845,7 @@ def initialize_logger(self): log_file = ( self.report_folder + os.path.sep - + config.get("LOG_FILE", "megalinter.log") + + config.get(self.request_id, "LOG_FILE", "megalinter.log") ) # Log console output in a file if not os.path.isdir(os.path.dirname(log_file)): @@ -796,57 +860,61 @@ def initialize_logger(self): ], ) - @staticmethod - def display_header(): + def display_header(self): # Header prints logging.info(utils.format_hyphens("")) logging.info(utils.format_hyphens("MegaLinter, by OX Security")) logging.info(utils.format_hyphens("")) logging.info( - " - Image Creation Date: " + config.get("BUILD_DATE", "No docker image") + " - Image Creation Date: " + + config.get(None, "BUILD_DATE", "No docker image") ) logging.info( - " - Image Revision: " + config.get("BUILD_REVISION", "No docker image") + " - Image Revision: " + + config.get(None, "BUILD_REVISION", "No docker image") ) logging.info( - " - Image Version: " + config.get("BUILD_VERSION", "No docker image") + " - Image Version: " + config.get(None, "BUILD_VERSION", "No docker image") ) logging.info(utils.format_hyphens("")) logging.info("The MegaLinter documentation can be found at:") logging.info(" - " + ML_DOC_URL) logging.info(utils.format_hyphens("")) logging.info(log_section_start("megalinter-init", "MegaLinter initialization")) - if os.environ.get("GITHUB_REPOSITORY", "") != "": + if config.get(None, "GITHUB_REPOSITORY", "") != "": logging.info( - "GITHUB_REPOSITORY: " + os.environ.get("GITHUB_REPOSITORY", "") + "GITHUB_REPOSITORY: " + config.get(None, "GITHUB_REPOSITORY", "") ) # logging.info("GITHUB_SHA: " + os.environ.get("GITHUB_SHA", "")) - logging.info("GITHUB_REF: " + os.environ.get("GITHUB_REF", "")) + logging.info("GITHUB_REF: " + config.get(None, "GITHUB_REF", "")) # logging.info("GITHUB_TOKEN: " + os.environ.get("GITHUB_TOKEN", "")) - logging.info("GITHUB_RUN_ID: " + os.environ.get("GITHUB_RUN_ID", "")) - logging.info("PAT: " + "set" if os.environ.get("PAT", "") != "" else "") + logging.info("GITHUB_RUN_ID: " + config.get(None, "GITHUB_RUN_ID", "")) + logging.info("PAT: " + "set" if config.get(None, "PAT", "") != "" else "") # Display config variables for debug mode - for name, value in sorted(config.get_config().items()): - logging.debug("" + name + "=" + str(value)) + for name, value in sorted(config.get_config(self.request_id).items()): + if name not in ["GITHUB_TOKEN", "PAT"]: + logging.debug("" + name + "=" + str(value)) + else: + logging.debug("" + name + "=**********") logging.debug(utils.format_hyphens("")) logging.info("") def check_results(self): - if "GITHUB_OUTPUT" in os.environ: - github_output_file = os.environ["GITHUB_OUTPUT"] + if config.exists(self.request_id, "GITHUB_OUTPUT"): + github_output_file = config.get(self.request_id, "GITHUB_OUTPUT") with open(github_output_file, "a", encoding="utf-8") as output_stream: output_stream.write( f"has_updated_sources={str(self.has_updated_sources)}\n" ) if self.status == "success": logging.info(c.green("✅ Successfully linted all files without errors")) - config.delete() + config.delete(self.request_id) self.check_updated_sources_failure() elif self.status == "warning": logging.warning( c.yellow("◬ Successfully linted all files, but with ignored errors") ) - config.delete() + config.delete(self.request_id) self.check_updated_sources_failure() else: logging.error(c.red("❌ Error(s) have been found during linting")) @@ -856,13 +924,13 @@ def check_results(self): ) logging.warning(f"More info at {ML_DOC_URL}/configuration/") if self.cli is True: - if config.get("DISABLE_ERRORS", "false") == "true": - config.delete() + if config.get(self.request_id, "DISABLE_ERRORS", "false") == "true": + config.delete(self.request_id) sys.exit(0) else: - config.delete() + config.delete(self.request_id) sys.exit(self.return_code) - config.delete() + config.delete(self.request_id) def check_updated_sources_failure(self): if self.has_updated_sources > 0 and self.fail_if_updated_sources is True: @@ -887,7 +955,9 @@ def manage_clean_git_repo(self): # Propose legacy versions users to upgrade def manage_upgrade_message(self): - mega_linter_version = config.get("BUILD_VERSION", "No docker image") + mega_linter_version = config.get( + self.request_id, "BUILD_VERSION", "No docker image" + ) if ( "insiders" in mega_linter_version or "v4" in mega_linter_version diff --git a/megalinter/Reporter.py b/megalinter/Reporter.py index 063f7373375..e10dec6a725 100644 --- a/megalinter/Reporter.py +++ b/megalinter/Reporter.py @@ -6,6 +6,8 @@ from typing import Optional +from megalinter.Linter import Linter + class Reporter: # Report definition @@ -16,7 +18,7 @@ class Reporter: # Constructor: Initialize Linter instance with name and config variables def __init__(self, params=None): self.processing_order = 0 - self.master = params["master"] + self.master: Linter = params["master"] self.report_folder = params["report_folder"] # Any reporter is inactive by default except if __init__ is overridden on sub class if not hasattr(self, "is_active"): diff --git a/megalinter/alpaca.py b/megalinter/alpaca.py index 3c67aac200b..5d829a64e29 100644 --- a/megalinter/alpaca.py +++ b/megalinter/alpaca.py @@ -5,7 +5,7 @@ # pylint: disable=E1111 def alpaca(): - print_alpaca = config.get("PRINT_ALPACA", "true") == "true" + print_alpaca = config.get(None, "PRINT_ALPACA", "true") == "true" if not print_alpaca: return diff --git a/megalinter/config.py b/megalinter/config.py index a827b2eef5c..b00f52945dd 100644 --- a/megalinter/config.py +++ b/megalinter/config.py @@ -8,26 +8,34 @@ import requests import yaml -CONFIG_DATA = None -CONFIG_SOURCE = None +RUN_CONFIGS = {} # type: ignore[var-annotated] -def init_config(workspace=None, params={}): - global CONFIG_DATA, CONFIG_SOURCE - if CONFIG_DATA is not None: - logging.debug(f"[config] Already initialized: {CONFIG_SOURCE}") +def init_config(request_id, workspace=None, params={}): + global RUN_CONFIGS + if request_id in RUN_CONFIGS: + logging.debug( + f"[config] Already initialized: {RUN_CONFIGS[request_id]['CONFIG_SOURCE']}" + ) return env = os.environ.copy() env_plus_params = env | params - if workspace is None and "MEGALINTER_CONFIG" not in os.environ: - set_config(env_plus_params) - CONFIG_SOURCE = "Environment variables only (no workspace)" - print(f"[config] {CONFIG_SOURCE}") + if workspace is None and "MEGALINTER_CONFIG" not in env_plus_params: + set_config(request_id, env_plus_params) + RUN_CONFIGS[request_id][ + "CONFIG_SOURCE" + ] = "Environment variables only (no workspace)" + print(f"[config] {RUN_CONFIGS[request_id]['CONFIG_SOURCE']}") return + else: + set_config(request_id, env_plus_params) + RUN_CONFIGS[request_id][ + "CONFIG_SOURCE" + ] = "TEMPORARY VAL THAT SHOULD NOT REMAIN" # Search for config file config_file = None - if "MEGALINTER_CONFIG" in os.environ: - config_file_name = os.environ.get("MEGALINTER_CONFIG") + if "MEGALINTER_CONFIG" in env_plus_params: + config_file_name = env_plus_params.get("MEGALINTER_CONFIG") if config_file_name.startswith("https://"): # Remote configuration file config_file = ( @@ -65,22 +73,27 @@ def init_config(workspace=None, params={}): else: # append config file variables to env variables, with priority to env variables runtime_config = config_data | env_plus_params - CONFIG_SOURCE = f"{config_file} + Environment variables" + RUN_CONFIGS[request_id][ + "CONFIG_SOURCE" + ] = f"{config_file} + Environment variables" else: runtime_config = env_plus_params - CONFIG_SOURCE = ( - f"Environment variables only (no config file found in {workspace})" - ) + RUN_CONFIGS[request_id][ + "CONFIG_SOURCE" + ] = f"Environment variables only (no config file found in {workspace})" # manage EXTENDS in configuration if "EXTENDS" in runtime_config: combined_config = {} - CONFIG_SOURCE = combine_config( - workspace, runtime_config, combined_config, CONFIG_SOURCE + RUN_CONFIGS[request_id]["CONFIG_SOURCE"] = combine_config( + workspace, + runtime_config, + combined_config, + RUN_CONFIGS[request_id]["CONFIG_SOURCE"], ) runtime_config = combined_config # Print & set config in cache - print(f"[config] {CONFIG_SOURCE}") - set_config(runtime_config) + print(f"[config] {RUN_CONFIGS[request_id]['CONFIG_SOURCE']}") + set_config(request_id, runtime_config) def combine_config(workspace, config, combined_config, config_source): @@ -112,23 +125,36 @@ def combine_config(workspace, config, combined_config, config_source): return config_source -def get_config(): - global CONFIG_DATA - if CONFIG_DATA is not None: - return CONFIG_DATA +def is_initialized_for(request_id): + global RUN_CONFIGS + if request_id in RUN_CONFIGS: + return True + return False + + +def get_config(request_id=None): + global RUN_CONFIGS + if request_id is not None and request_id in RUN_CONFIGS: + # Return request config + return RUN_CONFIGS[request_id] + elif request_id is not None: + raise Exception( + f"Internal error: there should be a config for request_id {request_id}" + ) else: + # Return ENV return os.environ.copy() -def set_config(runtime_config): - global CONFIG_DATA - CONFIG_DATA = runtime_config +def set_config(request_id, runtime_config): + global RUN_CONFIGS + RUN_CONFIGS[request_id] = runtime_config -def get(config_var=None, default=None): +def get(request_id, config_var=None, default=None): if config_var is None: - return get_config() - val = get_config().get(config_var, default) + return get_config(request_id) + val = get_config(request_id).get(config_var, default) # IF boolean, convert to "true" or "false" if isinstance(val, bool): if val is True: @@ -138,15 +164,42 @@ def get(config_var=None, default=None): return val -def set(config_var, value): - global CONFIG_DATA - assert CONFIG_DATA is not None, "Config has not been initialized yet !" - CONFIG_DATA[config_var] = value +def build_env(request_id, secured=True): + secured_env_variables = [] + if secured is True: + secured_env_variables = get_list( + request_id, + "SECURED_ENV_VARIABLES", + [ + "GITHUB_TOKEN", + "PAT", + "SYSTEM_ACCESSTOKEN", + "CI_JOB_TOKEN", + "GITLAB_ACCESS_TOKEN_MEGALINTER", + "GITLAB_CUSTOM_CERTIFICATE", + "WEBHOOK_REPORTER_BEARER_TOKEN", + ], + ) + env_dict = {} + for key, value in get_config(request_id).items(): + if key in secured_env_variables: + continue + elif not isinstance(value, str): + env_dict[key] = str(value) + else: + env_dict[key] = value + return env_dict + + +def set(request_id, config_var, value): + global RUN_CONFIGS + assert request_id in RUN_CONFIGS, "Config has not been initialized yet !" + RUN_CONFIGS[request_id][config_var] = value # Get list of elements from configuration. It can be list of strings or objects -def get_list(config_var, default=None): - var = get(config_var, None) +def get_list(request_id, config_var, default=None): + var = get(request_id, config_var, None) if var is not None: # List format if isinstance(var, list): @@ -162,8 +215,8 @@ def get_list(config_var, default=None): return default -def get_list_args(config_var, default=None): - var = get(config_var, None) +def get_list_args(request_id, config_var, default=None): + var = get(request_id, config_var, None) if var is not None: if isinstance(var, list): return var @@ -173,28 +226,31 @@ def get_list_args(config_var, default=None): return default -def set_value(config_var, val): - config = get_config() +def set_value(request_id, config_var, val): + config = get_config(request_id) config[config_var] = val - set_config(config) + set_config(request_id, config) -def exists(config_var): - return config_var in get_config() +def exists(request_id, config_var): + return config_var in get_config(request_id) -def copy(): - return get_config().copy() +def copy(request_id): + return get_config(request_id).copy() -def delete(key=None): - global CONFIG_DATA, CONFIG_SOURCE +def delete(request_id=None, key=None): + global RUN_CONFIGS + # Global delete (used for tests) + if request_id is None: + RUN_CONFIGS = {} + return if key is None: - CONFIG_DATA = None - CONFIG_SOURCE = None - logging.debug("Cleared MegaLinter runtime config") + del RUN_CONFIGS[request_id] + logging.debug("Cleared MegaLinter runtime config for request " + request_id) return - config = get_config() + config = get_config(request_id) if key in config: del config[key] - set_config(config) + set_config(request_id, config) diff --git a/megalinter/constants.py b/megalinter/constants.py index 5ff6f567989..29365f52dfc 100644 --- a/megalinter/constants.py +++ b/megalinter/constants.py @@ -5,7 +5,7 @@ ML_REPO = f"{ML_REPO_OWNER}/{ML_REPO_NAME}" ML_REPO_URL = f"https://github.com/{ML_REPO_OWNER}/{ML_REPO_NAME}" ML_DOC_URL_BASE = "https://megalinter.io/" -ML_VERSION = config.get("BUILD_VERSION", "latest").replace("v", "") +ML_VERSION = config.get(None, "BUILD_VERSION", "latest").replace("v", "") ML_DOC_URL = ML_DOC_URL_BASE + (ML_VERSION if len(ML_VERSION) > 1 else "latest") ML_REPO_ISSUES_URL = f"https://github.com/{ML_REPO_OWNER}/{ML_REPO_NAME}/issues" ML_DOC_URL_DESCRIPTORS_ROOT = f"{ML_DOC_URL}/descriptors" diff --git a/megalinter/descriptors/bicep.megalinter-descriptor.yml b/megalinter/descriptors/bicep.megalinter-descriptor.yml index 1225d2f5cae..a8ad4d7d8ab 100644 --- a/megalinter/descriptors/bicep.megalinter-descriptor.yml +++ b/megalinter/descriptors/bicep.megalinter-descriptor.yml @@ -31,6 +31,8 @@ linters: # Azure CLI az bicep build -f infra.bicep install: + apk: + - icu-libs dockerfile: - ARG BICEP_EXE='bicep' - ARG BICEP_URI='https://github.com/Azure/bicep/releases/latest/download/bicep-linux-musl-x64' diff --git a/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json b/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json index 6c1210b6bae..80041d24f53 100644 --- a/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json +++ b/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json @@ -11930,6 +11930,33 @@ "title": "SCALA_SCALAFIX: Custom config file path", "type": "string" }, + "SECURED_ENV_VARIABLES": { + "$id": "#/properties/SECURED_ENV_VARIABLES", + "default": [ + "GITHUB_TOKEN", + "PAT", + "SYSTEM_ACCESSTOKEN", + "CI_JOB_TOKEN", + "GITLAB_ACCESS_TOKEN_MEGALINTER", + "GITLAB_CUSTOM_CERTIFICATE", + "WEBHOOK_REPORTER_BEARER_TOKEN" + ], + "description": "List of secured environment variables to hide when calling linters", + "examples:": [[ + "GITHUB_TOKEN", + "PAT", + "SYSTEM_ACCESSTOKEN", + "CI_JOB_TOKEN", + "GITLAB_ACCESS_TOKEN_MEGALINTER", + "GITLAB_CUSTOM_CERTIFICATE", + "WEBHOOK_REPORTER_BEARER_TOKEN"] + ], + "items": { + "type": "string" + }, + "title": "Secured environment variables", + "type": "array" + }, "SHOW_ELAPSED_TIME": { "$id": "#/properties/SHOW_ELAPSED_TIME", "default": false, diff --git a/megalinter/flavor_factory.py b/megalinter/flavor_factory.py index 49ef8b397fd..33b458e6775 100644 --- a/megalinter/flavor_factory.py +++ b/megalinter/flavor_factory.py @@ -65,11 +65,11 @@ def list_megalinter_flavors(): def get_image_flavor(): - return config.get("MEGALINTER_FLAVOR", "all") + return config.get(None, "MEGALINTER_FLAVOR", "all") # Compare linters active for the current repo, and linters available in the current MegaLinter image flavor -def check_active_linters_match_flavor(active_linters): +def check_active_linters_match_flavor(active_linters, request_id): flavor = get_image_flavor() if flavor == "all": logging.debug('MegaLinter flavor is "all", no need to check match with linters') @@ -103,7 +103,14 @@ def check_active_linters_match_flavor(active_linters): "- ignore this message by setting config variable FLAVOR_SUGGESTIONS to false" ) # Stop the process if user wanted so in case of missing linters - if config.get("FAIL_IF_MISSING_LINTER_IN_FLAVOR", "") == "true": + if ( + config.get( + request_id, + "FAIL_IF_MISSING_LINTER_IN_FLAVOR", + "", + ) + == "true" + ): logging.error( 'Missing linter and FAIL_IF_MISSING_LINTER_IN_FLAVOR has been set to "true": Stop run' ) diff --git a/megalinter/linters/ArmLinter.py b/megalinter/linters/ArmLinter.py index ce33aa0e1a4..0ce15873ef8 100644 --- a/megalinter/linters/ArmLinter.py +++ b/megalinter/linters/ArmLinter.py @@ -9,10 +9,13 @@ class ArmLinter(Linter): - arm_ttk_psd1 = config.get("ARM_TTK_PSD1", "/usr/bin/arm-ttk") + arm_ttk_psd1 = "" # Build the CLI command to call to lint a file with a powershell script def build_lint_command(self, file=None): + self.arm_ttk_psd1 = config.get( + self.request_id, "ARM_TTK_PSD1", "/usr/bin/arm-ttk" + ) pwsh_script = ["Import-Module " + self.arm_ttk_psd1 + " ;"] if self.config_file is not None: pwsh_script += [ diff --git a/megalinter/linters/BashBashExecLinter.py b/megalinter/linters/BashBashExecLinter.py index 8a4ea17d768..12c63b56011 100644 --- a/megalinter/linters/BashBashExecLinter.py +++ b/megalinter/linters/BashBashExecLinter.py @@ -9,7 +9,10 @@ class BashBashExecLinter(megalinter.Linter): # To execute before linting files def before_lint_files(self): - if megalinter.config.get("ERROR_ON_MISSING_EXEC_BIT", "false") == "true": + if ( + megalinter.config.get(self.request_id, "ERROR_ON_MISSING_EXEC_BIT", "false") + == "true" + ): self.disable_errors = False else: self.disable_errors = True diff --git a/megalinter/linters/BicepLinter.py b/megalinter/linters/BicepLinter.py index 5bc37ac5154..e8e848d1123 100644 --- a/megalinter/linters/BicepLinter.py +++ b/megalinter/linters/BicepLinter.py @@ -2,15 +2,14 @@ """ Use Bicep to lint bicep files """ -import os -from megalinter import Linter +from megalinter import Linter, config class BicepLinter(Linter): # Build the CLI command to call to lint a file def build_lint_command(self, file=None): - os.environ["DOTNET_SYSTEM_GLOBALIZATION_INVARIANT"] = "1" + config.set(self.request_id, "DOTNET_SYSTEM_GLOBALIZATION_INVARIANT", "1") cmd = super().build_lint_command(file) diff --git a/megalinter/linters/CSpellLinter.py b/megalinter/linters/CSpellLinter.py index 9b743584f3d..8cb30ae5327 100644 --- a/megalinter/linters/CSpellLinter.py +++ b/megalinter/linters/CSpellLinter.py @@ -29,7 +29,8 @@ def build_lint_command(self, file=None) -> list: self.cli_lint_mode == "list_of_files" and self.files is not None and len(self.files) > 0 - and config.get("SPELL_CSPELL_ANALYZE_FILE_NAMES", "true") == "true" + and config.get(self.request_id, "SPELL_CSPELL_ANALYZE_FILE_NAMES", "true") + == "true" ): file_names_txt = "" for file_path in self.files: @@ -137,4 +138,6 @@ def complete_text_reporter_report(self, reporter_self): return additional_report.splitlines() def pre_test(self): - config.set_value("SPELL_CSPELL_FILE_EXTENSIONS", [".js", ".md"]) + config.set_value( + self.request_id, "SPELL_CSPELL_FILE_EXTENSIONS", [".js", ".md"] + ) diff --git a/megalinter/linters/MisspellLinter.py b/megalinter/linters/MisspellLinter.py index e39804f3fab..820d35576c9 100644 --- a/megalinter/linters/MisspellLinter.py +++ b/megalinter/linters/MisspellLinter.py @@ -8,4 +8,6 @@ class MisspellLinter(Linter): def pre_test(self): - config.set_value("SPELL_MISSPELL_FILE_EXTENSIONS", [".js", ".md"]) + config.set_value( + self.request_id, "SPELL_MISSPELL_FILE_EXTENSIONS", [".js", ".md"] + ) diff --git a/megalinter/linters/PowershellLinter.py b/megalinter/linters/PowershellLinter.py index 2ef22f8a865..96de5440395 100644 --- a/megalinter/linters/PowershellLinter.py +++ b/megalinter/linters/PowershellLinter.py @@ -36,7 +36,9 @@ def build_lint_command(self, file=None): if self.apply_fixes is True: file_encoding = config.get( - "POWERSHELL_POWERSHELL_FORMATTER_OUTPUT_ENCODING", "utf8" + self.request_id, + "POWERSHELL_POWERSHELL_FORMATTER_OUTPUT_ENCODING", + "utf8", ) pwsh_script[ diff --git a/megalinter/linters/ProselintLinter.py b/megalinter/linters/ProselintLinter.py index cfdabf070e2..99583d015e3 100644 --- a/megalinter/linters/ProselintLinter.py +++ b/megalinter/linters/ProselintLinter.py @@ -8,4 +8,6 @@ class ProselintLinter(Linter): def pre_test(self): - config.set_value("SPELL_PROSELINT_FILE_EXTENSIONS", [".js", ".md"]) + config.set_value( + self.request_id, "SPELL_PROSELINT_FILE_EXTENSIONS", [".js", ".md"] + ) diff --git a/megalinter/linters/RakuLinter.py b/megalinter/linters/RakuLinter.py index 72c51571569..bbcf26a8db6 100644 --- a/megalinter/linters/RakuLinter.py +++ b/megalinter/linters/RakuLinter.py @@ -8,6 +8,7 @@ import subprocess import megalinter +from megalinter import config class RakuLinter(megalinter.Linter): @@ -23,6 +24,7 @@ def before_lint_files(self): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, + env=config.build_env(self.request_id), ) return_code = process.returncode return_stdout = megalinter.utils.decode_utf8(process.stdout) diff --git a/megalinter/linters/SemgrepLinter.py b/megalinter/linters/SemgrepLinter.py index 27651c842c8..a92195d773b 100644 --- a/megalinter/linters/SemgrepLinter.py +++ b/megalinter/linters/SemgrepLinter.py @@ -16,7 +16,10 @@ def manage_activation(self, params): custom_rulesets = self.get_custom_rulesets() if ( len(custom_rulesets) == 0 - and len(config.get_list("REPOSITORY_SEMGREP_ARGUMENTS", [])) == 0 + and len( + config.get_list(self.request_id, "REPOSITORY_SEMGREP_ARGUMENTS", []) + ) + == 0 and "semgrep" not in utils.get_current_test_name(full_name=True) ): logging.info( @@ -45,13 +48,14 @@ def build_lint_command(self, file=None): return cmd def get_custom_rulesets(self): - if config.exists("REPOSITORY_SEMGREP_RULESETS"): + if config.exists(self.request_id, "REPOSITORY_SEMGREP_RULESETS"): # User defined rulesets - return config.get_list("REPOSITORY_SEMGREP_RULESETS") + return config.get_list(self.request_id, "REPOSITORY_SEMGREP_RULESETS") elif ( # security rulesets flavor_factory.get_image_flavor() in ["security", "none"] - or config.get("REPOSITORY_SEMGREP_RULESETS_TYPE", "") == "security" + or config.get(self.request_id, "REPOSITORY_SEMGREP_RULESETS_TYPE", "") + == "security" ): return [ "p/docker-compose", diff --git a/megalinter/linters/ShellcheckLinter.py b/megalinter/linters/ShellcheckLinter.py index e5e946909ae..6d5d3d07ecd 100644 --- a/megalinter/linters/ShellcheckLinter.py +++ b/megalinter/linters/ShellcheckLinter.py @@ -6,7 +6,7 @@ import logging import subprocess -from megalinter import Linter, utils +from megalinter import Linter, config, utils class ShellcheckLinter(Linter): @@ -21,6 +21,7 @@ def manage_sarif_output(self, return_stdout): stderr=subprocess.STDOUT, text=True, input=return_stdout + "\n", + env=config.build_env(self.request_id), ) return_code = process.returncode shellcheck_res_sarif = utils.decode_utf8(process.stdout) diff --git a/megalinter/linters/ValeLinter.py b/megalinter/linters/ValeLinter.py index 7824016fd86..9a2ef8e1e17 100644 --- a/megalinter/linters/ValeLinter.py +++ b/megalinter/linters/ValeLinter.py @@ -9,4 +9,4 @@ class ValeLinter(Linter): def pre_test(self): - config.set_value("SPELL_VALE_FILE_EXTENSIONS", [".js", ".md"]) + config.set_value(self.request_id, "SPELL_VALE_FILE_EXTENSIONS", [".js", ".md"]) diff --git a/megalinter/linters/XmlLintLinter.py b/megalinter/linters/XmlLintLinter.py index 1d2a91c6953..839912ae456 100644 --- a/megalinter/linters/XmlLintLinter.py +++ b/megalinter/linters/XmlLintLinter.py @@ -3,7 +3,6 @@ Use XmlLint to lint xml files http://xmlsoft.org/xmllint.html """ -import os from megalinter import Linter, config @@ -16,11 +15,14 @@ def build_lint_command(self, file=None): if ( self.apply_fixes is True and self.cli_lint_fix_arg_name is not None - and config.get("XML_XMLLINT_AUTOFORMAT", "false") == "true" + and config.get(self.request_id, "XML_XMLLINT_AUTOFORMAT", "false") == "true" ): if self.cli_lint_mode == "file": - os.environ["XMLLINT_INDENT"] = config.get("XML_XMLLINT_INDENT", " ") - + config.set( + self.request_id, + "XMLLINT_INDENT", + config.get(self.request_id, "XML_XMLLINT_INDENT", " "), + ) cmd += ["--output", f"{file}"] else: raise KeyError( @@ -29,5 +31,5 @@ def build_lint_command(self, file=None): return cmd def pre_test(self): - config.set_value("XML_XMLLINT_AUTOFORMAT", "true") - config.set_value("XML_XMLLINT_CLI_LINT_MODE", "file") + config.set_value(self.request_id, "XML_XMLLINT_AUTOFORMAT", "true") + config.set_value(self.request_id, "XML_XMLLINT_CLI_LINT_MODE", "file") diff --git a/megalinter/plugin_factory.py b/megalinter/plugin_factory.py index 234765a6ffd..881ee0dd01d 100644 --- a/megalinter/plugin_factory.py +++ b/megalinter/plugin_factory.py @@ -10,17 +10,17 @@ from megalinter import config, linter_factory, utils -def list_plugins(): - plugins = config.get_list("PLUGINS", []) +def list_plugins(request_id): + plugins = config.get_list(request_id, "PLUGINS", []) return plugins # Load & install plugins from external URLs -def initialize_plugins(): - plugins = list_plugins() +def initialize_plugins(request_id): + plugins = list_plugins(request_id) for plugin in plugins: descriptor_file = load_plugin(plugin) - install_plugin(descriptor_file) + install_plugin(descriptor_file, request_id) # Load plugin descriptor @@ -75,23 +75,23 @@ def load_plugin(plugin): # Run plugin installation routines -def install_plugin(descriptor_file): +def install_plugin(descriptor_file, request_id): descriptor = linter_factory.build_descriptor_info(descriptor_file) # Install descriptor level items if "install" in descriptor: - process_install(descriptor["install"]) + process_install(descriptor["install"], request_id) # Install linter level items if "linters" in descriptor: for linter_description in descriptor["linters"]: if "install" in linter_description: - process_install(linter_description["install"]) + process_install(linter_description["install"], request_id) logging.info( f"[Plugins] Successful initialization of {descriptor['descriptor_id']} plugins" ) # WARNING: works only with dockerfile and RUN instructions for now -def process_install(install): +def process_install(install, request_id): # Build commands from descriptor commands = [] # Dockerfile commands @@ -110,6 +110,7 @@ def process_install(install): stderr=subprocess.STDOUT, shell=True, executable=shutil.which("bash") if sys.platform == "win32" else "/bin/bash", + env=config.build_env(request_id), ) return_code = process.returncode stdout = utils.decode_utf8(process.stdout) diff --git a/megalinter/pre_post_factory.py b/megalinter/pre_post_factory.py index 3a4562ead59..5dc9bc72f9d 100644 --- a/megalinter/pre_post_factory.py +++ b/megalinter/pre_post_factory.py @@ -38,7 +38,7 @@ def run_linter_post_commands(mega_linter, linter): # Get commands from configuration def run_pre_post_commands(key, log_key, mega_linter): - pre_or_post_commands = config.get_list(key, None) + pre_or_post_commands = config.get_list(mega_linter.request_id, key, None) return run_commands(pre_or_post_commands, log_key, mega_linter) @@ -60,8 +60,13 @@ def run_command(command_info, log_key, mega_linter, linter=None): if command_info.get("cwd", "root") == "workspace": cwd = mega_linter.workspace command_info = complete_command(command_info) + subprocess_env = {**config.build_env(mega_linter.request_id, False)} add_in_logs( - linter, log_key, [f"{log_key} run: [{command_info['command']}] in cwd [{cwd}]"] + linter, + log_key, + [ + f"{log_key} run: [{command_info['command']}] in cwd [{cwd}]" + ], # TODO: Uncomment before merge ) # Run command process = subprocess.run( @@ -71,6 +76,7 @@ def run_command(command_info, log_key, mega_linter, linter=None): shell=True, cwd=os.path.realpath(cwd), executable=shutil.which("bash") if sys.platform == "win32" else "/bin/bash", + env=subprocess_env, ) return_code = process.returncode return_stdout = utils.decode_utf8(process.stdout) @@ -91,13 +97,8 @@ def run_command(command_info, log_key, mega_linter, linter=None): def complete_command(command_info): - # NPM dependencies case - if command_info["command"].startswith("npm install") or command_info[ - "command" - ].startswith("npm i"): - command_info["command"] = "cd /node-deps && " + command_info["command"] # Pip dependencies case - elif command_info.get("venv", None) is not None: + if command_info.get("venv", None) is not None: venv = command_info.get("venv") cmd = command_info["command"] command_info[ @@ -114,3 +115,13 @@ def add_in_logs(linter, log_key, lines): linter.log_lines_post += lines else: logging.info("\n".join(lines)) + + +def has_npm_or_yarn_commands(request_id): + config_dict = config.get(request_id) + for key, value in config_dict.items(): + if ("PRE_COMMANDS" in key or "POST_COMMANDS" in key) and ( + "npm" in value or "yarn" in value + ): + return True + return False diff --git a/megalinter/reporters/AzureCommentReporter.py b/megalinter/reporters/AzureCommentReporter.py index 029e08d1b65..8142328f4b2 100644 --- a/megalinter/reporters/AzureCommentReporter.py +++ b/megalinter/reporters/AzureCommentReporter.py @@ -26,18 +26,25 @@ class AzureCommentReporter(Reporter): scope = "mega-linter" def manage_activation(self): - if config.get("AZURE_COMMENT_REPORTER", "true") != "true": + if ( + config.get(self.master.request_id, "AZURE_COMMENT_REPORTER", "true") + != "true" + ): self.is_active = False - elif config.get("POST_AZURE_COMMENT", "true") == "true": # True by default + elif ( + config.get(self.master.request_id, "POST_AZURE_COMMENT", "true") == "true" + ): # True by default self.is_active = True def produce_report(self): # Post thread on Azure pull request - if config.get("SYSTEM_ACCESSTOKEN", "") != "": + if config.get(self.master.request_id, "SYSTEM_ACCESSTOKEN", "") != "": # Collect variables - SYSTEM_COLLECTIONURI = config.get("SYSTEM_COLLECTIONURI") + SYSTEM_COLLECTIONURI = config.get( + self.master.request_id, "SYSTEM_COLLECTIONURI" + ) SYSTEM_PULLREQUEST_PULLREQUESTID = config.get( - "SYSTEM_PULLREQUEST_PULLREQUESTID", "" + self.master.request_id, "SYSTEM_PULLREQUEST_PULLREQUESTID", "" ) if SYSTEM_PULLREQUEST_PULLREQUESTID == "": logging.info( @@ -47,13 +54,21 @@ def produce_report(self): + "branch-policies?view=azure-devops&tabs=browser#build-validation" ) return - SYSTEM_TEAMPROJECT = urllib.parse.quote(config.get("SYSTEM_TEAMPROJECT")) - BUILD_REPOSITORY_ID = config.get("BUILD_REPOSITORY_ID") - BUILD_BUILDID = config.get("BUILD_BUILDID", config.get("BUILD_BUILD_ID")) + SYSTEM_TEAMPROJECT = urllib.parse.quote( + config.get(self.master.request_id, "SYSTEM_TEAMPROJECT") + ) + BUILD_REPOSITORY_ID = config.get( + self.master.request_id, "BUILD_REPOSITORY_ID" + ) + BUILD_BUILDID = config.get( + self.master.request_id, + "BUILD_BUILDID", + config.get(self.master.request_id, "BUILD_BUILD_ID"), + ) # Build thread data AZURE_COMMENT_REPORTER_LINKS_TYPE = config.get( - "AZURE_COMMENT_REPORTER_LINKS_TYPE", "artifacts" + self.master.request_id, "AZURE_COMMENT_REPORTER_LINKS_TYPE", "artifacts" ) if AZURE_COMMENT_REPORTER_LINKS_TYPE == "artifacts": artifacts_url = ( @@ -72,7 +87,7 @@ def produce_report(self): } # Create connection to Azure API - access_token = config.get("SYSTEM_ACCESSTOKEN") + access_token = config.get(self.master.request_id, "SYSTEM_ACCESSTOKEN") credentials = BasicTokenAuthentication({"access_token": access_token}) connection = Connection( base_url=f"{SYSTEM_COLLECTIONURI}", diff --git a/megalinter/reporters/ConfigReporter.py b/megalinter/reporters/ConfigReporter.py index e3e1aa848a9..93f0eb407cb 100644 --- a/megalinter/reporters/ConfigReporter.py +++ b/megalinter/reporters/ConfigReporter.py @@ -25,12 +25,12 @@ def __init__(self, params=None): def manage_activation(self): if not utils.can_write_report_files(self.master): self.is_active = False - elif config.get("CONFIG_REPORTER", "true") == "false": + elif config.get(self.master.request_id, "CONFIG_REPORTER", "true") == "false": self.is_active = False def produce_report(self): config_report_folder_name = config.get( - "CONFIG_REPORTER_SUB_FOLDER", "IDE-config" + self.master.request_id, "CONFIG_REPORTER_SUB_FOLDER", "IDE-config" ) config_report_folder = ( f"{self.report_folder}{os.path.sep}{config_report_folder_name}" diff --git a/megalinter/reporters/ConsoleLinterReporter.py b/megalinter/reporters/ConsoleLinterReporter.py index 7a4ac8c84e6..e1b56ae7e12 100644 --- a/megalinter/reporters/ConsoleLinterReporter.py +++ b/megalinter/reporters/ConsoleLinterReporter.py @@ -19,14 +19,14 @@ def __init__(self, params=None): # Activate console output by default self.is_active = True self.report_type = "simple" - if config.get("OUTPUT_DETAIL", "") == "detailed": - self.report_type = "detailed" - if config.get("PRINT_ALL_FILES", "") == "true": - self.print_all_files = True super().__init__(params) def manage_activation(self): - if config.get("CONSOLE_REPORTER", "true") == "false": + if config.get(self.master.request_id, "OUTPUT_DETAIL", "") == "detailed": + self.report_type = "detailed" + if config.get(self.master.request_id, "PRINT_ALL_FILES", "") == "true": + self.print_all_files = True + if config.get(self.master.request_id, "CONSOLE_REPORTER", "true") == "false": self.is_active = False def produce_report(self): diff --git a/megalinter/reporters/ConsoleReporter.py b/megalinter/reporters/ConsoleReporter.py index 4ebabeb890a..47e6fa1c3d5 100644 --- a/megalinter/reporters/ConsoleReporter.py +++ b/megalinter/reporters/ConsoleReporter.py @@ -3,7 +3,6 @@ Output results in console """ import logging -import os import urllib import chalk as c @@ -26,7 +25,7 @@ def __init__(self, params=None): super().__init__(params) def manage_activation(self): - if config.get("CONSOLE_REPORTER", "true") == "false": + if config.get(self.master.request_id, "CONSOLE_REPORTER", "true") == "false": self.is_active = False def initialize(self): @@ -130,7 +129,7 @@ def produce_report(self): f"[flavors] Use the following link to request the new flavor: {new_flavor_url}" ) else: - build_version = os.environ.get("BUILD_VERSION", DEFAULT_RELEASE) + build_version = config.get(None, "BUILD_VERSION", DEFAULT_RELEASE) action_version = ( "v5" if "v5" in build_version diff --git a/megalinter/reporters/EmailReporter.py b/megalinter/reporters/EmailReporter.py index 8a33b351cf1..47edc66b38f 100644 --- a/megalinter/reporters/EmailReporter.py +++ b/megalinter/reporters/EmailReporter.py @@ -25,9 +25,11 @@ def __init__(self, params=None): super().__init__(params) def manage_activation(self): - if config.get("EMAIL_REPORTER", "true") != "true": + if config.get(self.master.request_id, "EMAIL_REPORTER", "true") != "true": self.is_active = False - elif config.get("EMAIL_REPORTER_EMAIL", "none") == "none": + elif ( + config.get(self.master.request_id, "EMAIL_REPORTER_EMAIL", "none") == "none" + ): logging.info( "To receive reports as email, please set variable EMAIL_REPORTER_EMAIL" ) @@ -37,7 +39,10 @@ def produce_report(self): # Skip report if no errors has been found if ( self.master.status == "success" - and config.get("EMAIL_REPORTER_SEND_SUCCESS", "false") == "true" + and config.get( + self.master.request_id, "EMAIL_REPORTER_SEND_SUCCESS", "false" + ) + == "true" and self.master.has_updated_sources is False ): logging.info( @@ -47,12 +52,20 @@ def produce_report(self): return # get server and email config values - smtp_host = config.get("EMAIL_REPORTER_SMTP_HOST", "smtp.gmail.com") - smtp_port = config.get("EMAIL_REPORTER_SMTP_PORT", 465) - recipients = config.get_list("EMAIL_REPORTER_EMAIL", []) - sender = config.get("EMAIL_REPORTER_SENDER", "megalinter@gmail.com") - smtp_username = config.get("EMAIL_REPORTER_SMTP_USERNAME", sender) - smtp_password = config.get("EMAIL_REPORTER_SMTP_PASSWORD", "") + smtp_host = config.get( + self.master.request_id, "EMAIL_REPORTER_SMTP_HOST", "smtp.gmail.com" + ) + smtp_port = config.get(self.master.request_id, "EMAIL_REPORTER_SMTP_PORT", 465) + recipients = config.get_list(self.master.request_id, "EMAIL_REPORTER_EMAIL", []) + sender = config.get( + self.master.request_id, "EMAIL_REPORTER_SENDER", "megalinter@gmail.com" + ) + smtp_username = config.get( + self.master.request_id, "EMAIL_REPORTER_SMTP_USERNAME", sender + ) + smtp_password = config.get( + self.master.request_id, "EMAIL_REPORTER_SMTP_PASSWORD", "" + ) # Skip report if SMTP password is not set if smtp_password == "": diff --git a/megalinter/reporters/FileIoReporter.py b/megalinter/reporters/FileIoReporter.py index c06ce535b12..556dc444dfc 100644 --- a/megalinter/reporters/FileIoReporter.py +++ b/megalinter/reporters/FileIoReporter.py @@ -23,14 +23,17 @@ def __init__(self, params=None): super().__init__(params) def manage_activation(self): - if config.get("FILEIO_REPORTER", "false") == "true": + if config.get(self.master.request_id, "FILEIO_REPORTER", "false") == "true": self.is_active = True def produce_report(self): # Skip report if no errors has been found if ( self.master.status == "success" - and config.get("FILEIO_REPORTER_SEND_SUCCESS", "false") == "true" + and config.get( + self.master.request_id, "FILEIO_REPORTER_SEND_SUCCESS", "false" + ) + == "true" and self.master.has_updated_sources is False ): logging.info( diff --git a/megalinter/reporters/GithubCommentReporter.py b/megalinter/reporters/GithubCommentReporter.py index a96f655f866..cddcfca17d2 100644 --- a/megalinter/reporters/GithubCommentReporter.py +++ b/megalinter/reporters/GithubCommentReporter.py @@ -22,10 +22,13 @@ class GithubCommentReporter(Reporter): issues_root = ML_REPO_URL + "/issues" def manage_activation(self): - if config.get("GITHUB_COMMENT_REPORTER", "true") != "true": + if ( + config.get(self.master.request_id, "GITHUB_COMMENT_REPORTER", "true") + != "true" + ): self.is_active = False elif ( - config.get("POST_GITHUB_COMMENT", "true") == "true" + config.get(self.master.request_id, "POST_GITHUB_COMMENT", "true") == "true" ): # Legacy - true by default self.is_active = True @@ -53,15 +56,21 @@ def comment_marker(self): def produce_report(self): # Post comment on GitHub pull request - if config.get("GITHUB_TOKEN", "") != "": - github_repo = config.get("GITHUB_REPOSITORY") - github_server_url = config.get("GITHUB_SERVER_URL", self.github_server_url) - github_api_url = config.get("GITHUB_API_URL", self.github_api_url) - run_id = config.get("GITHUB_RUN_ID") - sha = config.get("GITHUB_SHA") + if config.get(self.master.request_id, "GITHUB_TOKEN", "") != "": + github_repo = config.get(self.master.request_id, "GITHUB_REPOSITORY") + github_server_url = config.get( + self.master.request_id, "GITHUB_SERVER_URL", self.github_server_url + ) + github_api_url = config.get( + self.master.request_id, "GITHUB_API_URL", self.github_api_url + ) + run_id = config.get(self.master.request_id, "GITHUB_RUN_ID") + sha = config.get(self.master.request_id, "GITHUB_SHA") - if config.get("CI_ACTION_RUN_URL", "") != "": - action_run_url = config.get("CI_ACTION_RUN_URL", "") + if config.get(self.master.request_id, "CI_ACTION_RUN_URL", "") != "": + action_run_url = config.get( + self.master.request_id, "CI_ACTION_RUN_URL", "" + ) elif run_id is not None: action_run_url = ( f"{github_server_url}/{github_repo}/actions/runs/{run_id}" @@ -77,9 +86,9 @@ def produce_report(self): # Post comment on pull request if found github_auth = ( - config.get("PAT") - if config.get("PAT", "") != "" - else config.get("GITHUB_TOKEN") + config.get(self.master.request_id, "PAT") + if config.get(self.master.request_id, "PAT", "") != "" + else config.get(self.master.request_id, "GITHUB_TOKEN") ) g = github.Github(base_url=github_api_url, login_or_token=github_auth) repo = g.get_repo(github_repo) diff --git a/megalinter/reporters/GithubStatusReporter.py b/megalinter/reporters/GithubStatusReporter.py index 5c071452371..4a4d78432d1 100644 --- a/megalinter/reporters/GithubStatusReporter.py +++ b/megalinter/reporters/GithubStatusReporter.py @@ -23,35 +23,45 @@ def __init__(self, params=None): def manage_activation(self): # Disable status for each linter if MULTI_STATUS is 'false' - if config.exists("MULTI_STATUS") and config.get("MULTI_STATUS") == "true": + if ( + config.exists(self.master.request_id, "MULTI_STATUS") + and config.get(self.master.request_id, "MULTI_STATUS") == "true" + ): self.is_active = True - elif config.get("GITHUB_STATUS_REPORTER", "false") != "false": + elif ( + config.get(self.master.request_id, "GITHUB_STATUS_REPORTER", "false") + != "false" + ): self.is_active = True def produce_report(self): if ( - config.exists("GITHUB_REPOSITORY") - and config.exists("GITHUB_SHA") - and config.exists("GITHUB_TOKEN") + config.exists(self.master.request_id, "GITHUB_REPOSITORY") + and config.exists(self.master.request_id, "GITHUB_SHA") + and config.exists(self.master.request_id, "GITHUB_TOKEN") ): - github_repo = config.get("GITHUB_REPOSITORY") - github_server_url = config.get("GITHUB_SERVER_URL", self.github_server_url) - github_api_url = config.get("GITHUB_API_URL", self.github_api_url) - sha = config.get("GITHUB_SHA") - run_id = config.get("GITHUB_RUN_ID") + github_repo = config.get(self.master.request_id, "GITHUB_REPOSITORY") + github_server_url = config.get( + self.master.request_id, "GITHUB_SERVER_URL", self.github_server_url + ) + github_api_url = config.get( + self.master.request_id, "GITHUB_API_URL", self.github_api_url + ) + sha = config.get(self.master.request_id, "GITHUB_SHA") + run_id = config.get(self.master.request_id, "GITHUB_RUN_ID") success_msg = "No errors were found in the linting process" error_not_blocking = "Errors were detected but are considered not blocking" error_msg = f"Found {self.master.total_number_errors}, please check logs" url = f"{github_api_url}/repos/{github_repo}/statuses/{sha}" headers = { "accept": "application/vnd.github.v3+json", - "authorization": f"Bearer {config.get('GITHUB_TOKEN')}", + "authorization": f"Bearer {config.get(self.master.request_id,'GITHUB_TOKEN')}", "content-type": "application/json", } - if config.exists("GITHUB_RUN_ID"): + if config.exists(self.master.request_id, "GITHUB_RUN_ID"): target_url = f"{github_server_url}/{github_repo}/actions/runs/{run_id}" else: - target_url = config.get("GITHUB_TARGET_URL") + target_url = config.get(self.master.request_id, "GITHUB_TARGET_URL") description = ( success_msg if self.master.status == "success" and self.master.return_code == 0 diff --git a/megalinter/reporters/GitlabCommentReporter.py b/megalinter/reporters/GitlabCommentReporter.py index ca75bbb1e4c..e96f9ca1b6b 100644 --- a/megalinter/reporters/GitlabCommentReporter.py +++ b/megalinter/reporters/GitlabCommentReporter.py @@ -18,23 +18,35 @@ class GitlabCommentReporter(Reporter): gitlab_server_url = "https://gitlab.com" def manage_activation(self): - if config.get("GITLAB_COMMENT_REPORTER", "true") != "true": + if ( + config.get(self.master.request_id, "GITLAB_COMMENT_REPORTER", "true") + != "true" + ): self.is_active = False elif ( - config.get("POST_GITLAB_COMMENT", "true") == "true" + config.get(self.master.request_id, "POST_GITLAB_COMMENT", "true") == "true" ): # Legacy - true by default self.is_active = True def produce_report(self): # Post comment on Gitlab pull request - if config.get("CI_JOB_TOKEN", "") != "": - gitlab_repo = config.get("CI_PROJECT_NAME") - gitlab_project_id = config.get("CI_PROJECT_ID") - gitlab_merge_request_id = config.get("CI_MERGE_REQUEST_ID", "") + if config.get(self.master.request_id, "CI_JOB_TOKEN", "") != "": + gitlab_repo = config.get(self.master.request_id, "CI_PROJECT_NAME") + gitlab_project_id = config.get(self.master.request_id, "CI_PROJECT_ID") + gitlab_merge_request_id = config.get( + self.master.request_id, "CI_MERGE_REQUEST_ID", "" + ) if gitlab_merge_request_id == "": - if config.get("CI_OPEN_MERGE_REQUESTS", "") != "": + if ( + config.get(self.master.request_id, "CI_OPEN_MERGE_REQUESTS", "") + != "" + ): gitlab_merge_request_id = ( - config.get("CI_OPEN_MERGE_REQUESTS", "missing!missing") + config.get( + self.master.request_id, + "CI_OPEN_MERGE_REQUESTS", + "missing!missing", + ) .split(",")[0] .split("!")[1] ) @@ -44,24 +56,38 @@ def produce_report(self): ) return - gitlab_server_url = config.get("CI_SERVER_URL", self.gitlab_server_url) - action_run_url = config.get("CI_JOB_URL", "") + gitlab_server_url = config.get( + self.master.request_id, "CI_SERVER_URL", self.gitlab_server_url + ) + action_run_url = config.get(self.master.request_id, "CI_JOB_URL", "") p_r_msg = build_markdown_summary(self, action_run_url) # Build gitlab options gitlab_options = {} # auth token - if config.get("GITLAB_ACCESS_TOKEN_MEGALINTER", "") != "": + if ( + config.get(self.master.request_id, "GITLAB_ACCESS_TOKEN_MEGALINTER", "") + != "" + ): gitlab_options["private_token"] = config.get( - "GITLAB_ACCESS_TOKEN_MEGALINTER" + self.master.request_id, "GITLAB_ACCESS_TOKEN_MEGALINTER" ) else: - gitlab_options["job_token"] = config.get("CI_JOB_TOKEN") + gitlab_options["job_token"] = config.get( + self.master.request_id, "CI_JOB_TOKEN" + ) # Certificate management - gitlab_certificate_path = config.get("GITLAB_CERTIFICATE_PATH", "") - if config.get("GITLAB_CUSTOM_CERTIFICATE", "") != "": + gitlab_certificate_path = config.get( + self.master.request_id, "GITLAB_CERTIFICATE_PATH", "" + ) + if ( + config.get(self.master.request_id, "GITLAB_CUSTOM_CERTIFICATE", "") + != "" + ): # Certificate value defined in an ENV variable - cert_value = config.get("GITLAB_CUSTOM_CERTIFICATE") + cert_value = config.get( + self.master.request_id, "GITLAB_CUSTOM_CERTIFICATE" + ) gitlab_certificate_path = "/etc/ssl/certs/gitlab-cert.crt" with open(gitlab_certificate_path, "w", encoding="utf-8") as cert_file: cert_file.write(cert_value) @@ -99,7 +125,9 @@ def produce_report(self): try: mr = project.mergerequests.get(gitlab_merge_request_id) except gitlab.GitlabGetError: - gitlab_merge_request_id = config.get("CI_MERGE_REQUEST_IID", "none") + gitlab_merge_request_id = config.get( + self.master.request_id, "CI_MERGE_REQUEST_IID", "none" + ) try: mr = project.mergerequests.get(gitlab_merge_request_id) except gitlab.GitlabGetError as e: @@ -120,7 +148,11 @@ def produce_report(self): # List comments on merge request existing_comment = None if ( - config.get("GITLAB_COMMENT_REPORTER_OVERWRITE_COMMENT", "true") + config.get( + self.master.request_id, + "GITLAB_COMMENT_REPORTER_OVERWRITE_COMMENT", + "true", + ) == "true" ): try: diff --git a/megalinter/reporters/JsonReporter.py b/megalinter/reporters/JsonReporter.py index 216ab589bee..ee71cd7ce12 100644 --- a/megalinter/reporters/JsonReporter.py +++ b/megalinter/reporters/JsonReporter.py @@ -49,8 +49,6 @@ class JsonReporter(Reporter): def __init__(self, params=None): # Deactivate JSON output by default self.is_active = False - if config.get("JSON_REPORTER_OUTPUT_DETAIL", "simple") == "detailed": - self.report_type = "detailed" self.processing_order = ( 9999 # Run at last so the output is on the last console line ) @@ -59,13 +57,23 @@ def __init__(self, params=None): def manage_activation(self): if not utils.can_write_report_files(self.master): self.is_active = False - elif config.get("JSON_REPORTER", "false") == "true": + elif config.get(self.master.request_id, "JSON_REPORTER", "false") == "true": self.is_active = True + if ( + config.get( + self.master.request_id, "JSON_REPORTER_OUTPUT_DETAIL", "simple" + ) + == "detailed" + ): + self.report_type = "detailed" def produce_report(self): result_obj = copy.deepcopy(self.master) # Remove output data if result is simple (except if we are in debug mode) - if self.report_type == "simple" and config.get("LOG_LEVEL", "") != "DEBUG": + if ( + self.report_type == "simple" + and config.get(self.master.request_id, "LOG_LEVEL", "") != "DEBUG" + ): result_obj = self.filter_fields(result_obj, self.megalinter_fields) result_obj.linters = filter( lambda x: x.is_active is True, result_obj.linters @@ -91,7 +99,7 @@ def produce_report(self): result_json = json.dumps(result_json_obj, sort_keys=True, indent=4) # Write output file json_file_name = f"{self.report_folder}{os.path.sep}" + config.get( - "JSON_REPORTER_FILE_NAME", "mega-linter-report.json" + self.master.request_id, "JSON_REPORTER_FILE_NAME", "mega-linter-report.json" ) with open(json_file_name, "w", encoding="utf-8") as json_file: json_file.write(result_json) diff --git a/megalinter/reporters/SarifReporter.py b/megalinter/reporters/SarifReporter.py index 5f2dbd7e94a..a860c006ae6 100644 --- a/megalinter/reporters/SarifReporter.py +++ b/megalinter/reporters/SarifReporter.py @@ -31,7 +31,7 @@ def __init__(self, params=None): def manage_activation(self): if not utils.can_write_report_files(self.master): self.is_active = False - elif config.get("SARIF_REPORTER", "false") == "true": + elif config.get(self.master.request_id, "SARIF_REPORTER", "false") == "true": self.is_active = True def produce_report(self): @@ -42,18 +42,18 @@ def produce_report(self): "comment": "Generated by MegaLinter", "docUrl": ML_DOC_URL, "dockerImage": { - "buildDate": config.get("BUILD_DATE", ""), - "buildRevision": config.get("BUILD_REVISION", ""), - "buildVersion": config.get("BUILD_VERSION", ""), - "flavor": config.get("MEGALINTER_FLAVOR", "none"), - "singleLinter": config.get("SINGLE_LINTER", ""), + "buildDate": config.get(None, "BUILD_DATE", ""), + "buildRevision": config.get(None, "BUILD_REVISION", ""), + "buildVersion": config.get(None, "BUILD_VERSION", ""), + "flavor": config.get(None, "MEGALINTER_FLAVOR", "none"), + "singleLinter": config.get(None, "SINGLE_LINTER", ""), }, }, "runs": [], } # Check delete linter SARIF file if LOG_FILE=none keep_sarif_logs = True - if config.get("LOG_FILE", "") == "none": + if config.get(self.master.request_id, "LOG_FILE", "") == "none": keep_sarif_logs = False # Build unique SARIF file with all SARIF output files for linter in self.master.linters: @@ -99,7 +99,9 @@ def produce_report(self): result_json = normalize_log_string(result_json) # Write output file sarif_file_name = f"{self.report_folder}{os.path.sep}" + config.get( - "SARIF_REPORTER_FILE_NAME", DEFAULT_SARIF_REPORT_FILE_NAME + self.master.request_id, + "SARIF_REPORTER_FILE_NAME", + DEFAULT_SARIF_REPORT_FILE_NAME, ) if os.path.isfile(sarif_file_name): # Remove from previous run diff --git a/megalinter/reporters/TapReporter.py b/megalinter/reporters/TapReporter.py index a710268c8e1..f53e51724e7 100644 --- a/megalinter/reporters/TapReporter.py +++ b/megalinter/reporters/TapReporter.py @@ -16,19 +16,22 @@ class TapReporter(Reporter): def __init__(self, params=None): # report_type is tap by default self.report_type = "tap" - if config.get("OUTPUT_DETAIL", "") == "detailed": - self.report_type = "detailed" - if config.get("TAP_REPORTER_OUTPUT_DETAIL", "") == "detailed": - self.report_type = "detailed" super().__init__(params) def manage_activation(self): # Super-Linter legacy variables - output_format = config.get("OUTPUT_FORMAT", "") + if config.get(self.master.request_id, "OUTPUT_DETAIL", "") == "detailed": + self.report_type = "detailed" + if ( + config.get(self.master.request_id, "TAP_REPORTER_OUTPUT_DETAIL", "") + == "detailed" + ): + self.report_type = "detailed" + output_format = config.get(self.master.request_id, "OUTPUT_FORMAT", "") if output_format.startswith("tap"): self.is_active = True # MegaLinter vars (false by default) - elif config.get("TAP_REPORTER", "false") == "true": + elif config.get(self.master.request_id, "TAP_REPORTER", "false") == "true": self.is_active = True else: self.is_active = False @@ -67,7 +70,9 @@ def produce_report(self): file_tap_lines += detailed_lines tap_report_lines += file_tap_lines # Write TAP file - tap_report_sub_folder = config.get("TAP_REPORTER_SUB_FOLDER", "tap") + tap_report_sub_folder = config.get( + self.master.request_id, "TAP_REPORTER_SUB_FOLDER", "tap" + ) tap_file_name = ( f"{self.report_folder}{os.path.sep}" f"{tap_report_sub_folder}{os.path.sep}" diff --git a/megalinter/reporters/TextReporter.py b/megalinter/reporters/TextReporter.py index 30b66e1396c..984982353f9 100644 --- a/megalinter/reporters/TextReporter.py +++ b/megalinter/reporters/TextReporter.py @@ -17,20 +17,20 @@ class TextReporter(Reporter): def __init__(self, params=None): # report_type is simple by default self.report_type = "simple" - if config.get("OUTPUT_DETAIL", "") == "detailed": - self.report_type = "detailed" self.processing_order = -5 super().__init__(params) def manage_activation(self): - output_format = config.get("OUTPUT_FORMAT", "") + if config.get(self.master.request_id, "OUTPUT_DETAIL", "") == "detailed": + self.report_type = "detailed" + output_format = config.get(self.master.request_id, "OUTPUT_FORMAT", "") if not utils.can_write_report_files(self.master): self.is_active = False elif output_format.startswith("text"): # Super-Linter legacy variable self.is_active = True # MegaLinter vars (true by default) - elif config.get("TEXT_REPORTER", "true") != "true": + elif config.get(self.master.request_id, "TEXT_REPORTER", "true") != "true": self.is_active = False else: self.is_active = True @@ -90,7 +90,9 @@ def produce_report(self): # Complete lines text_report_lines += self.master.complete_text_reporter_report(self) # Write to file - text_report_sub_folder = config.get("TEXT_REPORTER_SUB_FOLDER", "linters_logs") + text_report_sub_folder = config.get( + self.master.request_id, "TEXT_REPORTER_SUB_FOLDER", "linters_logs" + ) text_file_name = ( f"{self.report_folder}{os.path.sep}" f"{text_report_sub_folder}{os.path.sep}" diff --git a/megalinter/reporters/UpdatedSourcesReporter.py b/megalinter/reporters/UpdatedSourcesReporter.py index 533c18b3b98..e4a5234699f 100644 --- a/megalinter/reporters/UpdatedSourcesReporter.py +++ b/megalinter/reporters/UpdatedSourcesReporter.py @@ -22,7 +22,10 @@ def __init__(self, params=None): def manage_activation(self): if not utils.can_write_report_files(self.master): self.is_active = False - elif config.get("UPDATED_SOURCES_REPORTER", "true") != "true": + elif ( + config.get(self.master.request_id, "UPDATED_SOURCES_REPORTER", "true") + != "true" + ): self.is_active = False def produce_report(self): @@ -30,7 +33,9 @@ def produce_report(self): # Copy updated files in report folder updated_files = utils.list_updated_files(self.master.github_workspace) logging.debug("Updated files :\n" + "\n -".join(updated_files)) - updated_dir = config.get("UPDATED_SOURCES_REPORTER_DIR", "updated_sources") + updated_dir = config.get( + self.master.request_id, "UPDATED_SOURCES_REPORTER_DIR", "updated_sources" + ) updated_sources_dir = f"{self.report_folder}{os.path.sep}{updated_dir}" for updated_file in updated_files: updated_file_clean = utils.normalize_log_string(updated_file) diff --git a/megalinter/reporters/WebHookLinterReporter.py b/megalinter/reporters/WebHookLinterReporter.py index 7ba7fc89092..5ef546321dd 100644 --- a/megalinter/reporters/WebHookLinterReporter.py +++ b/megalinter/reporters/WebHookLinterReporter.py @@ -26,10 +26,12 @@ def __init__(self, params=None): super().__init__(params) def manage_activation(self): - if config.get("WEBHOOK_REPORTER", "false") == "true": - if config.exists("WEBHOOK_REPORTER_URL"): + if config.get(self.master.request_id, "WEBHOOK_REPORTER", "false") == "true": + if config.exists(self.master.request_id, "WEBHOOK_REPORTER_URL"): self.is_active = True - self.hook_url = config.get("WEBHOOK_REPORTER_URL") + self.hook_url = config.get( + self.master.request_id, "WEBHOOK_REPORTER_URL" + ) else: logging.error( "You need to define WEBHOOK_REPORTER_URL to use WebHookReporter" @@ -44,10 +46,10 @@ def produce_report(self): "accept": "application/json", "content-type": "application/json", } - if config.exists("WEBHOOK_REPORTER_BEARER_TOKEN"): + if config.exists(self.master.request_id, "WEBHOOK_REPORTER_BEARER_TOKEN"): headers[ "authorization" - ] = f"Bearer {config.get('WEBHOOK_REPORTER_BEARER_TOKEN')}" + ] = f"Bearer {config.get(self.master.request_id,'WEBHOOK_REPORTER_BEARER_TOKEN')}" status_message = ( success_msg if self.master.status == "success" and self.master.return_code == 0 @@ -84,7 +86,7 @@ def produce_report(self): self.web_hook_data["outputSarif"] = json.load(linter_sarif_file) else: text_report_sub_folder = config.get( - "TEXT_REPORTER_SUB_FOLDER", "linters_logs" + self.master.request_id, "TEXT_REPORTER_SUB_FOLDER", "linters_logs" ) text_file_name = ( f"{self.report_folder}{os.path.sep}" diff --git a/megalinter/run.py b/megalinter/run.py index 87be99d4cf6..da72ceb02eb 100644 --- a/megalinter/run.py +++ b/megalinter/run.py @@ -4,9 +4,11 @@ """ +import uuid + import megalinter -linter = megalinter.Megalinter({"cli": True}) +linter = megalinter.Megalinter({"cli": True, "request_id": str(uuid.uuid1())}) # Guess who's there ? :) megalinter.alpaca() diff --git a/megalinter/server.py b/megalinter/server.py index 4aeb01a1795..b60e2137d80 100644 --- a/megalinter/server.py +++ b/megalinter/server.py @@ -16,11 +16,13 @@ from pydantic import BaseModel, Field from pygments import lexers -print("MegaLinter Server starting...") +print("MegaLinter Server starting…") logging.config.fileConfig("logging.conf", disable_existing_loggers=False) # type: ignore[attr-defined] logger = logging.getLogger(__name__) alpaca() -app = FastAPI(title="MegaLinter Server", version=config.get("BUILD_VERSION", "DEV")) +app = FastAPI( + title="MegaLinter Server", version=config.get(None, "BUILD_VERSION", "DEV") +) global running_process_number, max_running_process_number, ANALYSIS_EXECUTIONS running_process_number = 0 diff --git a/megalinter/tests/test_megalinter/LinterTestRoot.py b/megalinter/tests/test_megalinter/LinterTestRoot.py index dd7c8036602..23c06d94633 100644 --- a/megalinter/tests/test_megalinter/LinterTestRoot.py +++ b/megalinter/tests/test_megalinter/LinterTestRoot.py @@ -2,6 +2,7 @@ """ Unit tests for Linter class (and sub-classes) """ +import uuid from typing import Optional from megalinter import linter_factory, utilstest @@ -10,8 +11,9 @@ class LinterTestRoot: descriptor_id: Optional[str] = None linter_name: Optional[str] = None + request_id: str | None = None - def get_linter_instance(self): + def get_linter_instance(self, request_id): return linter_factory.build_linter( self.descriptor_id, self.linter_name, @@ -24,54 +26,66 @@ def get_linter_instance(self): "disable_errors_linters": [], "github_workspace": ".", "post_linter_status": True, + "request_id": request_id, }, ) def test_success(self): - utilstest.linter_test_setup() - linter = self.get_linter_instance() + self.request_id = str(uuid.uuid1()) + utilstest.linter_test_setup({"request_id": self.request_id}) + linter = self.get_linter_instance(self.request_id) linter.pre_test() utilstest.test_linter_success(linter, self) linter.post_test() def test_failure(self): - utilstest.linter_test_setup() - linter = self.get_linter_instance() + self.request_id = str(uuid.uuid1()) + utilstest.linter_test_setup({"request_id": self.request_id}) + linter = self.get_linter_instance(self.request_id) linter.pre_test() utilstest.test_linter_failure(linter, self) linter.post_test() def test_get_linter_version(self): - utilstest.linter_test_setup() - linter = self.get_linter_instance() + self.request_id = str(uuid.uuid1()) + utilstest.linter_test_setup({"request_id": self.request_id}) + linter = self.get_linter_instance(self.request_id) linter.pre_test() utilstest.test_get_linter_version(linter, self) linter.post_test() def test_get_linter_help(self): - utilstest.linter_test_setup() - linter = self.get_linter_instance() + self.request_id = str(uuid.uuid1()) + utilstest.linter_test_setup({"request_id": self.request_id}) + linter = self.get_linter_instance(self.request_id) linter.pre_test() utilstest.test_get_linter_help(linter, self) linter.post_test() def test_report_tap(self): - utilstest.linter_test_setup({"report_type": "tap"}) - linter = self.get_linter_instance() + self.request_id = str(uuid.uuid1()) + utilstest.linter_test_setup( + {"request_id": self.request_id, "report_type": "tap"} + ) + linter = self.get_linter_instance(self.request_id) linter.pre_test() utilstest.test_linter_report_tap(linter, self) linter.post_test() def test_report_sarif(self): - utilstest.linter_test_setup({"report_type": "SARIF"}) - linter = self.get_linter_instance() + self.request_id = str(uuid.uuid1()) + utilstest.linter_test_setup( + {"request_id": self.request_id, "report_type": "SARIF"} + ) + linter = self.get_linter_instance(self.request_id) linter.pre_test() utilstest.test_linter_report_sarif(linter, self) linter.post_test() def test_format_fix(self): - utilstest.linter_test_setup() - linter = self.get_linter_instance() + self.request_id = str(uuid.uuid1()) + utilstest.linter_test_setup({"request_id": self.request_id}) + linter = self.get_linter_instance(self.request_id) linter.pre_test() utilstest.test_linter_format_fix(linter, self) linter.post_test() diff --git a/megalinter/tests/test_megalinter/config_test.py b/megalinter/tests/test_megalinter/config_test.py index 465d573b77e..49e2060cf35 100644 --- a/megalinter/tests/test_megalinter/config_test.py +++ b/megalinter/tests/test_megalinter/config_test.py @@ -7,6 +7,7 @@ import os import re import unittest +import uuid from git import Repo from megalinter import config, utilstest @@ -48,17 +49,17 @@ def tearDown(self): def test_remote_config_success(self): changed_files = self.replace_branch_in_input_files() remote_config = self.test_folder + "remote/custom.mega-linter.yml" - os.environ["MEGALINTER_CONFIG"] = remote_config - config.init_config() - self.assertEqual("(custom)", config.get("FILTER_REGEX_INCLUDE")) + request_id = str(uuid.uuid1()) + config.init_config(request_id, None, {"MEGALINTER_CONFIG": remote_config}) + self.assertEqual("(custom)", config.get(request_id, "FILTER_REGEX_INCLUDE")) self.restore_branch_in_input_files(changed_files) def test_remote_config_error(self): changed_files = self.replace_branch_in_input_files() remote_config = self.test_folder + "custom.mega-linter-not-existing.yml" + request_id = str(uuid.uuid1()) try: - os.environ["MEGALINTER_CONFIG"] = remote_config - config.init_config() + config.init_config(request_id, None, {"MEGALINTER_CONFIG": remote_config}) except Exception as e: self.assertRegex( str(e), @@ -74,8 +75,9 @@ def test_remote_config_error(self): def test_local_config_extends_success(self): changed_files = self.replace_branch_in_input_files() local_config = "local.mega-linter.yml" - os.environ["MEGALINTER_CONFIG"] = local_config + request_id = str(uuid.uuid1()) config.init_config( + request_id, REPO_HOME_DEFAULT + os.path.sep + ".automation" @@ -84,17 +86,19 @@ def test_local_config_extends_success(self): + os.path.sep + "mega-linter-config-test" + os.path.sep - + "local_extends" + + "local_extends", + {"MEGALINTER_CONFIG": local_config}, ) - self.assertEqual("(local)", config.get("FILTER_REGEX_INCLUDE")) - self.assertEqual("false", config.get("SHOW_ELAPSED_TIME")) + self.assertEqual("(local)", config.get(request_id, "FILTER_REGEX_INCLUDE")) + self.assertEqual("false", config.get(request_id, "SHOW_ELAPSED_TIME")) self.restore_branch_in_input_files(changed_files) def test_local_config_extends_recurse_success(self): changed_files = self.replace_branch_in_input_files() local_config = "recurse.mega-linter.yml" - os.environ["MEGALINTER_CONFIG"] = local_config + request_id = str(uuid.uuid1()) config.init_config( + request_id, REPO_HOME_DEFAULT + os.path.sep + ".automation" @@ -103,27 +107,30 @@ def test_local_config_extends_recurse_success(self): + os.path.sep + "mega-linter-config-test" + os.path.sep - + "local_extends_recurse" + + "local_extends_recurse", + {"MEGALINTER_CONFIG": local_config}, ) - self.assertEqual("(local)", config.get("FILTER_REGEX_INCLUDE")) - self.assertEqual("false", config.get("SHOW_ELAPSED_TIME")) - self.assertEqual("dev", config.get("DEFAULT_BRANCH")) - self.assertEqual("DEBUG", config.get("LOG_LEVEL")) + self.assertEqual("(local)", config.get(request_id, "FILTER_REGEX_INCLUDE")) + self.assertEqual("false", config.get(request_id, "SHOW_ELAPSED_TIME")) + self.assertEqual("dev", config.get(request_id, "DEFAULT_BRANCH")) + self.assertEqual("DEBUG", config.get(request_id, "LOG_LEVEL")) self.restore_branch_in_input_files(changed_files) def test_local_config_extends_error(self): changed_files = self.replace_branch_in_input_files() local_config = "local-error.mega-linter.yml" - os.environ["MEGALINTER_CONFIG"] = local_config + request_id = str(uuid.uuid1()) try: config.init_config( + request_id, REPO_HOME_DEFAULT + os.path.sep + ".automation" + os.path.sep + "test" + os.path.sep - + "mega-linter-config-test" + + "mega-linter-config-test", + {"MEGALINTER_CONFIG": local_config}, ) except Exception as e: self.assertIn("No such file or directory", str(e)) @@ -133,21 +140,21 @@ def test_local_config_extends_error(self): def test_remote_config_extends_success(self): changed_files = self.replace_branch_in_input_files() remote_config = self.test_folder + "remote_extends/base.mega-linter.yml" - os.environ["MEGALINTER_CONFIG"] = remote_config - config.init_config() - self.assertEqual("(base)", config.get("FILTER_REGEX_INCLUDE")) - self.assertEqual("(extension2)", config.get("FILTER_REGEX_EXCLUDE")) - self.assertEqual("true", config.get("SHOW_ELAPSED_TIME")) + request_id = str(uuid.uuid1()) + config.init_config(request_id, None, {"MEGALINTER_CONFIG": remote_config}) + self.assertEqual("(base)", config.get(request_id, "FILTER_REGEX_INCLUDE")) + self.assertEqual("(extension2)", config.get(request_id, "FILTER_REGEX_EXCLUDE")) + self.assertEqual("true", config.get(request_id, "SHOW_ELAPSED_TIME")) self.restore_branch_in_input_files(changed_files) def test_remote_config_extends_success_2(self): changed_files = self.replace_branch_in_input_files() remote_config = self.test_folder + "remote_extends_2/base2.mega-linter.yml" - os.environ["MEGALINTER_CONFIG"] = remote_config - config.init_config() - self.assertEqual("(base)", config.get("FILTER_REGEX_INCLUDE")) - self.assertEqual("(extension2)", config.get("FILTER_REGEX_EXCLUDE")) - self.assertEqual("true", config.get("SHOW_ELAPSED_TIME")) + request_id = str(uuid.uuid1()) + config.init_config(request_id, None, {"MEGALINTER_CONFIG": remote_config}) + self.assertEqual("(base)", config.get(request_id, "FILTER_REGEX_INCLUDE")) + self.assertEqual("(extension2)", config.get(request_id, "FILTER_REGEX_EXCLUDE")) + self.assertEqual("true", config.get(request_id, "SHOW_ELAPSED_TIME")) self.restore_branch_in_input_files(changed_files) def test_remote_config_extends_error(self): @@ -155,10 +162,9 @@ def test_remote_config_extends_error(self): remote_config = ( self.test_folder + "remote_extends_error/base-error.mega-linter.yml" ) - os.environ["MEGALINTER_CONFIG"] = remote_config + request_id = str(uuid.uuid1()) try: - os.environ["MEGALINTER_CONFIG"] = remote_config - config.init_config() + config.init_config(request_id, None, {"MEGALINTER_CONFIG": remote_config}) except Exception as e: self.assertRegex( str(e), @@ -174,8 +180,9 @@ def test_remote_config_extends_error(self): def test_local_remote_config_extends_recurse_success(self): changed_files = self.replace_branch_in_input_files() local_config = "local.remote.mega-linter.yml" - os.environ["MEGALINTER_CONFIG"] = local_config + request_id = str(uuid.uuid1()) config.init_config( + request_id, REPO_HOME_DEFAULT + os.path.sep + ".automation" @@ -184,22 +191,26 @@ def test_local_remote_config_extends_recurse_success(self): + os.path.sep + "mega-linter-config-test" + os.path.sep - + "local_remote_extends_recurse" + + "local_remote_extends_recurse", + {"MEGALINTER_CONFIG": local_config}, ) - self.assertEqual("(base)", config.get("FILTER_REGEX_INCLUDE")) - self.assertEqual("(extension2)", config.get("FILTER_REGEX_EXCLUDE")) - self.assertEqual("true", config.get("SHOW_ELAPSED_TIME")) - self.assertEqual("dev", config.get("DEFAULT_BRANCH")) - self.assertEqual("DEBUG", config.get("LOG_LEVEL")) + self.assertEqual("(base)", config.get(request_id, "FILTER_REGEX_INCLUDE")) + self.assertEqual("(extension2)", config.get(request_id, "FILTER_REGEX_EXCLUDE")) + self.assertEqual("true", config.get(request_id, "SHOW_ELAPSED_TIME")) + self.assertEqual("dev", config.get(request_id, "DEFAULT_BRANCH")) + self.assertEqual("DEBUG", config.get(request_id, "LOG_LEVEL")) self.restore_branch_in_input_files(changed_files) def test_list_of_obj_as_env_var(self): - os.environ[ - "PRE_COMMANDS" - ] = '[{"cwd": "workspace", "command:": "echo \\"hello world\\""}]' - config.init_config() - pre_commands = config.get_list("PRE_COMMANDS", []) - del os.environ["PRE_COMMANDS"] + request_id = str(uuid.uuid1()) + config.init_config( + request_id, + None, + { + "PRE_COMMANDS": '[{"cwd": "workspace", "command:": "echo \\"hello world\\""}]' + }, + ) + pre_commands = config.get_list(request_id, "PRE_COMMANDS", []) self.assertTrue(len(pre_commands) > 0, "PRE_COMMANDS not loaded from ENV var") def replace_branch_in_input_files(self): diff --git a/megalinter/tests/test_megalinter/linters/powershell_powershell_formatter_test.py b/megalinter/tests/test_megalinter/linters/powershell_powershell_formatter_test.py index 0a4e879f711..afc87b62525 100644 --- a/megalinter/tests/test_megalinter/linters/powershell_powershell_formatter_test.py +++ b/megalinter/tests/test_megalinter/linters/powershell_powershell_formatter_test.py @@ -4,6 +4,8 @@ This class has been automatically @generated by .automation/build.py, please don't update it manually """ +import os +import unittest from unittest import TestCase from megalinter.tests.test_megalinter.LinterTestRoot import LinterTestRoot @@ -12,3 +14,35 @@ class powershell_powershell_formatter_test(TestCase, LinterTestRoot): descriptor_id = "POWERSHELL" linter_name = "powershell_formatter" + + def test_success(self): + self.check_if_another_test_suite() + super().test_success() + + def test_failure(self): + self.check_if_another_test_suite() + super().test_failure() + + def test_get_linter_version(self): + self.check_if_another_test_suite() + super().test_get_linter_version() + + def test_get_linter_help(self): + self.check_if_another_test_suite() + super().test_get_linter_help() + + def test_report_tap(self): + self.check_if_another_test_suite() + super().test_report_tap() + + def test_report_sarif(self): + self.check_if_another_test_suite() + super().test_report_sarif() + + def test_format_fix(self): + self.check_if_another_test_suite() + super().test_format_fix() + + def check_if_another_test_suite(self): + if os.environ.get("SINGLE_LINTER", "") == "POWERSHELL_POWERSHELL": + raise unittest.SkipTest("Skipped because unrelated test suite") diff --git a/megalinter/tests/test_megalinter/mega_linter_1_test.py b/megalinter/tests/test_megalinter/mega_linter_1_test.py index 09fd277f70f..cf7aa46f1ed 100644 --- a/megalinter/tests/test_megalinter/mega_linter_1_test.py +++ b/megalinter/tests/test_megalinter/mega_linter_1_test.py @@ -5,6 +5,7 @@ """ import os import unittest +import uuid import megalinter from megalinter import utilstest @@ -12,22 +13,34 @@ class mega_linter_1_test(unittest.TestCase): + def __init__(self, args) -> None: + self.request_id = str(uuid.uuid1()) + super().__init__(args) + def setUp(self): utilstest.linter_test_setup( { - "sub_lint_root": f"{os.path.sep}.automation{os.path.sep}test{os.path.sep}sample_project" + "request_id": self.request_id, + "sub_lint_root": f"{os.path.sep}.automation{os.path.sep}test{os.path.sep}sample_project", } ) def test_disable_language(self): - mega_linter, output = utilstest.call_mega_linter({"DISABLE": "GROOVY"}) + mega_linter, output = utilstest.call_mega_linter( + { + "DISABLE": "GROOVY,REPOSITORY,SPELL", + "request_id": self.request_id, + } + ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" ) utilstest.assert_is_skipped("GROOVY", output, self) def test_disable_language_legacy(self): - mega_linter, output = utilstest.call_mega_linter({"VALIDATE_GROOVY": "false"}) + mega_linter, output = utilstest.call_mega_linter( + {"VALIDATE_GROOVY": "false", "request_id": self.request_id} + ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" ) @@ -35,7 +48,7 @@ def test_disable_language_legacy(self): def test_disable_linter(self): mega_linter, output = utilstest.call_mega_linter( - {"DISABLE_LINTERS": "JAVASCRIPT_ES"} + {"DISABLE_LINTERS": "JAVASCRIPT_ES", "request_id": self.request_id} ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" @@ -46,7 +59,7 @@ def test_disable_linter(self): def test_disable_linter_legacy(self): mega_linter, output = utilstest.call_mega_linter( - {"VALIDATE_JAVASCRIPT_ES": "false"} + {"VALIDATE_JAVASCRIPT_ES": "false", "request_id": self.request_id} ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" @@ -57,7 +70,7 @@ def test_disable_linter_legacy(self): def test_enable_only_one_linter(self): mega_linter, output = utilstest.call_mega_linter( - {"ENABLE_LINTERS": "JAVASCRIPT_ES"} + {"ENABLE_LINTERS": "JAVASCRIPT_ES", "request_id": self.request_id} ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" @@ -69,7 +82,7 @@ def test_enable_only_one_linter(self): def test_enable_only_one_linter_legacy(self): mega_linter, output = utilstest.call_mega_linter( - {"VALIDATE_JAVASCRIPT_ES": "true"} + {"VALIDATE_JAVASCRIPT_ES": "true", "request_id": self.request_id} ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" @@ -80,7 +93,9 @@ def test_enable_only_one_linter_legacy(self): utilstest.assert_is_skipped("GROOVY", output, self) def test_enable_only_one_language(self): - mega_linter, output = utilstest.call_mega_linter({"ENABLE": "JAVASCRIPT"}) + mega_linter, output = utilstest.call_mega_linter( + {"ENABLE": "JAVASCRIPT", "request_id": self.request_id} + ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" ) @@ -91,7 +106,7 @@ def test_enable_only_one_language(self): def test_enable_only_one_language_legacy(self): mega_linter, output = utilstest.call_mega_linter( - {"VALIDATE_JAVASCRIPT": "true"} + {"VALIDATE_JAVASCRIPT": "true", "request_id": self.request_id} ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" @@ -103,6 +118,7 @@ def test_enable_only_one_language_legacy(self): def test_validate_all_code_base_false(self): megalinter.config.set_value( + self.request_id, "GITHUB_WORKSPACE", ( DEFAULT_DOCKER_WORKSPACE_DIR @@ -117,6 +133,7 @@ def test_validate_all_code_base_false(self): { "ENABLE_LINTERS": "PYTHON_PYLINT", "VALIDATE_ALL_CODEBASE": "false", + "request_id": self.request_id, } ) self.assertTrue( @@ -129,6 +146,7 @@ def test_override_linter_rules_path(self): "ENABLE_LINTERS": "JAVASCRIPT_ES", "LINTER_RULES_PATH": ".", "JAVASCRIPT_ES_CONFIG_FILE": ".eslintrc-custom.yml", + "request_id": self.request_id, } ) self.assertTrue( @@ -144,6 +162,7 @@ def test_override_linter_rules_path_remote(self): "ENABLE_LINTERS": "JAVASCRIPT_ES", "LINTER_RULES_PATH": f"https://raw.githubusercontent.com/{ML_REPO}/main" "/.automation/test/sample_project", + "request_id": self.request_id, } ) self.assertTrue( @@ -164,6 +183,7 @@ def test_override_linter_rules_path_remote_custom_file_name(self): "LINTER_RULES_PATH": f"https://raw.githubusercontent.com/{ML_REPO}/main/" ".automation/test/sample_project", "JAVASCRIPT_ES_CONFIG_FILE": ".eslintrc-custom.yml", + "request_id": self.request_id, } ) self.assertTrue( @@ -183,6 +203,7 @@ def test_override_linter_rules_path_remote_error(self): { "ENABLE_LINTERS": "JAVASCRIPT_ES", "LINTER_RULES_PATH": "https://raw.githubusercontent.com/notexisting/wesh", + "request_id": self.request_id, } ) self.assertTrue( @@ -202,6 +223,7 @@ def test_custom_config_on_language(self): "JAVASCRIPT_CONFIG_FILE": ".eslintrc-custom.yml", "JAVASCRIPT_FILTER_REGEX_INCLUDE": r"(.*_good_.*|.*\/good\/.*)", "JAVASCRIPT_FILTER_REGEX_EXCLUDE": r"(.*_bad_.*|.*\/bad\/.*)", + "request_id": self.request_id, } ) self.assertTrue( @@ -216,6 +238,7 @@ def test_general_include_exclude(self): "ENABLE_LINTERS": "JAVASCRIPT_ES", "FILTER_REGEX_INCLUDE": r"(.*_good_.*|.*\/good\/.*)", "FILTER_REGEX_EXCLUDE": r"(.*_bad_.*|.*\/bad\/.*)", + "request_id": self.request_id, } ) self.assertTrue( @@ -232,6 +255,7 @@ def test_custom_config_on_linter(self): "JAVASCRIPT_FILTER_REGEX_INCLUDE": r"(.*_good_.*|.*\/good\/.*)", "JAVASCRIPT_FILTER_REGEX_EXCLUDE": r"(.*_bad_.*|.*\/bad\/.*)", "MULTI_STATUS": "false", + "request_id": self.request_id, } ) self.assertTrue( @@ -248,6 +272,7 @@ def test_user_arguments_on_linter(self): "JAVASCRIPT_ES_ARGUMENTS": "--debug --env-info", "MULTI_STATUS": "false", "LOG_LEVEL": "DEBUG", + "request_id": self.request_id, } ) self.assertTrue( @@ -262,7 +287,11 @@ def test_alpaca(self): def test_new_flavor_suggestion(self): mega_linter, output = utilstest.call_mega_linter( - {"MULTI_STATUS": "false", "LOG_LEVEL": "DEBUG"} + { + "MULTI_STATUS": "false", + "LOG_LEVEL": "DEBUG", + "request_id": self.request_id, + } ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" @@ -270,7 +299,13 @@ def test_new_flavor_suggestion(self): self.assertEqual("new", mega_linter.flavor_suggestions[0]) def test_json_output(self): - mega_linter, output = utilstest.call_mega_linter({"JSON_REPORTER": "true"}) + mega_linter, output = utilstest.call_mega_linter( + { + "JSON_REPORTER": "true", + "request_id": self.request_id, + "ENABLE_LINTERS": "JAVASCRIPT_ES", + } + ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" ) @@ -284,7 +319,12 @@ def test_json_output(self): def test_json_output_detailed(self): mega_linter, output = utilstest.call_mega_linter( - {"JSON_REPORTER": "true", "JSON_REPORTER_OUTPUT_DETAIL": "detailed"} + { + "JSON_REPORTER": "true", + "JSON_REPORTER_OUTPUT_DETAIL": "detailed", + "ENABLE_LINTERS": "JAVASCRIPT_ES", + "request_id": self.request_id, + } ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" @@ -303,6 +343,7 @@ def test_tap_output_detailed(self): "ENABLE_LINTERS": "JAVASCRIPT_ES", "TAP_REPORTER": "true", "TAP_REPORTER_OUTPUT_DETAIL": "detailed", + "request_id": self.request_id, } ) self.assertTrue( @@ -321,7 +362,13 @@ def test_tap_output_detailed(self): ) def test_config_reporter(self): - mega_linter, output = utilstest.call_mega_linter({"CONFIG_REPORTER": "true"}) + mega_linter, output = utilstest.call_mega_linter( + { + "CONFIG_REPORTER": "true", + "request_id": self.request_id, + "ENABLE_LINTERS": "JAVASCRIPT_ES", + } + ) self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" ) @@ -336,8 +383,9 @@ def test_config_reporter(self): def test_override_cli_lint_mode(self): mega_linter, output = utilstest.call_mega_linter( { - "ENABLE": "YAML", + "ENABLE_LINTERS": "YAML_YAMLLINT", "YAML_YAMLLINT_CLI_LINT_MODE": "file", + "request_id": self.request_id, } ) self.assertTrue( @@ -365,6 +413,7 @@ def test_print_all_files_false_and_no_flavor_suggestion(self): "PRINT_ALL_FILES": "false", "MEGALINTER_FLAVOR": "javascript", "FLAVOR_SUGGESTIONS": "false", + "request_id": self.request_id, } ) self.assertTrue( @@ -380,6 +429,7 @@ def test_list_of_files_sent(self): "PRINT_ALL_FILES": "false", "MEGALINTER_FLAVOR": "javascript", "FLAVOR_SUGGESTIONS": "false", + "request_id": self.request_id, } ) self.assertTrue( @@ -396,6 +446,7 @@ def test_skip_cli_lint_mode(self): "MEGALINTER_FLAVOR": "javascript", "FLAVOR_SUGGESTIONS": "false", "SKIP_CLI_LINT_MODES": "list_of_files", + "request_id": self.request_id, } ) self.assertIn( diff --git a/megalinter/tests/test_megalinter/mega_linter_2_fixes_test.py b/megalinter/tests/test_megalinter/mega_linter_2_fixes_test.py index 64d70ac9dc5..3fa54ef0715 100644 --- a/megalinter/tests/test_megalinter/mega_linter_2_fixes_test.py +++ b/megalinter/tests/test_megalinter/mega_linter_2_fixes_test.py @@ -4,85 +4,21 @@ """ import os -import time import unittest +import uuid from megalinter import utilstest class mega_linter_2_fixes_test(unittest.TestCase): + def __init__(self, args) -> None: + self.request_id = str(uuid.uuid1()) + super().__init__(args) + def setUp(self): utilstest.linter_test_setup( { - "sub_lint_root": f"{os.path.sep}.automation{os.path.sep}test{os.path.sep}sample_project_fixes" - } - ) - - def test_1_apply_fixes_on_one_linter(self): - mega_linter, output = utilstest.call_mega_linter( - { - "APPLY_FIXES": "JAVASCRIPT_PRETTIER", - "LOG_LEVEL": "DEBUG", - "MULTI_STATUS": "false", - "JAVASCRIPT_DEFAULT_STYLE": "prettier", - "DISABLE_LINTERS": "TERRAFORM_KICS,REPOSITORY_GITLEAKS,REPOSITORY_TRIVY," - "JSON_V8R,YAML_V8R,MARKDOWN_MARKDOWN_LINK_CHECK,TERRAFORM_CHECKOV", + "request_id": self.request_id, + "sub_lint_root": f"{os.path.sep}.automation{os.path.sep}test{os.path.sep}sample_project_fixes", } ) - self.assertTrue( - len(mega_linter.linters) > 0, "Linters have been created and run" - ) - self.assertIn("Linted [JAVASCRIPT] files", output) - time.sleep(5) - utilstest.assert_file_has_been_updated("javascript_for_fixes_1.js", True, self) - utilstest.assert_file_has_been_updated("env_for_fixes_1.env", False, self) - - def test_2_apply_fixes_on_all_linters(self): - mega_linter, output = utilstest.call_mega_linter( - { - "APPLY_FIXES": "all", - "LOG_LEVEL": "DEBUG", - "MULTI_STATUS": "false", - "JAVASCRIPT_DEFAULT_STYLE": "prettier", - "DISABLE_LINTERS": "TERRAFORM_KICS,REPOSITORY_GITLEAKS,REPOSITORY_TRIVY," - "JSON_V8R,YAML_V8R,MARKDOWN_MARKDOWN_LINK_CHECK,TERRAFORM_CHECKOV", - } - ) - self.assertTrue( - len(mega_linter.linters) > 0, "Linters have been created and run" - ) - self.assertIn("Linted [JAVASCRIPT] files", output) - time.sleep(5) - # Check fixable files has been updated - fixable_files = [ - "bash_for_fixes_1.sh", - "csharp_for_fixes_1.cs", - "env_for_fixes_1.env", - "groovy_for_fixes_1.groovy", - "javascript_for_fixes_1.js", - "kotlin_for_fixes_1.kt", - "markdown_for_fixes_1.md", - "python_for_fixes_1.py", - # "rst_for_fixes_1.rst", - "ruby_for_fixes_1.rb", - "spell_for_fixes_1.js", - # "scala_for_fixes_1.scala", - # "snakemake_for_fixes_1.smk", - "vbdotnet_for_fixes_1.vb", - "yaml_for_fixes_1.yml", - ] - # updated_dir = config.get("UPDATED_SOURCES_REPORTER_DIR", "updated_sources") - # updated_sources_dir = f"{mega_linter.report_folder}{os.path.sep}{updated_dir}" - for fixable_file in fixable_files: - # Check linters applied updates - utilstest.assert_file_has_been_updated(fixable_file, True, self) - # Check UpdatedSourcesReporter result - # file_name = ( - # updated_sources_dir - # + os.path.sep - # + fixable_file.replace(DEFAULT_DOCKER_WORKSPACE_DIR, "") - # ) - # self.assertTrue( - # os.path.isfile(file_name), - # f"File {file_name} not found in UpdatedSources report", - # ) diff --git a/megalinter/tests/test_megalinter/mega_linter_3_sarif_test.py b/megalinter/tests/test_megalinter/mega_linter_3_sarif_test.py index 38a476636d3..d8b3b1673d3 100644 --- a/megalinter/tests/test_megalinter/mega_linter_3_sarif_test.py +++ b/megalinter/tests/test_megalinter/mega_linter_3_sarif_test.py @@ -25,10 +25,15 @@ class mega_linter_3_sarif_test(unittest.TestCase): + def __init__(self, args) -> None: + self.request_id = str(uuid.uuid1()) + super().__init__(args) + def setUp(self): utilstest.linter_test_setup( { - "sub_lint_root": f"{os.path.sep}.automation{os.path.sep}test{os.path.sep}sample_project_sarif" + "request_id": self.request_id, + "sub_lint_root": f"{os.path.sep}.automation{os.path.sep}test{os.path.sep}sample_project_sarif", } ) @@ -38,8 +43,9 @@ def test_sarif_output(self): "APPLY_FIXES": "false", "LOG_LEVEL": "DEBUG", "MULTI_STATUS": "false", - "ENABLE_LINTERS": "JAVASCRIPT_ES,REPOSITORY_TRIVY,REPOSITORY_GITLEAKS,PYTHON_BANDIT,TERRAFORM_KICS", + "ENABLE_LINTERS": "JAVASCRIPT_ES,PYTHON_BANDIT", "SARIF_REPORTER": "true", + "request_id": self.request_id, } ) self.assertTrue( @@ -55,7 +61,7 @@ def test_sarif_output(self): def test_sarif_fix(self): # Create megalinter - mega_linter = MegaLinter.Megalinter() + mega_linter = MegaLinter.Megalinter({"request_id": uuid.uuid1()}) # Create sample linters sarif_dir = ( root diff --git a/megalinter/tests/test_megalinter/plugins_test.py b/megalinter/tests/test_megalinter/plugins_test.py index 4920482c8c1..36d42882459 100644 --- a/megalinter/tests/test_megalinter/plugins_test.py +++ b/megalinter/tests/test_megalinter/plugins_test.py @@ -5,16 +5,22 @@ """ import os import unittest +import uuid from megalinter import utilstest from megalinter.constants import ML_REPO class plugins_test(unittest.TestCase): + def __init__(self, args) -> None: + self.request_id = str(uuid.uuid1()) + super().__init__(args) + def setUp(self): utilstest.linter_test_setup( { - "sub_lint_root": f"{os.path.sep}.automation{os.path.sep}test{os.path.sep}mega-linter-plugin-test" + "request_id": self.request_id, + "sub_lint_root": f"{os.path.sep}.automation{os.path.sep}test{os.path.sep}mega-linter-plugin-test", } ) @@ -32,6 +38,8 @@ def test_load_plugin_success(self): "LOG_LEVEL": "DEBUG", "MULTI_STATUS": "false", "GITHUB_COMMENT_REPORTER": "false", + "DISABLE": "REPOSITORY,SPELL", + "request_id": self.request_id, } ) self.assertTrue( @@ -48,6 +56,8 @@ def test_load_local_plugin_success(self): "LOG_LEVEL": "DEBUG", "MULTI_STATUS": "false", "GITHUB_COMMENT_REPORTER": "false", + "DISABLE": "REPOSITORY,SPELL", + "request_id": self.request_id, } ) self.assertTrue( @@ -65,6 +75,8 @@ def test_load_local_plugin_fail(self): "LOG_LEVEL": "DEBUG", "MULTI_STATUS": "false", "GITHUB_COMMENT_REPORTER": "false", + "DISABLE": "REPOSITORY,SPELL", + "request_id": self.request_id, } ) except Exception as e: @@ -80,6 +92,8 @@ def test_load_local_plugin_read_fail(self): "LOG_LEVEL": "DEBUG", "MULTI_STATUS": "false", "GITHUB_COMMENT_REPORTER": "false", + "DISABLE": "REPOSITORY,SPELL", + "request_id": self.request_id, } ) except Exception as e: @@ -94,6 +108,8 @@ def test_load_plugin_http_error(self): "LOG_LEVEL": "DEBUG", "MULTI_STATUS": "false", "GITHUB_COMMENT_REPORTER": "false", + "DISABLE": "REPOSITORY,SPELL", + "request_id": self.request_id, } ) except Exception as e: @@ -108,6 +124,8 @@ def test_load_plugin_host_url_error_1(self): "LOG_LEVEL": "DEBUG", "MULTI_STATUS": "false", "GITHUB_COMMENT_REPORTER": "false", + "DISABLE": "REPOSITORY,SPELL", + "request_id": self.request_id, } ) except Exception as e: @@ -126,6 +144,8 @@ def test_load_plugin_file_name_error(self): "LOG_LEVEL": "DEBUG", "MULTI_STATUS": "false", "GITHUB_COMMENT_REPORTER": "false", + "DISABLE": "REPOSITORY,SPELL", + "request_id": self.request_id, } ) except Exception as e: @@ -142,6 +162,8 @@ def test_load_plugin_format_error_2(self): "LOG_LEVEL": "DEBUG", "MULTI_STATUS": "false", "GITHUB_COMMENT_REPORTER": "false", + "DISABLE": "REPOSITORY,SPELL", + "request_id": self.request_id, } ) except Exception as e: diff --git a/megalinter/tests/test_megalinter/pre_post_test.py b/megalinter/tests/test_megalinter/pre_post_test.py index c74fd4c85ca..6d38817f21c 100644 --- a/megalinter/tests/test_megalinter/pre_post_test.py +++ b/megalinter/tests/test_megalinter/pre_post_test.py @@ -5,14 +5,20 @@ """ import os import unittest +import uuid from megalinter import utilstest class PrePostTest(unittest.TestCase): + def __init__(self, args) -> None: + self.request_id = str(uuid.uuid1()) + super().__init__(args) + def setUp(self): utilstest.linter_test_setup( { + "request_id": self.request_id, "sub_lint_root": f"{os.path.sep}.automation{os.path.sep}test{os.path.sep}pre-post-test", "required_config_file": True, } @@ -24,6 +30,7 @@ def test_pre_post_success(self): "MULTI_STATUS": "false", "GITHUB_COMMENT_REPORTER": "false", "LOG_LEVEL": "DEBUG", + "request_id": self.request_id, } ) self.assertTrue( diff --git a/megalinter/utils.py b/megalinter/utils.py index 95fec59e57b..ecc816bf642 100644 --- a/megalinter/utils.py +++ b/megalinter/utils.py @@ -68,7 +68,7 @@ ] -def get_excluded_directories(): +def get_excluded_directories(request_id): default_excluded_dirs = [ "__pycache__", ".git", @@ -80,10 +80,12 @@ def get_excluded_directories(): ".terraform", ".terragrunt-cache", "node_modules", - config.get("REPORT_OUTPUT_FOLDER", "megalinter-reports"), + config.get(request_id, "REPORT_OUTPUT_FOLDER", "megalinter-reports"), ] - excluded_dirs = config.get_list("EXCLUDED_DIRECTORIES", default_excluded_dirs) - excluded_dirs += config.get_list("ADDITIONAL_EXCLUDED_DIRECTORIES", []) + excluded_dirs = config.get_list( + request_id, "EXCLUDED_DIRECTORIES", default_excluded_dirs + ) + excluded_dirs += config.get_list(request_id, "ADDITIONAL_EXCLUDED_DIRECTORIES", []) return set(excluded_dirs) @@ -235,11 +237,13 @@ def list_active_reporters_for_scope(scope, reporter_init_params): return reporters -def check_activation_rules(activation_rules, _linter): +def check_activation_rules(activation_rules, linter): active = False for rule in activation_rules: if rule["type"] == "variable": - value = config.get(rule["variable"], rule["default_value"]) + value = config.get( + linter.request_id, rule["variable"], rule["default_value"] + ) if value == rule["expected_value"]: active = True else: diff --git a/megalinter/utils_reporter.py b/megalinter/utils_reporter.py index 3249969a320..7bb78d6615b 100644 --- a/megalinter/utils_reporter.py +++ b/megalinter/utils_reporter.py @@ -125,7 +125,7 @@ def build_markdown_summary(reporter_self, action_run_url): " if you use a MegaLinter flavor:" + os.linesep ) for suggestion in reporter_self.master.flavor_suggestions: - build_version = os.environ.get("BUILD_VERSION", DEFAULT_RELEASE) + build_version = config.get(None, "BUILD_VERSION", DEFAULT_RELEASE) action_version = "v5" if len(build_version) > 20 else build_version action_path = ( f"{ML_REPO}/flavors/{suggestion['flavor']}@{action_version}" @@ -136,7 +136,12 @@ def build_markdown_summary(reporter_self, action_run_url): ) p_r_msg += os.linesep # Link to ox - if config.get("REPORTERS_MARKDOWN_TYPE", "advanced") == "simple": + if ( + config.get( + reporter_self.master.request_id, "REPORTERS_MARKDOWN_TYPE", "advanced" + ) + == "simple" + ): p_r_msg += ( os.linesep + "MegaLinter is graciously provided by [OX Security]" @@ -168,7 +173,10 @@ def get_linter_doc_url(linter): def log_section_start(section_key: str, section_title: str): - if "CI" in os.environ and config.get("CONSOLE_REPORTER_SECTIONS", "true") == "true": + if ( + "CI" in os.environ + and config.get(None, "CONSOLE_REPORTER_SECTIONS", "true") == "true" + ): if is_github_actions(): return f"::group::{section_title} (expand for details)" elif is_gitlab_ci(): @@ -182,7 +190,10 @@ def log_section_start(section_key: str, section_title: str): def log_section_end(section_key): - if "CI" in os.environ and config.get("CONSOLE_REPORTER_SECTIONS", "true") == "true": + if ( + "CI" in os.environ + and config.get(None, "CONSOLE_REPORTER_SECTIONS", "true") == "true" + ): if is_github_actions(): return "::endgroup::" elif is_gitlab_ci(): @@ -205,7 +216,7 @@ def is_azure_pipelines() -> bool: # Convert SARIF into human readable text -def convert_sarif_to_human(sarif_in) -> str: +def convert_sarif_to_human(sarif_in, request_id) -> str: sarif_fmt_command = "sarif-fmt" process = subprocess.run( sarif_fmt_command, @@ -213,6 +224,7 @@ def convert_sarif_to_human(sarif_in) -> str: stderr=subprocess.STDOUT, text=True, input=sarif_in + "\n", + env=config.build_env(request_id), ) return_code = process.returncode output = utils.decode_utf8(process.stdout) diff --git a/megalinter/utils_sarif.py b/megalinter/utils_sarif.py index f226558837d..d5d1ddb63e7 100644 --- a/megalinter/utils_sarif.py +++ b/megalinter/utils_sarif.py @@ -53,9 +53,12 @@ def normalize_sarif_files(linter): ) # In case SARIF is active, and default workspace is set, clear that from sarif files - default_workspace = config.get("DEFAULT_WORKSPACE") + default_workspace = config.get(linter.request_id, "DEFAULT_WORKSPACE") if ( - config.get("SARIF_REPORTER_NORMALIZE_LINTERS_OUTPUT", True) == "true" + config.get( + linter.request_id, "SARIF_REPORTER_NORMALIZE_LINTERS_OUTPUT", True + ) + == "true" and default_workspace ): clear_default_workspace_prefix( diff --git a/megalinter/utilstest.py b/megalinter/utilstest.py index a31a6be74ee..f8136961eea 100644 --- a/megalinter/utilstest.py +++ b/megalinter/utilstest.py @@ -11,7 +11,7 @@ import unittest import uuid from datetime import datetime -from distutils.dir_util import copy_tree +from shutil import copytree from git import Repo from megalinter import Megalinter, config, utils @@ -49,36 +49,10 @@ def get_root_dir(): # Define env variables before any test case def linter_test_setup(params=None): - for key in [ - "APPLY_FIXES", - "CLEAR_REPORT_FOLDER", - "ENABLE", - "ENABLE_LINTERS", - "DISABLE", - "DISABLE_LINTERS", - "MEGALINTER_CONFIG", - "EXTENDS", - "FILTER_REGEX_INCLUDE", - "FILTER_REGEX_EXCLUDE", - "IGNORE_GITIGNORED_FILES", - "IGNORE_GENERATED_FILES", - "SHOW_ELAPSED_TIME", - "UPDATED_SOURCES_REPORTER", - "MEGALINTER_FLAVOR", - "FLAVOR_SUGGESTIONS", - "DISABLE_ERRORS", - "SARIF_REPORTER", - "LOG_FILE", - "REPORT_OUTPUT_FOLDER", - "OUTPUT_FOLDER", - "REPOSITORY_SEMGREP_RULESETS_TYPE", - "REPOSITORY_SEMGREP_RULESETS", - ]: - if key in os.environ: - del os.environ[key] config.delete() if params is None: - params = {} + params = {"request_id": str(uuid.uuid1())} + request_id = params["request_id"] # Root to lint sub_lint_root = ( params["sub_lint_root"] @@ -96,61 +70,61 @@ def linter_test_setup(params=None): raise Exception( f"[test] There should be a .mega-linter.yml file in test folder {config_file_path}" ) - config.init_config(workspace) + config.init_config(request_id, workspace) # Ignore report folder - config.set_value("FILTER_REGEX_EXCLUDE", r"\/megalinter-reports\/") + config.set_value(request_id, "FILTER_REGEX_EXCLUDE", r"\/megalinter-reports\/") # TAP Output deactivated by default - config.set_value("OUTPUT_FORMAT", "text") - config.set_value("OUTPUT_DETAIL", "detailed") - config.set_value("PLUGINS", "") - config.set_value("GITHUB_STATUS_REPORTER", "false") - config.set_value("IGNORE_GITIGNORED_FILES", "true") - config.set_value("VALIDATE_ALL_CODEBASE", "true") - config.set_value("CLEAR_REPORT_FOLDER", "true") + config.set_value(request_id, "OUTPUT_FORMAT", "text") + config.set_value(request_id, "OUTPUT_DETAIL", "detailed") + config.set_value(request_id, "PLUGINS", "") + config.set_value(request_id, "GITHUB_STATUS_REPORTER", "false") + config.set_value(request_id, "IGNORE_GITIGNORED_FILES", "true") + config.set_value(request_id, "VALIDATE_ALL_CODEBASE", "true") + config.set_value(request_id, "CLEAR_REPORT_FOLDER", "true") if params.get("additional_test_variables"): for env_var_key, env_var_value in params.get( "additional_test_variables" ).items(): - config.set_value(env_var_key, env_var_value) + config.set_value(request_id, env_var_key, env_var_value) # Root path of files to lint config.set_value( + request_id, "DEFAULT_WORKSPACE", ( - config.get("DEFAULT_WORKSPACE") + sub_lint_root - if config.exists("DEFAULT_WORKSPACE") - and os.path.isdir(config.get("DEFAULT_WORKSPACE") + sub_lint_root) + config.get(request_id, "DEFAULT_WORKSPACE") + sub_lint_root + if config.exists(request_id, "DEFAULT_WORKSPACE") + and os.path.isdir( + config.get(request_id, "DEFAULT_WORKSPACE") + sub_lint_root + ) else root_dir + sub_lint_root ), ) - assert os.path.isdir(config.get("DEFAULT_WORKSPACE")), ( + assert os.path.isdir(config.get(request_id, "DEFAULT_WORKSPACE")), ( "DEFAULT_WORKSPACE " - + config.get("DEFAULT_WORKSPACE") + + config.get(request_id, "DEFAULT_WORKSPACE") + " is not a valid folder" ) def print_output(output): - if config.exists("OUTPUT_DETAIL") and config.get("OUTPUT_DETAIL") == "detailed": + if ( + config.exists(None, "OUTPUT_DETAIL") + and config.get(None, "OUTPUT_DETAIL") == "detailed" + ): for line in output.splitlines(): print(line) def call_mega_linter(env_vars): - prev_environ = config.copy() usage_stdout = io.StringIO() + request_id = env_vars["request_id"] with contextlib.redirect_stdout(usage_stdout): # Set env variables for env_var_key, env_var_value in env_vars.items(): - config.set_value(env_var_key, env_var_value) + config.set_value(request_id, env_var_key, env_var_value) # Call linter - mega_linter = Megalinter() + mega_linter = Megalinter({"request_id": request_id}) mega_linter.run() - # Set back env variable previous values - for env_var_key, env_var_value in env_vars.items(): - if env_var_key in prev_environ: - config.set_value(env_var_key, prev_environ[env_var_key]) - else: - config.delete(env_var_key) output = usage_stdout.getvalue().strip() print_output(output) return mega_linter, output @@ -165,7 +139,9 @@ def test_linter_success(linter, test_self): ): raise unittest.SkipTest("Linter has been disabled") test_folder = linter.test_folder - workspace = config.get("DEFAULT_WORKSPACE") + os.path.sep + test_folder + workspace = ( + config.get(linter.request_id, "DEFAULT_WORKSPACE") + os.path.sep + test_folder + ) # Special cases when files must be copied in a temp directory before being linted if os.path.isdir(workspace + os.path.sep + "good"): workspace = workspace + os.path.sep + "good" @@ -181,7 +157,8 @@ def test_linter_success(linter, test_self): "REPORT_OUTPUT_FOLDER": tmp_report_folder, "LOG_LEVEL": "DEBUG", "ENABLE_LINTERS": linter.name, - "PRINT_ALL_FILES": True, + "PRINT_ALL_FILES": "true", + "request_id": test_self.request_id, } env_vars.update(linter.test_variables) mega_linter, output = call_mega_linter(env_vars) @@ -222,7 +199,9 @@ def test_linter_failure(linter, test_self): ): raise unittest.SkipTest("Linter or test has been disabled") test_folder = linter.test_folder - workspace = config.get("DEFAULT_WORKSPACE") + os.path.sep + test_folder + workspace = ( + config.get(linter.request_id, "DEFAULT_WORKSPACE") + os.path.sep + test_folder + ) if os.path.isdir(workspace + os.path.sep + "bad"): workspace = workspace + os.path.sep + "bad" workspace = manage_copy_sources(workspace) @@ -242,6 +221,7 @@ def test_linter_failure(linter, test_self): "REPORT_OUTPUT_FOLDER": tmp_report_folder, "LOG_LEVEL": "DEBUG", "ENABLE_LINTERS": linter.name, + "request_id": test_self.request_id, } env_vars_failure.update(linter.test_variables) mega_linter, output = call_mega_linter(env_vars_failure) @@ -302,7 +282,7 @@ def test_linter_failure(linter, test_self): def manage_copy_sources(workspace): if os.path.isfile(workspace + os.path.sep + "test_copy_in_tmp_folder"): tmp_sources_folder = tempfile.gettempdir() + os.path.sep + str(uuid.uuid4()) - copy_tree(workspace, tmp_sources_folder) + copytree(workspace, tmp_sources_folder) workspace = tmp_sources_folder return workspace @@ -434,7 +414,9 @@ def test_linter_report_tap(linter, test_self): ): raise unittest.SkipTest("Linter has been disabled") test_folder = linter.test_folder - workspace = config.get("DEFAULT_WORKSPACE") + os.path.sep + test_folder + workspace = ( + config.get(linter.request_id, "DEFAULT_WORKSPACE") + os.path.sep + test_folder + ) assert os.path.isdir(workspace), f"Test folder {workspace} is not existing" expected_file_name = "" # Identify expected report if defined @@ -462,6 +444,7 @@ def test_linter_report_tap(linter, test_self): "OUTPUT_DETAIL": "detailed", "REPORT_OUTPUT_FOLDER": tmp_report_folder, "ENABLE_LINTERS": linter.name, + "request_id": test_self.request_id, } env_vars.update(linter.test_variables) mega_linter, _output = call_mega_linter(env_vars) @@ -525,7 +508,9 @@ def test_linter_report_sarif(linter, test_self): ): raise unittest.SkipTest("SARIF is not configured for this linter") test_folder = linter.test_folder - workspace = config.get("DEFAULT_WORKSPACE") + os.path.sep + test_folder + workspace = ( + config.get(linter.request_id, "DEFAULT_WORKSPACE") + os.path.sep + test_folder + ) assert os.path.isdir(workspace), f"Test folder {workspace} is not existing" # Call linter tmp_report_folder = tempfile.gettempdir() + os.path.sep + str(uuid.uuid4()) @@ -536,6 +521,7 @@ def test_linter_report_sarif(linter, test_self): "ENABLE_LINTERS": linter.name, "LOG_LEVEL": "DEBUG", "LOG_FILE": "megalinter.log", + "request_id": test_self.request_id, } env_vars.update(linter.test_variables) mega_linter, _output = call_mega_linter(env_vars) @@ -610,7 +596,9 @@ def test_linter_format_fix(linter, test_self): ): raise unittest.SkipTest("Linter doesn't format and can't apply fixes") test_folder = linter.test_folder - workspace = config.get("DEFAULT_WORKSPACE") + os.path.sep + test_folder + workspace = ( + config.get(linter.request_id, "DEFAULT_WORKSPACE") + os.path.sep + test_folder + ) # Special cases when files must be copied in a temp directory before being linted if os.path.isdir(workspace + os.path.sep + "fix"): workspace = workspace + os.path.sep + "fix" @@ -649,7 +637,8 @@ def test_linter_format_fix(linter, test_self): "REPORT_OUTPUT_FOLDER": tmp_report_folder, "LOG_LEVEL": "DEBUG", "ENABLE_LINTERS": linter.name, - "PRINT_ALL_FILES": True, + "PRINT_ALL_FILES": "true", + "request_id": test_self.request_id, } env_vars.update(linter.test_variables) mega_linter, output = call_mega_linter(env_vars)