From f2aecd4f8402fa4ab755e4bf9b3f43c5b5182d0e Mon Sep 17 00:00:00 2001 From: dorschw <81086590+dorschw@users.noreply.github.com> Date: Tue, 19 Mar 2024 10:42:07 +0200 Subject: [PATCH] Validate content paths (#4084) --- .changelog/4084.yml | 4 + .github/workflows/on-push.yml | 27 ++ .pre-commit-config.yaml | 4 +- .pre-commit-hooks.yaml | 6 + demisto_sdk/commands/common/constants.py | 1 + demisto_sdk/scripts/validate_content_path.py | 431 ++++++++++++++++++ .../tests/validate_content_path_test.py | 291 ++++++++++++ poetry.lock | 23 +- pyproject.toml | 3 +- 9 files changed, 776 insertions(+), 14 deletions(-) create mode 100644 .changelog/4084.yml create mode 100644 demisto_sdk/scripts/validate_content_path.py create mode 100644 demisto_sdk/tests/validate_content_path_test.py diff --git a/.changelog/4084.yml b/.changelog/4084.yml new file mode 100644 index 00000000000..db50464920f --- /dev/null +++ b/.changelog/4084.yml @@ -0,0 +1,4 @@ +changes: +- description: Added the `validate-content-path` **pre-commit** hook + type: feature +pr_number: 4084 diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 50ed62a9765..f4300864ad8 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -322,6 +322,33 @@ jobs: artifacts-path-dir: content/test-pre-commit-command artifact-name: test-demisto-sdk-pre-commit-command-artifacts + test-validate-content-path: + runs-on: ubuntu-latest + name: Test validate-content-path + steps: + - name: Checkout SDK + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Checkout Content + uses: actions/checkout@v4 + with: + fetch-depth: 0 + repository: demisto/content + path: content + + - name: Setup Python Environment + uses: ./.github/actions/setup_environment + with: + python-version: "3.10" + + - name: Validate content master paths + run: | + source $(poetry env info --path)/bin/activate + validate-content-path validate-all content --skip-depth-one-file --skip-depth-one-folder --skip-depth-zero-file --skip-integration-script-file-name --skip-integration-script-file-type --skip-markdown + + test-graph-commands: runs-on: ubuntu-latest name: Test Graph Commands diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d72cba40f18..71d5741b461 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -145,7 +145,7 @@ repos: - ruamel-yaml-clib==0.2.7 ; platform_python_implementation == "CPython" and python_version < "3.11" and python_version >= "3.8" - ruamel-yaml==0.17.22 ; python_version >= "3.8" and python_version < "3.11" - setuptools==67.7.2 ; python_version >= "3.8" and python_version < "3.11" - - shellingham==1.5.0.post1 ; python_version >= "3.8" and python_version < "3.11" + - shellingham==1.5.4 ; python_version >= "3.8" and python_version < "3.11" - six==1.16.0 ; python_version >= "3.8" and python_version < "3.11" - slack-sdk==3.21.3 ; python_version >= "3.8" and python_version < "3.11" - smmap==5.0.0 ; python_version >= "3.8" and python_version < "3.11" @@ -157,7 +157,7 @@ repos: - tomli==2.0.1 ; python_version >= "3.8" and python_version < "3.11" - tqdm==4.65.0 ; python_version >= "3.8" and python_version < "3.11" - typed-ast==1.5.4 ; python_version >= "3.8" and python_version < "3.11" - - typer[all]==0.7.0 ; python_version >= "3.8" and python_version < "3.11" + - typer[all]==0.9.0 ; python_version >= "3.8" and python_version < "3.11" - types-chardet==5.0.4.5 ; python_version >= "3.8" and python_version < "3.11" - types-cryptography==3.3.23.2 ; python_version >= "3.8" and python_version < "3.11" - types-dateparser==1.1.4.9 ; python_version >= "3.8" and python_version < "3.11" diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index be27ae97fd7..565b0c8efc2 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -78,3 +78,9 @@ language: python require_serial: true pass_filenames: false + +- id: validate-content-path + name: validate-content-path + entry: validate-content-path + language: python + pass_filenames: true diff --git a/demisto_sdk/commands/common/constants.py b/demisto_sdk/commands/common/constants.py index f81f4831d44..2f2ee7545ff 100644 --- a/demisto_sdk/commands/common/constants.py +++ b/demisto_sdk/commands/common/constants.py @@ -923,6 +923,7 @@ class FileType(str, Enum): PACKS_CONTRIBUTORS_FILE_NAME = "CONTRIBUTORS.json" AUTHOR_IMAGE_FILE_NAME = "Author_image.png" PACKS_FOLDER = "Packs" +GIT_IGNORE_FILE_NAME = ".gitignore" CONF_JSON_FILE_NAME = "conf.json" diff --git a/demisto_sdk/scripts/validate_content_path.py b/demisto_sdk/scripts/validate_content_path.py new file mode 100644 index 00000000000..181cfbd30fe --- /dev/null +++ b/demisto_sdk/scripts/validate_content_path.py @@ -0,0 +1,431 @@ +from abc import ABC +from pathlib import Path +from typing import ClassVar, List, Sequence + +import typer +from more_itertools import split_at +from tqdm import tqdm +from typing_extensions import Annotated + +from demisto_sdk.commands.common.constants import ( + AUTHOR_IMAGE_FILE_NAME, + CLASSIFIERS_DIR, + CONNECTIONS_DIR, + CORRELATION_RULES_DIR, + DASHBOARDS_DIR, + DOC_FILES_DIR, + GENERIC_DEFINITIONS_DIR, + GENERIC_MODULES_DIR, + GENERIC_TYPES_DIR, + GIT_IGNORE_FILE_NAME, + INCIDENT_FIELDS_DIR, + INCIDENT_TYPES_DIR, + INDICATOR_FIELDS_DIR, + INDICATOR_TYPES_DIR, + JOBS_DIR, + LAYOUT_RULES_DIR, + LAYOUTS_DIR, + LISTS_DIR, + MAPPERS_DIR, + MODELING_RULES_DIR, + PACKS_CONTRIBUTORS_FILE_NAME, + PACKS_FOLDER, + PACKS_PACK_IGNORE_FILE_NAME, + PACKS_PACK_META_FILE_NAME, + PACKS_README_FILE_NAME, + PACKS_WHITELIST_FILE_NAME, + PARSING_RULES_DIR, + PLAYBOOKS_DIR, + PRE_PROCESS_RULES_DIR, + RELEASE_NOTES_DIR, + REPORTS_DIR, + TEST_PLAYBOOKS_DIR, + TESTS_AND_DOC_DIRECTORIES, + TRIGGER_DIR, + WIDGETS_DIR, + WIZARDS_DIR, + XSIAM_DASHBOARDS_DIR, + XSIAM_REPORTS_DIR, +) +from demisto_sdk.commands.common.logger import logger, logging_setup +from demisto_sdk.commands.content_graph.common import ContentType + +ZERO_DEPTH_FILES = frozenset( + ( + GIT_IGNORE_FILE_NAME, + PACKS_PACK_IGNORE_FILE_NAME, + PACKS_WHITELIST_FILE_NAME, + AUTHOR_IMAGE_FILE_NAME, + PACKS_CONTRIBUTORS_FILE_NAME, + PACKS_README_FILE_NAME, + PACKS_PACK_META_FILE_NAME, + ) +) + +DEPTH_ONE_FOLDERS = ( + set(ContentType.folders()) | set(TESTS_AND_DOC_DIRECTORIES) | {RELEASE_NOTES_DIR} +).difference( + ( + "Packs", + "BaseContents", + "BaseNodes", + "BasePlaybooks", + "BaseScripts", + "TestScripts", + "CommandOrScripts", + ) +) + +DEPTH_ONE_FOLDERS_ALLOWED_TO_CONTAIN_FILES = frozenset( + ( + PLAYBOOKS_DIR, + TEST_PLAYBOOKS_DIR, + REPORTS_DIR, + DASHBOARDS_DIR, + INCIDENT_FIELDS_DIR, + INCIDENT_TYPES_DIR, + INDICATOR_FIELDS_DIR, + INDICATOR_TYPES_DIR, + GENERIC_TYPES_DIR, + GENERIC_MODULES_DIR, + GENERIC_DEFINITIONS_DIR, + LAYOUTS_DIR, + CLASSIFIERS_DIR, + MAPPERS_DIR, + CONNECTIONS_DIR, + RELEASE_NOTES_DIR, + DOC_FILES_DIR, + JOBS_DIR, + PRE_PROCESS_RULES_DIR, + LISTS_DIR, + PARSING_RULES_DIR, + MODELING_RULES_DIR, + CORRELATION_RULES_DIR, + XSIAM_DASHBOARDS_DIR, + XSIAM_REPORTS_DIR, + TRIGGER_DIR, + WIDGETS_DIR, + WIZARDS_DIR, + LAYOUT_RULES_DIR, + *TESTS_AND_DOC_DIRECTORIES, + ) +) + +DIRS_ALLOWING_SPACE_IN_FILENAMES = frozenset( + TESTS_AND_DOC_DIRECTORIES + [TEST_PLAYBOOKS_DIR] +) +app = typer.Typer() + + +class InvalidPathException(Exception, ABC): + message: ClassVar[str] + + +class SpacesInFileNameError(InvalidPathException): + message = "File name contains spaces." + + +class InvalidDepthZeroFile(InvalidPathException): + message = "The file cannot be saved directly under the pack folder." + + +class InvalidDepthOneFolder(InvalidPathException): + message = "The name of the first level folder under the pack is not allowed." + + +class InvalidDepthOneFile(InvalidPathException): + message = "The folder containing this file cannot directly contain files. Add another folder under it." + + +class InvalidLayoutFileName(InvalidPathException): + message = "The Layout folder can only contain JSON files, with names starting with `layout-` or `layoutscontainer-`" + + +class InvalidIntegrationScriptFileName(InvalidPathException): + message = "This file's name must start with the name of its parent folder." + + +class InvalidIntegrationScriptFileType(InvalidPathException): + message = "This file type is not allowed under this folder." + + +class InvalidIntegrationScriptMarkdownFileName(InvalidPathException): + message = ( + "This file's name must either be (parent folder)_description.md, or README.md" + ) + + +class InvalidCommandExampleFile(InvalidPathException): + message = "This file's name must be command_examples" + + +class ExemptedPath(Exception, ABC): + message: ClassVar[str] + + +class PathOutsidePacks(ExemptedPath): + message = "Path is not under Packs" + + +class PathIsFolder(ExemptedPath): + message = "Folder paths are not validated" + + +class PathUnderDeprecatedContent(ExemptedPath): + message = "Paths under DeprecatedContent are not validated." + + +class PathIsUnified(ExemptedPath): + message = "Paths of unified content items are not validated." + + +def _validate(path: Path) -> None: + """Runs the logic and raises exceptions on skipped/errorneous paths""" + logger.debug(f"checking {path}") + if path.is_dir(): + raise PathIsFolder + if PACKS_FOLDER not in path.parts: + raise PathOutsidePacks + + if "Tests" in path.parts and (path.parts).index("Tests") < (path.parts).index( + PACKS_FOLDER + ): # if Tests comes before Packs, it's not a real content path + raise PathOutsidePacks + + parts_before_packs, parts_after_packs = tuple( + split_at(path.parts, lambda v: v == PACKS_FOLDER, maxsplit=1) + ) + + if parts_after_packs[0] in {"DeprecatedContent", "D2"}: # Pack name + """ + This set neither does nor should contain all names of deprecated packs. + D2 is unique with the files it has, so it is explicitly mentioned here. + Avoid extending this set beyond these values. + """ + raise PathUnderDeprecatedContent + + parts_inside_pack = parts_after_packs[1:] # everything after Packs/ + depth = len(parts_inside_pack) - 1 + + if depth == 0: # file is directly under pack + if path.name not in ZERO_DEPTH_FILES: + raise InvalidDepthZeroFile + return # following checks assume the depth>0, so we stop here + + if (first_level_folder := parts_inside_pack[0]) not in DEPTH_ONE_FOLDERS: + raise InvalidDepthOneFolder + + if " " in path.stem and set(parts_after_packs).isdisjoint( + DIRS_ALLOWING_SPACE_IN_FILENAMES + ): + raise SpacesInFileNameError + + if depth == 1: # Packs/myPack// + _exempt_unified_files(path, first_level_folder) # Raises PathIsUnified + + if first_level_folder not in DEPTH_ONE_FOLDERS_ALLOWED_TO_CONTAIN_FILES: + # Packs/MyPack/SomeFolderThatShouldntHaveFilesDirectly/ + raise InvalidDepthOneFile + + if first_level_folder == LAYOUTS_DIR and not ( + path.stem.startswith(("layout-", "layoutscontainer-")) + and path.suffix == ".json" + ): + raise InvalidLayoutFileName + + if depth == 2 and first_level_folder in { + ContentType.INTEGRATION.as_folder, + ContentType.SCRIPT.as_folder, + }: + _validate_integration_script_file(path, parts_after_packs) + + +def _validate_integration_script_file(path: Path, parts_after_packs: Sequence[str]): + """Only use from _validate""" + parent = path.parent.name + + if path.suffix == ".png": + if path.stem != f"{parent}_image": + raise InvalidIntegrationScriptFileName + + elif path.suffix in {".yml", ".js"}: + if path.stem != parent: + raise InvalidIntegrationScriptFileName + + elif path.suffix == ".ps1": + if path.stem not in {parent, f"{parent}.Tests"}: + raise InvalidIntegrationScriptFileName + + elif path.suffix == ".py": + if path.stem not in { + parent, + f"{parent}_test", + "conftest", + ".vulture_whitelist", + }: + raise InvalidIntegrationScriptFileName + + elif path.suffix == ".md": + if path.stem not in {"README", f"{parent}_description"}: + raise InvalidIntegrationScriptMarkdownFileName + + elif not path.suffix: + if path.stem in {"command_examples", ".pylintrc"}: + return + if ( + path.stem == "LICENSE" + and parts_after_packs[0] == "FireEye-Detection-on-Demand" + ): + # Decided to exempt this pack only from using LICENSE files. + return + if "command" in path.stem and "example" in path.stem: + # `command example`, `commands examples` and other single/plural, delimiters permutations + raise InvalidCommandExampleFile + raise InvalidIntegrationScriptFileName + + elif path.suffix not in { # remaining supported suffixes + ".png", + ".svg", + ".txt", + }: + raise InvalidIntegrationScriptFileType + + +def _exempt_unified_files(path: Path, first_level_folder: str): + """Raises PathIsUnified when necessary. Only use from _validate""" + for prefix, folder in ( + ("script", ContentType.SCRIPT), + ("integration", ContentType.INTEGRATION), + ): + if ( + first_level_folder == folder.as_folder + and path.name.startswith(f"{prefix}-") + and (path.suffix in {".md", ".yml"}) # these fail validate-all + ): + # old, unified format, e.g. Packs/myPack/Scripts/script-foo.yml + raise PathIsUnified + + +def validate( + path: Path, + github_action: bool, + skip_depth_one_file: bool = False, + skip_depth_one_folder: bool = False, + skip_depth_zero_file: bool = False, + skip_integration_script_file_name: bool = False, + skip_integration_script_file_type: bool = False, + skip_markdown: bool = False, +) -> bool: + """Validate a path, returning a boolean answer after handling skip/error exceptions""" + try: + _validate(path) + logger.debug(f"[green]{path} is valid[/green]") + return True + + except InvalidPathException as e: + for exception_type, skip in { + # Allows gradual application + InvalidDepthOneFile: skip_depth_one_file, + InvalidDepthOneFolder: skip_depth_one_folder, + InvalidDepthZeroFile: skip_depth_zero_file, + InvalidIntegrationScriptFileName: skip_integration_script_file_name, + InvalidIntegrationScriptFileType: skip_integration_script_file_type, + InvalidIntegrationScriptMarkdownFileName: skip_markdown, + }.items(): + if isinstance(e, exception_type) and skip: + logger.warning(f"skipping {path} ({e.message})") + return True + + if github_action: + print( # noqa: T201 + f"::error file={path},line=1,endLine=1,title=Invalid Path::{e.message}" + ) + else: + logger.error(f"Invalid {path}: {e.message}") + return False + + except ExemptedPath as e: + logger.debug(f"Skipped {path}: {e.message}") + return True + + except Exception: + logger.exception(f"Error checking {path}") + return False + + +@app.command(name="validate") +def validate_paths( + paths: Annotated[ + List[Path], typer.Argument(exists=True, file_okay=True, dir_okay=True) + ], + github_action: Annotated[bool, typer.Option(envvar="GITHUB_ACTIONS")] = False, + skip_depth_one_file: bool = False, + skip_depth_one_folder: bool = False, + skip_depth_zero_file: bool = False, + skip_integration_script_file_name: bool = False, + skip_integration_script_file_type: bool = False, + skip_markdown: bool = False, +) -> None: + """Validate given paths""" + if not all( + ( + validate( + path, + github_action, + skip_depth_one_file=skip_depth_one_file, + skip_depth_one_folder=skip_depth_one_folder, + skip_depth_zero_file=skip_depth_zero_file, + skip_integration_script_file_name=skip_integration_script_file_name, + skip_integration_script_file_type=skip_integration_script_file_type, + skip_markdown=skip_markdown, + ) + for path in paths + ) + ): + raise typer.Exit(1) + + +@app.command(name="validate-all") +def validate_all( + content_path: Annotated[Path, typer.Argument(dir_okay=True, file_okay=False)], + skip_depth_one_file: bool = False, + skip_depth_one_folder: bool = False, + skip_depth_zero_file: bool = False, + skip_integration_script_file_name: bool = False, + skip_integration_script_file_type: bool = False, + skip_markdown: bool = False, +): + """ + Used in the SDK CI for testing compatibility with content. + Skip arguments will be removed in future versions. + """ + logger.info(f"Content path: {content_path.resolve()}") + paths = sorted(content_path.rglob("*")) + invalid = len( + [ + path + for path in tqdm(paths) + if path.is_file() + and not validate( + path, + github_action=False, + skip_depth_one_file=skip_depth_one_file, + skip_depth_one_folder=skip_depth_one_folder, + skip_depth_zero_file=skip_depth_zero_file, + skip_integration_script_file_name=skip_integration_script_file_name, + skip_integration_script_file_type=skip_integration_script_file_type, + skip_markdown=skip_markdown, + ) + ] + ) + valid = (total := len(paths)) - invalid + logger.info(f"{total=},[green]{valid=}[/green],[red]{invalid=}[/red]") + + +def main(): + logging_setup() + app() + + +if __name__ == "__main__": + main() diff --git a/demisto_sdk/tests/validate_content_path_test.py b/demisto_sdk/tests/validate_content_path_test.py new file mode 100644 index 00000000000..dd81119b1c4 --- /dev/null +++ b/demisto_sdk/tests/validate_content_path_test.py @@ -0,0 +1,291 @@ +from pathlib import Path + +import pytest + +from demisto_sdk.commands.common.constants import ( + CONTENT_ENTITIES_DIRS, + INTEGRATIONS_DIR, + LAYOUTS_DIR, + PACKS_FOLDER, +) +from demisto_sdk.scripts.validate_content_path import ( + DEPTH_ONE_FOLDERS, + DEPTH_ONE_FOLDERS_ALLOWED_TO_CONTAIN_FILES, + DIRS_ALLOWING_SPACE_IN_FILENAMES, + ZERO_DEPTH_FILES, + InvalidDepthOneFile, + InvalidDepthOneFolder, + InvalidDepthZeroFile, + InvalidIntegrationScriptFileName, + InvalidIntegrationScriptFileType, + InvalidLayoutFileName, + PathIsFolder, + PathIsUnified, + PathUnderDeprecatedContent, + SpacesInFileNameError, + _validate, +) + + +def test_content_entities_dir_length(): + """ + This test is here so we don't forget to update FOLDERS_ALLOWED_TO_CONTAIN_FILES when adding/removing content types. + If this test failed, it's likely you modified either CONTENT_ENTITIES_DIRS or FOLDERS_ALLOWED_TO_CONTAIN_FILES. + Update the test values accordingly. + """ + assert len(set(DEPTH_ONE_FOLDERS_ALLOWED_TO_CONTAIN_FILES)) == 34 + + # change this one if you added a content item folder that can't have files directly under it + assert ( + len( + DEPTH_ONE_FOLDERS_ALLOWED_TO_CONTAIN_FILES.intersection( + CONTENT_ENTITIES_DIRS + ) + ) + == 26 + ) + + +folders_not_allowed_to_contain_files = ( + set(CONTENT_ENTITIES_DIRS) | DEPTH_ONE_FOLDERS +).difference(DEPTH_ONE_FOLDERS_ALLOWED_TO_CONTAIN_FILES) + +DUMMY_PACK_PATH = Path("content", "Packs", "myPack") + + +@pytest.mark.parametrize("file_name", ZERO_DEPTH_FILES) +def test_depth_zero_pass(file_name: str): + """ + Given + A file name which is allowed directly under the pack + When + Running validate_path + Then + Make sure the validation passes + """ + _validate(Path(PACKS_FOLDER, "MyPack", file_name)) + + +@pytest.mark.parametrize("file_name", ("foo.py", "bar.md")) +def test_depth_zero_fail(file_name: str): + """ + Given + A file name which is NOT allowed directly under the pack + When + Running validate_path + Then + Make sure the validation raises InvalidDepthZeroFile + """ + assert file_name not in ZERO_DEPTH_FILES # sanity + with pytest.raises(InvalidDepthZeroFile): + _validate(Path(PACKS_FOLDER, "MyPack", file_name)) + + +def test_first_level_folder_fail(): + """ + Given + A name of a folder, which is NOT allowed as a first-level folder + When + Running validate_path on a file created directly under the folder + Then + Make sure the validation raises InvalidDepthOneFolder + """ + assert (folder_name := "folder_name") not in DEPTH_ONE_FOLDERS + with pytest.raises(InvalidDepthOneFolder): + _validate(Path(DUMMY_PACK_PATH, folder_name, "file")) + with pytest.raises(InvalidDepthOneFolder): + _validate(Path(DUMMY_PACK_PATH, folder_name, "nested", "very nested", "file")) + + +@pytest.mark.parametrize("folder", DEPTH_ONE_FOLDERS) +def test_depth_one_pass(folder: str): + """ + Given + A name of a folder, which IS allowed as a first-level folder + When + Running validate_path on a file created indirectly under it + Then + Make sure the validation does not raise InvalidDepthOneFileError + """ + assert folder in DEPTH_ONE_FOLDERS + try: + _validate(Path(DUMMY_PACK_PATH, folder, "nested", "file")) + _validate(Path(DUMMY_PACK_PATH, folder, "nested", "nested_deeper", "file")) + except (InvalidIntegrationScriptFileType, InvalidIntegrationScriptFileName): + # In Integration/script, InvalidIntegrationScriptFileType will be raised but is irrelevant for this test. + pass + + +@pytest.mark.parametrize("folder", folders_not_allowed_to_contain_files) +def test_depth_one_fail(folder: str): + """ + Given + A name of a folder, which may NOT contain files directly + When + Running validate_path on a file created directly under the folder + Then + Make sure InvalidDepthOneFileError is raised + """ + with pytest.raises(InvalidDepthOneFile): + _validate(DUMMY_PACK_PATH / folder / "file") + + +@pytest.mark.parametrize( + "path", + ( + pytest.param( + Path("Packs/myPack/Scripts/script-foo.yml"), + id="Unified script (yml)", + ), + pytest.param( + Path("Packs/myPack/Scripts/script-foo.md"), + id="Unified script (md)", + ), + pytest.param( + Path("Packs/myPack/Integrations/integration-foo.yml"), + id="Unified integration (yml)", + ), + pytest.param( + Path("Packs/myPack/Integrations/integration-foo.md"), + id="Unified integration (md)", + ), + ), +) +def test_unified_conten(path: Path): + """ + Given + A file under a path under UnifiedContent + When + Running validate_path on the path + Then + Make sure the validation raises PathIsUnified + """ + with pytest.raises(PathIsUnified): + _validate(path) + + +@pytest.mark.parametrize( + "path", + ( + "foo", + "foo/bar", + "foo/bar.py", + "Integrations/myIntegration.yml", + "Integrations/myIntegration/myIntegration.py", + "Integrations/myIntegration/myIntegration.yml", + ), +) +def test_deprecatedcontent(path: str): + with pytest.raises(PathUnderDeprecatedContent): + _validate(Path("Packs/DeprecatedContent", path)) + + +def test_first_level_folders_subset(): + assert DEPTH_ONE_FOLDERS_ALLOWED_TO_CONTAIN_FILES.issubset(DEPTH_ONE_FOLDERS) + + +def test_dir(repo): + """ + Given + A repo + When + Calling validate_path on a folder path + Then + Make sure it raises the apporpiate exception + """ + pack = repo.create_pack("myPack") + integration = pack.create_integration() + with pytest.raises(PathIsFolder): + _validate(Path(pack.path)) + + with pytest.raises(PathIsFolder): + _validate(Path(integration.path)) + + +DUMMY_INTEGRATION_NAME = "MyIntegration" +DUMMY_INTEGRATION_PATH = DUMMY_PACK_PATH / INTEGRATIONS_DIR / DUMMY_INTEGRATION_NAME +MALFORMED_DUMMY_INTEGRATION_NAME = DUMMY_INTEGRATION_NAME + "-" + + +def test_space_invalid(): + with pytest.raises(SpacesInFileNameError): + _validate(DUMMY_INTEGRATION_PATH / "foo bar.yml") + + +@pytest.mark.parametrize("path", DIRS_ALLOWING_SPACE_IN_FILENAMES) +def test_space_valid(path): + """Make sure files under""" + _validate(DUMMY_INTEGRATION_PATH / path / "foo bar.yml") + + +@pytest.mark.parametrize( + "file_name", + [ + f"{MALFORMED_DUMMY_INTEGRATION_NAME}.yml", + f"{MALFORMED_DUMMY_INTEGRATION_NAME}.png", + f"{MALFORMED_DUMMY_INTEGRATION_NAME}.py", + f"{MALFORMED_DUMMY_INTEGRATION_NAME}.js", + f"{MALFORMED_DUMMY_INTEGRATION_NAME}.ps1", + f"{MALFORMED_DUMMY_INTEGRATION_NAME}.Tests.ps1", + f"{DUMMY_INTEGRATION_NAME}_Test.py", + f"{DUMMY_INTEGRATION_NAME}_tests.py", + f"{DUMMY_INTEGRATION_NAME.upper()}.py", + "README", + "{DUMMY_INTEGRATION_NAME}_description", + ], +) +def test_integration_script_file_invalid(file_name: str): + with pytest.raises(InvalidIntegrationScriptFileName): + _validate(DUMMY_INTEGRATION_PATH / file_name) + + +@pytest.mark.parametrize( + "file_name", + [ + f"{DUMMY_INTEGRATION_NAME}.yml", + f"{DUMMY_INTEGRATION_NAME}_image.png", + f"{DUMMY_INTEGRATION_NAME}.py", + f"{DUMMY_INTEGRATION_NAME}.js", + f"{DUMMY_INTEGRATION_NAME}.ps1", + f"{DUMMY_INTEGRATION_NAME}.Tests.ps1", + f"{DUMMY_INTEGRATION_NAME}_test.py", + "conftest.py", + ".vulture_whitelist.py", + "README.md", + f"{DUMMY_INTEGRATION_NAME}_description.md", + "command_examples", + ".pylintrc", + ], +) +def test_integration_script_file_valid(file_name: str): + _validate(DUMMY_INTEGRATION_PATH / file_name) + + +@pytest.mark.parametrize( + "file_name", + ( + "layouts-.json", + "not-layout-.json", + "not-layoutscontainer-.json", + "foo.json", + "Layout-.json", + "Layoutscontainer-.json", + "layout_.json", + "layout-foo.NOTjson", + "layoutscontainer-foo.NOTjson", + ), +) +def test_layout_invalid(file_name: str): + with pytest.raises(InvalidLayoutFileName): + _validate(DUMMY_PACK_PATH / LAYOUTS_DIR / file_name) + + +@pytest.mark.parametrize( + "file_name", + ( + "layout-foo.json", + "layoutscontainer-.json", + ), +) +def test_layout_file_valid(file_name: str): + _validate(DUMMY_PACK_PATH / LAYOUTS_DIR / file_name) diff --git a/poetry.lock b/poetry.lock index b8e9e814f96..5f768550408 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -4211,13 +4211,13 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( [[package]] name = "shellingham" -version = "1.5.0.post1" +version = "1.5.4" description = "Tool to Detect Surrounding Shell" optional = false python-versions = ">=3.7" files = [ - {file = "shellingham-1.5.0.post1-py2.py3-none-any.whl", hash = "sha256:368bf8c00754fd4f55afb7bbb86e272df77e4dc76ac29dbcbb81a59e9fc15744"}, - {file = "shellingham-1.5.0.post1.tar.gz", hash = "sha256:823bc5fb5c34d60f285b624e7264f4dda254bc803a3774a147bf99c0e3004a28"}, + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, ] [[package]] @@ -4466,26 +4466,27 @@ files = [ [[package]] name = "typer" -version = "0.7.0" +version = "0.9.0" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.6" files = [ - {file = "typer-0.7.0-py3-none-any.whl", hash = "sha256:b5e704f4e48ec263de1c0b3a2387cd405a13767d2f907f44c1a08cbad96f606d"}, - {file = "typer-0.7.0.tar.gz", hash = "sha256:ff797846578a9f2a201b53442aedeb543319466870fbe1c701eab66dd7681165"}, + {file = "typer-0.9.0-py3-none-any.whl", hash = "sha256:5d96d986a21493606a358cae4461bd8cdf83cbf33a5aa950ae629ca3b51467ee"}, + {file = "typer-0.9.0.tar.gz", hash = "sha256:50922fd79aea2f4751a8e0408ff10d2662bd0c8bbfa84755a699f3bada2978b2"}, ] [package.dependencies] click = ">=7.1.1,<9.0.0" colorama = {version = ">=0.4.3,<0.5.0", optional = true, markers = "extra == \"all\""} -rich = {version = ">=10.11.0,<13.0.0", optional = true, markers = "extra == \"all\""} +rich = {version = ">=10.11.0,<14.0.0", optional = true, markers = "extra == \"all\""} shellingham = {version = ">=1.3.0,<2.0.0", optional = true, markers = "extra == \"all\""} +typing-extensions = ">=3.7.4.3" [package.extras] -all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] doc = ["cairosvg (>=2.5.2,<3.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pillow (>=9.3.0,<10.0.0)"] -test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<8.0.0)", "pytest-cov (>=2.10.0,<5.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<4.0.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<8.0.0)", "pytest-cov (>=2.10.0,<5.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<4.0.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] [[package]] name = "types-chardet" @@ -5271,4 +5272,4 @@ generate-unit-tests = ["klara"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "a3400e41efef65dce0aaf0cb95c4dac48e7485ad58fc77e94ca3b25acebc037b" +content-hash = "dc160b5845c37e0adb592accc1aad10261e87427ff0e3f41059fefc7894cfe74" diff --git a/pyproject.toml b/pyproject.toml index c5936d5340a..ab4129f271e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ update-additional-dependencies = "demisto_sdk.scripts.update_additional_dependen sdk-changelog = "demisto_sdk.scripts.changelog.changelog:main" merge-coverage-report = "demisto_sdk.scripts.merge_coverage_report:main" merge-pytest-reports = "demisto_sdk.scripts.merge_pytest_reports:main" +validate-content-path = "demisto_sdk.scripts.validate_content_path:main" validate-conf-json = "demisto_sdk.scripts.validate_conf_json:main" init-validation = "demisto_sdk.scripts.init_validation_script:main" validate-deleted-files = "demisto_sdk.scripts.validate_deleted_files:main" @@ -101,7 +102,7 @@ types-decorator = "^5.1.8" types-mock = "^4.0.15" types-setuptools = ">=65.6.0.1,<68.0.0.0" types-ujson = "^5.6.0.0" -typer = {extras = ["all"], version = ">=0.6.1,<0.8.0"} +typer = {extras = ["all"], version = "^0.9.0"} types-pkg-resources = "^0.1.3" types-toml = "^0.10.8.7" packaging = "^23.1"