allowed paths

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
 ## Unreleased
+* Added a **validate** step checking for misplaced files (i.e. directly under `Integrations`, or under mistyped folder name)
 * Fixed an issue where **validate** falsely detected backwards-compatibility issues, and prevented adding the `marketplaces` key to content items.
 * Calling **format** with the `-d` flag now removes test playbooks testing the deprecated content from conf.json.
 * Fixed an issue where in some cases the **split** command did not remove pack version note from the script.

TestSuite/pack.py

@@ -4,10 +4,15 @@
 from demisto_sdk.commands.common.constants import (
     CORRELATION_RULES_DIR,
     DEFAULT_IMAGE_BASE64,
+    INDICATOR_TYPE_PREFIX,
+    LAYOUT_CONTAINER_PREFIX,
+    LAYOUT_PREFIX,
     LAYOUT_RULES_DIR,
     MODELING_RULES_DIR,
     PARSING_RULES_DIR,
+    REPORT_PREFIX,
     TRIGGER_DIR,
+    WIDGET_PREFIX,
     XDRC_TEMPLATE_DIR,
     XSIAM_DASHBOARDS_DIR,
     XSIAM_REPORTS_DIR,
@@ -87,6 +92,7 @@ def __init__(self, packs_dir: Path, name: str, repo):
         self.wizards: List[Wizard] = list()
         self.xdrc_templates: List[XDRCTemplate] = list()
         self.layout_rules: List[LayoutRule] = list()
+        self.connections: List[JSONBased] = list()
 
         # Create base pack
         self._pack_path = packs_dir / self.name
@@ -156,6 +162,9 @@ def __init__(self, packs_dir: Path, name: str, repo):
         self._lists_path = self._pack_path / "Lists"
         self._lists_path.mkdir()
 
+        self._connections_path = self._pack_path / "Connections"
+        self._connections_path.mkdir()
+
         self._parsing_rules_path = self._pack_path / PARSING_RULES_DIR
         self._parsing_rules_path.mkdir()
 
@@ -380,8 +389,9 @@ def create_indicator_field(self, name, content: dict = None) -> JSONBased:
         self.indicator_fields.append(indicator_field)
         return indicator_field
 
-    def create_indicator_type(self, name, content: dict = None) -> JSONBased:
-        prefix = "reputation"
+    def create_indicator_type(
+        self, name, content: dict = None, prefix=INDICATOR_TYPE_PREFIX
+    ) -> JSONBased:
         indicator_type = self._create_json_based(
             name, prefix, content, dir_path=self._indicator_types
         )
@@ -442,34 +452,42 @@ def create_job(
         self.jobs.append(job)
         return job
 
-    def create_layout(self, name, content: dict = None) -> JSONBased:
-        prefix = "layout"
+    def create_layout(
+        self,
+        name,
+        content: dict = None,
+        prefix: str = LAYOUT_PREFIX,
+    ) -> JSONBased:
         layout = self._create_json_based(
             name, prefix, content, dir_path=self._layout_path
         )
         self.layouts.append(layout)
         return layout
 
-    def create_layoutcontainer(self, name, content: Optional[dict] = None) -> JSONBased:
-        if not content:
-            content = {"group": "default"}
-        prefix = "layoutscontainer"
+    def create_layoutcontainer(
+        self, name, content: dict = None, prefix: str = LAYOUT_CONTAINER_PREFIX
+    ) -> JSONBased:
         layoutcontainer = self._create_json_based(
             name, prefix, content, dir_path=self._layout_path
         )
         self.layoutcontainers.append(layoutcontainer)
         return layoutcontainer
 
-    def create_report(self, name, content: dict = None) -> JSONBased:
-        prefix = "report"
+    def create_report(
+        self, name, content: dict = None, prefix=REPORT_PREFIX
+    ) -> JSONBased:
         report = self._create_json_based(
             name, prefix, content, dir_path=self._report_path
         )
         self.reports.append(report)
         return report
 
-    def create_widget(self, name, content: dict = None) -> JSONBased:
-        prefix = "widget"
+    def create_widget(
+        self,
+        name,
+        content: dict = None,
+        prefix: str = WIDGET_PREFIX,
+    ) -> JSONBased:
         widget = self._create_json_based(
             name, prefix, content, dir_path=self._widget_path
         )
@@ -505,14 +523,20 @@ def create_wizard(
         self.wizards.append(wizard)
         return wizard
 
-    def create_list(self, name, content: dict = None) -> JSONBased:
-        prefix = "list"
+    def create_list(self, name, content: dict = None, prefix="list") -> JSONBased:
         list_item = self._create_json_based(
             name, prefix, content, dir_path=self._lists_path
         )
         self.lists.append(list_item)
         return list_item
 
+    def create_connection(self, name: str, content: dict = None) -> JSONBased:
+        connection = self._create_json_based(
+            name, "", content, dir_path=self._connections_path
+        )
+        self.connections.append(connection)
+        return connection
+
     def create_playbook(
         self,
         name: Optional[str] = None,

demisto_sdk/commands/common/constants.py

@@ -28,9 +28,10 @@
 CLASSIFIERS_DIR = "Classifiers"
 MAPPERS_DIR = "Classifiers"
 CONNECTIONS_DIR = "Connections"
+RELEASE_NOTES_DIR = "ReleaseNotes"
+
 PACKS_DIR = "Packs"
 TOOLS_DIR = "Tools"
-RELEASE_NOTES_DIR = "ReleaseNotes"
 TESTS_DIR = "Tests"
 DOC_FILES_DIR = "doc_files"
 DOCUMENTATION_DIR = "Documentation"
@@ -48,6 +49,75 @@
 XDRC_TEMPLATE_DIR = "XDRCTemplates"
 LAYOUT_RULES_DIR = "LayoutRules"
 
+FIRST_LEVEL_FOLDERS = {
+    CLASSIFIERS_DIR,
+    CONNECTIONS_DIR,
+    CORRELATION_RULES_DIR,
+    DASHBOARDS_DIR,
+    GENERIC_DEFINITIONS_DIR,
+    GENERIC_FIELDS_DIR,
+    GENERIC_MODULES_DIR,
+    GENERIC_TYPES_DIR,
+    INCIDENT_FIELDS_DIR,
+    INCIDENT_TYPES_DIR,
+    INDICATOR_FIELDS_DIR,
+    INDICATOR_TYPES_DIR,
+    INTEGRATIONS_DIR,
+    LAYOUTS_DIR,
+    LISTS_DIR,
+    MAPPERS_DIR,
+    MODELING_RULES_DIR,
+    PRE_PROCESS_RULES_DIR,
+    PARSING_RULES_DIR,
+    PLAYBOOKS_DIR,
+    RELEASE_NOTES_DIR,
+    REPORTS_DIR,
+    SCRIPTS_DIR,
+    TEST_PLAYBOOKS_DIR,
+    TOOLS_DIR,
+    TRIGGER_DIR,
+    WIDGETS_DIR,
+    WIZARDS_DIR,
+    XDRC_TEMPLATE_DIR,
+    XSIAM_DASHBOARDS_DIR,
+    XSIAM_REPORTS_DIR,
+    DOC_FILES_DIR,
+    LAYOUT_RULES_DIR,
+    JOBS_DIR,
+}
+
+FIRST_LEVEL_FOLDERS_ALLOWED_TO_CONTAIN_FILES = {
+    PLAYBOOKS_DIR,
+    TEST_PLAYBOOKS_DIR,
+    REPORTS_DIR,
+    DASHBOARDS_DIR,
+    INCIDENT_FIELDS_DIR,
+    INCIDENT_TYPES_DIR,
+    INDICATOR_FIELDS_DIR,
+    INDICATOR_TYPES_DIR,
+    GENERIC_TYPES_DIR,
+    GENERIC_MODULES_DIR,
+    GENERIC_DEFINITIONS_DIR,
+    LAYOUTS_DIR,
+    CLASSIFIERS_DIR,
+    MAPPERS_DIR,
+    CONNECTIONS_DIR,
+    RELEASE_NOTES_DIR,
+    DOC_FILES_DIR,
+    JOBS_DIR,
+    PRE_PROCESS_RULES_DIR,
+    LISTS_DIR,
+    PARSING_RULES_DIR,
+    MODELING_RULES_DIR,
+    CORRELATION_RULES_DIR,
+    XSIAM_DASHBOARDS_DIR,
+    XSIAM_REPORTS_DIR,
+    TRIGGER_DIR,
+    WIDGETS_DIR,
+    WIZARDS_DIR,
+    LAYOUT_RULES_DIR,
+}
+
 # NAMES OF ENTITIES
 
 SCRIPT = "script"
@@ -801,6 +871,11 @@ class FileType(str, Enum):
 MODELING_RULE_NAME_SUFFIX = "Modeling Rule"
 XDRC_TEMPLATE_PREFIX = "xdrctemplate"
 LAYOUT_RULE_PREFIX = "layoutrule"
+LAYOUT_CONTAINER_PREFIX = "layoutscontainer"
+INDICATOR_TYPE_PREFIX = "reputation"
+REPORT_PREFIX = "report"
+LAYOUT_PREFIX = "layout"
+WIDGET_PREFIX = "widget"
 
 # Pack Unique Files
 PACKS_WHITELIST_FILE_NAME = ".secrets-ignore"
@@ -813,6 +888,9 @@ class FileType(str, Enum):
 PACKS_FOLDER = "Packs"
 
 CONF_JSON_FILE_NAME = "conf.json"
+GIT_IGNORE_FILE_NAME = ".gitignore"
+PIPFILE_FILE_NAME = "Pipfile"
+PIPFILE_LOCK_FILE_NAME = "Pipfile.lock"
 
 PYTHON_TEST_REGEXES = [PACKS_SCRIPT_TEST_PY_REGEX, PACKS_INTEGRATION_TEST_PY_REGEX]
 
@@ -1841,4 +1919,28 @@ class ParameterType(Enum):
 
 NATIVE_IMAGE_DOCKER_NAME = "demisto/py3-native"
 
+PACK_ROOT_FILE_NAMES = {
+    GIT_IGNORE_FILE_NAME,
+    PACKS_PACK_IGNORE_FILE_NAME,
+    PACKS_WHITELIST_FILE_NAME,
+    AUTHOR_IMAGE_FILE_NAME,
+    PACKS_CONTRIBUTORS_FILE_NAME,
+    PIPFILE_FILE_NAME,
+    PIPFILE_LOCK_FILE_NAME,
+    PACKS_README_FILE_NAME,
+    PACKS_PACK_META_FILE_NAME,
+}
+
+PACK_ROOT_FILE_NAMES = {
+    GIT_IGNORE_FILE_NAME,
+    PACKS_PACK_IGNORE_FILE_NAME,
+    PACKS_WHITELIST_FILE_NAME,
+    AUTHOR_IMAGE_FILE_NAME,
+    PACKS_CONTRIBUTORS_FILE_NAME,
+    PIPFILE_FILE_NAME,
+    PIPFILE_LOCK_FILE_NAME,
+    PACKS_README_FILE_NAME,
+    PACKS_PACK_META_FILE_NAME,
+}
+
 FORMATTING_SCRIPT = "indicator-format"

demisto_sdk/commands/common/errors.py

@@ -16,6 +16,7 @@
     MODULES,
     PACK_METADATA_DESC,
     PACK_METADATA_NAME,
+    PACK_ROOT_FILE_NAMES,
     PARSING_RULE_ID_SUFFIX,
     PARSING_RULE_NAME_SUFFIX,
     RELIABILITY_PARAMETER_NAMES,
@@ -41,6 +42,8 @@
     "BA116",
     "BA119",
     "BA124",
+    "BA120",  # mostly for editing old (unified) content
+    "BA121",  # mostly for editing old (unified) content
     "DS107",
     "GF102",
     "IF100",
@@ -131,7 +134,7 @@
     "deprecated": ["ST", "BC", "BA", "IN127", "IN128", "PB104", "SC101"],
 }
 
-ERROR_CODE = {
+ERROR_CODE: dict = {
     # BA - Basic
     "wrong_version": {
         "code": "BA100",
@@ -210,6 +213,26 @@
         "code": "BA119",
         "related_field": "",
     },
+    "files_not_allowed_directly_under_this_folder": {
+        "code": "BA120",
+        "ui_applicable": False,
+        "related_field": "",
+    },
+    "invalid_first_level_folder": {
+        "code": "BA121",
+        "ui_applicable": False,
+        "related_field": "",
+    },
+    "file_not_allowed_at_pack_root": {
+        "code": "BA122",
+        "ui_applicable": False,
+        "related_field": "",
+    },
+    "file_not_allowed_outside_pack": {
+        "code": "BA123",
+        "ui_applicable": False,
+        "related_field": "",
+    },
     "missing_unit_test_file": {
         "code": "BA124",
         "related_field": "",
@@ -3336,6 +3359,41 @@ def copyright_section_in_python_error(line_nums):
             f"in lines: {line_nums}. Copyright section cannot be part of script."
         )
 
+    @staticmethod
+    @error_code_decorator
+    def files_not_allowed_directly_under_this_folder(file_path: Path):
+        return (
+            f"Files are not allowed directly under the {file_path.parent.name} folder. "
+            f"Please move {file_path.name} to a suitable folder. "
+            "See https://github.com/demisto/content for reference."
+        )
+
+    @staticmethod
+    @error_code_decorator
+    def invalid_first_level_folder(file_path: Path):
+        return (
+            f"The {file_path.parent.name} folder is not allowed directly under the pack's folder ({file_path.parent.parent.name})."
+            f"Please match the structure to other folders in the repo, or see https://github.com/demisto/content as reference."
+        )
+
+    @staticmethod
+    @error_code_decorator
+    def file_not_allowed_at_pack_root(file_path: Path):
+        allowed_files_string = ", ".join(sorted(PACK_ROOT_FILE_NAMES))
+        return (
+            f"The {file_path.name} file is saved directly under the {file_path.parent.name} pack folder.\n"
+            f"Only the following file names are allowed at the pack root: {allowed_files_string}.\n"
+            f"Please match the structure to other folders in the repo, or see https://github.com/demisto/content as reference."
+        )
+
+    @staticmethod
+    @error_code_decorator
+    def file_not_allowed_outside_pack(file_path: Path):
+        return (
+            f"{file_path.name} is saved directly under the Packs folder, "
+            "Please move it into a pack. See https://github.com/demisto/content as reference."
+        )
+
     @staticmethod
     def pack_readme_image_relative_path_error(path):
         return (

demisto_sdk/commands/common/tests/update_id_set_test.py

@@ -1581,7 +1581,7 @@ def test_process_layoutscontainer__excluding_from_marketplace_by_layout_type(
         pack = repo.create_pack(name=f"DummyPack-{layout_type}")
         layout = pack.create_layoutcontainer(
             "Reut",
-            {
+            content={
                 "id": "Reut",
                 "group": layout_type,
                 "detailsV2": {},
@@ -1616,7 +1616,7 @@ def test_process_layoutscontainer__marketplace_mismatch(repo):
         pack = repo.create_pack(name="DummyPack")
         layout = pack.create_layoutcontainer(
             "Itay",
-            {
+            content={
                 "id": "Itay",
                 "group": "indicator",
                 "detailsV2": {},