Add Github organization collaborators fetcher (#38)

* Add GitHub repo collaborators fetcher

.gitignore

@@ -102,3 +102,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+# Local Mac files
+.DS_Store

CHANGES.md

@@ -1,3 +1,7 @@
+# [0.8.0](https://github.com/ComplianceAsCode/auditree-arboretum/releases/tag/v0.8.0)
+
+- [ADDED] GitHub org collaborators fetcher added to `Permissions`.
+
 # [0.7.1](https://github.com/ComplianceAsCode/auditree-arboretum/releases/tag/v0.7.1)
 
 - [FIXED] Github issues fetcher now uses `states` option matching the README writeup.

arboretum/__init__.py

@@ -14,4 +14,4 @@
 # limitations under the License.
 """Arboretum - Checking your compliance & security posture, continuously."""
 
-__version__ = '0.7.1'
+__version__ = '0.8.0'

arboretum/common/constants.py

@@ -41,3 +41,6 @@
 
 # Zenhub public API
 ZH_API_ROOT = 'https://api.zenhub.com'
+
+# Default affiliation type to fetch github repo collaborators
+GH_ALL_COLLABORATORS = ['all']

arboretum/permissions/README.md

@@ -16,7 +16,84 @@ how to include the fetchers and checks from this library in your downstream proj
 
 ## Fetchers
 
-Fetchers coming soon...
+### Organization Integrity (Repository Collaborators)
+
+* Class: [GithubOrgCollaboratorsFetcher][gh-org-fetcher]
+* Purpose: Writes the details of collaborators in Github organizations to the evidence locker. This fetcher class is only meant for use with Github or Github Enterprise organizations.
+* Behavior: For each Github organization specified, an evidence file per collaborator type (affiliation) is stored in
+the locker containing collaborator details for the specified repositories in the organization. The default is to
+retrieve all collaborators by affiliation from all repositories in each specified Github organization.  TTL is set to 1
+day.
+* Configuration elements:
+  * `org.permissions.org_integrity.orgs`
+     * Required
+     * List of dictionaries each containing Github organization retrieval configuration.
+        * `url`
+           * Required
+           * String in the form of `"https://github.com/my-org"` or `"https://github.<company>.com/my-org"`.
+           * Use to define the Github organization url to use.
+        * `repos`
+           * Optional
+           * List of strings in the form of `["my-repo", "my-other-repo"]`.
+           * Defaults to all repositories in the organization.
+           * Use if looking to filter collaborator evidence to a subset of repositories in the organization otherwise do not include.
+        * `collaborator_types`
+           * Optional
+           * List of strings in the form of `["all", "direct", "outside"]`.
+           * Valid list element values are `"all"`, `"direct"`, `"outside"`.
+           * Defaults to `["all"]`.
+           * Use if looking to override retrieval of all collaborators otherwise do not include.
+* Example configuration:
+
+  ```json
+  {
+    "org": {
+      "permissions": {
+        "org_integrity": {
+          "orgs": [
+            {
+              "url": "https://github.com/my-org-1"
+            },
+            {
+              "url": "https://github.my-company.com/my-org-2",
+              "collaborator_types": ["direct", "outside"],
+              "repos": ["repo1", "repo2"]
+            }
+          ]
+        }
+      }
+    }
+  }
+  ```
+
+* Required credentials:
+  * `github` or `github_enterprise` credentials with [appropriate permissions to the repositories][repository-permissions] are required for this fetcher to successfully retrieve evidence.
+     * `username`: The Github user used to run the fetcher.
+     * `token`: The Github user access token used to run the fetcher.
+  * Example credentials file entry:
+
+     ```ini
+     [github]
+     username=gh-user-name
+     token=gh-access-token
+     ```
+
+     or
+
+     ```ini
+     [github_enterprise]
+     username=ghe-user-name
+     token=ghe-access-token
+     ```
+
+  * NOTE: These credentials are also needed for basic configuration of the
+  Auditree framework. The expectation is that the same credentials are used for all Github interactions.
+
+* Import statement:
+
+  ```python
+  from arboretum.permissions.fetchers.github.fetch_org_collaborators import GithubOrgCollaboratorsFetcher
+  ```
 
 ## Checks
 
@@ -25,3 +102,5 @@ Checks coming soon...
 [auditree-framework]: https://github.com/ComplianceAsCode/auditree-framework
 [auditree-framework documentation]: https://complianceascode.github.io/auditree-framework/
 [usage]: https://github.com/ComplianceAsCode/auditree-arboretum#usage
+[gh-org-fetcher]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/permissions/fetchers/github/fetch_org_collaborators.py
+[repository-permissions]: https://docs.github.com/en/free-pro-team@latest/github/setting-up-and-managing-organizations-and-teams/repository-permission-levels-for-an-organization

arboretum/permissions/fetchers/github/fetch_org_collaborators.py

@@ -0,0 +1,65 @@
+# -*- mode:python; coding:utf-8 -*-
+# Copyright (c) 2021 IBM Corp. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Github organization collaborators fetcher."""
+
+import json
+
+from arboretum.common.constants import GH_ALL_COLLABORATORS
+
+from compliance.evidence import DAY, RawEvidence, raw_evidence
+from compliance.fetch import ComplianceFetcher
+from compliance.utils.data_parse import get_sha256_hash
+from compliance.utils.services.github import Github
+
+
+class GithubOrgCollaboratorsFetcher(ComplianceFetcher):
+    """Fetch collaborators from GH organization repositories."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Initialize the fetcher object with configuration settings."""
+        cls.gh_pool = {}
+        return cls
+
+    def fetch_gh_org_collaborators(self):
+        """Fetch collaborators from GH organization repositories."""
+        for config in self.config.get('org.permissions.org_integrity.orgs'):
+            host, org = config['url'].rsplit('/', 1)
+            for aff in config.get('collaborator_types', GH_ALL_COLLABORATORS):
+                url_hash = get_sha256_hash([config['url']], 10)
+                json_file = f'gh_{aff}_collaborators_{url_hash}.json'
+                path = ['permissions', json_file]
+                description = (
+                    f'{aff.title()} collaborators of the {org} GH org'
+                )
+                self.config.add_evidences(
+                    [RawEvidence(path[1], path[0], DAY, description)]
+                )
+                with raw_evidence(self.locker, '/'.join(path)) as evidence:
+                    if evidence:
+                        if host not in self.gh_pool:
+                            self.gh_pool[host] = Github(base_url=host)
+                        if not config.get('repos'):
+                            repos = self.gh_pool[host].paginate_api(
+                                f'orgs/{org}/repos'
+                            )
+                            config['repos'] = [repo['name'] for repo in repos]
+                        collabs = {}
+                        for repo in config['repos']:
+                            collabs_url = f'repos/{org}/{repo}/collaborators'
+                            collabs[repo] = self.gh_pool[host].paginate_api(
+                                collabs_url, affiliation=aff
+                            )
+                        evidence.set_content(json.dumps(collabs))

devel.json

@@ -25,6 +25,20 @@
         }
       }
     },
+    "permissions": {
+      "org_integrity": {
+        "orgs": [
+          {
+            "url": "https://github.com/my-org-1"
+          },
+          {
+            "url": "https://github.my-company.com/my-org-2",
+            "collaborator_types": ["direct", "outside"],
+            "repos": ["repo1", "repo2"]
+          }
+        ]
+      }
+    },
     "issue_mgmt": {
       "github": [
         {