Custom Projects List Feature

README.md

@@ -67,6 +67,9 @@ python3 follow_repos_by_search_term.py <LANGUAGE> <SEARCH_TERM> <CUSTOM_LIST_NAM
 
 # Finds top repositories that have a minimum 500 stars and use the provided programming language.
 python3 follow_top_repos_by_star_count.py <LANGUAGE> <CUSTOM_LIST_NAME>(optional)
+
+# Unfollows all projects you're currently following that are not in a custom list.
+python3 unfollow_all_followed_projects.py
 ```
 
 ## The Custom Projects Lists Feature
@@ -104,7 +107,7 @@ LGTM can't move projects that are being processed into custom lists. To resolve
 
 > The <CACHED_FILE_NAME> can't be processed at this time because a project build is still in progress.
 
-If you receive this error, wait a few hours and run the script again. 
+If you receive this error, wait a few hours and run the script again.
 
 ## Legal
 

auto_sort_projects.py

@@ -71,7 +71,7 @@
     project_list_name = gh_org_to_project_list_name[org]
     project_list_id = site.get_or_create_project_list(project_list_name)
     for project in org_to_projects[org]:
-        if project.is_protoproject:
+        if project.is_protoproject():
             print('Unable to add project to project list since it is a protoproject. %s' % project)
             continue
         site.load_into_project_list(project_list_id, [project.key])

follow_repos_by_search_term.py

@@ -1,5 +1,5 @@
 from typing import List
-from lgtm import LGTMSite
+from lgtm import LGTMSite, LGTMDataFilters
 
 import utils.cacher
 import utils.github_dates
@@ -28,7 +28,8 @@ def find_and_save_projects_to_lgtm(language: str, search_term: str) -> List[str]
     for date_range in utils.github_dates.generate_dates():
         repos = github.search_repositories(query=f'stars:>5 language:{language} fork:false created:{date_range} {search_term}')
 
-        # TODO: This occasionally returns requests.exceptions.ConnectionError which is annoying as hell. It would be nice if we built in exception handling.
+        # TODO: This occasionally returns requests.exceptions.ConnectionError which is annoying as hell.
+        # It would be nice if we built in exception handling.
         for repo in repos:
             # Github has rate limiting in place hence why we add a sleep here. More info can be found here:
             # https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting
@@ -39,18 +40,11 @@ def find_and_save_projects_to_lgtm(language: str, search_term: str) -> List[str]
 
             saved_project = save_project_to_lgtm(site, repo.full_name)
 
-            # TODO: This process is duplicated elsewhere and should be under one location
-            # We only save realProjects to the cache since those are the only
-            # ones we can actually process.
-            if "realProject" in saved_project:
-                saved_project_name = saved_project['realProject'][0]['displayName']
-                saved_project_id = saved_project['realProject'][0]['key']
-                saved_project_data.append(f'{saved_project_name},{saved_project_id},realProject')
-
-            if "protoproject" in saved_project:
-                saved_project_name = saved_project['protoproject']['displayName']
-                saved_project_id = saved_project['protoproject']['key']
-                saved_project_data.append(f'{saved_project_name},{saved_project_id},protoproject')
+            simple_project = LGTMDataFilters.build_simple_project(saved_project)
+
+            if simple_project.is_valid_project:
+                saved_data = f'{simple_project.display_name},{simple_project.key},{simple_project.project_type}'
+                saved_project_data.append(saved_data)
 
     return saved_project_data
 

follow_top_repos_by_star_count.py

@@ -1,5 +1,5 @@
 from typing import List
-from lgtm import LGTMSite
+from lgtm import LGTMSite, LGTMDataFilters
 
 import utils.github_dates
 import utils.github_api
@@ -37,17 +37,11 @@ def find_and_save_projects_to_lgtm(language: str) -> List[str]:
                 continue
 
             saved_project = save_project_to_lgtm(site, repo.full_name)
+            simple_project = LGTMDataFilters.build_simple_project(saved_project)
 
-            # TODO: This process is duplicated elsewhere and should be under one location
-            if "realProject" in saved_project:
-                saved_project_name = saved_project['realProject'][0]['displayName']
-                saved_project_id = saved_project['realProject'][0]['key']
-                saved_project_data.append(f'{saved_project_name},{saved_project_id},realProject')
-
-            if "protoproject" in saved_project:
-                saved_project_name = saved_project['protoproject']['displayName']
-                saved_project_id = saved_project['protoproject']['key']
-                saved_project_data.append(f'{saved_project_name},{saved_project_id},protoproject')
+            if simple_project.is_valid_project:
+                saved_data = f'{simple_project.display_name},{simple_project.key},{simple_project.project_type}'
+                saved_project_data.append(saved_data)
 
     return saved_project_data
 

lgtm.py

@@ -38,7 +38,7 @@ def _make_lgtm_get(self, url: str) -> dict:
 
     def get_my_projects(self) -> List[dict]:
         '''
-        Returns a user's projects.
+        Returns a user's followed projects that are not in a custom list.
 
                 Returns:
                         data (List[dict]): Response data from LGTM
@@ -134,7 +134,7 @@ def force_rebuild_all_proto_projects(self):
         org_to_projects = LGTMDataFilters.org_to_ids(self.get_my_projects())
         for org in org_to_projects:
             for project in org_to_projects[org]:
-                if not project.is_protoproject:
+                if not project.is_protoproject():
                     continue
                 self.force_rebuild_project(project)
 
@@ -186,15 +186,15 @@ def unfollow_proto_repository_by_id(self, project_id: str):
         self._make_lgtm_post(url, data)
 
     def unfollow_repository(self, simple_project: 'SimpleProject'):
-        url = "https://lgtm.com/internal_api/v0.2/unfollowProject" if not simple_project.is_protoproject \
+        url = "https://lgtm.com/internal_api/v0.2/unfollowProject" if not simple_project.is_protoproject() \
             else "https://lgtm.com/internal_api/v0.2/unfollowProtoproject"
         data = simple_project.make_post_data()
         self._make_lgtm_post(url, data)
 
     def unfollow_repository_by_org(self, org: str, include_protoproject: bool = False):
         projects_under_org = self.get_my_projects_under_org(org)
         for project in projects_under_org:
-            if not include_protoproject and project.is_protoproject:
+            if not include_protoproject and project.is_protoproject():
                 print("Not unfollowing project since it is a protoproject. %s" % project)
                 continue
             print('Unfollowing project %s' % project.display_name)
@@ -280,17 +280,24 @@ def create_from_file() -> 'LGTMSite':
 
 
 @dataclass
+# TODO: this SimpleProject is no longer 'simple'. Some refactoring here could be nice.
 class SimpleProject:
     display_name: str
     key: str
-    is_protoproject: bool
+    project_type: str
+    is_valid_project: bool
+    org: str
+    state: str
 
     def make_post_data(self):
-        data_dict_key = 'protoproject_key' if self.is_protoproject else 'project_key'
+        data_dict_key = 'protoproject_key' if self.is_protoproject() else 'project_key'
         return {
             data_dict_key: self.key
         }
 
+    def is_protoproject(self):
+        # The values for project_type should be hardcoded in one central location
+        return self.project_type == "protoproject"
 
 class LGTMDataFilters:
 
@@ -302,43 +309,17 @@ def org_to_ids(projects: List[Dict]) -> Dict[str, List[SimpleProject]]:
         """
         org_to_ids = {}
         for project in projects:
-            org: str
-            display_name: str
-            key: str
-            is_protoproject: bool
-            if 'protoproject' in project:
-                the_project = project['protoproject']
-                if 'https://github.com/' not in the_project['cloneUrl']:
-                    # Not really concerned with BitBucket right now
-                    continue
-                display_name = the_project['displayName']
-                org = display_name.split('/')[0]
-                key = the_project['key']
-                is_protoproject = True
-            elif 'realProject' in project:
-
-                the_project = project['realProject'][0]
-                if the_project['repoProvider'] != 'github_apps':
-                    # Not really concerned with BitBucket right now
-                    continue
-                org = str(the_project['slug']).split('/')[1]
-                display_name = the_project['displayName']
-                key = the_project['key']
-                is_protoproject = False
-            else:
-                raise KeyError('\'realProject\' nor \'protoproject\' in %s' % str(project))
+            simple_project = LGTMDataFilters.build_simple_project(project)
+            if not simple_project.is_valid_project:
+                continue
 
             ids_list: List[SimpleProject]
-            if org in org_to_ids:
-                ids_list = org_to_ids[org]
+            if simple_project.org in org_to_ids:
+                ids_list = org_to_ids[simple_project.org]
             else:
                 ids_list = []
-                org_to_ids[org] = ids_list
-            ids_list.append(SimpleProject(
-                display_name=display_name,
-                key=key,
-                is_protoproject=is_protoproject
-            ))
+                org_to_ids[simple_project.org] = ids_list
+            ids_list.append(simple_project)
 
         return org_to_ids
 
@@ -348,3 +329,45 @@ def extract_project_under_org(org: str, projects_sorted: Dict[str, List[SimplePr
             print('org %s not found in projects list' % org)
             return []
         return projects_sorted[org]
+
+    @staticmethod
+    def build_simple_project(project: dict) -> SimpleProject:
+        org: str
+        display_name: str
+        key: str
+        project_type: str
+        is_valid_project: bool = True
+        state: str = ""
+
+        if 'protoproject' in project:
+            the_project = project['protoproject']
+            if 'https://github.com/' not in the_project['cloneUrl']:
+                # Not really concerned with BitBucket right now
+                is_valid_project = False
+            display_name = the_project['displayName']
+            state = the_project['state']
+            org = display_name.split('/')[0]
+            key = the_project['key']
+            project_type = 'protoproject'
+        elif 'realProject' in project:
+            the_project = project['realProject'][0]
+            if the_project['repoProvider'] != 'github_apps':
+                # Not really concerned with BitBucket right now
+                is_valid_project = False
+            org = str(the_project['slug']).split('/')[1]
+            display_name = the_project['displayName']
+            key = the_project['key']
+            project_type = "realProject"
+        else:
+            # We raise this in cases where we can't intrepret the data we get
+            # back from LGTM.
+            is_valid_project = False
+
+        return SimpleProject(
+            display_name=display_name,
+            key=key,
+            project_type=project_type,
+            is_valid_project=is_valid_project,
+            org=org,
+            state=state
+        )

unfollow_all_followed_projects.py

@@ -0,0 +1,11 @@
+from lgtm import LGTMSite, SimpleProject, LGTMDataFilters
+import time
+
+site = LGTMSite.create_from_file()
+
+projects = site.get_my_projects()
+
+for project in projects:
+    simple_project = LGTMDataFilters.build_simple_project(project)
+    if simple_project.is_valid_project:
+        site.unfollow_repository(simple_project)