update

#!/usr/bin/env python3

#  Licensed under the Apache License, Version 2.0 (the "License"); you may
#  not use this file except in compliance with the License. You may obtain
#  a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#  License for the specific language governing permissions and limitations
#  under the License.

import _thread
import argparse
import json
import multiprocessing
import os
import queue
import shutil
import subprocess
import sys
import threading

import requests

"""
Clone/Fetch all openstack repos from opendev.org.

Pulls the following orgs: openstack openstack-dev and openstack-infra
"""

# TODO(jogo): generalize beyond just openstack

repos_not_on_master = []
repo_errors = []


class GitException(Exception):
    pass


class Project(object):
    def __init__(self, repo, recurse=False, is_os=True):
        if (is_os):
            # OpenStack repo
            self.full_name = repo
            self.org, self.name = self._split_org_name()
            self.git_uri = self._git_uri()
        else:
            # Assume a GitHub repo
            # the other repo which should be dict and have these keys
            self.full_name = repo['full_name']
            self.org, self.name = repo['owner']['login'], repo['name']
            # NOTE: Use html_url here because of the GitHub recommendation
            # https://help.github.com/articles/which-remote-url-should-i-use/
            self.git_uri = repo['html_url']
        self.recurse = recurse

    def _split_org_name(self):
        try:
            org, name = self.full_name.split('/')
        except ValueError:
            print("Unable to split '%s' on '/'" % self.full_name)
            raise
        return org, name

    def _git_uri(self):
        # Convert to git://opendev.org/openstack/tempest
        return "https://opendev.org/%s/%s" % (self.org, self.name)

    def __eq__(self, other):
        return self.full_name == other.full_name

    def call(self, command, cwd=None):
        """Wrapper around subprocess.

        Make sure subprocess exits on early termination.
        Collects stdout, stderr
        raise error on non zero exit.
        """
        p = subprocess.Popen(command.split(' '),
                             cwd=cwd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             start_new_session=True)
        stdout, stderr = p.communicate()
        if p.returncode != 0:
            print(stderr)
            raise GitException()
        return stdout

    def _update(self, git_dir, to_print):
        # get notes, fail silently
        # Not using notes, and trying to speed things up
        # command = ("git fetch origin "
        #            "refs/notes/review:refs/notes/review")
        # try:
        #     self.call(command, git_dir)
        # except GitException:
        #     pass

        # If current branch is master or main, do git pull
        command = "git rev-parse --abbrev-ref HEAD"
        out = self.call(command, git_dir)
        if "master\n" == out.decode() or "main\n" == out.decode():
            # Don't do a git pull if local changes
            command = "git diff --quiet".split(' ')
            if subprocess.call(command, cwd=git_dir) == 0:
                # if needs a merge, fail
                command = "git pull --ff-only"
                out = self.call(command, git_dir)
                to_print.append(out.decode())
            else:
                command = "git fetch origin"
                self.call(command, git_dir)
        else:
            # make a note of the current branch
            branch = out.decode().strip()
            to_print.append("Branch: '%s'" % branch)
            repos_not_on_master.append(git_dir)
            command = "git fetch origin"
            self.call(command, git_dir)

    # TODO(jogo): fix output with concurrency
    def git(self):
        """Git Update/clone a project"""
        to_print = []
        git_dir = os.path.join(self.org, self.name)

        if os.path.exists(git_dir):
            # If git_dir exists just run git pull/fetch
            to_print.append("Project: %s" % git_dir)
            try:
                self._update(git_dir, to_print)
            except GitException:
                repo_errors.append(git_dir)

        elif not self.org in ignore_org:
            print(self.git_uri)
            try:
                command = "git clone %s %s" % (self.git_uri, git_dir)
                if self.recurse:
                    command += ' --recursive'
                out = self.call(command)
                to_print.append(out.decode())
            except GitException:
                repo_errors.append(git_dir)

        for line in to_print:
            if line.rstrip() == '':
                continue
            print("%s" % line.rstrip())


def _sanity_check(projects):
    """Make sure the correct org directories already exist."""
    orgs = set()

    for project in projects:
        if not project.org in ignore_org:
            orgs.add(project.org)

    for org in orgs:
        if not os.path.isdir(org):
            print("Directory '%s' does not exist, see help" % org)
            sys.exit(1)


def _get_orphaned(projects):
    # go over non ignored orgs and look for orphaned repos
    orgs = set()
    orphaned = []

    for project in projects:
        orgs.add(project.org)
    for org in orgs:
        if not os.path.isdir(org):
            # directory doesn't exist
            continue
        directories = [os.path.join(org, name) for name in
                       os.listdir(org) if
                       os.path.isdir(os.path.join(org, name))]
        for directory in directories:
            if Project(directory) not in projects:
                orphaned.append(directory)
    return orphaned


def _print_issues(projects):
    if len(repos_not_on_master) > 0:
        print("Repositories not on master or main branch:")
    for repo in repos_not_on_master:
        print("- %s" % repo)

    if len(repo_errors) > 0:
        print("Errors pulling the following repositories:")
    for repo in repo_errors:
        print("- %s" % repo)

    orphaned = _get_orphaned(projects)
    if orphaned:
        print("the following directories have been orphaned (no upstream):")
        for orphan in orphaned:
            print("- %s" % orphan)


def skip_project(project_name):
    """skip special projects

    Skip All-Users, API-Projects, All-Projects project Since we aren't
    interested in it
    """
    names = ["All-Users", "API-Projects", "All-Projects"]
    return project_name in names


def main(create_org_dir, delete_orphaned, concurrency, recurse=False,
         repos_url='https://review.opendev.org:443/projects/', ignore_org_arg=[]):
    global ignore_org
    ignore_org = ignore_org_arg
    def worker():
        """Thread worker."""
        while True:
            try:
                q.get().git()
                q.task_done()
            except Exception:
                # If exception on worker, exit
                _thread.interrupt_main()
                raise

    q = queue.Queue()
    num_worker_threads = multiprocessing.cpu_count()
    if concurrency:
        num_worker_threads = concurrency

    projects = []
    projects_json = ""
    is_os = True

    if (repos_url == 'https://review.opendev.org:443/projects/'):
        # List of all openstack repos
        r = requests.get("https://review.opendev.org:443/projects/")
        # strip off first few chars because 'the JSON response body starts with a
        # magic prefix line that must be stripped before feeding the rest of the
        # response body to a JSON parser'
        # https://review.opendev.org/Documentation/rest-api.html
        projects_json = r.text[4:]
    elif (repos_url.find('https://api.github.com/') == 0):
        # List of all the specified github repos
        is_os = False
        r = requests.get(repos_url)
        # Assume that there's no trick for this URL
        projects_json = r.text
    else:
        print("Unsupported repos_url: %s" % repos_url)
        sys.exit(1)

    for name in json.loads(projects_json):
        if is_os and skip_project(name):
            continue
        projects.append(Project(name, recurse, is_os))

    if not create_org_dir:
        _sanity_check(projects)

    if delete_orphaned:
        orphaned = _get_orphaned(projects)
        for orphan in orphaned:
            print("deleting repo: %s" % orphan)
            shutil.rmtree(orphan)
        sys.exit(0)

    try:
        for project in projects:
            q.put(project)

        print("Using %d threads" % num_worker_threads)
        for _ in range(num_worker_threads):
            t = threading.Thread(target=worker, daemon=True)
            t.start()

        q.join()
    except (KeyboardInterrupt, SystemExit):
        print("Exiting")
    finally:
        _print_issues(projects)


if __name__ == "__main__":
    parser = argparse.ArgumentParser("Clone OpenStack repos")
    parser.add_argument('--force', '-f', action='store_true',
                        help='create organization directories if not found')
    parser.add_argument('--delete', '-d', action='store_true',
                        help='Just delete orphaned repos')
    parser.add_argument('--concurrency', '-c', type=int,
                        help='The number of workers to use when running in'
                             'parallel. By default this is the number of cpus')
    parser.add_argument('--recurse-submodule', '-r', action='store_true',
                        dest='recurse',
                        help='For projects with git submodules perform a '
                             'recursive clone to also clone the submodules')
    parser.add_argument('--repos-url', '-u', type=str,
                        default='https://review.opendev.org:443/projects/',
                        help='The URL of repos that should return JSON')
    parser.add_argument('--ignore-org', '-i', type=str,
                        default=['stackforge', 'stackforge-attic',
                                 'openstack-attic', 'skyline', 'pypa', 'pyca',
                                 'openinfra', 'openinfralabs'], nargs='*',
                        help='Organizations to ignore')
    args = parser.parse_args()
    main(args.force, args.delete, args.concurrency, args.recurse,
         args.repos_url, args.ignore_org)