[WIP] refactoring of install action for '--requirements' and 'download'

ansible_galaxy/install.py

@@ -10,90 +10,14 @@
 from ansible_galaxy import exceptions
 from ansible_galaxy import installed_repository_db
 from ansible_galaxy.models.install_destination import InstallDestinationInfo
-from ansible_galaxy.models.repository_spec import FetchMethods, RepositorySpec
+from ansible_galaxy.models.repository_spec import FetchMethods
 
 log = logging.getLogger(__name__)
 
 # This should probably be a state machine for stepping through the install states
 # See actions.install.install_collection for a sketch of the states
 
 
-def find(fetcher):
-    """find/discover info about the content"""
-
-    find_results = fetcher.find()
-
-    return find_results
-
-
-# def fetch(fetcher, collection):
-#    pass
-
-def fetch(fetcher, repository_spec, find_results):
-    """download the archive and side effect set self._archive_path to where it was downloaded to.
-
-    MUST be called after self.find()."""
-
-    log.debug('Fetching repository_spec=%s', repository_spec)
-
-    try:
-        # FIXME: note that ignore_certs for the galaxy
-        # server(galaxy_context.server['ignore_certs'])
-        # does not really imply that the repo archive download should ignore certs as well
-        # (galaxy api server vs cdn) but for now, we use the value for both
-        fetch_results = fetcher.fetch(find_results=find_results)
-    except exceptions.GalaxyDownloadError as e:
-        log.exception(e)
-
-        # TODO: having to keep fetcher state for tracking fetcher.remote_resource and/or cleanup
-        #       is kind of annoying.These methods may need to be in a class. Or maybe
-        #       the GalaxyDownloadError shoud/could have any info.
-        blurb = 'Failed to fetch the content archive "%s": %s'
-        log.error(blurb, fetcher.remote_resource, e)
-
-        # reraise, currently handled in main
-        # TODO: if we support more than one archive per invocation, will need to accumulate errors
-        #       if we want to skip some of them
-        raise
-
-    return fetch_results
-
-
-def repository_spec_from_find_results(find_results,
-                                      requirement_spec):
-    '''Create a new RepositorySpec with updated info from fetch_results.
-
-    Evolves repository_spec to match fetch results.'''
-
-    # TODO: do we still need to check the fetched version against the spec version?
-    #       We do, since the unspecific version is None, so fetched versions wont match
-    #       so we need a new repository_spec for install.
-    # TODO: this is more or less a verify/validate step or state transition
-    content_data = find_results.get('content', {})
-    resolved_version = content_data.get('version')
-
-    log.debug('version_spec "%s" for %s was requested and was resolved to version "%s"',
-              requirement_spec.version_spec, requirement_spec.label,
-              resolved_version)
-
-    # In theory, a fetch can return a different namespace/name than the one request. This
-    # is for things like server side aliases.
-    resolved_name = content_data.get('fetched_name', requirement_spec.name)
-    resolved_namespace = content_data.get('content_namespace', requirement_spec.namespace)
-
-    # Build a RepositorySpec based on RequirementSpec and the extra info resolved in find()
-    spec_data = attr.asdict(requirement_spec)
-
-    del spec_data['version_spec']
-
-    spec_data['version'] = resolved_version
-    spec_data['namespace'] = resolved_namespace
-    spec_data['name'] = resolved_name
-
-    repository_spec = RepositorySpec.from_dict(spec_data)
-    return repository_spec
-
-
 def install(galaxy_context,
             fetcher,
             fetch_results,

ansible_galaxy/repository.py

@@ -144,6 +144,7 @@ def load_from_dir(content_dir, namespace_path, namespace, name, installed=True):
     # TODO: make existence of a galaxy.yml and a MANIFEST.json mutual exclude and raise an exception for that case
 
     col_info = None
+
     # MANIFEST.json is higher prec than galaxy.yml
     if galaxy_yml_data:
         col_info = galaxy_yml_data

ansible_galaxy/repository_spec.py

@@ -1,7 +1,8 @@
 import logging
 
-from ansible_galaxy import repository_spec_parse
+import attr
 
+from ansible_galaxy import repository_spec_parse
 from ansible_galaxy.models.repository_spec import RepositorySpec
 
 
@@ -21,3 +22,38 @@ def repository_spec_from_string(repository_spec_string, namespace_override=None,
                           spec_string=spec_data.get('spec_string'),
                           fetch_method=spec_data.get('fetch_method'),
                           src=spec_data.get('src'))
+
+
+def repository_spec_from_find_results(find_results,
+                                      requirement_spec):
+    '''Create a new RepositorySpec with updated info from fetch_results.
+
+    Evolves repository_spec to match fetch results.'''
+
+    # TODO: do we still need to check the fetched version against the spec version?
+    #       We do, since the unspecific version is None, so fetched versions wont match
+    #       so we need a new repository_spec for install.
+    # TODO: this is more or less a verify/validate step or state transition
+    content_data = find_results.get('content', {})
+    resolved_version = content_data.get('version')
+
+    log.debug('version_spec "%s" for %s was requested and was resolved to version "%s"',
+              requirement_spec.version_spec, requirement_spec.label,
+              resolved_version)
+
+    # In theory, a fetch can return a different namespace/name than the one request. This
+    # is for things like server side aliases.
+    resolved_name = content_data.get('fetched_name', requirement_spec.name)
+    resolved_namespace = content_data.get('content_namespace', requirement_spec.namespace)
+
+    # Build a RepositorySpec based on RequirementSpec and the extra info resolved in find()
+    spec_data = attr.asdict(requirement_spec)
+
+    del spec_data['version_spec']
+
+    spec_data['version'] = resolved_version
+    spec_data['namespace'] = resolved_namespace
+    spec_data['name'] = resolved_name
+
+    repository_spec = RepositorySpec.from_dict(spec_data)
+    return repository_spec

ansible_galaxy/requirements.py

@@ -1,19 +1,71 @@
 import logging
 
+from ansible_galaxy import collection_artifact
+from ansible_galaxy import download
+from ansible_galaxy.models.repository_spec import FetchMethods
 from ansible_galaxy.models.requirement import Requirement, RequirementOps, RequirementScopes
 from ansible_galaxy.models.requirement_spec import RequirementSpec
-from ansible_galaxy.repository_spec_parse import spec_data_from_string
+from ansible_galaxy import repository_spec_parse
 
 log = logging.getLogger(__name__)
 
 
+def requirements_from_strings(repository_spec_strings,
+                              namespace_override=None,
+                              editable=False):
+    requirements_list = []
+
+    for repository_spec_string in repository_spec_strings:
+        fetch_method = \
+            repository_spec_parse.choose_repository_fetch_method(repository_spec_string,
+                                                                 editable=editable)
+        log.debug('fetch_method: %s', fetch_method)
+
+        if fetch_method == FetchMethods.LOCAL_FILE:
+            # Since we only know this is a local file we vaguely recognize, we have to
+            # open it up to get any more details. We _could_ attempt to parse the file
+            # name, but that rarely ends well. Filename could also be arbitrary for downloads
+            # from remote urls ('mazer install http://myci.example.com/somebuildjob/latest' etc)
+            spec_data = collection_artifact.load_data_from_collection_artifact(repository_spec_string)
+            spec_data['fetch_method'] = fetch_method
+        elif fetch_method == FetchMethods.REMOTE_URL:
+            # download the url
+            # hope it is a collection artifact and use load_data_from_collection_artifact() for the
+            # rest of the repo_spec data
+            log.debug('repository_spec_string: %s', repository_spec_string)
+
+            tmp_downloaded_path = download.fetch_url(repository_spec_string,
+                                                     # This is for random remote_urls, so always validate_certs
+                                                     validate_certs=True)
+            spec_data = collection_artifact.load_data_from_collection_artifact(tmp_downloaded_path)
+
+            # pretend like this is a local_file install now
+            spec_data['fetch_method'] = FetchMethods.LOCAL_FILE
+        else:
+            spec_data = repository_spec_parse.spec_data_from_string(repository_spec_string,
+                                                                    namespace_override=namespace_override,
+                                                                    editable=editable)
+
+            spec_data['fetch_method'] = fetch_method
+
+        log.debug('spec_data: %s', spec_data)
+
+        req_spec = RequirementSpec.from_dict(spec_data)
+
+        req = Requirement(repository_spec=None, op=RequirementOps.EQ, requirement_spec=req_spec)
+
+        requirements_list.append(req)
+
+    return requirements_list
+
+
 def from_dependencies_dict(dependencies_dict, namespace_override=None, editable=False, repository_spec=None):
     '''Build a list of Requirement objects from the 'dependencies' item in galaxy.yml'''
     reqs = []
     for req_label, req_version_spec in dependencies_dict.items():
-        req_spec_data = spec_data_from_string(req_label,
-                                              namespace_override=namespace_override,
-                                              editable=editable)
+        req_spec_data = repository_spec_parse.spec_data_from_string(req_label,
+                                                                    namespace_override=namespace_override,
+                                                                    editable=editable)
         req_spec_data['version_spec'] = req_version_spec
 
         log.debug('req_spec_data: %s', req_spec_data)

ansible_galaxy_cli/cli/galaxy.py

@@ -273,17 +273,14 @@ def execute_install(self):
         self.log.debug('self.options: %s', self.options)
 
         galaxy_context = self._get_galaxy_context(self.options, self.config)
-        requested_spec_strings = self.args
 
         # TODO: build requirement_specs from requested_collection_specs strings
-        rc = install.install_repository_specs_loop(galaxy_context,
-                                                   editable=self.options.editable_install,
-                                                   repository_spec_strings=requested_spec_strings,
-                                                   namespace_override=self.options.namespace,
-                                                   display_callback=self.display,
-                                                   ignore_errors=self.options.ignore_errors,
-                                                   no_deps=self.options.no_deps,
-                                                   force_overwrite=self.options.force)
+        rc = install.run(galaxy_context,
+                         requirement_spec_strings=self.args,
+                         display_callback=self.display,
+                         ignore_errors=self.options.ignore_errors,
+                         no_deps=self.options.no_deps,
+                         force_overwrite=self.options.force)
 
         return rc