Add --lockfile option to use a collections lockfile

Add support for 'collection lockfiles'.
Add 'list --lockfile' to output a collections lockfile to stdout.

A collections lock specifies a set of collections to install
and the version specs to match.

Fixes ansible#173

Add collections_lock and collections_lockfile models.

Add ansible_galaxy.collections_lockfile with a yaml loader
for a collections lockfile.

Update cli and install action to use the collections lockfile.

Update README.md with lockfile info

To reproduce an existing installed collection path, use like:

  mazer list --lockfile  > collections_lockfile.yml
  mazer install --collections-path /tmp/somenewplace --collections-lock collections_lockfile.yml

To create a lockfile that matches current versions exactly, add
the '--frozen' flag:

  mazer list --lockfile --frozen

Example output for `mazer list --lockfile`:

alikins.collection_inspect: '*'
example.foobar: '*'

Example output for `mazer list --lockfile --frozen`:

alikins.collection_inspect: ==1.2.3
example.foobar: ==0.0.9

The contents of  collections lock file is a yaml file,
containing a dictionary. The dictitionary is the same
format as the 'dependencies' dict in galaxy.yml.

ie, The keys are collection labels (the namespace and name
dot separated ala 'alikins.collection_inspect').

The values is a version spec string. ie "*" or "==1.0.0".

Example contents of a collections lock file:

alikins.collection_inspect: "1.0.0"
alikins.collection_ntp: ">0.0.1,!=0.0.2"

README.md

@@ -98,6 +98,77 @@ The install option **'--editable'** or the short **'-e'** can be used.
 
 Note that **'--namespace'** option is required.
 
+### Install collections specified in a collections lockfile
+
+Mazer supports specifying a list of collections to be installed
+from a file (a 'collections lockfile').
+
+To install collections specified in a lockfile, use the
+**'--lockfile' option of the 'install' subcommand**:
+
+```
+$ mazer install --lockfile collections_lockfile.yml
+```
+
+### Generate a collections lockfile based on installed collections
+
+To create a collections lockfile representing the currently installed
+collections:
+
+```
+$ mazer list --lockfile
+```
+
+To create a lockfile that matches current versions exactly, add
+the **'--frozen'** flag:
+
+```
+$ mazer list --lockfile --frozen
+```
+
+To reproduce an existing installed collection path, redirect the 'list --lockfile'
+output to a file and use that file with 'install --collections-lock':
+
+```
+$ mazer list --lockfile  > collections_lockfile.yml
+$ mazer install --collections-path /tmp/somenewplace --lockfile collections_lockfile.yml
+```
+
+#### Collections lockfile format
+
+The contents of  collections lock file is a yaml file, containing a dictionary.
+
+The dictionary is the same format as the 'dependencies' dict in
+galaxy.yml.
+
+The keys are collection labels (the namespace and the name
+dot separated ala 'alikins.collection_inspect').
+
+The values are a version spec string. For ex, `*` or "==1.0.0".
+
+Example contents of a collections lockfile:
+
+``` yaml
+alikins.collection_inspect: "*"
+alikins.collection_ntp: "*"
+```
+
+Example contents of a collections lockfile specifying
+version specs:
+
+``` yaml
+alikins.collection_inspect: "1.0.0"
+alikins.collection_ntp: ">0.0.1,!=0.0.2"
+```
+
+Example contents of a collections lockfile specifying
+exact "frozen" versions:
+
+``` yaml
+alikins.collection_inspect: "1.0.0"
+alikins.collection_ntp: "2.3.4"
+```
+
 ### Building ansible content collection artifacts with 'mazer build'
 
 In the future, galaxy will support importing and ansible content collection

ansible_galaxy/actions/install.py

@@ -2,14 +2,17 @@
 import pprint
 
 from ansible_galaxy import collection_artifact
+from ansible_galaxy import collections_lockfile
 from ansible_galaxy import display
 from ansible_galaxy import download
 from ansible_galaxy import exceptions
 from ansible_galaxy import install
 from ansible_galaxy import installed_repository_db
 from ansible_galaxy import matchers
 from ansible_galaxy import repository_spec_parse
+from ansible_galaxy import requirements
 from ansible_galaxy.fetch import fetch_factory
+from ansible_galaxy.models.collections_lock import CollectionsLock
 from ansible_galaxy.models.repository_spec import FetchMethods
 from ansible_galaxy.models.requirement import Requirement, RequirementOps
 from ansible_galaxy.models.requirement_spec import RequirementSpec
@@ -95,11 +98,27 @@ def install_repositories_matching_repository_specs(galaxy_context,
                                 force_overwrite=force_overwrite)
 
 
+def load_collections_lockfile(lockfile_path):
+    try:
+        log.debug('Opening the collections lockfile %s', lockfile_path)
+        with open(lockfile_path, 'r') as lffd:
+            return collections_lockfile.load(lffd)
+
+    except EnvironmentError as exc:
+        log.exception(exc)
+
+        msg = 'Error opening the collections lockfile "%s": %s' % (lockfile_path, exc)
+        log.error(msg)
+
+        raise exceptions.GalaxyClientError(msg)
+
+
 # FIXME: probably pass the point where passing around all the data to methods makes sense
 #        so probably needs a stateful class here
 def install_repository_specs_loop(galaxy_context,
                                   repository_spec_strings=None,
                                   requirements_list=None,
+                                  collections_lockfile_path=None,
                                   editable=False,
                                   namespace_override=None,
                                   display_callback=None,
@@ -152,6 +171,19 @@ def install_repository_specs_loop(galaxy_context,
 
         requirements_list.append(req)
 
+    log.debug('collections_lockfile_path: %s', collections_lockfile_path)
+
+    if collections_lockfile_path:
+        # load collections lockfile as if the 'dependencies' dict from a collection_info
+        collections_lockfile = load_collections_lockfile(collections_lockfile_path)
+
+        dependencies_list = requirements.from_dependencies_dict(collections_lockfile.dependencies)
+
+        # Create the CollectionsLock for the validators
+        collections_lock = CollectionsLock(dependencies=dependencies_list)
+
+        requirements_list.extend(collections_lock.dependencies)
+
     log.debug('requirements_list: %s', requirements_list)
 
     while True:

ansible_galaxy/actions/list.py

@@ -4,13 +4,37 @@
 from ansible_galaxy import installed_content_item_db
 from ansible_galaxy import installed_repository_db
 from ansible_galaxy import matchers
+from ansible_galaxy import yaml_persist
 
 log = logging.getLogger(__name__)
 
 
+def format_as_lockfile(repo_list, lockfile_freeze=False):
+    '''For a given repo_list, return the string content of the lockfile that matches'''
+
+    if not repo_list:
+        return ''
+
+    collections_deps = {}
+    for repo_item in repo_list:
+        label = "{installed_repository.repository_spec.label}".format(**repo_item)
+        version_spec = '*'
+
+        if lockfile_freeze:
+            version_spec = "=={installed_repository.repository_spec.version}".format(**repo_item)
+
+        collections_deps[label] = version_spec
+
+    buf = yaml_persist.safe_dump(collections_deps, None, default_flow_style=False)
+
+    return buf.strip()
+
+
 def _list(galaxy_context,
           repository_spec_match_filter=None,
           list_content=False,
+          lockfile_format=False,
+          lockfile_freeze=False,
           display_callback=None):
 
     log.debug('list_content: %s', list_content)
@@ -55,6 +79,12 @@ def _list(galaxy_context,
                      'installed_repository': installed_repository}
         repo_list.append(repo_dict)
 
+    if lockfile_format:
+        output = format_as_lockfile(repo_list, lockfile_freeze=lockfile_freeze)
+        display_callback(output)
+
+        return repo_list
+
     for repo_item in repo_list:
         repo_msg = "repo={installed_repository.repository_spec.label}, type=repository, version={installed_repository.repository_spec.version}"
         display_callback(repo_msg.format(**repo_item))
@@ -84,12 +114,16 @@ def _list(galaxy_context,
 def list_action(galaxy_context,
                 repository_spec_match_filter=None,
                 list_content=False,
+                lockfile_format=False,
+                lockfile_freeze=False,
                 display_callback=None):
     '''Run _list action and return an exit code suitable for process exit'''
 
     _list(galaxy_context,
           repository_spec_match_filter=repository_spec_match_filter,
           list_content=list_content,
+          lockfile_format=lockfile_format,
+          lockfile_freeze=lockfile_freeze,
           display_callback=display_callback)
 
     return 0

ansible_galaxy/collections_lockfile.py

@@ -0,0 +1,26 @@
+import logging
+
+import yaml
+
+from ansible_galaxy import exceptions
+from ansible_galaxy.models.collections_lockfile import CollectionsLockfile
+
+log = logging.getLogger(__name__)
+
+
+# TODO: replace with a generic version for cases
+#       where SomeClass(**dict_from_yaml) works
+def load(data_or_file_object):
+    data_dict = yaml.safe_load(data_or_file_object)
+
+    log.debug('data_dict: %s', data_dict)
+
+    try:
+        collections_lockfile = CollectionsLockfile(dependencies=data_dict)
+    except ValueError:
+        raise
+    except Exception as exc:
+        log.exception(exc)
+        raise exceptions.GalaxyClientError("Error parsing collections lockfile: %s" % str(exc))
+
+    return collections_lockfile

ansible_galaxy/models/collections_lock.py

@@ -0,0 +1,9 @@
+
+import attr
+
+
+@attr.s(frozen=True)
+class CollectionsLock(object):
+    '''The collections "lock" (ie, manifest) used with --colleections-lock'''
+
+    dependencies = attr.ib(factory=list, validator=attr.validators.instance_of(list))

ansible_galaxy/models/collections_lockfile.py

@@ -0,0 +1,11 @@
+from ansible_galaxy.utils import attr_utils
+import attr
+
+
+@attr.s(frozen=True)
+class CollectionsLockfile(object):
+    '''Represents the data in a collections lock yaml file.'''
+
+    dependencies = attr.ib(factory=dict,
+                           validator=attr.validators.instance_of(dict),
+                           converter=attr_utils.convert_none_to_empty_dict)

ansible_galaxy_cli/cli/galaxy.py

@@ -92,6 +92,8 @@ def set_action(self):
             self.parser.set_usage("usage: %prog info [options] repo_name[,version]")
         elif self.action == "install":
             self.parser.set_usage("usage: %prog install [options] [collection_name(s)[,version] | collection_artifact_file(s)]")
+            self.parser.add_option('-l', '--lockfile', dest='collections_lockfile',
+                                   help='A collections lockfile listing collections to install')
             self.parser.add_option('-g', '--global', dest='global_install', action='store_true',
                                    help='Install content to the path containing your global or system-wide content. The default is the '
                                    'global_collections_path configured in your mazer.yml file (/usr/share/ansible/content, if not configured)')
@@ -107,7 +109,11 @@ def set_action(self):
             self.parser.set_usage("usage: %prog remove repo1 repo2 ...")
         elif self.action == "list":
             self.parser.set_usage("usage: %prog list [repo_name]")
-            self.parser.add_option('--content', dest='list_content', default=False, action='store_true', help="List each content item type in a repo")
+            self.parser.add_option('--content', dest='list_content', default=False, action='store_true', help="List each content item type in a collection")
+            self.parser.add_option('--lockfile', dest='list_lockfile_format', default=False, action='store_true',
+                                   help="List installed collections in collections lockfile format")
+            self.parser.add_option('--freeze', dest='list_lockfile_freeze', default=False, action='store_true',
+                                   help="List installed collections in collections lockfile format with frozen versions")
         elif self.action == "version":
             self.parser.set_usage("usage: %prog version")
 
@@ -279,6 +285,7 @@ def execute_install(self):
         rc = install.install_repository_specs_loop(galaxy_context,
                                                    editable=self.options.editable_install,
                                                    repository_spec_strings=requested_spec_strings,
+                                                   collections_lockfile_path=self.options.collections_lockfile,
                                                    namespace_override=self.options.namespace,
                                                    display_callback=self.display,
                                                    ignore_errors=self.options.ignore_errors,
@@ -334,6 +341,8 @@ def execute_list(self):
         return list_action.list_action(galaxy_context,
                                        repository_spec_match_filter=match_filter,
                                        list_content=list_content,
+                                       lockfile_format=self.options.list_lockfile_format,
+                                       lockfile_freeze=self.options.list_lockfile_freeze,
                                        display_callback=self.display)
 
     def execute_version(self):

tests/ansible_galaxy/actions/test_list.py

@@ -54,3 +54,31 @@ def test_list_no_content_dir(galaxy_context):
     # TODO: list should probably return non-zero if galaxy_context.collections_path doesnt exist,
     #       but should probaly initially check that when creating galaxy_context
     assert res == 0
+
+
+def test_format_as_lockfile_empty():
+    repo_list = []
+    res = list_action.format_as_lockfile(repo_list)
+    log.debug('res: |%s|', res)
+    assert res == ''
+
+
+def test_format_as_lockfile(mocker):
+    repo_list = []
+    mock_installed = mocker.MagicMock()
+    mock_installed.repository_spec.version = '1.2.3'
+    mock_installed.repository_spec.label = 'testns.testcollection'
+    mock_installed2 = mocker.MagicMock()
+    mock_installed2.repository_spec.version = '0.0.1'
+    mock_installed2.repository_spec.label = 'example.randomjunk'
+
+    repo_dict = {'content_items': {},
+                 'installed_repository': mock_installed
+                 }
+    repo_list.append(repo_dict)
+    repo_list.append({'content_items': {},
+                      'installed_repository': mock_installed2})
+    res = list_action.format_as_lockfile(repo_list)
+    log.debug('res: |%s|', res)
+
+    assert 'testns.testcollection' in res

tests/ansible_galaxy/models/test_collections_lock_model.py

@@ -0,0 +1,23 @@
+import logging
+
+import pytest
+
+from ansible_galaxy.models import collections_lock
+
+log = logging.getLogger(__name__)
+
+
+def test_init():
+    res = collections_lock.CollectionsLock()
+
+    assert isinstance(res, collections_lock.CollectionsLock)
+    assert isinstance(res.dependencies, list)
+    assert res.dependencies == []
+
+
+def test_init_deps_dict():
+    not_a_list = {'foo': 'bar'}
+    with pytest.raises(TypeError, match='.*dependencies.*') as exc_info:
+        collections_lock.CollectionsLock(dependencies=not_a_list)
+
+    log.debug('exc_info: %s', exc_info)

tests/ansible_galaxy/models/test_collections_lockfile_model.py

@@ -0,0 +1,31 @@
+import logging
+
+import pytest
+
+from ansible_galaxy.models import collections_lockfile
+
+log = logging.getLogger(__name__)
+
+
+def test_init():
+    res = collections_lockfile.CollectionsLockfile()
+
+    assert isinstance(res, collections_lockfile.CollectionsLockfile)
+    assert isinstance(res.dependencies, dict)
+    assert res.dependencies == {}
+
+
+def test_init_deps_none():
+    res = collections_lockfile.CollectionsLockfile(dependencies=None)
+
+    assert isinstance(res, collections_lockfile.CollectionsLockfile)
+    assert isinstance(res.dependencies, dict)
+    assert res.dependencies == {}
+
+
+def test_init_deps_dict():
+    not_a_dict = ['foo', 'bar']
+    with pytest.raises(TypeError, match='.*dependencies.*') as exc_info:
+        collections_lockfile.CollectionsLockfile(dependencies=not_a_dict)
+
+    log.debug('exc_info: %s', exc_info)