diff --git a/README.md b/README.md index 66aefecc..4338d0ef 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,77 @@ The install option **'--editable'** or the short **'-e'** can be used. Note that **'--namespace'** option is required. +### Install collections specified in a collections lockfile + +Mazer supports specifying a list of collections to be installed +from a file (a 'collections lockfile'). + +To install collections specified in a lockfile, use the +**'--lockfile' option of the 'install' subcommand**: + +``` +$ mazer install --lockfile collections_lockfile.yml +``` + +### Generate a collections lockfile based on installed collections + +To create a collections lockfile representing the currently installed +collections: + +``` +$ mazer list --lockfile +``` + +To create a lockfile that matches current versions exactly, add +the **'--frozen'** flag: + +``` +$ mazer list --lockfile --frozen +``` + +To reproduce an existing installed collection path, redirect the 'list --lockfile' +output to a file and use that file with 'install --collections-lock': + +``` +$ mazer list --lockfile > collections_lockfile.yml +$ mazer install --collections-path /tmp/somenewplace --lockfile collections_lockfile.yml +``` + +#### Collections lockfile format + +The contents of collections lock file is a yaml file, containing a dictionary. + +The dictionary is the same format as the 'dependencies' dict in +galaxy.yml. + +The keys are collection labels (the namespace and the name +dot separated ala 'alikins.collection_inspect'). + +The values are a version spec string. For ex, `*` or "==1.0.0". + +Example contents of a collections lockfile: + +``` yaml +alikins.collection_inspect: "*" +alikins.collection_ntp: "*" +``` + +Example contents of a collections lockfile specifying +version specs: + +``` yaml +alikins.collection_inspect: "1.0.0" +alikins.collection_ntp: ">0.0.1,!=0.0.2" +``` + +Example contents of a collections lockfile specifying +exact "frozen" versions: + +``` yaml +alikins.collection_inspect: "1.0.0" +alikins.collection_ntp: "2.3.4" +``` + ### Building ansible content collection artifacts with 'mazer build' In the future, galaxy will support importing and ansible content collection diff --git a/ansible_galaxy/actions/install.py b/ansible_galaxy/actions/install.py index e17701a9..70f67cfb 100644 --- a/ansible_galaxy/actions/install.py +++ b/ansible_galaxy/actions/install.py @@ -2,6 +2,7 @@ import pprint from ansible_galaxy import collection_artifact +from ansible_galaxy import collections_lockfile from ansible_galaxy import display from ansible_galaxy import download from ansible_galaxy import exceptions @@ -9,7 +10,9 @@ from ansible_galaxy import installed_repository_db from ansible_galaxy import matchers from ansible_galaxy import repository_spec_parse +from ansible_galaxy import requirements from ansible_galaxy.fetch import fetch_factory +from ansible_galaxy.models.collections_lock import CollectionsLock from ansible_galaxy.models.repository_spec import FetchMethods from ansible_galaxy.models.requirement import Requirement, RequirementOps from ansible_galaxy.models.requirement_spec import RequirementSpec @@ -95,11 +98,27 @@ def install_repositories_matching_repository_specs(galaxy_context, force_overwrite=force_overwrite) +def load_collections_lockfile(lockfile_path): + try: + log.debug('Opening the collections lockfile %s', lockfile_path) + with open(lockfile_path, 'r') as lffd: + return collections_lockfile.load(lffd) + + except EnvironmentError as exc: + log.exception(exc) + + msg = 'Error opening the collections lockfile "%s": %s' % (lockfile_path, exc) + log.error(msg) + + raise exceptions.GalaxyClientError(msg) + + # FIXME: probably pass the point where passing around all the data to methods makes sense # so probably needs a stateful class here def install_repository_specs_loop(galaxy_context, repository_spec_strings=None, requirements_list=None, + collections_lockfile_path=None, editable=False, namespace_override=None, display_callback=None, @@ -152,6 +171,19 @@ def install_repository_specs_loop(galaxy_context, requirements_list.append(req) + log.debug('collections_lockfile_path: %s', collections_lockfile_path) + + if collections_lockfile_path: + # load collections lockfile as if the 'dependencies' dict from a collection_info + collections_lockfile = load_collections_lockfile(collections_lockfile_path) + + dependencies_list = requirements.from_dependencies_dict(collections_lockfile.dependencies) + + # Create the CollectionsLock for the validators + collections_lock = CollectionsLock(dependencies=dependencies_list) + + requirements_list.extend(collections_lock.dependencies) + log.debug('requirements_list: %s', requirements_list) while True: diff --git a/ansible_galaxy/actions/list.py b/ansible_galaxy/actions/list.py index b63fd04e..27b8b85b 100644 --- a/ansible_galaxy/actions/list.py +++ b/ansible_galaxy/actions/list.py @@ -4,13 +4,37 @@ from ansible_galaxy import installed_content_item_db from ansible_galaxy import installed_repository_db from ansible_galaxy import matchers +from ansible_galaxy import yaml_persist log = logging.getLogger(__name__) +def format_as_lockfile(repo_list, lockfile_freeze=False): + '''For a given repo_list, return the string content of the lockfile that matches''' + + if not repo_list: + return '' + + collections_deps = {} + for repo_item in repo_list: + label = "{installed_repository.repository_spec.label}".format(**repo_item) + version_spec = '*' + + if lockfile_freeze: + version_spec = "=={installed_repository.repository_spec.version}".format(**repo_item) + + collections_deps[label] = version_spec + + buf = yaml_persist.safe_dump(collections_deps, None, default_flow_style=False) + + return buf.strip() + + def _list(galaxy_context, repository_spec_match_filter=None, list_content=False, + lockfile_format=False, + lockfile_freeze=False, display_callback=None): log.debug('list_content: %s', list_content) @@ -55,6 +79,12 @@ def _list(galaxy_context, 'installed_repository': installed_repository} repo_list.append(repo_dict) + if lockfile_format: + output = format_as_lockfile(repo_list, lockfile_freeze=lockfile_freeze) + display_callback(output) + + return repo_list + for repo_item in repo_list: repo_msg = "repo={installed_repository.repository_spec.label}, type=repository, version={installed_repository.repository_spec.version}" display_callback(repo_msg.format(**repo_item)) @@ -84,12 +114,16 @@ def _list(galaxy_context, def list_action(galaxy_context, repository_spec_match_filter=None, list_content=False, + lockfile_format=False, + lockfile_freeze=False, display_callback=None): '''Run _list action and return an exit code suitable for process exit''' _list(galaxy_context, repository_spec_match_filter=repository_spec_match_filter, list_content=list_content, + lockfile_format=lockfile_format, + lockfile_freeze=lockfile_freeze, display_callback=display_callback) return 0 diff --git a/ansible_galaxy/collections_lockfile.py b/ansible_galaxy/collections_lockfile.py new file mode 100644 index 00000000..84c2bb1a --- /dev/null +++ b/ansible_galaxy/collections_lockfile.py @@ -0,0 +1,26 @@ +import logging + +import yaml + +from ansible_galaxy import exceptions +from ansible_galaxy.models.collections_lockfile import CollectionsLockfile + +log = logging.getLogger(__name__) + + +# TODO: replace with a generic version for cases +# where SomeClass(**dict_from_yaml) works +def load(data_or_file_object): + data_dict = yaml.safe_load(data_or_file_object) + + log.debug('data_dict: %s', data_dict) + + try: + collections_lockfile = CollectionsLockfile(dependencies=data_dict) + except ValueError: + raise + except Exception as exc: + log.exception(exc) + raise exceptions.GalaxyClientError("Error parsing collections lockfile: %s" % str(exc)) + + return collections_lockfile diff --git a/ansible_galaxy/models/collections_lock.py b/ansible_galaxy/models/collections_lock.py new file mode 100644 index 00000000..da4f5ea7 --- /dev/null +++ b/ansible_galaxy/models/collections_lock.py @@ -0,0 +1,9 @@ + +import attr + + +@attr.s(frozen=True) +class CollectionsLock(object): + '''The collections "lock" (ie, manifest) used with --colleections-lock''' + + dependencies = attr.ib(factory=list, validator=attr.validators.instance_of(list)) diff --git a/ansible_galaxy/models/collections_lockfile.py b/ansible_galaxy/models/collections_lockfile.py new file mode 100644 index 00000000..e056025d --- /dev/null +++ b/ansible_galaxy/models/collections_lockfile.py @@ -0,0 +1,11 @@ +from ansible_galaxy.utils import attr_utils +import attr + + +@attr.s(frozen=True) +class CollectionsLockfile(object): + '''Represents the data in a collections lock yaml file.''' + + dependencies = attr.ib(factory=dict, + validator=attr.validators.instance_of(dict), + converter=attr_utils.convert_none_to_empty_dict) diff --git a/ansible_galaxy_cli/cli/galaxy.py b/ansible_galaxy_cli/cli/galaxy.py index 57fd3f41..6bc80860 100644 --- a/ansible_galaxy_cli/cli/galaxy.py +++ b/ansible_galaxy_cli/cli/galaxy.py @@ -92,6 +92,8 @@ def set_action(self): self.parser.set_usage("usage: %prog info [options] repo_name[,version]") elif self.action == "install": self.parser.set_usage("usage: %prog install [options] [collection_name(s)[,version] | collection_artifact_file(s)]") + self.parser.add_option('-l', '--lockfile', dest='collections_lockfile', + help='A collections lockfile listing collections to install') self.parser.add_option('-g', '--global', dest='global_install', action='store_true', help='Install content to the path containing your global or system-wide content. The default is the ' 'global_collections_path configured in your mazer.yml file (/usr/share/ansible/content, if not configured)') @@ -107,7 +109,11 @@ def set_action(self): self.parser.set_usage("usage: %prog remove repo1 repo2 ...") elif self.action == "list": self.parser.set_usage("usage: %prog list [repo_name]") - self.parser.add_option('--content', dest='list_content', default=False, action='store_true', help="List each content item type in a repo") + self.parser.add_option('--content', dest='list_content', default=False, action='store_true', help="List each content item type in a collection") + self.parser.add_option('--lockfile', dest='list_lockfile_format', default=False, action='store_true', + help="List installed collections in collections lockfile format") + self.parser.add_option('--freeze', dest='list_lockfile_freeze', default=False, action='store_true', + help="List installed collections in collections lockfile format with frozen versions") elif self.action == "version": self.parser.set_usage("usage: %prog version") @@ -279,6 +285,7 @@ def execute_install(self): rc = install.install_repository_specs_loop(galaxy_context, editable=self.options.editable_install, repository_spec_strings=requested_spec_strings, + collections_lockfile_path=self.options.collections_lockfile, namespace_override=self.options.namespace, display_callback=self.display, ignore_errors=self.options.ignore_errors, @@ -334,6 +341,8 @@ def execute_list(self): return list_action.list_action(galaxy_context, repository_spec_match_filter=match_filter, list_content=list_content, + lockfile_format=self.options.list_lockfile_format, + lockfile_freeze=self.options.list_lockfile_freeze, display_callback=self.display) def execute_version(self): diff --git a/tests/ansible_galaxy/actions/test_list.py b/tests/ansible_galaxy/actions/test_list.py index bad6d674..9727b1f8 100644 --- a/tests/ansible_galaxy/actions/test_list.py +++ b/tests/ansible_galaxy/actions/test_list.py @@ -54,3 +54,31 @@ def test_list_no_content_dir(galaxy_context): # TODO: list should probably return non-zero if galaxy_context.collections_path doesnt exist, # but should probaly initially check that when creating galaxy_context assert res == 0 + + +def test_format_as_lockfile_empty(): + repo_list = [] + res = list_action.format_as_lockfile(repo_list) + log.debug('res: |%s|', res) + assert res == '' + + +def test_format_as_lockfile(mocker): + repo_list = [] + mock_installed = mocker.MagicMock() + mock_installed.repository_spec.version = '1.2.3' + mock_installed.repository_spec.label = 'testns.testcollection' + mock_installed2 = mocker.MagicMock() + mock_installed2.repository_spec.version = '0.0.1' + mock_installed2.repository_spec.label = 'example.randomjunk' + + repo_dict = {'content_items': {}, + 'installed_repository': mock_installed + } + repo_list.append(repo_dict) + repo_list.append({'content_items': {}, + 'installed_repository': mock_installed2}) + res = list_action.format_as_lockfile(repo_list) + log.debug('res: |%s|', res) + + assert 'testns.testcollection' in res diff --git a/tests/ansible_galaxy/models/test_collections_lock_model.py b/tests/ansible_galaxy/models/test_collections_lock_model.py new file mode 100644 index 00000000..1f8911e3 --- /dev/null +++ b/tests/ansible_galaxy/models/test_collections_lock_model.py @@ -0,0 +1,23 @@ +import logging + +import pytest + +from ansible_galaxy.models import collections_lock + +log = logging.getLogger(__name__) + + +def test_init(): + res = collections_lock.CollectionsLock() + + assert isinstance(res, collections_lock.CollectionsLock) + assert isinstance(res.dependencies, list) + assert res.dependencies == [] + + +def test_init_deps_dict(): + not_a_list = {'foo': 'bar'} + with pytest.raises(TypeError, match='.*dependencies.*') as exc_info: + collections_lock.CollectionsLock(dependencies=not_a_list) + + log.debug('exc_info: %s', exc_info) diff --git a/tests/ansible_galaxy/models/test_collections_lockfile_model.py b/tests/ansible_galaxy/models/test_collections_lockfile_model.py new file mode 100644 index 00000000..d171f391 --- /dev/null +++ b/tests/ansible_galaxy/models/test_collections_lockfile_model.py @@ -0,0 +1,31 @@ +import logging + +import pytest + +from ansible_galaxy.models import collections_lockfile + +log = logging.getLogger(__name__) + + +def test_init(): + res = collections_lockfile.CollectionsLockfile() + + assert isinstance(res, collections_lockfile.CollectionsLockfile) + assert isinstance(res.dependencies, dict) + assert res.dependencies == {} + + +def test_init_deps_none(): + res = collections_lockfile.CollectionsLockfile(dependencies=None) + + assert isinstance(res, collections_lockfile.CollectionsLockfile) + assert isinstance(res.dependencies, dict) + assert res.dependencies == {} + + +def test_init_deps_dict(): + not_a_dict = ['foo', 'bar'] + with pytest.raises(TypeError, match='.*dependencies.*') as exc_info: + collections_lockfile.CollectionsLockfile(dependencies=not_a_dict) + + log.debug('exc_info: %s', exc_info) diff --git a/tests/ansible_galaxy/test_collections_lockfile.py b/tests/ansible_galaxy/test_collections_lockfile.py new file mode 100644 index 00000000..82c7ebb6 --- /dev/null +++ b/tests/ansible_galaxy/test_collections_lockfile.py @@ -0,0 +1,76 @@ +import logging +import os + +import pytest + +from ansible_galaxy import collections_lockfile +from ansible_galaxy import exceptions +from ansible_galaxy.models.collections_lockfile import CollectionsLockfile + +log = logging.getLogger(__name__) + + +EXAMPLE_LOCKFILE_DIR = os.path.join(os.path.dirname(__file__), '../', 'data', 'collection_lockfiles') + + +def test_load(): + lockfile_path = os.path.join(EXAMPLE_LOCKFILE_DIR, + 'test_1.yml') + + with open(lockfile_path, 'r') as lfd: + lockfile = collections_lockfile.load(lfd) + + assert isinstance(lockfile, CollectionsLockfile) + assert 'alikins.collection_inspect' in lockfile.dependencies + assert 'alikins.collection_ntp' in lockfile.dependencies + assert lockfile.dependencies['alikins.collection_inspect'] == '>=0.0.1' + + +def test_load_frozen(): + lockfile_path = os.path.join(EXAMPLE_LOCKFILE_DIR, + 'frozen.yml') + + with open(lockfile_path, 'r') as lfd: + lockfile = collections_lockfile.load(lfd) + assert isinstance(lockfile, CollectionsLockfile) + assert 'alikins.collection_inspect' in lockfile.dependencies + assert 'alikins.collection_ntp' in lockfile.dependencies + assert lockfile.dependencies['alikins.collection_inspect'] == '==1.0.0' + assert lockfile.dependencies['alikins.collection_ntp'] == '==2.0.0' + + +def test_load_floating(): + lockfile_path = os.path.join(EXAMPLE_LOCKFILE_DIR, + 'floating.yml') + + with open(lockfile_path, 'r') as lfd: + lockfile = collections_lockfile.load(lfd) + assert isinstance(lockfile, CollectionsLockfile) + assert 'alikins.collection_inspect' in lockfile.dependencies + assert 'alikins.collection_ntp' in lockfile.dependencies + assert lockfile.dependencies['alikins.collection_inspect'] == '*' + assert lockfile.dependencies['alikins.collection_ntp'] == '*' + + +def test_load_explicit_start(): + lockfile_path = os.path.join(EXAMPLE_LOCKFILE_DIR, + 'explicit_start.yml') + + with open(lockfile_path, 'r') as lfd: + lockfile = collections_lockfile.load(lfd) + assert isinstance(lockfile, CollectionsLockfile) + assert 'alikins.collection_inspect' in lockfile.dependencies + assert 'alikins.collection_ntp' in lockfile.dependencies + assert lockfile.dependencies['alikins.collection_inspect'] == '*' + assert lockfile.dependencies['example2.name'] == '>=2.3.4,!=1.0.0' + + +def test_load_not_dict(): + lockfile_path = os.path.join(EXAMPLE_LOCKFILE_DIR, + 'not_dict.yml') + + with open(lockfile_path, 'r') as lfd: + with pytest.raises(exceptions.GalaxyClientError) as exc_info: + collections_lockfile.load(lfd) + + log.debug('exc_info: %s', exc_info) diff --git a/tests/data/collection_lockfiles/explicit_start.yml b/tests/data/collection_lockfiles/explicit_start.yml new file mode 100644 index 00000000..c0b51a0a --- /dev/null +++ b/tests/data/collection_lockfiles/explicit_start.yml @@ -0,0 +1,5 @@ +--- +alikins.collection_inspect: "*" +alikins.collection_ntp: ">=1.0.0" +example1.name: 1.0.0 +example2.name: ">=2.3.4,!=1.0.0" diff --git a/tests/data/collection_lockfiles/floating.yml b/tests/data/collection_lockfiles/floating.yml new file mode 100644 index 00000000..bd399a1b --- /dev/null +++ b/tests/data/collection_lockfiles/floating.yml @@ -0,0 +1,2 @@ +alikins.collection_inspect: "*" +alikins.collection_ntp: "*" diff --git a/tests/data/collection_lockfiles/frozen.yml b/tests/data/collection_lockfiles/frozen.yml new file mode 100644 index 00000000..8899fa7f --- /dev/null +++ b/tests/data/collection_lockfiles/frozen.yml @@ -0,0 +1,2 @@ +alikins.collection_inspect: "==1.0.0" +alikins.collection_ntp: "==2.0.0" diff --git a/tests/data/collection_lockfiles/key_not_string.yml b/tests/data/collection_lockfiles/key_not_string.yml new file mode 100644 index 00000000..ea490e6a --- /dev/null +++ b/tests/data/collection_lockfiles/key_not_string.yml @@ -0,0 +1,4 @@ +- "foo" +- "bar": + alikins.collection_inspect: "*" + alikins.collection_ntp: ">=1.0.0" diff --git a/tests/data/collection_lockfiles/not_dict.yml b/tests/data/collection_lockfiles/not_dict.yml new file mode 100644 index 00000000..e88ff768 --- /dev/null +++ b/tests/data/collection_lockfiles/not_dict.yml @@ -0,0 +1,3 @@ +--- +- "foo" +- "bar": diff --git a/tests/data/collection_lockfiles/test_1.yml b/tests/data/collection_lockfiles/test_1.yml new file mode 100644 index 00000000..5fbcf43f --- /dev/null +++ b/tests/data/collection_lockfiles/test_1.yml @@ -0,0 +1,2 @@ +alikins.collection_inspect: ">=0.0.1" +alikins.collection_ntp: ">=0.0.1,!=0.0.2"