Add Yaml environment source

doc/dynamic-environments/configuration.mkd

@@ -509,3 +509,100 @@ This will create the following directory structure:
 |-- app1_production  # app1 data repository, production branch
 |-- app1_develop     # app1 data repository, develop branch
 ```
+
+Experimental Features
+--------
+
+### YAML Environment Source
+
+Dynamically deploying Puppet content based on the state of version control repositories can be powerful and efficient for development workflows. The linkage however is not advantageous when trying to build precision controls over deployment of previously-developed and tested content.
+
+The YAML environment source type allows for a clear separation of tooling between development workflow, and deployment workflow. Development workflow creates new commits in the version control system. Deployment workflow consumes them.
+
+To use the YAML environment source, configure r10k's sources with at least one entry using the yaml type.
+
+```yaml
+# r10k.yaml
+---
+sources:
+  puppet:
+    type: yaml
+    basedir: /etc/puppetlabs/code/environments
+    config: /etc/puppetlabs/r10k/environments.yaml # default
+
+```
+
+When using the YAML source type, every environment is enumerated in a single yaml file. Each environment specifies a type, source, and version (typically a Git ref) to deploy. In the following example, two environments are defined, which are identical to each other.
+
+```yaml
+---
+production:
+  type: git
+  remote: [email protected]:puppetlabs/control-repo.git
+  ref: 8820892
+
+development:
+  type: git
+  remote: [email protected]:puppetlabs/control-repo.git
+  ref: 8820892
+```
+
+### Environment Modules
+
+The environment modules feature allows module content to be attached to an environment at environment definition time. This happens before modules specified in a Puppetfile are attached to an environment, which does not happen until deploy time. Environment module implementation depends on the environment source type.
+
+For the YAML environment source type, attach modules to an environment by specifying a modules key for the environment, and providing a hash of modules to attach. Each module accepts the same arguments accepted by the `mod` method in a Puppetfile.
+
+The example below includes two Forge modules and one module sourced from a Git repository. The two environments are almost identical. However, a new version of the stdlib module has been deployed in development (6.2.0), that has not yet been deployed to production.
+
+```yaml
+---
+production:
+  type: git
+  remote: [email protected]:puppetlabs/control-repo.git
+  ref: 8820892
+  modules:
+    puppetlabs-stdlib: 6.0.0
+    puppetlabs-concat: 6.1.0
+    reidmv-xampl:
+      git: https://github.com/reidmv/reidmv-xampl.git
+      ref: 62d07f2
+
+development:
+  type: git
+  remote: [email protected]:puppetlabs/control-repo.git
+  ref: 8820892
+  modules:
+    puppetlabs-stdlib: 6.2.0
+    puppetlabs-concat: 6.1.0
+    reidmv-xampl:
+      git: https://github.com/reidmv/reidmv-xampl.git
+      ref: 62d07f2
+```
+
+### Bare Environment Type
+
+A "control repository" typically contains a hiera.yaml, an environment.conf, a manifests/site.pp file, and a few other things. However, none of these are strictly necessary for an environment to be functional if modules can be deployed to it.
+
+The bare environment type allows sources that support environment modules to operate without a control repo being required. Modules can be deployed directly.
+
+```yaml
+---
+production:
+  type: bare
+  modules:
+    puppetlabs-stdlib: 6.0.0
+    puppetlabs-concat: 6.1.0
+    reidmv-xampl:
+      git: https://github.com/reidmv/reidmv-xampl.git
+      ref: 62d07f2
+
+development:
+  type: bare
+  modules:
+    puppetlabs-stdlib: 6.0.0
+    puppetlabs-concat: 6.1.0
+    reidmv-xampl:
+      git: https://github.com/reidmv/reidmv-xampl.git
+      ref: 62d07f2
+```

lib/r10k/action/deploy/environment.rb

@@ -140,14 +140,14 @@ def visit_puppetfile(puppetfile)
         end
 
         def visit_module(mod)
-          logger.info _("Deploying Puppetfile content %{mod_path}") % {mod_path: mod.path}
+          logger.info _("Deploying %{origin} content %{path}") % {origin: mod.origin, path: mod.path}
           mod.sync(force: @force)
         end
 
         def write_environment_info!(environment, started_at, success)
           module_deploys = []
           begin
-            environment.puppetfile.modules.each do |mod|
+            environment.modules.each do |mod|
               name = mod.name
               version = mod.version
               sha = mod.repo.head rescue nil

lib/r10k/environment.rb

@@ -1,6 +1,36 @@
 module R10K
   module Environment
+    def self.factory
+      @factory ||= R10K::KeyedFactory.new
+    end
+
+    def self.register(key, klass)
+      factory.register(key, klass)
+    end
+
+    def self.retrieve(key)
+      factory.retrieve(key)
+    end
+
+    def self.generate(type, name, basedir, dirname, options = {})
+      factory.generate(type, name, basedir, dirname, options)
+    end
+
+    def self.from_hash(name, hash)
+      R10K::Util::SymbolizeKeys.symbolize_keys!(hash)
+
+      basedir = hash.delete(:basedir)
+      dirname = hash.delete(:dirname) || name
+
+      type = hash.delete(:type)
+      type = type.is_a?(String) ? type.to_sym : type
+
+      generate(type, name, basedir, dirname, hash)
+    end
+
     require 'r10k/environment/base'
+    require 'r10k/environment/with_modules'
+    require 'r10k/environment/bare'
     require 'r10k/environment/git'
     require 'r10k/environment/svn'
   end

lib/r10k/environment/bare.rb

@@ -0,0 +1,16 @@
+class R10K::Environment::Bare < R10K::Environment::WithModules
+
+  R10K::Environment.register(:bare, self)
+
+  def sync
+    path.mkpath
+  end
+
+  def status
+    :not_applicable
+  end
+
+  def signature
+    'bare-default'
+  end
+end

lib/r10k/environment/git.rb

@@ -6,10 +6,14 @@
 # This class implements an environment based on a Git branch.
 #
 # @since 1.3.0
-class R10K::Environment::Git < R10K::Environment::Base
+class R10K::Environment::Git < R10K::Environment::WithModules
 
   include R10K::Logging
 
+  R10K::Environment.register(:git, self)
+  # Register git as the default environment type
+  R10K::Environment.register(nil, self)
+
   # @!attribute [r] remote
   #   @return [String] The URL to the remote git repository
   attr_reader :remote
@@ -66,15 +70,12 @@ def signature
 
   include R10K::Util::Purgeable
 
-  def managed_directories
-    [@full_path]
-  end
-
   # Returns an array of the full paths to all the content being managed.
   # @note This implements a required method for the Purgeable mixin
   # @return [Array<String>]
   def desired_contents
     desired = [File.join(@full_path, '.git')]
     desired += @repo.tracked_paths.map { |entry| File.join(@full_path, entry) }
+    desired += super
   end
 end

lib/r10k/environment/svn.rb

@@ -9,6 +9,8 @@ class R10K::Environment::SVN < R10K::Environment::Base
 
   include R10K::Logging
 
+  R10K::Environment.register(:svn, self)
+
   # @!attribute [r] remote
   #   @return [String] The URL to the remote SVN branch to check out
   attr_reader :remote

lib/r10k/environment/with_modules.rb

@@ -0,0 +1,139 @@
+require 'r10k/logging'
+require 'r10k/util/purgeable'
+
+# This abstract base class implements an environment that can include module
+# content
+#
+# @since 3.4.0
+class R10K::Environment::WithModules < R10K::Environment::Base
+
+  include R10K::Logging
+
+  # @!attribute [r] moduledir
+  #   @return [String] The directory to install environment-defined modules
+  #     into (default: #{basedir}/modules)
+  attr_reader :moduledir
+
+  # Initialize the given environment.
+  #
+  # @param name [String] The unique name describing this environment.
+  # @param basedir [String] The base directory where this environment will be created.
+  # @param dirname [String] The directory name for this environment.
+  # @param options [Hash] An additional set of options for this environment.
+  #
+  # @param options [String] :moduledir The path to install modules to
+  # @param options [Hash] :modules Modules to add to the environment
+  def initialize(name, basedir, dirname, options = {})
+    super(name, basedir, dirname, options)
+
+    @managed_content = {}
+    @modules = []
+    @moduledir = case options[:moduledir]
+                 when nil
+                   File.join(@basedir, @dirname, 'modules')
+                 when File.absolute_path(options[:moduledir])
+                   options.delete(:moduledir)
+                 else
+                   File.join(@basedir, @dirname, options.delete(:moduledir))
+                 end
+
+    modhash = options.delete(:modules)
+    load_modules(modhash) unless modhash.nil?
+  end
+
+  # @return [Array<R10K::Module::Base>] All modules associated with this environment.
+  #   Modules may originate from either:
+  #     - The r10k environment object
+  #     - A Puppetfile in the environment's content
+  def modules
+    return @modules if @puppetfile.nil?
+
+    @puppetfile.load unless @puppetfile.loaded?
+    @modules + @puppetfile.modules
+  end
+
+  def accept(visitor)
+    visitor.visit(:environment, self) do
+      @modules.each do |mod|
+        mod.accept(visitor)
+      end
+
+      puppetfile.accept(visitor)
+      validate_no_module_conflicts
+    end
+  end
+
+  def load_modules(module_hash)
+    module_hash.each do |name, args|
+      add_module(name, args)
+    end
+  end
+
+  # @param [String] name
+  # @param [*Object] args
+  def add_module(name, args)
+    if args.is_a?(Hash)
+      # symbolize keys in the args hash
+      args = args.inject({}) { |memo,(k,v)| memo[k.to_sym] = v; memo }
+    end
+
+    if args.is_a?(Hash) && install_path = args.delete(:install_path)
+      install_path = resolve_install_path(install_path)
+      validate_install_path(install_path, name)
+    else
+      install_path = @moduledir
+    end
+
+    # Keep track of all the content this environment is managing to enable purging.
+    @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
+
+    mod = R10K::Module.new(name, install_path, args, self.name)
+    mod.origin = 'Environment'
+
+    @managed_content[install_path] << mod.name
+    @modules << mod
+  end
+
+  def validate_no_module_conflicts
+    @puppetfile.load unless @puppetfile.loaded?
+    conflicts = (@modules + @puppetfile.modules)
+                .group_by { |mod| mod.name }
+                .select { |_, v| v.size > 1 }
+                .map(&:first)
+    unless conflicts.empty?
+      msg = _('Puppetfile cannot contain module names defined by environment %{name}') % {name: self.name}
+      msg += ' '
+      msg += _("Remove the conflicting definitions of the following modules: %{conflicts}" % { conflicts: conflicts.join(' ') })
+      raise R10K::Error.new(msg)
+    end
+  end
+
+  include R10K::Util::Purgeable
+
+  # Returns an array of the full paths that can be purged.
+  # @note This implements a required method for the Purgeable mixin
+  # @return [Array<String>]
+  def managed_directories
+    [@full_path]
+  end
+
+  # Returns an array of the full paths of filenames that should exist. Files
+  # inside managed_directories that are not listed in desired_contents will
+  # be purged.
+  # @note This implements a required method for the Purgeable mixin
+  # @return [Array<String>]
+  def desired_contents
+    list = @managed_content.keys
+    list += @managed_content.flat_map do |install_path, modnames|
+      modnames.collect { |name| File.join(install_path, name) }
+    end
+  end
+
+  def purge_exclusions
+    super + @managed_content.flat_map do |install_path, modnames|
+      modnames.map do |name|
+        File.join(install_path, name, '**', '*')
+      end
+    end
+  end
+end

lib/r10k/module/base.rb

@@ -31,6 +31,10 @@ class R10K::Module::Base
   #   @return [R10K::Environment, nil] The parent environment of the module
   attr_reader :environment
 
+  # @!attribute [rw] origin
+  #   @return [String] Where the module was sourced from. E.g., "Puppetfile"
+  attr_accessor :origin
+
   # There's been some churn over `author` vs `owner` and `full_name` over
   # `title`, so in the short run it's easier to support both and deprecate one
   # later.
@@ -47,6 +51,7 @@ def initialize(title, dirname, args, environment=nil)
     @owner, @name = parse_title(@title)
     @path = Pathname.new(File.join(@dirname, @name))
     @environment = environment
+    @origin = 'external' # Expect Puppetfile or R10k::Environment to set this to a specific value
   end
 
   # @deprecated

lib/r10k/module/forge.rb

@@ -75,7 +75,11 @@ def expected_version
 
   # @return [String] The version of the currently installed module
   def current_version
-    @metadata ? @metadata.version : nil
+    if insync?
+      (@metadata ||= @metadata_file.read).nil? ? nil : @metadata.version
+    else
+      nil
+    end
   end
 
   alias version current_version