Add exec environment source

CHANGELOG.mkd

@@ -3,6 +3,7 @@ CHANGELOG
 
 Unreleased
 ----
+- Add exec environment source type. The exec source type allows for the implementation of external environment sources
 
 3.4.1
 ----

doc/dynamic-environments/configuration.mkd

@@ -578,6 +578,20 @@ remote: [email protected]:puppetlabs/control-repo.git
 ref: 8820892
 ```
 
+### Exec environment Source
+
+The exec environment source runs an external command which is expected to return on stdout content compatible with the YAML environment source data format. The command may return the data in JSON or YAML form. The exec environment source is similar in purpose to Puppet's exec node terminus, used to implement external node classifiers (ENCs). R10k's exec source type allows the the implementation of external environment sources.
+
+```yaml
+# r10k.yaml
+---
+sources:
+  puppet:
+    type: exec
+    basedir: /etc/puppetlabs/code/environments
+    command: /usr/local/bin/r10k-environments.sh
+```
+
 ### Environment Modules
 
 The environment modules feature allows module content to be attached to an environment at environment definition time. This happens before modules specified in a Puppetfile are attached to an environment, which does not happen until deploy time. Environment module implementation depends on the environment source type.

lib/r10k/source.rb

@@ -37,5 +37,6 @@ def self.from_hash(name, hash)
     require 'r10k/source/svn'
     require 'r10k/source/yaml'
     require 'r10k/source/yamldir'
+    require 'r10k/source/exec'
   end
 end

lib/r10k/source/exec.rb

@@ -0,0 +1,51 @@
+require 'r10k/util/subprocess'
+require 'json'
+require 'yaml'
+
+class R10K::Source::Exec < R10K::Source::Hash
+  R10K::Source.register(:exec, self)
+
+  def initialize(name, basedir, options = {})
+    unless @command = options[:command]
+      raise ConfigError, _('Environment source %{name} missing required parameter: command') % {name: name}
+    end
+
+    # We haven't set the environments option yet. We will do that by
+    # overloading the #environments method.
+    super(name, basedir, options)
+  end
+
+  def environments_hash
+    @environments_hash ||= set_environments_hash(run_environments_command)
+  end
+
+  private
+
+  def run_environments_command
+    subproc = R10K::Util::Subprocess.new([@command])
+    subproc.raise_on_fail = true
+    subproc.logger = self.logger
+    procresult = subproc.execute
+
+    begin
+      environments = JSON.parse(procresult.stdout)
+    rescue JSON::ParserError => json_err
+      begin
+        environments = YAML.safe_load(procresult.stdout)
+      rescue Psych::SyntaxError => yaml_err
+        raise R10K::Error, _("Error parsing command output for exec source %{name}:\n" \
+                             "Not valid JSON: %{j_msg}\n" \
+                             "Not valid YAML: %{y_msg}\n" \
+                             "Stdout:\n%{out}") % {name: name, j_msg: json_err.message, y_msg: yaml_err.message, out: procresult.stdout}
+      end
+    end
+
+    unless R10K::Source::Hash.valid_environments_hash?(environments)
+      raise R10K::Error, _("Environment source %{name} command %{cmd} did not return valid environment data.\n" \
+                           'Returned: %{data}') % {name: name, cmd: @command, data: environments}
+    end
+
+    # Return the resulting environments hash
+    environments
+  end
+end

lib/r10k/source/hash.rb

@@ -120,6 +120,16 @@
 #
 class R10K::Source::Hash < R10K::Source::Base
 
+  include R10K::Logging
+
+  # @param hash [Hash] A hash to validate.
+  # @return [Boolean] False if the hash is obviously invalid. A true return
+  #   means _maybe_ it's valid.
+  def self.valid_environments_hash?(hash)
+    # TODO: more robust schema valiation
+    hash.is_a?(Hash)
+  end
+
   # @param name [String] The identifier for this source.
   # @param basedir [String] The base directory where the generated environments will be created.
   # @param options [Hash] An additional set of options for this source. The
@@ -131,18 +141,33 @@ class R10K::Source::Hash < R10K::Source::Base
   # @option options [Hash] :environments The hash definition of environments
   def initialize(name, basedir, options = {})
     super(name, basedir, options)
+  end
 
-    @environments_hash = options.delete(:environments) || {}
-
-    @environments_hash.keys.each do |name|
-      R10K::Util::SymbolizeKeys.symbolize_keys!(@environments_hash[name])
-      @environments_hash[name][:basedir] = basedir
-      @environments_hash[name][:dirname] = R10K::Environment::Name.new(name, {prefix: @prefix, source: @name}).dirname
+  # Set the environment hash for the source. The environment hash is what the
+  # source uses to generate enviroments.
+  # @param hash [Hash] The hash to sanitize and use as the source's environments.
+  #   Should be formatted for use with R10K::Environment#from_hash.
+  def set_environments_hash(hash)
+    @environments_hash = hash.reduce({}) do |memo,(name,opts)|
+      R10K::Util::SymbolizeKeys.symbolize_keys!(opts)
+      memo.merge({ 
+        name => opts.merge({
+          :basedir => @basedir,
+          :dirname => R10K::Environment::Name.new(name, {prefix: @prefix, source: @name}).dirname
+        })
+      })
     end
   end
 
+  # Return the sanitized environments hash for this source. The environments
+  # hash should contain objects formatted for use with R10K::Environment#from_hash.
+  # If the hash does not exist it will be built based on @options.
+  def environments_hash
+    @environments_hash ||= set_environments_hash(@options.fetch(:environments, {}))
+  end
+
   def environments
-    @environments ||= @environments_hash.map do |name, hash|
+    @environments ||= environments_hash.map do |name, hash|
       R10K::Environment.from_hash(name, hash)
     end
   end

spec/unit/source/exec_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+require 'r10k/source'
+require 'json'
+require 'yaml'
+
+describe R10K::Source::Exec do
+
+  let(:environments_hash) do
+    {
+      'production' => {
+        'remote'  => 'https://git.example.com/puppet/control-repo.git',
+        'ref'     => 'release-141',
+        'modules' => {
+          'puppetlabs-stdlib' => '6.1.0',
+          'puppetlabs-ntp' => '8.1.0',
+          'example-myapp1' => {
+            'git' => 'https://git.example.com/puppet/example-myapp1.git',
+            'ref' => 'v1.3.0'
+          }
+        }
+      },
+      'development' => {
+        'remote'  => 'https://git.example.com/puppet/control-repo.git',
+        'ref'     => 'master',
+        'modules' => {
+          'puppetlabs-stdlib' => '6.1.0',
+          'puppetlabs-ntp' => '8.1.0',
+          'example-myapp1' => {
+            'git' => 'https://git.example.com/puppet/example-myapp1.git',
+            'ref' => 'v1.3.1'
+          }
+        }
+      }
+    }
+  end
+
+  describe 'initialize' do
+    context 'with a valid command' do
+      context 'that produces valid output' do
+        it 'accepts json' do
+          allow_any_instance_of(R10K::Util::Subprocess)
+            .to receive(:execute)
+            .and_return(double('result', stdout: environments_hash.to_json))
+
+          source = described_class.new('execsource', '/some/nonexistent/dir', command: '/path/to/command')
+          expect(source.environments.map(&:name)).to contain_exactly('production', 'development')
+        end
+
+        it 'accepts yaml' do
+          allow_any_instance_of(R10K::Util::Subprocess)
+            .to receive(:execute)
+            .and_return(double('result', stdout: environments_hash.to_yaml))
+
+          source = described_class.new('execsource', '/some/nonexistent/dir', command: '/path/to/command')
+          expect(source.environments.map(&:name)).to contain_exactly('production', 'development')
+        end
+
+      end
+
+      context 'that produces invalid output' do
+        it 'raises an error for non-json, non-yaml data' do
+          allow_any_instance_of(R10K::Util::Subprocess)
+            .to receive(:execute)
+            .and_return(double('result', stdout: "one:\ntwo\n"))
+
+          source = described_class.new('execsource', '/some/nonexistent/dir', command: '/path/to/command')
+          expect { source.environments }.to raise_error(/Error parsing command output/)
+        end
+
+        it 'raises an error for yaml data that is not a hash' do
+          allow_any_instance_of(R10K::Util::Subprocess)
+            .to receive(:execute)
+            .and_return(double('result', stdout: "[one, two]"))
+
+          source = described_class.new('execsource', '/some/nonexistent/dir', command: '/path/to/command')
+          expect { source.environments }.to raise_error(R10K::Error, /Environment source execsource.*did not return valid environment data.*one.*two.*/m)
+        end
+      end
+    end
+  end
+end

spec/unit/source/hash_spec.rb

@@ -2,6 +2,14 @@
 require 'r10k/source'
 
 describe R10K::Source::Hash do
+
+  describe '.valid_environments_hash?' do
+    it "rejects strings" do
+      expect(R10K::Source::Hash.valid_environments_hash?('200 OK'))
+        .to eq false
+    end
+  end
+
   let(:environments_hash) do
     {
       'production' => {

spec/unit/source/yaml_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+require 'r10k/source'
+
+describe R10K::Source::Yaml do
+
+  let(:environments_hash) do
+    {
+      'production' => {
+        'remote'  => 'https://git.example.com/puppet/control-repo.git',
+        'ref'     => 'release-141',
+        'modules' => {
+          'puppetlabs-stdlib' => '6.1.0',
+          'puppetlabs-ntp' => '8.1.0',
+          'example-myapp1' => {
+            'git' => 'https://git.example.com/puppet/example-myapp1.git',
+            'ref' => 'v1.3.0'
+          }
+        }
+      },
+      'development' => {
+        'remote'  => 'https://git.example.com/puppet/control-repo.git',
+        'ref'     => 'master',
+        'modules' => {
+          'puppetlabs-stdlib' => '6.1.0',
+          'puppetlabs-ntp' => '8.1.0',
+          'example-myapp1' => {
+            'git' => 'https://git.example.com/puppet/example-myapp1.git',
+            'ref' => 'v1.3.1'
+          }
+        }
+      }
+    }
+  end
+
+  describe "with valid yaml file" do
+    it "produces environments" do
+      allow(YAML).to receive(:load_file).with('/envs.yaml').and_return(environments_hash)
+      source = described_class.new('yamlsource', '/some/nonexistent/dir', config: '/envs.yaml')
+      expect(source.environments.map(&:name)).to contain_exactly('production', 'development')
+    end
+  end
+end