Beatless has now his default list of settings

.travis.yml

@@ -99,6 +99,16 @@ jobs:
       go: $GO_VERSION
       stage: test
 
+    # Beatless
+    - os: linux
+      env: TARGETS="-C x-pack/beatless testsuite"
+      go: $GO_VERSION
+      stage: test
+    - os: osx
+      env: TARGETS="TEST_ENVIRONMENT=0 -C x-pack/beatless testsuite"
+      go: $GO_VERSION
+      stage: test
+
     # Generators
     - os: linux
       env: TARGETS="-C generator/metricbeat test"

CHANGELOG-developer.asciidoc

@@ -51,3 +51,4 @@ The list below covers the major changes between 6.3.0 and master only.
 - Libbeat provides a new function `cmd.GenRootCmdWithSettings` that should be preferred over deprecated functions
   `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850]
 - Set current year in generator templates. {pull}8396[8396]
+- You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449]

NOTICE.txt

@@ -115,6 +115,53 @@ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+--------------------------------------------------------------------
+Dependency: github.com/aws/aws-lambda-go
+Version: =v1.6.0
+Revision: 2d482ef09017ae953b1e8d5a6ddac5b696663a3c
+License type (autodetected): Apache-2.0
+./vendor/github.com/aws/aws-lambda-go/LICENSE:
+--------------------------------------------------------------------
+Apache License 2.0
+
+
+--------------------------------------------------------------------
+Dependency: github.com/aws/aws-lambda-go
+Version: =v1.6.0
+Revision: 2d482ef09017ae953b1e8d5a6ddac5b696663a3c
+License type (autodetected): MIT
+./vendor/github.com/aws/aws-lambda-go/LICENSE-LAMBDACODE:
+--------------------------------------------------------------------
+MIT No Attribution
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+--------------------------------------------------------------------
+Dependency: github.com/aws/aws-sdk-go-v2
+Version: v2.0.0-preview.4
+Revision: ff1a530c31507c97cf5edbee226e604ca08661cc
+License type (autodetected): Apache-2.0
+./vendor/github.com/aws/aws-sdk-go-v2/LICENSE.txt:
+--------------------------------------------------------------------
+Apache License 2.0
+
+-------NOTICE.txt-----
+AWS SDK for Go
+Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 
+Copyright 2014-2015 Stripe, Inc.
+
 --------------------------------------------------------------------
 Dependency: github.com/bsm/sarama-cluster
 Revision: 7e67d87a6b3f83fe08c096fd084691bd9dca112f
@@ -691,6 +738,15 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+--------------------------------------------------------------------
+Dependency: github.com/go-ini/ini
+Revision: fa25069db393aecc09b71267d0489b357781c860
+License type (autodetected): Apache-2.0
+./vendor/github.com/go-ini/ini/LICENSE:
+--------------------------------------------------------------------
+Apache License 2.0
+
+
 --------------------------------------------------------------------
 Dependency: github.com/go-ole/go-ole
 Revision: de8695c8edbf8236f30d6e1376e20b198a028d42
@@ -1362,6 +1418,15 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+--------------------------------------------------------------------
+Dependency: github.com/jmespath/go-jmespath
+Revision: c2b33e8439af944379acbdd9c3a5fe0bc44bd8a5
+License type (autodetected): Apache-2.0
+./vendor/github.com/jmespath/go-jmespath/LICENSE:
+--------------------------------------------------------------------
+Apache License 2.0
+
+
 --------------------------------------------------------------------
 Dependency: github.com/joeshaw/multierror
 Revision: 69b34d4ec901851247ae7e77d33909caf9df99ed
@@ -2974,40 +3039,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 --------------------------------------------------------------------
 Dependency: gopkg.in/yaml.v2
 Revision: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b
-License type (autodetected): MIT
-./vendor/gopkg.in/yaml.v2/LICENSE.libyaml:
+License type (autodetected): Apache-2.0
+./vendor/gopkg.in/yaml.v2/LICENSE:
 --------------------------------------------------------------------
-The following files were ported to Go from C files of libyaml, and thus
-are still covered by their original copyright and license:
-
-    apic.go
-    emitterc.go
-    parserc.go
-    readerc.go
-    scannerc.go
-    writerc.go
-    yamlh.go
-    yamlprivateh.go
-
-Copyright (c) 2006 Kirill Simonov
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+Apache License 2.0
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
 
 --------------------------------------------------------------------
 Dependency: gopkg.in/yaml.v2

dev-tools/generate_notice.py

@@ -7,6 +7,8 @@
 import json
 import csv
 import re
+import pdb
+import copy
 
 
 def read_file(filename):
@@ -59,7 +61,8 @@ def gather_dependencies(vendor_dirs, overrides=None):
                     print("WARNING: No version information found for: {}".format(lib_path))
                     lib = {"path": lib_path}
                 else:
-                    lib = lib_search[0]
+                    lib = copy.deepcopy(lib_search[0])
+
                 lib["license_file"] = os.path.join(root, filename)
 
                 lib["license_contents"] = read_file(lib["license_file"])
@@ -79,16 +82,24 @@ def gather_dependencies(vendor_dirs, overrides=None):
             # don't walk down into another vendor dir
             if "vendor" in dirs:
                 dirs.remove("vendor")
+
     return dependencies
 
 
+# Allow to skip files that could match the `LICENSE` pattern but does not have any license information.
+SKIP_FILES = [
+    # AWS lambda go defines that some part of the code is APLv2 and other on a MIT Modified license.
+    "./vendor/github.com/aws/aws-lambda-go/LICENSE-SUMMARY"
+]
+
+
 def get_licenses(folder):
     """
     Get a list of license files from a given directory.
     """
     licenses = []
     for filename in sorted(os.listdir(folder)):
-        if filename.startswith("LICENSE") and "docs" not in filename:
+        if filename.startswith("LICENSE") and "docs" not in filename and os.path.join(folder, filename) not in SKIP_FILES:
             licenses.append(filename)
         elif filename.startswith("APLv2"):  # gorhill/cronexpr
             licenses.append(filename)
@@ -241,6 +252,19 @@ def create_notice(filename, beat, copyright, vendor_dirs, csvfile, overrides=Non
 permit persons to whom the Software is furnished to do so, subject to
 the following conditions:
     """),
+    re.sub(r"\s+", " ", """Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+    """),
 ]
 
 BSD_LICENSE_CONTENTS = [

libbeat/cfgfile/cfgfile.go

@@ -37,7 +37,7 @@ var (
 	testConfig  = flag.Bool("configtest", false, "Test configuration and exit.")
 
 	// Additional default settings, that must be available for variable expansion
-	defaults = mustNewConfigFrom(map[string]interface{}{
+	defaults = common.MustNewConfigFrom(map[string]interface{}{
 		"path": map[string]interface{}{
 			"home":   ".", // to be initialized by beat
 			"config": "${path.home}",
@@ -63,14 +63,6 @@ func init() {
 	makePathFlag("path.logs", "Logs path")
 }
 
-func mustNewConfigFrom(from interface{}) *common.Config {
-	cfg, err := common.NewConfigFrom(from)
-	if err != nil {
-		panic(err)
-	}
-	return cfg
-}
-
 // ChangeDefaultCfgfileFlag replaces the value and default value for the `-c`
 // flag so that it reflects the beat name.
 func ChangeDefaultCfgfileFlag(beatName string) error {
@@ -105,7 +97,7 @@ func HandleFlags() error {
 // structure. If path is empty this method reads from the configuration
 // file specified by the '-c' command line flag.
 func Read(out interface{}, path string) error {
-	config, err := Load(path)
+	config, err := Load(path, nil)
 	if err != nil {
 		return err
 	}
@@ -116,7 +108,7 @@ func Read(out interface{}, path string) error {
 // Load reads the configuration from a YAML file structure. If path is empty
 // this method reads from the configuration file specified by the '-c' command
 // line flag.
-func Load(path string) (*common.Config, error) {
+func Load(path string, beatOverrides *common.Config) (*common.Config, error) {
 	var config *common.Config
 	var err error
 
@@ -142,13 +134,22 @@ func Load(path string) (*common.Config, error) {
 		return nil, err
 	}
 
-	config, err = common.MergeConfigs(
-		defaults,
-		config,
-		overwrites,
-	)
-	if err != nil {
-		return nil, err
+	if beatOverrides != nil {
+		config, err = common.MergeConfigs(
+			defaults,
+			beatOverrides,
+			config,
+			overwrites,
+		)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		config, err = common.MergeConfigs(
+			defaults,
+			config,
+			overwrites,
+		)
 	}
 
 	config.PrintDebugf("Complete configuration loaded:")

libbeat/cmd/instance/beat.go

@@ -229,8 +229,8 @@ func NewBeat(name, indexPrefix, v string) (*Beat, error) {
 	return &Beat{Beat: b}, nil
 }
 
-// init does initialization of things common to all actions (read confs, flags)
-func (b *Beat) Init() error {
+// InitWithSettings does initialization of things common to all actions (read confs, flags)
+func (b *Beat) InitWithSettings(settings Settings) error {
 	err := b.handleFlags()
 	if err != nil {
 		return err
@@ -240,13 +240,20 @@ func (b *Beat) Init() error {
 		return err
 	}
 
-	if err := b.configure(); err != nil {
+	if err := b.configure(settings); err != nil {
 		return err
 	}
 
 	return nil
 }
 
+// Init does initialization of things common to all actions (read confs, flags)
+//
+// Deprecated: use InitWithSettings
+func (b *Beat) Init() error {
+	return b.InitWithSettings(Settings{})
+}
+
 // BeatConfig returns config section for this beat
 func (b *Beat) BeatConfig() (*common.Config, error) {
 	configName := strings.ToLower(b.Info.Beat)
@@ -323,7 +330,7 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error {
 	defer logp.Sync()
 	defer logp.Info("%s stopped.", b.Info.Beat)
 
-	err := b.Init()
+	err := b.InitWithSettings(settings)
 	if err != nil {
 		return err
 	}
@@ -504,10 +511,10 @@ func (b *Beat) handleFlags() error {
 // config reads the configuration file from disk, parses the common options
 // defined in BeatConfig, initializes logging, and set GOMAXPROCS if defined
 // in the config. Lastly it invokes the Config method implemented by the beat.
-func (b *Beat) configure() error {
+func (b *Beat) configure(settings Settings) error {
 	var err error
 
-	cfg, err := cfgfile.Load("")
+	cfg, err := cfgfile.Load("", settings.ConfigOverrides)
 	if err != nil {
 		return fmt.Errorf("error loading config file: %v", err)
 	}

libbeat/cmd/instance/settings.go

@@ -20,14 +20,16 @@ package instance
 import (
 	"github.com/spf13/pflag"
 
+	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/monitoring/report"
 )
 
 // Settings contains basic settings for any beat to pass into GenRootCmd
 type Settings struct {
-	Name        string
-	IndexPrefix string
-	Version     string
-	Monitoring  report.Settings
-	RunFlags    *pflag.FlagSet
+	Name            string
+	IndexPrefix     string
+	Version         string
+	Monitoring      report.Settings
+	RunFlags        *pflag.FlagSet
+	ConfigOverrides *common.Config
 }

libbeat/common/cli/cli.go

@@ -45,3 +45,14 @@ func RunWith(
 		}
 	}
 }
+
+// GetEnvOr return the value of the environment variable if the value is set, if its not set it will
+// return the default value.
+//
+// Note: if the value is set but it is an empty string we will return the empty string.
+func GetEnvOr(name, def string) string {
+	if env, ok := os.LookupEnv(name); ok {
+		return env
+	}
+	return def
+}

libbeat/feature/feature.go

@@ -23,7 +23,7 @@ import (
 
 // Registry is the global plugin registry, this variable is meant to be temporary to move all the
 // internal factory to receive a context that include the current beat registry.
-var Registry = newRegistry()
+var Registry = NewRegistry()
 
 // Featurable implements the description of a feature.
 type Featurable interface {