Merge remote-tracking branch 'origin' into feature/beatless

CHANGELOG.asciidoc

@@ -41,17 +41,21 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Remove unix-like permission checks on Windows, so files can be opened. {issue}7849[7849]
 - Deregister pipeline loader callback when inputsRunner is stopped. {pull}7893[7893]
 - Replace index patterns in TSVB visualizations. {pull}7929[7929]
+- Fixed Support `add_docker_metadata` in Windows by identifying systems' path separator. {issue}7797[7797]
+- Add backoff support to x-pack monitoring outputs. {issue}7966[7966]
 
 *Auditbeat*
 
 - Fixed a crash in the file_integrity module under Linux. {issue}7753[7753]
 - Fixed a data race in the file_integrity module. {issue}8009[8009]
 - Fixed a deadlock in the file_integrity module. {pull}8027[8027]
+- Fixed the RPM by designating the config file as configuration data in the RPM spec. {issue}8075[8075]
 
 *Filebeat*
 
 - Fixed a memory leak when harvesters are closed. {pull}7820[7820]
 - Fix date format in Mongodb Ingest pipeline. {pull}7974[7974]
+- Mark the TCP and UDP input as GA. {pull}8125[8125]
 
 *Heartbeat*
 
@@ -63,6 +67,9 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Fixed a panic when the kvm module cannot establish a connection to libvirtd. {issue}7792[7792].
 - Recover metrics for old apache versions removed by mistake on #6450. {pull}7871[7871]
 - Add missing namespace field in http server metricset {pull}7890[7890]
+- Fixed the RPM by designating the modules.d config files as configuration data in the RPM spec. {issue}8075[8075]
+- Fixed the location of the modules.d dir in Deb and RPM packages. {issue}8104[8104]
+- Add docker diskio stats on Windows. {issue}6815[6815] {pull}8126[8126]
 
 *Packetbeat*
 
@@ -85,6 +92,8 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Make kubernetes autodiscover ignore events with empty container IDs {pull}7971[7971]
 - Add DNS processor with support for performing reverse lookups on IP addresses. {issue}7770[7770]
 - Implement CheckConfig in RunnerFactory to make autodiscover check configs {pull}7961[7961]
+- Count HTTP 429 responses in the elasticsearch output {pull}8056[8056]
+- Report configured queue type. {pull}8091[8091]
 
 *Auditbeat*
 
@@ -104,6 +113,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Add `metircs` metricset to MongoDB module. {pull}7611[7611]
 - Add fields for mermory fragmentation, memory allocator stats, copy on write, master-slave status, and active defragmentation to `info` metricset of Redis module. {pull}7695[7695]
 - Add experimental socket summary metricset to system module {pull}6782[6782]
+- Increase ignore_above for system.process.cmdline to 2048. {pull}8101[8100]
 
 *Packetbeat*
 

Makefile

@@ -83,7 +83,7 @@ check: python-env
 
 .PHONY: check-headers
 check-headers:
-	@go get github.com/elastic/go-licenser
+	@go get -u github.com/elastic/go-licenser
 	@go-licenser -d -exclude x-pack
 	@go-licenser -d -license Elastic x-pack
 

auditbeat/auditbeat.reference.yml

@@ -1148,6 +1148,17 @@ logging.files:
   # The default is 50.
   #bulk_max_size: 50
 
+  # The number of seconds to wait before trying to reconnect to Elasticsearch
+  # after a network error. After waiting backoff.init seconds, the Beat
+  # tries to reconnect. If the attempt fails, the backoff timer is increased
+  # exponentially up to backoff.max. After a successful connection, the backoff
+  # timer is reset. The default is 1s.
+  #backoff.init: 1s
+
+  # The maximum number of seconds to wait before attempting to connect to
+  # Elasticsearch after a network error. The default is 60s.
+  #backoff.max: 60s
+
   # Configure http request timeout before failing an request to Elasticsearch.
   #timeout: 90
 

auditbeat/magefile.go

@@ -169,6 +169,7 @@ func customizePackaging() {
 			Mode:   0600,
 			Source: "{{.PackageDir}}/auditbeat.yml",
 			Dep:    generateShortConfig,
+			Config: true,
 		}
 		referenceConfig = mage.PackageFile{
 			Mode:   0644,

auditbeat/module/file_integrity/monitor/monitor.go

@@ -21,6 +21,10 @@ import (
 	"github.com/fsnotify/fsnotify"
 )
 
+const (
+	moduleName = "file_integrity"
+)
+
 // Watcher is an interface for a file watcher akin to fsnotify.Watcher
 // with an additional Start method.
 type Watcher interface {

auditbeat/module/file_integrity/monitor/recursive.go

@@ -24,6 +24,8 @@ import (
 	"github.com/fsnotify/fsnotify"
 	"github.com/joeshaw/multierror"
 	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/libbeat/logp"
 )
 
 type recursiveWatcher struct {
@@ -33,6 +35,7 @@ type recursiveWatcher struct {
 	done    chan bool
 	addC    chan string
 	addErrC chan error
+	log     *logp.Logger
 }
 
 func newRecursiveWatcher(inner *fsnotify.Watcher) *recursiveWatcher {
@@ -42,6 +45,7 @@ func newRecursiveWatcher(inner *fsnotify.Watcher) *recursiveWatcher {
 		eventC:  make(chan fsnotify.Event, 1),
 		addC:    make(chan string),
 		addErrC: make(chan error),
+		log:     logp.NewLogger(moduleName),
 	}
 }
 
@@ -101,6 +105,8 @@ func (watcher *recursiveWatcher) addRecursive(path string) error {
 		}
 		return err
 	})
+	watcher.log.Debugw("Added recursive watch", "path", path)
+
 	if err != nil {
 		errs = append(errs, errors.Wrapf(err, "failed to walk path '%s'", path))
 	}
@@ -147,33 +153,47 @@ func (watcher *recursiveWatcher) forwardEvents() error {
 			}
 			switch event.Op {
 			case fsnotify.Create:
-				if err := watcher.addRecursive(event.Name); err != nil {
-					watcher.inner.Errors <- errors.Wrapf(err, "unable to recurse path '%s'", event.Name)
+				err := watcher.addRecursive(event.Name)
+				if err != nil {
+					watcher.inner.Errors <- errors.Wrapf(err, "failed to add created path '%s'", event.Name)
 				}
-				watcher.tree.Visit(event.Name, PreOrder, func(path string, _ bool) error {
+				err = watcher.tree.Visit(event.Name, PreOrder, func(path string, _ bool) error {
 					watcher.deliver(fsnotify.Event{
 						Name: path,
 						Op:   event.Op,
 					})
 					return nil
 				})
+				if err != nil {
+					watcher.inner.Errors <- errors.Wrapf(err, "failed to visit created path '%s'", event.Name)
+				}
 
 			case fsnotify.Remove:
-				watcher.tree.Visit(event.Name, PostOrder, func(path string, _ bool) error {
+				err := watcher.tree.Visit(event.Name, PostOrder, func(path string, _ bool) error {
 					watcher.deliver(fsnotify.Event{
 						Name: path,
 						Op:   event.Op,
 					})
 					return nil
 				})
-				watcher.tree.Remove(event.Name)
+				if err != nil {
+					watcher.inner.Errors <- errors.Wrapf(err, "failed to visit removed path '%s'", event.Name)
+				}
+
+				err = watcher.tree.Remove(event.Name)
+				if err != nil {
+					watcher.inner.Errors <- errors.Wrapf(err, "failed to visit removed path '%s'", event.Name)
+				}
 
 			// Handling rename (move) as a special case to give this recursion
 			// the same semantics as macOS FSEvents:
 			// - Removal of a dir notifies removal for all files inside it
 			// - Moving a dir away sends only one notification for this dir
 			case fsnotify.Rename:
-				watcher.tree.Remove(event.Name)
+				err := watcher.tree.Remove(event.Name)
+				if err != nil {
+					watcher.inner.Errors <- errors.Wrapf(err, "failed to remove path '%s'", event.Name)
+				}
 				fallthrough
 
 			default:

auditbeat/tests/system/test_file_integrity.py

@@ -154,13 +154,17 @@ def test_recursive(self):
             self.wait_log_contains(escape_path(dirs[0]), max_timeout=30, ignore_case=True)
             self.wait_log_contains("\"recursive\": true")
 
+            # auditbeat_test/subdir/
             subdir = os.path.join(dirs[0], "subdir")
             os.mkdir(subdir)
+            # auditbeat_test/subdir/file.txt
             file1 = os.path.join(subdir, "file.txt")
             self.create_file(file1, "hello world!")
 
+            # auditbeat_test/subdir/other/
             subdir2 = os.path.join(subdir, "other")
             os.mkdir(subdir2)
+            # auditbeat_test/subdir/other/more.txt
             file2 = os.path.join(subdir2, "more.txt")
             self.create_file(file2, "")
 

dev-tools/mage/pkg.go

@@ -96,21 +96,55 @@ func (b packageBuilder) Build() error {
 		b.Spec.Name, b.Type, b.Platform.Name)
 }
 
+type testPackagesParams struct {
+	HasModules  bool
+	HasModulesD bool
+}
+
+// TestPackagesOption defines a option to the TestPackages target.
+type TestPackagesOption func(params *testPackagesParams)
+
+// WithModules enables modules folder contents testing
+func WithModules() func(params *testPackagesParams) {
+	return func(params *testPackagesParams) {
+		params.HasModules = true
+	}
+}
+
+// WithModulesD enables modules.d folder contents testing
+func WithModulesD() func(params *testPackagesParams) {
+	return func(params *testPackagesParams) {
+		params.HasModulesD = true
+	}
+}
+
 // TestPackages executes the package tests on the produced binaries. These tests
 // inspect things like file ownership and mode.
-func TestPackages() error {
+func TestPackages(options ...TestPackagesOption) error {
+	params := testPackagesParams{}
+	for _, opt := range options {
+		opt(&params)
+	}
+
 	fmt.Println(">> Testing package contents")
 	goTest := sh.OutCmd("go", "test")
 
 	var args []string
 	if mg.Verbose() {
 		args = append(args, "-v")
 	}
-	args = append(args,
-		MustExpand("{{ elastic_beats_dir }}/dev-tools/packaging/package_test.go"),
-		"-files",
-		MustExpand("{{.PWD}}/build/distributions/*"),
-	)
+
+	args = append(args, MustExpand("{{ elastic_beats_dir }}/dev-tools/packaging/package_test.go"))
+
+	if params.HasModules {
+		args = append(args, "--modules")
+	}
+
+	if params.HasModulesD {
+		args = append(args, "--modules.d")
+	}
+
+	args = append(args, "-files", MustExpand("{{.PWD}}/build/distributions/*"))
 
 	if out, err := goTest(args...); err != nil {
 		if !mg.Verbose() {

dev-tools/packaging/package_test.go

@@ -26,6 +26,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"flag"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -49,13 +50,16 @@ const (
 var (
 	configFilePattern      = regexp.MustCompile(`.*beat\.yml|apm-server\.yml`)
 	manifestFilePattern    = regexp.MustCompile(`manifest.yml`)
-	modulesDirPattern      = regexp.MustCompile(`modules.d/$`)
-	modulesFilePattern     = regexp.MustCompile(`modules.d/.+`)
+	modulesDirPattern      = regexp.MustCompile(`module/.+`)
+	modulesDDirPattern     = regexp.MustCompile(`modules.d/$`)
+	modulesDFilePattern    = regexp.MustCompile(`modules.d/.+`)
 	systemdUnitFilePattern = regexp.MustCompile(`/lib/systemd/system/.*\.service`)
 )
 
 var (
-	files = flag.String("files", "../build/distributions/*/*", "filepath glob containing package files")
+	files    = flag.String("files", "../build/distributions/*/*", "filepath glob containing package files")
+	modules  = flag.Bool("modules", false, "check modules folder contents")
+	modulesd = flag.Bool("modules.d", false, "check modules.d folder contents")
 )
 
 func TestRPM(t *testing.T) {
@@ -101,6 +105,8 @@ func checkRPM(t *testing.T, file string) {
 	checkManifestPermissions(t, p)
 	checkManifestOwner(t, p)
 	checkModulesPermissions(t, p)
+	checkModulesPresent(t, "/usr/share", p)
+	checkModulesDPresent(t, "/etc/", p)
 	checkModulesOwner(t, p)
 	checkSystemdUnitPermissions(t, p)
 }
@@ -116,6 +122,8 @@ func checkDeb(t *testing.T, file string, buf *bytes.Buffer) {
 	checkConfigOwner(t, p)
 	checkManifestPermissions(t, p)
 	checkManifestOwner(t, p)
+	checkModulesPresent(t, "./usr/share", p)
+	checkModulesDPresent(t, "./etc/", p)
 	checkModulesPermissions(t, p)
 	checkModulesOwner(t, p)
 	checkSystemdUnitPermissions(t, p)
@@ -131,6 +139,8 @@ func checkTar(t *testing.T, file string) {
 	checkConfigPermissions(t, p)
 	checkConfigOwner(t, p)
 	checkManifestPermissions(t, p)
+	checkModulesPresent(t, "", p)
+	checkModulesDPresent(t, "", p)
 	checkModulesPermissions(t, p)
 	checkModulesOwner(t, p)
 }
@@ -144,6 +154,8 @@ func checkZip(t *testing.T, file string) {
 
 	checkConfigPermissions(t, p)
 	checkManifestPermissions(t, p)
+	checkModulesPresent(t, "", p)
+	checkModulesDPresent(t, "", p)
 	checkModulesPermissions(t, p)
 }
 
@@ -216,13 +228,13 @@ func checkManifestOwner(t *testing.T, p *packageFile) {
 func checkModulesPermissions(t *testing.T, p *packageFile) {
 	t.Run(p.Name+" modules.d file permissions", func(t *testing.T) {
 		for _, entry := range p.Contents {
-			if modulesFilePattern.MatchString(entry.File) {
+			if modulesDFilePattern.MatchString(entry.File) {
 				mode := entry.Mode.Perm()
 				if expectedModuleFileMode != mode {
 					t.Errorf("file %v has wrong permissions: expected=%v actual=%v",
 						entry.File, expectedModuleFileMode, mode)
 				}
-			} else if modulesDirPattern.MatchString(entry.File) {
+			} else if modulesDDirPattern.MatchString(entry.File) {
 				mode := entry.Mode.Perm()
 				if expectedModuleDirMode != mode {
 					t.Errorf("file %v has wrong permissions: expected=%v actual=%v",
@@ -237,7 +249,7 @@ func checkModulesPermissions(t *testing.T, p *packageFile) {
 func checkModulesOwner(t *testing.T, p *packageFile) {
 	t.Run(p.Name+" modules.d file owner", func(t *testing.T) {
 		for _, entry := range p.Contents {
-			if modulesFilePattern.MatchString(entry.File) || modulesDirPattern.MatchString(entry.File) {
+			if modulesDFilePattern.MatchString(entry.File) || modulesDDirPattern.MatchString(entry.File) {
 				if expectedConfigUID != entry.UID {
 					t.Errorf("file %v should be owned by user %v, owner=%v", entry.File, expectedConfigGID, entry.UID)
 				}
@@ -268,6 +280,37 @@ func checkSystemdUnitPermissions(t *testing.T, p *packageFile) {
 	})
 }
 
+// Verify that modules folder is present and has module files in
+func checkModulesPresent(t *testing.T, prefix string, p *packageFile) {
+	if *modules {
+		checkModules(t, "modules", prefix, modulesDirPattern, p)
+	}
+}
+
+// Verify that modules.d folder is present and has module files in
+func checkModulesDPresent(t *testing.T, prefix string, p *packageFile) {
+	if *modulesd {
+		checkModules(t, "modules.d", prefix, modulesDFilePattern, p)
+	}
+}
+
+func checkModules(t *testing.T, name, prefix string, r *regexp.Regexp, p *packageFile) {
+	t.Run(fmt.Sprintf("%s %s contents", p.Name, name), func(t *testing.T) {
+		minExpectedModules := 4
+		total := 0
+		for _, entry := range p.Contents {
+			if strings.HasPrefix(entry.File, prefix) && r.MatchString(entry.File) {
+				total++
+			}
+		}
+
+		if total < minExpectedModules {
+			t.Errorf("not enough modules found under %s: actual=%d, expected>=%d",
+				name, total, minExpectedModules)
+		}
+	})
+}
+
 // Helpers
 
 type packageFile struct {

dev-tools/set_version

@@ -28,7 +28,7 @@ def replace_in_file(filename, varname, version):
     with open(filename, 'r') as f:
         for line in f:
             if line.startswith("const " + varname):
-                new_lines.append('const {} = "{}"'.format(varname, version))
+                new_lines.append('const {} = "{}"\n'.format(varname, version))
             else:
                 new_lines.append(line)