Updating fork from remote master

.appveyor.yml

@@ -41,7 +41,6 @@ install:
   - set PATH=C:\tools\mingw64\bin;%GOROOT%\bin;%PATH%
   - set PATH=%GOPATH%\bin;%PATH%
   - go get github.com/pierrre/gotestcover
-  - set GO15VENDOREXPERIMENT=1
   - go version
   - go env
   # Download the PyWin32 installer if it is not cached.

.travis.yml

@@ -23,6 +23,8 @@ env:
     - TARGETS="-C libbeat crosscompile"
     - TARGETS="-C metricbeat crosscompile"
     - TARGETS="-C winlogbeat crosscompile"
+    - TARGETS="-C generate/metricbeat/metricset test"
+    - TARGETS="-C generate/beat test"
   global:
     # Cross-compile for amd64 only to speed up testing.
     - GOX_FLAGS="-arch amd64"
@@ -44,6 +46,10 @@ matrix:
       env: TARGETS="-C libbeat testsuite"
     - os: osx
       env: TARGETS="-C metricbeat testsuite"
+    - os: osx
+      env: TARGETS="-C generate/metricbeat/metricset test"
+    - os: osx
+      env: TARGETS="-C generate/beat test"
   fast_finish: true
   allow_failures:
     - env: TARGETS="-C libbeat crosscompile"
@@ -57,6 +63,11 @@ addons:
       - libpcap-dev
       - geoip-database
 
+before_script:
+  - wget https://github.com/Masterminds/glide/releases/download/0.10.2/glide-0.10.2-linux-386.tar.gz -O /tmp/glide.tar.gz
+  - tar -xvf /tmp/glide.tar.gz
+  - export PATH=$PATH:$PWD/linux-386
+
 before_install:
   # Update to most recent docker version
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then

CHANGELOG.asciidoc

@@ -22,16 +22,22 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha3...master[Check the HEAD d
 
 *Filebeat*
 
+- Stop following symlink. Symlinks are now ignored: {pull}1686[1686]
+
 *Winlogbeat*
 
 
 ==== Bugfixes
 
 *Affecting all Beats*
+- Reset backoff factor on partial ACK. {issue}1803[1803]
+- Fix beats load balancer deadlock if max_retries: -1 or publish_async is enabled in filebeat. {issue}1829[1829]
 
 *Metricbeat*
+- Fix the CPU values returned for each core. {issue}1863[1863]
 
 *Packetbeat*
+- Add missing nil-check to memcached GapInStream handler. {issue}1162[1162]
 
 *Topbeat*
 
@@ -79,7 +85,7 @@ https://github.com/elastic/beats/compare/v5.0.0alpha2...v5.0.0-alpha3[View commi
 *Affecting all Beats*
 
 - All configuration settings under `shipper:` are moved to be top level configuration settings. I.e.
-  `shipper.name:` becomes `name:` in the configuration file. #1570
+  `shipper.name:` becomes `name:` in the configuration file. {pull}1570[1570]
 
 *Topbeat*
 

Dockerfile

@@ -16,8 +16,6 @@ RUN set -x \
 
 COPY libbeat/scripts/docker-entrypoint.sh /entrypoint.sh
 
-ENV GO15VENDOREXPERIMENT=1
-
 RUN mkdir -p /etc/pki/tls/certs
 COPY testing/environments/docker/logstash/pki/tls/certs/logstash.crt /etc/pki/tls/certs/logstash.crt
 

Makefile

@@ -39,6 +39,8 @@ update:
 .PHONY: clean
 clean:
 	$(foreach var,$(PROJECTS),$(MAKE) -C $(var) clean || exit 1;)
+	$(MAKE) -C generate/beat clean
+	$(MAKE) -C generate/metricbeat/metricset clean
 
 .PHONY: check
 check:

dev-tools/deploy

@@ -13,13 +13,17 @@ def main():
 
     dir = os.path.dirname(os.path.realpath(__file__))
     os.chdir(dir + "/packer")
+    print("Getting dependencies")
     check_call("make deps", shell=True)
     check_call("make clean", shell=True)
+    print("Start building Docker images.")
     check_call("make images", shell=True)
+    print("Done building Docker images.")
     if args.no_snapshot:
         check_call("make SNAPSHOT=no", shell=True)
     else:
         check_call("make SNAPSHOT=yes", shell=True)
+    print("All done")
 
 if __name__ == "__main__":
     main()

dev-tools/export_dashboards.py

@@ -16,7 +16,7 @@ def ExportDashboards(es, beat, kibana_index, output_directory):
     for doc in res['hits']['hits']:
 
         if not reg_exp.match(doc["_source"]["title"]):
-            print "Ignore dashboard", doc["_source"]["title"]
+            print("Ignore dashboard", doc["_source"]["title"])
             continue
 
         # save dashboard

dev-tools/packer/Makefile

@@ -9,6 +9,7 @@ all: packetbeat/deb packetbeat/rpm packetbeat/darwin packetbeat/win packetbeat/b
 
 .PHONY: packetbeat metricbeat
 packetbeat metricbeat: build/upload
+	@echo Cross-compiling $@
 	# cross compile on ubuntu
 	docker run --rm \
 		-v $(abspath build):/build \
@@ -36,6 +37,7 @@ packetbeat metricbeat: build/upload
 
 .PHONY: filebeat
 filebeat:
+	@echo Cross-compiling $@
 	# pure go cross compiling doesn't need C compilers but
 	# we do it using the same docker instances for simplicity
 	docker run --rm \
@@ -53,6 +55,7 @@ filebeat:
 
 .PHONY: winlogbeat
 winlogbeat:
+	@echo Cross-compiling $@
 	# Winlogbeat is pure go and only needs Windows
 	docker run --rm \
 		-v $(abspath build):/build \

dev-tools/packer/README.md

@@ -71,26 +71,26 @@ For example, here are the artifacts created for Filebeat:
 
 ```
 filebeat-5.0.0-amd64.deb
-filebeat-5.0.0-darwin-x64.tar.gz
+filebeat-5.0.0-darwin-x86_64.tar.gz
 filebeat-5.0.0-i386.deb
 filebeat-5.0.0-i686.rpm
-filebeat-5.0.0-linux-x64.tar.gz
 filebeat-5.0.0-linux-x86.tar.gz
-filebeat-5.0.0-windows-x64.zip
+filebeat-5.0.0-linux-x86_64.tar.gz
 filebeat-5.0.0-windows-x86.zip
+filebeat-5.0.0-windows-x86_64.zip
 filebeat-5.0.0-x86_64.rpm
 ```
 
 And the SNAPSHOT versions:
 
 ```
 filebeat-5.0.0-SNAPSHOT-amd64.deb
+filebeat-5.0.0-SNAPSHOT-darwin-x86_64.tar.gz
 filebeat-5.0.0-SNAPSHOT-i386.deb
 filebeat-5.0.0-SNAPSHOT-i686.rpm
-filebeat-5.0.0-SNAPSHOT-x86_64.rpm
-filebeat-5.0.0-SNAPSHOT-darwin-x64.tar.gz
-filebeat-5.0.0-SNAPSHOT-linux-x64.tar.gz
 filebeat-5.0.0-SNAPSHOT-linux-x86.tar.gz
-filebeat-5.0.0-SNAPSHOT-windows-x64.zip
+filebeat-5.0.0-SNAPSHOT-linux-x86_64.tar.gz
 filebeat-5.0.0-SNAPSHOT-windows-x86.zip
+filebeat-5.0.0-SNAPSHOT-windows-x86_64.zip
+filebeat-5.0.0-SNAPSHOT-x86_64.rpm
 ```

dev-tools/packer/archs/amd64.yml

@@ -1,5 +1,5 @@
 arch: amd64
 deb_arch: amd64
 rpm_arch: x86_64
-bin_arch: x64
-win_arch: x64
+bin_arch: x86_64
+win_arch: x86_64

dev-tools/packer/docker/xgo-image-deb6/base/sources.list

@@ -1,3 +1,2 @@
-deb http://snapshot.debian.org/archive/debian/20160229T214851Z squeeze main
-deb http://snapshot.debian.org/archive/debian/20160229T214851Z squeeze-updates main
-deb http://snapshot.debian.org/archive/debian/20160229T214851Z squeeze-lts main
+deb http://archive.debian.org/debian/ squeeze main contrib
+deb http://archive.debian.org/debian/ squeeze-lts main

dev-tools/packer/docker/xgo-image/base/build.sh

@@ -51,8 +51,6 @@ fi
 set -e
 
 cd $WORKING_DIRECTORY
-export GOPATH=$GOPATH:`pwd`/Godeps/_workspace
-export GO15VENDOREXPERIMENT=1
 
 # Switch over the code-base to another checkout if requested
 if [ "$REPO_REMOTE" != "" ]; then
@@ -138,7 +136,7 @@ for TARGET in $TARGETS; do
 		GOOS=linux GOARCH=386 CGO_ENABLED=${CGO_ENABLED} go get -d ./$PACK
 		sh -c "GOOS=linux GOARCH=386 CGO_ENABLED=${CGO_ENABLED} go build $V $R $LDARGS -o $NAME-linux-386$R ./$PACK"
 		built_targets=$((built_targets+1))
-	fi	
+	fi
 	if ([ $XGOOS == "." ] || [ $XGOOS == "linux" ]) && ([ $XGOARCH == "." ] || [ $XGOARCH == "arm" ]); then
 		echo "Compiling $PACK for linux/arm..."
 		CC=arm-linux-gnueabi-gcc CXX=rm-linux-gnueabi-g++ HOST=arm-linux PREFIX=/usr/local/arm $BUILD_DEPS /deps $LIST_DEPS
@@ -164,7 +162,7 @@ for TARGET in $TARGETS; do
 
 		# Build the requested windows binaries
 		if [ $XGOARCH == "." ] || [ $XGOARCH == "amd64" ]; then
-			echo "Compiling for windows-$PLATFORM/amd64..."
+			echo "Compiling $PACK for windows-$PLATFORM/amd64..."
 			CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="$CGO_NTDEF" CXXFLAGS="$CGO_NTDEF" HOST=x86_64-w64-mingw32 PREFIX=/usr/x86_64-w64-mingw32 $BUILD_DEPS /deps $LIST_DEPS
 			export PKG_CONFIG_PATH=/usr/x86_64-w64-mingw32/lib/pkgconfig
 
@@ -174,7 +172,7 @@ for TARGET in $TARGETS; do
 		fi
 
 		if [ $XGOARCH == "." ] || [ $XGOARCH == "386" ]; then
-			echo "Compiling for windows-$PLATFORM/386..."
+			echo "Compiling $PACK for windows-$PLATFORM/386..."
 			CC=i686-w64-mingw32-gcc CXX=i686-w64-mingw32-g++ CFLAGS="$CGO_NTDEF" CXXFLAGS="$CGO_NTDEF" HOST=i686-w64-mingw32 PREFIX=/usr/i686-w64-mingw32 $BUILD_DEPS /deps $LIST_DEPS
 			export PKG_CONFIG_PATH=/usr/i686-w64-mingw32/lib/pkgconfig
 
@@ -183,7 +181,7 @@ for TARGET in $TARGETS; do
 			built_targets=$((built_targets+1))
 		fi
 	fi
-	
+
 	# Check and build for OSX targets
 	if ([ $XGOOS == "." ] || [ $XGOOS == "darwin" ]) && ([ $XGOARCH == "." ] || [ $XGOARCH == "amd64" ]); then
 		echo "Compiling $PACK for darwin/amd64..."
@@ -204,7 +202,7 @@ done
 
 # The binary files are the last created files
 echo "Moving $built_targets $PACK binaries to host folder..."
-ls -t | head -n $built_targets 
+ls -t | head -n $built_targets
 cp `ls -t | head -n $built_targets ` /build
 
 echo "Build process completed"

dev-tools/packer/platforms/darwin/run.sh.j2

@@ -10,18 +10,18 @@ if [ "$SNAPSHOT" = "yes" ]; then
     VERSION="${VERSION}-SNAPSHOT"
 fi
 
-mkdir /{{.beat_name}}-${VERSION}-darwin-x64
-cp -a homedirs/{{.beat_name}}/. /{{.beat_name}}-${VERSION}-darwin-x64/
-cp {{.beat_name}}-darwin-amd64 /{{.beat_name}}-${VERSION}-darwin-x64/{{.beat_name}}
-cp {{.beat_name}}-darwin.yml /{{.beat_name}}-${VERSION}-darwin-x64/{{.beat_name}}.yml
-cp {{.beat_name}}-darwin.full.yml /{{.beat_name}}-${VERSION}-darwin-x64/{{.beat_name}}.full.yml
-cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-darwin-x64/
-cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-darwin-x64/
+mkdir /{{.beat_name}}-${VERSION}-darwin-x86_64
+cp -a homedirs/{{.beat_name}}/. /{{.beat_name}}-${VERSION}-darwin-x86_64/
+cp {{.beat_name}}-darwin-amd64 /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}
+cp {{.beat_name}}-darwin.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.yml
+cp {{.beat_name}}-darwin.full.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.full.yml
+cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-darwin-x86_64/
+cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-darwin-x86_64/
 
 mkdir -p upload/{{.beat_name}}
-tar czvf upload/{{.beat_name}}/{{.beat_name}}-${VERSION}-darwin-x64.tar.gz /{{.beat_name}}-${VERSION}-darwin-x64
-echo "Created upload/{{.beat_name}}/{{.beat_name}}-${VERSION}-darwin-x64.tar.gz"
+tar czvf upload/{{.beat_name}}/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz /{{.beat_name}}-${VERSION}-darwin-x86_64
+echo "Created upload/{{.beat_name}}/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz"
 
 cd upload/{{.beat_name}}
-sha1sum {{.beat_name}}-${VERSION}-darwin-x64.tar.gz > {{.beat_name}}-${VERSION}-darwin-x64.tar.gz.sha1.txt
-echo "Created upload/{{.beat_name}}/{{.beat_name}}-${VERSION}-darwin-x64.tar.gz.sha1.txt"
+sha1sum {{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz > {{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz.sha1.txt
+echo "Created upload/{{.beat_name}}/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz.sha1.txt"

dev-tools/packer/scripts/Makefile

@@ -6,21 +6,26 @@ packer_absdir=$(shell dirname ${makefile_abspath})
 
 
 %/deb: % build/god-linux-386 build/god-linux-amd64 fpm-image
+	echo Creating DEB packages for $(@D)
 	ARCH=386 BEAT=$(@D) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/debian/build.sh
 	ARCH=amd64 BEAT=$(@D) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/debian/build.sh
 
 %/rpm: % build/god-linux-386 build/god-linux-amd64 fpm-image
+	echo Creating RPM packages for $(@D)
 	ARCH=386 BEAT=$(@D) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/centos/build.sh
 	ARCH=amd64 BEAT=$(@D) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/centos/build.sh
 
 %/darwin: %
+	echo Creating Darwin packages for $(@D)
 	ARCH=amd64 BEAT=$(@D) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/darwin/build.sh
 
 %/win: %
+	echo Creating Darwin packages for $(@D)
 	ARCH=386 BEAT=$(@D) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/windows/build.sh
 	ARCH=amd64 BEAT=$(@D) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/windows/build.sh
 
 %/bin: %
+	echo Creating Linux packages for $(@D)
 	ARCH=386 BEAT=$(@D) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/binary/build.sh
 	ARCH=amd64 BEAT=$(@D) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/binary/build.sh
 

dev-tools/packer/version.yml

@@ -1 +1 @@
-version: "5.0.0-alpha3"
+version: "5.0.0-alpha4"

filebeat/.gitignore

@@ -1,7 +1,5 @@
 /etc/filebeat.dev.yml
-.idea
 .vagrant
-/data/
 /docs/html_docs
 
 filebeat

filebeat/crawler/crawler.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	"github.com/elastic/beats/filebeat/input"
+	"github.com/elastic/beats/filebeat/prospector"
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
 )
@@ -24,7 +25,7 @@ import (
 type Crawler struct {
 	// Registrar object to persist the state
 	Registrar   *Registrar
-	prospectors []*Prospector
+	prospectors []*prospector.Prospector
 	wg          sync.WaitGroup
 }
 
@@ -42,7 +43,7 @@ func (c *Crawler) Start(prospectorConfigs []*common.Config, eventChan chan *inpu
 	// Prospect the globs/paths given on the command line and launch harvesters
 	for _, prospectorConfig := range prospectorConfigs {
 
-		prospector, err := NewProspector(prospectorConfig, states, eventChan)
+		prospector, err := prospector.NewProspector(prospectorConfig, states, eventChan)
 		if err != nil {
 			return fmt.Errorf("Error in initing prospector: %s", err)
 		}
@@ -55,7 +56,7 @@ func (c *Crawler) Start(prospectorConfigs []*common.Config, eventChan chan *inpu
 	for i, p := range c.prospectors {
 		c.wg.Add(1)
 
-		go func(id int, prospector *Prospector) {
+		go func(id int, prospector *prospector.Prospector) {
 			defer func() {
 				c.wg.Done()
 				logp.Debug("crawler", "Prospector %v stopped", id)