Merge pull request #2 from elastic/master

Updating from remote master

.appveyor.yml

@@ -8,12 +8,10 @@ os: Windows Server 2012 R2
 environment:
   GOROOT: c:\go1.6.2
   GOPATH: c:\gopath
-  PYWIN_DL: https://downloads.sourceforge.net/project/pywin32/pywin32/Build%20219/pywin32-219.win32-py2.7.exe
+  PYWIN_DL: https://beats-files.s3.amazonaws.com/deps/pywin32-220.win32-py2.7.exe
   matrix:
   - PROJ: github.com\elastic\beats\filebeat
     BEAT: filebeat
-  - PROJ: github.com\elastic\beats\topbeat
-    BEAT: topbeat
   - PROJ: github.com\elastic\beats\winlogbeat
     BEAT: winlogbeat
 

.travis.yml

@@ -16,7 +16,6 @@ env:
   matrix:
     - TARGETS="check"
     - TARGETS="-C libbeat testsuite"
-    - TARGETS="-C topbeat testsuite"
     - TARGETS="-C filebeat testsuite"
     - TARGETS="-C packetbeat testsuite"
     - TARGETS="-C metricbeat testsuite"
@@ -101,6 +100,5 @@ after_success:
   # Copy full.cov to coverage.txt because codecov.io requires this file
   - test -f filebeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f filebeat/build/coverage/full.cov
   - test -f packetbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f packetbeat/build/coverage/full.cov
-  - test -f topbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f topbeat/build/coverage/full.cov
   - test -f libbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f libbeat/build/coverage/full.cov
   - test -f metricbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f metricbeat/build/coverage/full.cov

CHANGELOG.asciidoc

@@ -8,15 +8,22 @@
 // Template, add newest changes here
 
 === Beats version HEAD
-https://github.com/elastic/beats/compare/v5.0.0-alpha3...master[Check the HEAD diff]
+https://github.com/elastic/beats/compare/v5.0.0-alpha4...master[Check the HEAD diff]
 
 ==== Breaking changes
 
 *Affecting all Beats*
 
+- Rename the `filters` section to `processors`. {pull}1944[1944]
+- Introduce the condition with `when` in the processor configuration. {pull}1949[1949]
+- The Elasticsearch template is now loaded by default. {pull}1993[1993]
+- The Redis output `index` setting is renamed to `key`. `index` still works but it's deprecated. {pull}2077[2077]
+- The undocumented file output `index` setting was removed. Use `filename` instead. {pull}2077[2077]
+
 *Metricbeat*
 
 *Packetbeat*
+- Set `enabled` ` in `packetbeat.protocols.icmp` configuration to `true` by default. {pull}1988[1988]
 
 *Topbeat*
 
@@ -30,38 +37,54 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha3...master[Check the HEAD d
 ==== Bugfixes
 
 *Affecting all Beats*
-- Reset backoff factor on partial ACK. {issue}1803[1803]
-- Fix beats load balancer deadlock if max_retries: -1 or publish_async is enabled in filebeat. {issue}1829[1829]
+- Fix sync publisher PublishEvents return value if client is closed concurrently. {pull}2046[2046]
 
 *Metricbeat*
-- Fix the CPU values returned for each core. {issue}1863[1863]
+
+- Do not send zero values when no value was present in the source. {issue}1972[1972]
 
 *Packetbeat*
-- Add missing nil-check to memcached GapInStream handler. {issue}1162[1162]
 
 *Topbeat*
 
 *Filebeat*
 
+- Fix potential data loss between filebeat restarts, reporting unpublished lines as published. {issue}2041[2041]
+- Fix open file handler issue {issue}2028[2028] {pull}2020[2020]
+- Fix filtering of JSON events when using integers in conditions {issue}2038[2038]
+
 *Winlogbeat*
+- Fix potential data loss between winlogbeat restarts, reporting unpublished lines as published. {issue}2041[2041]
 
 ==== Added
 
-
 *Affecting all Beats*
 
+- Periodically log internal metrics. {pull}1955[1955]
+- Add enabled setting to all output modules. {pull}1987[1987]
+- Command line flag -c can be used multiple times. {pull}1985[1985]
+- Add OR/AND/NOT to the condition associated with the processors. {pull}1983[1983]
+- Add '-E' CLI flag for overwriting single config options via command line. {pull}1986[1986]
+- Choose the mapping template file based on the Elasticsearch version. {pull}1993[1993]
+- Check stdout being available when console output is configured. {issue}2035[2035]
+- Add enabled setting to packetbeat protocols. {pull}1988[1988]
+- Add enabled setting to packetbeat network flows configuration. {pull}1988[1988]
+
 *Metricbeat*
+- Add pgid field to process information. {pull} 2021[2021]
 
 *Packetbeat*
 
 *Topbeat*
 
 *Filebeat*
+- Introduce close_removed and close_renamed harvester options {issue}1600[1600]
+- Introduce close_eof harvester option {issue}1600[1600]
+- Add clean_removed config option {issue}1600[1600]
 
 *Winlogbeat*
 
 
-
 ==== Deprecated
 
 *Affecting all Beats*
@@ -71,14 +94,67 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha3...master[Check the HEAD d
 *Topbeat*
 
 *Filebeat*
+- Deprecate close_older option and replace it with close_inactive {issue}2051[2051]
+- Deprecate force_close_files option and replace it with close_removed and close_renamed {issue}1600[1600]
+
 
 *Winlogbeat*
 
 ////////////////////////////////////////////////////////////
 
+[[release-notes-5.0.0-alpha4]]
+=== Beats version 5.0.0-alpha4
+https://github.com/elastic/beats/compare/v5.0.0-alpha3...v5.0.0-alpha4[View commits]
+
+==== Breaking changes
+
+*Affecting all Beats*
+
+- The topology_expire option of the Elasticserach output was removed. {pull}1907[1907]
+
+*Filebeat*
+
+- Stop following symlink. Symlinks are now ignored: {pull}1686[1686]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Reset backoff factor on partial ACK. {issue}1803[1803]
+- Fix beats load balancer deadlock if max_retries: -1 or publish_async is enabled in filebeat. {issue}1829[1829]
+- Fix logstash output with pipelining mode enabled not reconnecting. {issue}1876[1876]
+- Empty configuration sections become merge-able with variables containing full path. {pull}1900[1900]
+- Fix error message about required fields missing not printing the missing field name. {pull}1900[1900]
+
+*Metricbeat*
+
+- Fix the CPU values returned for each core. {issue}1863[1863]
+
+*Packetbeat*
+
+- Add missing nil-check to memcached GapInStream handler. {issue}1162[1162]
+- Fix NFSv4 Operation returning the first found first-class operation available in compound requests. {pull}1821[1821]
+- Fix TCP overlapping segments not being handled correctly. {pull}1898[1898]
+
+*Winlogbeat*
+
+- Fix issue with rendering forwarded event log records. {pull}1891[1891]
+
+==== Added
+
+*Affecting all Beats*
+
+- Improve error message if compiling regular expression from config files fails. {pull}1900[1900]
+- Compression support in the Elasticsearch output. {pull}1835[1835]
+
+*Metricbeat*
+
+- Add MongoDB module. {pull}1837[1837]
+
+
 [[release-notes-5.0.0-alpha3]]
 === Beats version 5.0.0-alpha3
-https://github.com/elastic/beats/compare/v5.0.0alpha2...v5.0.0-alpha3[View commits]
+https://github.com/elastic/beats/compare/v5.0.0-alpha2...v5.0.0-alpha3[View commits]
 
 ==== Breaking changes
 
@@ -148,7 +224,7 @@ https://github.com/elastic/beats/compare/v1.2.2...v1.2.3[View commits]
 
 [[release-notes-5.0.0-alpha2]]
 === Beats version 5.0.0-alpha2
-https://github.com/elastic/beats/compare/v5.0.0alpha1...v5.0.0-alpha2[View commits]
+https://github.com/elastic/beats/compare/v5.0.0-alpha1...v5.0.0-alpha2[View commits]
 
 ==== Breaking changes
 

CONTRIBUTING.md

@@ -101,6 +101,15 @@ Running the testsuite has the following requirements:
 * Docker >=1.10.0
 * Docker-compose >= 1.7.0
 
+
+## Documentation
+
+The documentation for each beat can be found in {beatname}/docs and are based on asciidoc. After every change of the
+docs it should be verified that the docs are still building to not break the automated docs build. To build the docs
+run `make docs`. In case you want to preview the docs for a beat, run `make docs-preview` inside the beats folder.
+This will automatically open your browser with the docs for preview.
+
+
 ## Dependencies
 
 To manage the `vendor/` folder we use

Makefile

@@ -1,13 +1,14 @@
 BUILD_DIR=build
 COVERAGE_DIR=${BUILD_DIR}/coverage
-BEATS=packetbeat topbeat filebeat winlogbeat metricbeat
+BEATS=packetbeat filebeat winlogbeat metricbeat
 PROJECTS=libbeat ${BEATS}
 
-# Runs complete testsuites (unit, system, integration) for all beats,
-# with coverage and race detection.
+# Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection.
+# Also it builds the docs and the generators
 .PHONY: testsuite
 testsuite:
 	$(foreach var,$(PROJECTS),$(MAKE) -C $(var) testsuite || exit 1;)
+	#$(MAKE) -C generate test
 
 # Runs unit and system tests without coverage and race detection.
 .PHONY: test
@@ -27,7 +28,6 @@ coverage-report:
 	# Collects all coverage files and skips top line with mode
 	-tail -q -n +2 ./filebeat/${COVERAGE_DIR}/*.cov >> ./${COVERAGE_DIR}/full.cov
 	-tail -q -n +2 ./packetbeat/${COVERAGE_DIR}/*.cov >> ./${COVERAGE_DIR}/full.cov
-	-tail -q -n +2 ./topbeat/${COVERAGE_DIR}/*.cov >> ./${COVERAGE_DIR}/full.cov
 	-tail -q -n +2 ./winlogbeat/${COVERAGE_DIR}/*.cov >> ./${COVERAGE_DIR}/full.cov
 	-tail -q -n +2 ./libbeat/${COVERAGE_DIR}/*.cov >> ./${COVERAGE_DIR}/full.cov
 	go tool cover -html=./${COVERAGE_DIR}/full.cov -o ${COVERAGE_DIR}/full.html
@@ -38,9 +38,15 @@ update:
 
 .PHONY: clean
 clean:
+	rm -rf build
 	$(foreach var,$(PROJECTS),$(MAKE) -C $(var) clean || exit 1;)
-	$(MAKE) -C generate/beat clean
-	$(MAKE) -C generate/metricbeat/metricset clean
+	$(MAKE) -C generate clean
+
+# Cleans up the vendor directory from unnecessary files
+# This should always be run after updating the dependencies
+.PHONY: clean-vendor
+clean-vendor:
+	sh scripts/clean_vendor.sh
 
 .PHONY: check
 check:
@@ -60,3 +66,7 @@ beats-dashboards:
 	mkdir -p build
 	$(foreach var,$(PROJECTS),cp -r $(var)/etc/kibana/ build/dashboards  || exit 1;)
 
+# Builds the documents for each beat
+.PHONY: docs
+docs:
+	sh libbeat/scripts/build_docs.sh ${PROJECTS}

README.md

@@ -17,7 +17,6 @@ following folders:
 Folder  | Description
 --- | ---
 [libbeat](https://github.com/elastic/beats/tree/master/libbeat) | The Go framework for creating new Beats
-[Topbeat](https://github.com/elastic/beats/tree/master/topbeat) | Like 'top' but inserting the data into Elasticsearch
 [Packetbeat](https://github.com/elastic/beats/tree/master/packetbeat) | Tap into your wire data
 [Filebeat](https://github.com/elastic/beats/tree/master/filebeat) | Lightweight log forwarder to Logstash & Elasticsearch
 [Winlogbeat](https://github.com/elastic/beats/tree/master/winlogbeat) | Sends Windows Event logs

dev-tools/.beatconfig

@@ -1,5 +1,4 @@
 packetbeat-/packetbeat-
 filebeat-/filebeat-
-topbeat-/topbeat-
 winlogonbeat-/winlogonbeat-
-logstash-/logstash-
+logstash-/logstash-

dev-tools/export_dashboards.py

@@ -104,11 +104,11 @@ def main():
                         help="Elasticsearch URL. E.g. http://localhost:9200",
                         default="http://localhost:9200")
     parser.add_argument("--beat",
-                        help="Beat name e.g. topbeat",
+                        help="Beat name e.g. metricbeat",
                         required=True)
     parser.add_argument("--index",
                         help="Elasticsearch index for the Beat data. "
-                        "E.g. topbeat-*")
+                        "E.g. metricbeat-*")
     parser.add_argument("--kibana",
                         help="Elasticsearch index for the Kibana dashboards. "
                         "E.g. .kibana",

dev-tools/import_dashboards.sh

@@ -122,7 +122,7 @@ if [ -f ${BEAT_CONFIG} ]; then
 fi
 # Failsafe
 if [ -z ${SED_STRING} ]; then
-  SED_STRING="s/packetbeat-/packetbeat-/g;s/filebeat-/filebeat-/g;s/topbeat-/topbeat-/g;s/winlogonbeat-/winlogonbeat-/g"
+  SED_STRING="s/packetbeat-/packetbeat-/g;s/filebeat-/filebeat-/g;s/metricbeat-/metricbeat-/g;s/winlogonbeat-/winlogonbeat-/g"
 fi
 
 if [ -d /tmp ]; then

dev-tools/packer/platforms/windows/run.sh.j2

@@ -10,20 +10,20 @@ if [ "$SNAPSHOT" = "yes" ]; then
     VERSION="${VERSION}-SNAPSHOT"
 fi
 
-mkdir /{{.beat_name}}-${VERSION}-windows
-cp -a homedirs/{{.beat_name}}/. /{{.beat_name}}-${VERSION}-windows/
-cp {{.beat_name}}-windows-{{.arch}}.exe /{{.beat_name}}-${VERSION}-windows/{{.beat_name}}.exe
+mkdir /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}
+cp -a homedirs/{{.beat_name}}/. /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
+cp {{.beat_name}}-windows-{{.arch}}.exe /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.exe
 unix2dos {{.beat_name}}-win.yml
-cp {{.beat_name}}-win.yml /{{.beat_name}}-${VERSION}-windows/{{.beat_name}}.yml
-cp {{.beat_name}}-win.full.yml /{{.beat_name}}-${VERSION}-windows/{{.beat_name}}.full.yml
-cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-windows/
-cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-windows/
-cp install-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows/
-cp uninstall-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows/
+cp {{.beat_name}}-win.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.yml
+cp {{.beat_name}}-win.full.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.full.yml
+cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
+cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
+cp install-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
+cp uninstall-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
 
 mkdir -p upload/{{.beat_name}}
-zip -r upload/{{.beat_name}}/{{.beat_name}}-${VERSION}-windows-{{.win_arch}}.zip /{{.beat_name}}-${VERSION}-windows
-echo "Created upload/{{.beat_name}}/{{.beat_name}}-${VERSION}-windows.zip"
+zip -r upload/{{.beat_name}}/{{.beat_name}}-${VERSION}-windows-{{.win_arch}}.zip /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}
+echo "Created upload/{{.beat_name}}/{{.beat_name}}-${VERSION}-windows-{{.win_arch}}.zip"
 
 cd upload/{{.beat_name}}
 sha1sum {{.beat_name}}-${VERSION}-windows-{{.win_arch}}.zip > {{.beat_name}}-${VERSION}-windows-{{.win_arch}}.zip.sha1.txt

dev-tools/packer/version.yml

@@ -1 +1 @@
-version: "5.0.0-alpha4"
+version: "5.0.0-alpha5"