[Auditbeat] Add system module (#9546)

Adds the system module to Auditbeat, with four metricsets: host, process, socket, and user. A fifth metricset - packages - is disabled for now.

Host collects general host information, e.g. boottime, timezone, OS, network interfaces. Processes collects information about currently running, started, and stopped processes. Socket collects information about open sockets. User detects new users, deleted users, changes to users (e.g. groups), and - as a special distinct category - password changes.

.travis.yml

@@ -62,6 +62,10 @@ jobs:
       env: TARGETS="-C auditbeat crosscompile"
       go: $GO_VERSION
       stage: test
+    - os: linux
+      env: TARGETS="-C x-pack/auditbeat testsuite"
+      go: $GO_VERSION
+      stage: test
 
     # Libbeat
     - os: linux

CHANGELOG.asciidoc

@@ -97,6 +97,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 
 *Auditbeat*
 
+- Add system module. {pull}9546[9546]
+
 *Filebeat*
 
 - Added `detect_null_bytes` selector to detect null bytes from a io.reader. {pull}9210[9210]

Makefile

@@ -13,11 +13,15 @@ REVIEWDOG_OPTIONS?=-diff "git diff master"
 REVIEWDOG_REPO=github.com/haya14busa/reviewdog/cmd/reviewdog
 XPACK_SUFFIX=x-pack/
 
+# PROJECTS_XPACK_PKG is a list of Beats that have independent packaging support
+# in the x-pack directory (rather than having the OSS build produce both sets
+# of artifacts). This will be removed once we complete the transition.
+PROJECTS_XPACK_PKG=x-pack/auditbeat
 # PROJECTS_XPACK_MAGE is a list of Beats whose primary build logic is based in
 # Mage. For compatibility with CI testing these projects support a subset of the
 # makefile targets. After all Beats converge to primarily using Mage we can
 # remove this and treat all sub-projects the same.
-PROJECTS_XPACK_MAGE=x-pack/filebeat x-pack/metricbeat
+PROJECTS_XPACK_MAGE=x-pack/filebeat x-pack/metricbeat $(PROJECTS_XPACK_PKG)
 
 # Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection.
 # Also it builds the docs and the generators
@@ -156,8 +160,8 @@ snapshot:
 # Builds a release.
 .PHONY: release
 release: beats-dashboards
-	@$(foreach var,$(BEATS),$(MAKE) -C $(var) release || exit 1;)
-	@$(foreach var,$(BEATS), \
+	@$(foreach var,$(BEATS) $(PROJECTS_XPACK_PKG),$(MAKE) -C $(var) release || exit 1;)
+	@$(foreach var,$(BEATS) $(PROJECTS_XPACK_PKG), \
       test -d $(var)/build/distributions && test -n "$$(ls $(var)/build/distributions)" || exit 0; \
       mkdir -p build/distributions/$(subst $(XPACK_SUFFIX),'',$(var)) && mv -f $(var)/build/distributions/* build/distributions/$(subst $(XPACK_SUFFIX),'',$(var))/ || exit 1;)
 

Vagrantfile

@@ -37,7 +37,7 @@ cmd /c mklink /d C:\\Gopath\\src\\github.com\\elastic\\beats \\\\vboxsvr\\vagran
 
 echo "Installing gvm to manage go version"
 [Net.ServicePointManager]::SecurityProtocol = "tls12"
-Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.0.5/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
+Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.1.0/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
 C:\Windows\System32\gvm.exe --format=powershell #{GO_VERSION} | Invoke-Expression
 go version
 
@@ -72,8 +72,9 @@ SCRIPT
 $linuxGvmProvision = <<SCRIPT
 mkdir -p ~/bin
 if [ ! -e "~/bin/gvm" ]; then
-  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.0.5/gvm-linux-amd64
+  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.1.0/gvm-linux-amd64
   chmod +x ~/bin/gvm
+  ~/bin/gvm $GO_VERSION
   echo 'export GOPATH=$HOME/go' >> ~/.bash_profile
   echo 'export PATH=$HOME/bin:$GOPATH/bin:$PATH' >> ~/.bash_profile
   echo 'eval "$(gvm #{GO_VERSION})"' >> ~/.bash_profile

auditbeat/Dockerfile

@@ -1,17 +1,12 @@
 FROM golang:1.11.3
-MAINTAINER Nicolas Ruflin <[email protected]>
 
-RUN set -x && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends \
-         netcat python-pip virtualenv && \
-    apt-get clean
+RUN \
+    apt-get update \
+      && apt-get install -y --no-install-recommends \
+         python-pip \
+         virtualenv \
+      && rm -rf /var/lib/apt/lists/*
 
+RUN pip install --upgrade pip
 RUN pip install --upgrade setuptools
-
-# Setup work environment
-ENV AUDITBEAT_PATH /go/src/github.com/elastic/beats/auditbeat
-
-RUN mkdir -p $AUDITBEAT_PATH/build/coverage
-WORKDIR $AUDITBEAT_PATH
-HEALTHCHECK CMD exit 0
+RUN pip install --upgrade docker-compose==1.21.0

auditbeat/Makefile

@@ -3,36 +3,12 @@ BEAT_TITLE=Auditbeat
 SYSTEM_TESTS=true
 TEST_ENVIRONMENT?=true
 GOX_OS?=linux windows ## @Building List of all OS to be supported by "make crosscompile".
-DEV_OS?=linux
 ES_BEATS?=..
+EXCLUDE_COMMON_UPDATE_TARGET=true
 
 # Path to the libbeat Makefile
 include ${ES_BEATS}/libbeat/scripts/Makefile
 
-# Collects all dependencies and then calls update
-.PHONY: collect
-collect: collect-docs configs kibana
-
-# Collects all module configs
-.PHONY: configs
-configs: python-env
-	@cat ${ES_BEATS}/auditbeat/_meta/common.p1.yml \
-		<(go run scripts/generate_config.go -os ${DEV_OS} -concat) \
-		${ES_BEATS}/auditbeat/_meta/common.p2.yml > _meta/beat.yml
-	@cat ${ES_BEATS}/auditbeat/_meta/common.reference.yml \
-		<(go run scripts/generate_config.go -os ${DEV_OS} -ref -concat) > _meta/beat.reference.yml
-
-# Collects all module docs
-.PHONY: collect-docs
-collect-docs: python-env
-	@rm -rf docs/modules
-	@mkdir -p docs/modules
-	@go run scripts/generate_config.go -os linux
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/auditbeat/scripts/docs_collector.py --beat ${BEAT_NAME}
-
-# Collects all module dashboards
-.PHONY: kibana
-kibana:
-	@-rm -rf _meta/kibana.generated
-	@mkdir -p _meta/kibana.generated
-	@-cp -pr module/*/_meta/kibana/* _meta/kibana.generated
+.PHONY: update
+update: mage
+	mage update

auditbeat/auditbeat.yml

@@ -48,7 +48,6 @@ auditbeat.modules:
   - /etc
 
 
-
 #==================== Elasticsearch template setting ==========================
 setup.template.settings:
   index.number_of_shards: 3

auditbeat/core/eventmod.go

@@ -30,4 +30,10 @@ func AddDatasetToEvent(module, metricSet string, event *mb.Event) {
 	}
 
 	event.RootFields.Put("event.module", module)
+
+	// Modules without "datasets" should set their module and metricset names
+	// to the same value then this will omit the event.dataset field.
+	if module != metricSet {
+		event.RootFields.Put("event.dataset", metricSet)
+	}
 }

auditbeat/docker-compose.yml

@@ -4,10 +4,12 @@ services:
     build: ${PWD}/.
     depends_on:
       - proxy_dep
-    env_file:
-      - ${PWD}/build/test.env
     working_dir: /go/src/github.com/elastic/beats/auditbeat
     environment:
+      - ES_HOST=elasticsearch
+      - ES_PORT=9200
+      - ES_USER=beats
+      - ES_PASS=testing
       - KIBANA_HOST=kibana
       - KIBANA_PORT=5601
     volumes: