diff --git a/LICENSE.txt b/LICENSE.txt index edfa6086a817..e601d4382ad6 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,13 +1,13 @@ -Copyright (c) 2012–2017 Elastic +Source code in this repository is variously licensed under the Apache License +Version 2.0, an Apache compatible license, or the Elastic License. Outside of +the "x-pack" folder, source code in a given file is licensed under the Apache +License Version 2.0, unless otherwise noted at the beginning of the file or a +LICENSE file present in the directory subtree declares a separate license. +Within the "x-pack" folder, source code in a given file is licensed under the +Elastic License, unless otherwise noted at the beginning of the file or a +LICENSE file present in the directory subtree declares a separate license. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +The build produces two sets of binaries - one set that falls under the Elastic +License and another set that falls under Apache License Version 2.0. The +binaries that contain `-oss` in the artifact name are licensed under the Apache +License Version 2.0. diff --git a/Makefile b/Makefile index 8dff139f3823..ea1f5beb6298 100644 --- a/Makefile +++ b/Makefile @@ -110,13 +110,13 @@ docs: @$(foreach var,$(PROJECTS),BUILD_DIR=${BUILD_DIR} $(MAKE) -C $(var) docs || exit 1;) sh ./script/build_docs.sh dev-guide github.com/elastic/beats/docs/devguide ${BUILD_DIR} -.PHONY: package -package: update beats-dashboards - @$(foreach var,$(BEATS),SNAPSHOT=$(SNAPSHOT) $(MAKE) -C $(var) package || exit 1;) +.PHONY: package-all +package-all: update beats-dashboards + @$(foreach var,$(BEATS),SNAPSHOT=$(SNAPSHOT) $(MAKE) -C $(var) package-all || exit 1;) @echo "Start building the dashboards package" @mkdir -p build/upload/ - @BUILD_DIR=${BUILD_DIR} SNAPSHOT=$(SNAPSHOT) $(MAKE) -C dev-tools/packer package-dashboards ${BUILD_DIR}/upload/build_id.txt + @BUILD_DIR=${BUILD_DIR} UPLOAD_DIR=${BUILD_DIR}/upload SNAPSHOT=$(SNAPSHOT) $(MAKE) -C dev-tools/packer package-dashboards ${BUILD_DIR}/upload/build_id.txt @mv build/upload build/dashboards-upload @# Copy build files over to top build directory diff --git a/README.md b/README.md index dfb8dbb3182b..aa735422e023 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ framework for creating Beats, and all the officially supported Beats: Beat | Description --- | --- +[Auditbeat](https://github.com/elastic/beats/tree/master/auditbeat) | Collect your Linux audit framework data and monitor the integrity of your files. [Filebeat](https://github.com/elastic/beats/tree/master/filebeat) | Tails and ships log files [Heartbeat](https://github.com/elastic/beats/tree/master/heartbeat) | Ping remote services for availability [Metricbeat](https://github.com/elastic/beats/tree/master/metricbeat) | Fetches sets of metrics from the operating system and services @@ -26,9 +27,9 @@ Beat | Description [Winlogbeat](https://github.com/elastic/beats/tree/master/winlogbeat) | Fetches and ships Windows Event logs In addition to the above Beats, which are officially supported by -[Elastic](https://elastic.co), the -community has created a set of other Beats that make use of libbeat but live -outside of this Github repository. We maintain a list of community Beats +[Elastic](https://elastic.co), the community has created a set of other Beats +that make use of libbeat but live outside of this Github repository. We maintain +a list of community Beats [here](https://www.elastic.co/guide/en/beats/libbeat/master/community-beats.html). ## Documentation and Getting Started @@ -37,6 +38,7 @@ You can find the documentation and getting started guides for each of the Beats on the [elastic.co site](https://www.elastic.co/guide/): * [Beats platform](https://www.elastic.co/guide/en/beats/libbeat/current/index.html) +* [Auditbeat](https://www.elastic.co/guide/en/beats/auditbeat/current/index.html) * [Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/index.html) * [Heartbeat](https://www.elastic.co/guide/en/beats/heartbeat/current/index.html) * [Metricbeat](https://www.elastic.co/guide/en/beats/metricbeat/current/index.html) @@ -64,8 +66,8 @@ create your own Beat. Please start by reading our [CONTRIBUTING](CONTRIBUTING.md) file. If you are creating a new Beat, you don't need to submit the code to this -repository. You can simply start working in a new repository and make use of -the libbeat packages, by following our [developer +repository. You can simply start working in a new repository and make use of the +libbeat packages, by following our [developer guide](https://www.elastic.co/guide/en/beats/libbeat/current/new-beat.html). After you have a working prototype, open a pull request to add your Beat to the list of [community @@ -73,5 +75,5 @@ Beats](https://github.com/elastic/beats/blob/master/libbeat/docs/communitybeats. ## Building Beats from the Source -See our [CONTRIBUTING](CONTRIBUTING.md) file for information about setting up your dev -environment to build Beats from the source. +See our [CONTRIBUTING](CONTRIBUTING.md) file for information about setting up +your dev environment to build Beats from the source. diff --git a/dev-tools/deploy b/dev-tools/deploy index f09ae357ee48..812a003439d2 100755 --- a/dev-tools/deploy +++ b/dev-tools/deploy @@ -16,9 +16,9 @@ def main(): check_call("make clean", shell=True) print("Done building Docker images.") if args.no_snapshot: - check_call("make SNAPSHOT=no package", shell=True) + check_call("make SNAPSHOT=no package-all", shell=True) else: - check_call("make SNAPSHOT=yes package", shell=True) + check_call("make SNAPSHOT=yes package-all", shell=True) print("All done") if __name__ == "__main__": diff --git a/dev-tools/packer/Makefile b/dev-tools/packer/Makefile index 542e0171b7c7..1c157d5d1a3c 100644 --- a/dev-tools/packer/Makefile +++ b/dev-tools/packer/Makefile @@ -12,32 +12,32 @@ beat_abspath=${BEATS_GOPATH}/src/${BEAT_PATH} %/deb: ${BUILD_DIR}/god-linux-386 ${BUILD_DIR}/god-linux-amd64 fpm-image echo Creating DEB packages for $(@D) - ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/debian/build.sh - ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/debian/build.sh + ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} UPLOAD_DIR=${UPLOAD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/debian/build.sh + ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} UPLOAD_DIR=${UPLOAD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/debian/build.sh %/rpm: ${BUILD_DIR}/god-linux-386 ${BUILD_DIR}/god-linux-amd64 fpm-image echo Creating RPM packages for $(@D) - ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/centos/build.sh - ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/centos/build.sh + ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} UPLOAD_DIR=${UPLOAD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/centos/build.sh + ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} UPLOAD_DIR=${UPLOAD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/centos/build.sh %/darwin: echo Creating Darwin packages for $(@D) - ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/darwin/build.sh + ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} UPLOAD_DIR=${UPLOAD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/darwin/build.sh %/win: echo Creating Darwin packages for $(@D) - ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/windows/build.sh - ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/windows/build.sh + ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} UPLOAD_DIR=${UPLOAD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/windows/build.sh + ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} UPLOAD_DIR=${UPLOAD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/windows/build.sh %/bin: echo Creating Linux packages for $(@D) - ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/binary/build.sh - ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/binary/build.sh + ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} UPLOAD_DIR=${UPLOAD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/binary/build.sh + ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} UPLOAD_DIR=${UPLOAD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/binary/build.sh .PHONY: package-dashboards package-dashboards: echo Creating the Dashboards package - BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/dashboards/build.sh + UPLOAD_DIR=${UPLOAD_DIR} BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/dashboards/build.sh .PHONY: deps deps: @@ -61,11 +61,11 @@ go-daemon-image: ${BUILD_DIR}/god-linux-386 ${BUILD_DIR}/god-linux-amd64: docker run --rm -v ${BUILD_DIR}:/build tudorg/go-daemon -${BUILD_DIR}/upload: - mkdir -p ${BUILD_DIR}/upload +${UPLOAD_DIR}: + mkdir -p ${UPLOAD_DIR} -${BUILD_DIR}/upload/build_id.txt: - echo $(BUILDID) > ${BUILD_DIR}/upload/build_id.txt +${UPLOAD_DIR}/build_id.txt: + echo $(BUILDID) > ${UPLOAD_DIR}/build_id.txt # Build the image required for package-upload. .PHONY: deb-rpm-s3 diff --git a/dev-tools/packer/platforms/binary/build.sh b/dev-tools/packer/platforms/binary/build.sh index f968cbaf1a7f..6603b8102976 100755 --- a/dev-tools/packer/platforms/binary/build.sh +++ b/dev-tools/packer/platforms/binary/build.sh @@ -12,7 +12,7 @@ cat ${BUILD_DIR}/package.yml ${ARCHDIR}/archs/$ARCH.yml > ${BUILD_DIR}/settings- gotpl ${BASEDIR}/run.sh.j2 < ${BUILD_DIR}/settings-$runid.yml > ${BUILD_DIR}/run-$runid.sh chmod +x ${BUILD_DIR}/run-$runid.sh -docker run --rm -v ${BUILD_DIR}:/build \ +docker run --rm -v ${BUILD_DIR}:/build -v ${UPLOAD_DIR}:/upload \ -e BUILDID=$BUILDID -e SNAPSHOT=$SNAPSHOT -e RUNID=$runid \ tudorg/fpm /build/run-$runid.sh diff --git a/dev-tools/packer/platforms/binary/run.sh.j2 b/dev-tools/packer/platforms/binary/run.sh.j2 index 5a8e06379575..f057dcbf5206 100644 --- a/dev-tools/packer/platforms/binary/run.sh.j2 +++ b/dev-tools/packer/platforms/binary/run.sh.j2 @@ -21,10 +21,9 @@ cp ${BEATS_YML_NAME}.reference.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch} cp fields.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/ cp -a modules.d-linux/ /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/modules.d || true -mkdir -p upload -tar czvf upload/{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}} -echo "Created upload/{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz" +tar czvf /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-linux-{{.bin_arch}}.tar.gz /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}} +echo "Created /upload/{{.beat_name}}{{.beat_pkg_suffix}}--${VERSION}-linux-{{.bin_arch}}.tar.gz" -cd upload -sha512sum {{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz > {{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz.sha512 -echo "Created upload/{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz.sha512" +cd /upload +sha512sum {{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-linux-{{.bin_arch}}.tar.gz > {{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-linux-{{.bin_arch}}.tar.gz.sha512 +echo "Created /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-linux-{{.bin_arch}}.tar.gz.sha512" diff --git a/dev-tools/packer/platforms/centos/build.sh b/dev-tools/packer/platforms/centos/build.sh index 57d02fb53f75..6fcd4fc9c6ea 100755 --- a/dev-tools/packer/platforms/centos/build.sh +++ b/dev-tools/packer/platforms/centos/build.sh @@ -16,7 +16,7 @@ gotpl ${BASEDIR}/systemd.j2 < ${BUILD_DIR}/settings-$runid.yml > ${BUILD_DIR}/$r gotpl ${BASEDIR}/beatname.sh.j2 < ${BUILD_DIR}/settings-$runid.yml > ${BUILD_DIR}/beatname-$runid.sh chmod +x ${BUILD_DIR}/beatname-$runid.sh -docker run --rm -v ${BUILD_DIR}:/build \ +docker run --rm -v ${BUILD_DIR}:/build -v ${UPLOAD_DIR}:/upload \ -e BUILDID=$BUILDID -e SNAPSHOT=$SNAPSHOT -e RUNID=$runid \ tudorg/fpm /build/run-$runid.sh diff --git a/dev-tools/packer/platforms/centos/run.sh.j2 b/dev-tools/packer/platforms/centos/run.sh.j2 index 018900146a09..0ea1bba8e00d 100644 --- a/dev-tools/packer/platforms/centos/run.sh.j2 +++ b/dev-tools/packer/platforms/centos/run.sh.j2 @@ -26,7 +26,7 @@ RPM_VERSION=`echo ${VERSION} | sed 's/-/_/g'` # create rpm FPM_ARGS=( --force -s dir -t rpm - -n {{.beat_pkg_name}} -v ${RPM_VERSION} + -n {{.beat_pkg_name}}{{.beat_pkg_suffix}} -v ${RPM_VERSION} --architecture {{.rpm_arch}} --vendor "{{.beat_vendor}}" --license "{{.beat_license}}" @@ -52,11 +52,10 @@ fi fpm "${FPM_ARGS[@]}" # rename so that the filename respects semver rules -mkdir -p upload -mv {{.beat_pkg_name}}-${RPM_VERSION}-1.{{.rpm_arch}}.rpm upload/{{.beat_name}}-${VERSION}-{{.rpm_arch}}.rpm -echo "Created upload/{{.beat_name}}-${VERSION}-{{.rpm_arch}}.rpm" +mv {{.beat_pkg_name}}{{.beat_pkg_suffix}}-${RPM_VERSION}-1.{{.rpm_arch}}.rpm /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-{{.rpm_arch}}.rpm +echo "Created /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-{{.rpm_arch}}.rpm" # create sha512 file -cd upload -sha512sum {{.beat_name}}-${VERSION}-{{.rpm_arch}}.rpm > {{.beat_name}}-${VERSION}-{{.rpm_arch}}.rpm.sha512 -echo "Created upload/{{.beat_name}}-${VERSION}-{{.rpm_arch}}.rpm.sha512" +cd /upload +sha512sum {{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-{{.rpm_arch}}.rpm > {{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-{{.rpm_arch}}.rpm.sha512 +echo "Created /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-{{.rpm_arch}}.rpm.sha512" diff --git a/dev-tools/packer/platforms/darwin/build.sh b/dev-tools/packer/platforms/darwin/build.sh index 78d23025081c..d0340f265b69 100755 --- a/dev-tools/packer/platforms/darwin/build.sh +++ b/dev-tools/packer/platforms/darwin/build.sh @@ -12,7 +12,7 @@ cat ${BUILD_DIR}/package.yml ${ARCHDIR}/archs/$ARCH.yml > ${BUILD_DIR}/settings- gotpl ${BASEDIR}/run.sh.j2 < ${BUILD_DIR}/settings-$runid.yml > ${BUILD_DIR}/run-$runid.sh chmod +x ${BUILD_DIR}/run-$runid.sh -docker run --rm -v ${BUILD_DIR}:/build \ +docker run --rm -v ${BUILD_DIR}:/build -v ${UPLOAD_DIR}:/upload \ -e BUILDID=$BUILDID -e SNAPSHOT=$SNAPSHOT -e RUNID=$runid \ tudorg/fpm /build/run-$runid.sh diff --git a/dev-tools/packer/platforms/darwin/run.sh.j2 b/dev-tools/packer/platforms/darwin/run.sh.j2 index 0238950cb31c..42a48220f0df 100644 --- a/dev-tools/packer/platforms/darwin/run.sh.j2 +++ b/dev-tools/packer/platforms/darwin/run.sh.j2 @@ -18,10 +18,9 @@ cp {{.beat_name}}-darwin.reference.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/ cp fields.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/ cp -a modules.d-darwin/ /{{.beat_name}}-${VERSION}-darwin-x86_64/modules.d || true -mkdir -p upload -tar czvf upload/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz /{{.beat_name}}-${VERSION}-darwin-x86_64 -echo "Created upload/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz" +tar czvf /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-darwin-x86_64.tar.gz /{{.beat_name}}-${VERSION}-darwin-x86_64 +echo "Created /upload/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz" -cd upload -sha512sum {{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz > {{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz.sha512 -echo "Created upload/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz.sha512" +cd /upload +sha512sum {{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-darwin-x86_64.tar.gz > {{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-darwin-x86_64.tar.gz.sha512 +echo "Created /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-darwin-x86_64.tar.gz.sha512" diff --git a/dev-tools/packer/platforms/dashboards/build.sh b/dev-tools/packer/platforms/dashboards/build.sh index 8051fbb2d19c..8d427f1c42a8 100755 --- a/dev-tools/packer/platforms/dashboards/build.sh +++ b/dev-tools/packer/platforms/dashboards/build.sh @@ -11,7 +11,7 @@ cat ${ARCHDIR}/version.yml > ${BUILD_DIR}/settings-$runid.yml gotpl ${BASEDIR}/run.sh.j2 < ${BUILD_DIR}/settings-$runid.yml > ${BUILD_DIR}/run-$runid.sh chmod +x ${BUILD_DIR}/run-$runid.sh -docker run --rm -v ${BUILD_DIR}:/build \ +docker run --rm -v ${BUILD_DIR}:/build -v ${UPLOAD_DIR}:/upload \ -e BUILDID=$BUILDID -e SNAPSHOT=$SNAPSHOT -e RUNID=$runid -e BEAT_NAME=$BEAT_NAME \ tudorg/fpm /build/run-$runid.sh diff --git a/dev-tools/packer/platforms/dashboards/run.sh.j2 b/dev-tools/packer/platforms/dashboards/run.sh.j2 index 494a9b204638..96d272aaf2a2 100644 --- a/dev-tools/packer/platforms/dashboards/run.sh.j2 +++ b/dev-tools/packer/platforms/dashboards/run.sh.j2 @@ -14,10 +14,9 @@ mkdir /${BEAT_NAME:-beats}-dashboards-${VERSION} cp -a dashboards/. /${BEAT_NAME:-beats}-dashboards-${VERSION}/ echo "$BUILDID" > /${BEAT_NAME:-beats}-dashboards-${VERSION}/.build_hash.txt -mkdir -p upload -zip -r upload/${BEAT_NAME:-beats}-dashboards-${VERSION}.zip /${BEAT_NAME:-beats}-dashboards-${VERSION} -echo "Created upload/${BEAT_NAME:-beats}-dashboards-${VERSION}.zip" +zip -r /upload/${BEAT_NAME:-beats}-dashboards-${VERSION}.zip /${BEAT_NAME:-beats}-dashboards-${VERSION} +echo "Created /upload/${BEAT_NAME:-beats}-dashboards-${VERSION}.zip" -cd upload +cd /upload sha512sum ${BEAT_NAME:-beats}-dashboards-${VERSION}.zip > ${BEAT_NAME:-beats}-dashboards-${VERSION}.zip.sha512 -echo "Created upload/${BEAT_NAME:-beats}-dashboards-${VERSION}.zip.sha512" +echo "Created /upload/${BEAT_NAME:-beats}-dashboards-${VERSION}.zip.sha512" diff --git a/dev-tools/packer/platforms/debian/build.sh b/dev-tools/packer/platforms/debian/build.sh index 0093a15a30ee..be79fac72b35 100755 --- a/dev-tools/packer/platforms/debian/build.sh +++ b/dev-tools/packer/platforms/debian/build.sh @@ -16,7 +16,7 @@ gotpl ${BASEDIR}/systemd.j2 < ${BUILD_DIR}/settings-$runid.yml > ${BUILD_DIR}/$r gotpl ${BASEDIR}/beatname.sh.j2 < ${BUILD_DIR}/settings-$runid.yml > ${BUILD_DIR}/beatname-$runid.sh chmod +x ${BUILD_DIR}/beatname-$runid.sh -docker run --rm -v ${BUILD_DIR}:/build \ +docker run --rm -v ${BUILD_DIR}:/build -v ${UPLOAD_DIR}:/upload \ -e BUILDID=$BUILDID -e SNAPSHOT=$SNAPSHOT -e RUNID=$runid \ tudorg/fpm /build/run-$runid.sh diff --git a/dev-tools/packer/platforms/debian/run.sh.j2 b/dev-tools/packer/platforms/debian/run.sh.j2 index 23f63624d723..a88afd1e09fa 100644 --- a/dev-tools/packer/platforms/debian/run.sh.j2 +++ b/dev-tools/packer/platforms/debian/run.sh.j2 @@ -23,9 +23,9 @@ BEATS_YML_NAME="{{.beat_name}}-linux-{{.arch}}" # create deb FPM_ARGS=( --force -s dir -t deb - -n {{.beat_pkg_name}} -v ${VERSION} + -n {{.beat_pkg_name}}{{.beat_pkg_suffix}} -v ${VERSION} --vendor "{{.beat_vendor}}" - --license "{{.beat_license}}" + --license $(echo {{.beat_license}} | tr " " "-") --architecture {{.deb_arch}} --description "{{.beat_description}}" --url {{.beat_url}} @@ -49,11 +49,10 @@ fi fpm "${FPM_ARGS[@]}" # move and rename to use the elastic conventions -mkdir -p upload -mv {{.beat_pkg_name}}_${VERSION}_{{.deb_arch}}.deb upload/{{.beat_name}}-${VERSION}-{{.deb_arch}}.deb -echo "Created upload/{{.beat_name}}-${VERSION}-{{.deb_arch}}.deb" +mv {{.beat_pkg_name}}{{.beat_pkg_suffix}}_${VERSION}_{{.deb_arch}}.deb /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-{{.deb_arch}}.deb +echo "Created /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-{{.deb_arch}}.deb" # create sha512 file -cd upload -sha512sum {{.beat_name}}-${VERSION}-{{.deb_arch}}.deb > {{.beat_name}}-${VERSION}-{{.deb_arch}}.deb.sha512 -echo "Created upload/{{.beat_name}}-${VERSION}-{{.deb_arch}}.deb.sha512" +cd /upload +sha512sum {{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-{{.deb_arch}}.deb > {{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-{{.deb_arch}}.deb.sha512 +echo "Created /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-{{.deb_arch}}.deb.sha512" diff --git a/dev-tools/packer/platforms/windows/build.sh b/dev-tools/packer/platforms/windows/build.sh index c52a9e2369ea..8b5e169e077b 100755 --- a/dev-tools/packer/platforms/windows/build.sh +++ b/dev-tools/packer/platforms/windows/build.sh @@ -14,7 +14,7 @@ gotpl ${BASEDIR}/install-service.ps1.j2 < ${BUILD_DIR}/settings-$runid.yml > ${B gotpl ${BASEDIR}/uninstall-service.ps1.j2 < ${BUILD_DIR}/settings-$runid.yml > ${BUILD_DIR}/uninstall-service-$BEAT.ps1 chmod +x ${BUILD_DIR}/run-$runid.sh -docker run --rm -v ${BUILD_DIR}:/build \ +docker run --rm -v ${BUILD_DIR}:/build -v ${UPLOAD_DIR}:/upload \ -e BUILDID=$BUILDID -e SNAPSHOT=$SNAPSHOT -e RUNID=$runid \ tudorg/fpm /build/run-$runid.sh diff --git a/dev-tools/packer/platforms/windows/run.sh.j2 b/dev-tools/packer/platforms/windows/run.sh.j2 index b7c2258e6317..d57fb15e0ab2 100644 --- a/dev-tools/packer/platforms/windows/run.sh.j2 +++ b/dev-tools/packer/platforms/windows/run.sh.j2 @@ -21,10 +21,9 @@ cp -a modules.d-win/ /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/modules.d cp install-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ cp uninstall-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ -mkdir -p upload -zip -r upload/{{.beat_name}}-${VERSION}-windows-{{.win_arch}}.zip /{{.beat_name}}-${VERSION}-windows-{{.win_arch}} -echo "Created upload/{{.beat_name}}-${VERSION}-windows-{{.win_arch}}.zip" +zip -r /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-windows-{{.win_arch}}.zip /{{.beat_name}}-${VERSION}-windows-{{.win_arch}} +echo "Created /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-windows-{{.win_arch}}.zip" -cd upload -sha512sum {{.beat_name}}-${VERSION}-windows-{{.win_arch}}.zip > {{.beat_name}}-${VERSION}-windows-{{.win_arch}}.zip.sha512 -echo "Created upload/{{.beat_name}}-${VERSION}-windows-{{.win_arch}}.zip.sha512" +cd /upload +sha512sum {{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-windows-{{.win_arch}}.zip > {{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-windows-{{.win_arch}}.zip.sha512 +echo "Created /upload/{{.beat_name}}{{.beat_pkg_suffix}}-${VERSION}-windows-{{.win_arch}}.zip.sha512" diff --git a/dev-tools/packer/xgo-scripts/before_build.sh b/dev-tools/packer/xgo-scripts/before_build.sh index daedd2a1ed86..6d746678cc9b 100755 --- a/dev-tools/packer/xgo-scripts/before_build.sh +++ b/dev-tools/packer/xgo-scripts/before_build.sh @@ -45,7 +45,7 @@ PREFIX=$PREFIX make before-build # Add data to the home directory mkdir -p $PREFIX/homedir -make install-home HOME_PREFIX=$PREFIX/homedir +make install-home HOME_PREFIX=$PREFIX/homedir LICENSE_FILE=${LICENSE_FILE} if [ -n "BUILDID" ]; then echo "$BUILDID" > $PREFIX/homedir/.build_hash.txt diff --git a/generator/beat/{beat}/Makefile b/generator/beat/{beat}/Makefile index 314b65cb2978..a45607534052 100644 --- a/generator/beat/{beat}/Makefile +++ b/generator/beat/{beat}/Makefile @@ -23,7 +23,7 @@ setup: copy-vendor copy-vendor: mkdir -p vendor/github.com/elastic/ cp -R ${BEAT_GOPATH}/src/github.com/elastic/beats vendor/github.com/elastic/ - rm -rf vendor/github.com/elastic/beats/.git + rm -rf vendor/github.com/elastic/beats/.git vendor/github.com/elastic/beats/x-pack .PHONY: git-init git-init: diff --git a/generator/metricbeat/{beat}/Makefile b/generator/metricbeat/{beat}/Makefile index eab3101c323c..80d6cb981542 100644 --- a/generator/metricbeat/{beat}/Makefile +++ b/generator/metricbeat/{beat}/Makefile @@ -22,7 +22,7 @@ setup: copy-vendor copy-vendor: mkdir -p vendor/github.com/elastic/ cp -R ${GOPATH}/src/github.com/elastic/beats vendor/github.com/elastic/ - rm -rf vendor/github.com/elastic/beats/.git + rm -rf vendor/github.com/elastic/beats/.git vendor/github.com/elastic/beats/x-pack # This is called by the beats packer before building starts .PHONY: before-build diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index ae21bec8e69e..32c8d36656ed 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -15,7 +15,8 @@ ES_BEATS?=..## @community_beat Must be set to ./vendor/github.com/elastic/beats. GOPACKAGES?=$(shell go list ${BEAT_PATH}/... | grep -v /vendor/ | grep -v /scripts/cmd/ ) PACKER_TEMPLATES_DIR?=${ES_BEATS}/dev-tools/packer ## @Building Directory of templates that are used by "make package" NOTICE_FILE?=../NOTICE.txt -LICENSE_FILE?=../LICENSE.txt +LICENSE_FILE?=../licenses/APACHE-LICENSE-2.0.txt +ELASTIC_LICENSE_FILE?=../licenses/ELASTIC-LICENSE.txt space:=$() # comma:=, @@ -31,6 +32,8 @@ SHELL=bash ES_HOST?="elasticsearch" PWD=$(shell pwd) BUILD_DIR?=$(shell pwd)/build +PKG_BUILD_DIR?=$(BUILD_DIR)/package${PKG_SUFFIX} +PKG_UPLOAD_DIR?=$(BUILD_DIR)/upload COVERAGE_DIR?=${BUILD_DIR}/coverage COVERAGE_TOOL?=${BEAT_GOPATH}/bin/gotestcover COVERAGE_TOOL_REPO?=github.com/elastic/beats/vendor/github.com/pierrre/gotestcover @@ -391,7 +394,7 @@ install-home: install -m 644 ${NOTICE_FILE} ${HOME_PREFIX}/; \ fi if [ -a ${LICENSE_FILE} ]; then \ - install -m 644 ${LICENSE_FILE} ${HOME_PREFIX}/; \ + install -m 644 ${LICENSE_FILE} ${HOME_PREFIX}/LICENSE.txt; \ fi if [ -d _meta/module.generated ]; then \ install -d -m 755 ${HOME_PREFIX}/module; \ @@ -411,7 +414,7 @@ prepare-package: -v $(abspath ${ES_BEATS}/dev-tools/packer/xgo-scripts):/scripts \ -v $(abspath ${PACKER_TEMPLATES_DIR}):/templates \ -v $(abspath ../):/source \ - -v $(BUILD_DIR):/build \ + -v $(PKG_BUILD_DIR):/build \ -e PUREGO="yes" \ -e PACK=${BEAT_NAME} \ -e BEFORE_BUILD=before_build.sh \ @@ -421,6 +424,7 @@ prepare-package: -e ES_BEATS=${ES_BEATS} \ -e BEAT_PATH=${BEAT_PATH} \ -e BEAT_NAME=${BEAT_NAME} \ + -e LICENSE_FILE=${LICENSE_FILE} \ ${BEATS_BUILDER_IMAGE} # Prepares for packaging. Builds binaries with cgo @@ -432,7 +436,7 @@ prepare-package-cgo: -v $(abspath ${ES_BEATS}/dev-tools/packer/xgo-scripts):/scripts \ -v $(abspath ${PACKER_TEMPLATES_DIR}):/templates \ -v $(abspath ../):/source \ - -v $(BUILD_DIR):/build \ + -v $(PKG_BUILD_DIR):/build \ -e PACK=${BEAT_NAME} \ -e BEFORE_BUILD=before_build.sh \ -e SOURCE=/source \ @@ -441,14 +445,15 @@ prepare-package-cgo: -e ES_BEATS=${ES_BEATS} \ -e BEAT_PATH=${BEAT_PATH} \ -e BEAT_NAME=${BEAT_NAME} \ + -e LICENSE_FILE=${LICENSE_FILE} \ ${BEATS_BUILDER_IMAGE} # linux builds on older debian for compatibility docker run --rm \ - -v ${BUILD_DIR}:/build \ -v $(abspath ${ES_BEATS}/dev-tools/packer/xgo-scripts):/scripts \ -v $(abspath ${PACKER_TEMPLATES_DIR}):/templates \ -v $(abspath ..):/source \ + -v ${PKG_BUILD_DIR}:/build \ -e PACK=${BEAT_NAME} \ -e BEFORE_BUILD=before_build.sh \ -e SOURCE=/source \ @@ -457,6 +462,7 @@ prepare-package-cgo: -e ES_BEATS=${ES_BEATS} \ -e BEAT_PATH=${BEAT_PATH} \ -e BEAT_NAME=${BEAT_NAME} \ + -e LICENSE_FILE=${LICENSE_FILE} \ ${BEATS_BUILDER_DEB_IMAGE} # Prepares images for packaging @@ -466,26 +472,28 @@ package-setup: .PHONY: package package: ## @packaging Create binary packages for the beat. -package: clean update package-setup - +package: update package-setup echo "Start building packages for ${BEAT_NAME}" - mkdir -p ${BUILD_DIR}/upload + rm -rf ${PKG_BUILD_DIR} + mkdir -p ${PKG_BUILD_DIR} + mkdir -p ${PKG_UPLOAD_DIR} # Generates the package.yml file with all information needed to create packages - echo "beat_name: ${BEAT_NAME}" > ${BUILD_DIR}/package.yml - echo "beat_url: ${BEAT_URL}" >> ${BUILD_DIR}/package.yml - echo "beat_repo: ${BEAT_PATH}" >> ${BUILD_DIR}/package.yml - echo "beat_pkg_name: ${BEAT_PACKAGE_NAME}" >> ${BUILD_DIR}/package.yml - echo "beat_description: ${BEAT_DESCRIPTION}" >> ${BUILD_DIR}/package.yml - echo "beat_vendor: ${BEAT_VENDOR}" >> ${BUILD_DIR}/package.yml - echo "beat_license: ${BEAT_LICENSE}" >> ${BUILD_DIR}/package.yml - echo "beat_doc_url: ${BEAT_DOC_URL}" >> ${BUILD_DIR}/package.yml + echo "beat_name: ${BEAT_NAME}" > ${PKG_BUILD_DIR}/package.yml + echo "beat_url: ${BEAT_URL}" >> ${PKG_BUILD_DIR}/package.yml + echo "beat_repo: ${BEAT_PATH}" >> ${PKG_BUILD_DIR}/package.yml + echo "beat_pkg_name: ${BEAT_PACKAGE_NAME}" >> ${PKG_BUILD_DIR}/package.yml + echo "beat_pkg_suffix: '${PKG_SUFFIX}'" >> ${PKG_BUILD_DIR}/package.yml + echo "beat_description: ${BEAT_DESCRIPTION}" >> ${PKG_BUILD_DIR}/package.yml + echo "beat_vendor: ${BEAT_VENDOR}" >> ${PKG_BUILD_DIR}/package.yml + echo "beat_license: ${BEAT_LICENSE}" >> ${PKG_BUILD_DIR}/package.yml + echo "beat_doc_url: ${BEAT_DOC_URL}" >> ${PKG_BUILD_DIR}/package.yml if [ -a version.yml ]; then \ - cat version.yml >> ${BUILD_DIR}/package.yml; \ + cat version.yml >> ${PKG_BUILD_DIR}/package.yml; \ else \ - cat ${ES_BEATS}/dev-tools/packer/version.yml >> ${BUILD_DIR}/package.yml; \ + cat ${ES_BEATS}/dev-tools/packer/version.yml >> ${PKG_BUILD_DIR}/package.yml; \ fi if [ $(CGO) = true ]; then \ @@ -494,10 +502,23 @@ package: clean update package-setup $(MAKE) prepare-package; \ fi - SNAPSHOT=${SNAPSHOT} BUILDID=${BUILDID} BEAT_PATH=${BEAT_PATH} BUILD_DIR=${BUILD_DIR} $(MAKE) -C ${ES_BEATS}/dev-tools/packer ${PACKAGES} ${BUILD_DIR}/upload/build_id.txt + SNAPSHOT=${SNAPSHOT} BUILDID=${BUILDID} BEAT_PATH=${BEAT_PATH} BUILD_DIR=${PKG_BUILD_DIR} UPLOAD_DIR=${PKG_UPLOAD_DIR} $(MAKE) -C ${ES_BEATS}/dev-tools/packer ${PACKAGES} ${BUILD_DIR}/upload/build_id.txt $(MAKE) fix-permissions echo "Finished packages for ${BEAT_NAME}" +# Packages the Beat without Elastic X-Pack content (OSS only). +.PHONY: package-oss +package-oss: + @$(MAKE) PKG_SUFFIX=-oss package + +# Packages the Beat with Elastic X-Pack content. +.PHONY: package-elastic +package-elastic: + @$(MAKE) BEAT_LICENSE="Elastic License" LICENSE_FILE=$(ELASTIC_LICENSE_FILE) package + +.PHONY: package-all +package-all: package-elastic package-oss + package-dashboards: package-setup mkdir -p ${BUILD_DIR} cp -r _meta/kibana ${BUILD_DIR}/dashboards diff --git a/licenses/APACHE-LICENSE-2.0.txt b/licenses/APACHE-LICENSE-2.0.txt new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/licenses/APACHE-LICENSE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/licenses/ELASTIC-LICENSE.txt b/licenses/ELASTIC-LICENSE.txt new file mode 100644 index 000000000000..7376ffc3ff10 --- /dev/null +++ b/licenses/ELASTIC-LICENSE.txt @@ -0,0 +1,223 @@ +ELASTIC LICENSE AGREEMENT + +PLEASE READ CAREFULLY THIS ELASTIC LICENSE AGREEMENT (THIS "AGREEMENT"), WHICH +CONSTITUTES A LEGALLY BINDING AGREEMENT AND GOVERNS ALL OF YOUR USE OF ALL OF +THE ELASTIC SOFTWARE WITH WHICH THIS AGREEMENT IS INCLUDED ("ELASTIC SOFTWARE") +THAT IS PROVIDED IN OBJECT CODE FORMAT, AND, IN ACCORDANCE WITH SECTION 2 BELOW, +CERTAIN OF THE ELASTIC SOFTWARE THAT IS PROVIDED IN SOURCE CODE FORMAT. BY +INSTALLING OR USING ANY OF THE ELASTIC SOFTWARE GOVERNED BY THIS AGREEMENT, YOU +ARE ASSENTING TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE +WITH SUCH TERMS AND CONDITIONS, YOU MAY NOT INSTALL OR USE THE ELASTIC SOFTWARE +GOVERNED BY THIS AGREEMENT. IF YOU ARE INSTALLING OR USING THE SOFTWARE ON +BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL +AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF +SUCH ENTITY. + +Posted Date: April 20, 2018 + +This Agreement is entered into by and between Elasticsearch BV ("Elastic") and +You, or the legal entity on behalf of whom You are acting (as applicable, +"You"). + +1. OBJECT CODE END USER LICENSES, RESTRICTIONS AND THIRD PARTY OPEN SOURCE +SOFTWARE + + 1.1 Object Code End User License. Subject to the terms and conditions of + Section 1.2 of this Agreement, Elastic hereby grants to You, AT NO CHARGE and + for so long as you are not in breach of any provision of this Agreement, a + License to the Basic Features and Functions of the Elastic Software. + + 1.2 Reservation of Rights; Restrictions. As between Elastic and You, Elastic + and its licensors own all right, title and interest in and to the Elastic + Software, and except as expressly set forth in Sections 1.1, and 2.1 of this + Agreement, no other license to the Elastic Software is granted to You under + this Agreement, by implication, estoppel or otherwise. You agree not to: (i) + reverse engineer or decompile, decrypt, disassemble or otherwise reduce any + Elastic Software provided to You in Object Code, or any portion thereof, to + Source Code, except and only to the extent any such restriction is prohibited + by applicable law, (ii) except as expressly permitted in this Agreement, + prepare derivative works from, modify, copy or use the Elastic Software Object + Code or the Commercial Software Source Code in any manner; (iii) except as + expressly permitted in Section 1.1 above, transfer, sell, rent, lease, + distribute, sublicense, loan or otherwise transfer, Elastic Software Object + Code, in whole or in part, to any third party; (iv) use Elastic Software + Object Code for providing time-sharing services, any software-as-a-service, + service bureau services or as part of an application services provider or + other service offering (collectively, "SaaS Offering") where obtaining access + to the Elastic Software or the features and functions of the Elastic Software + is a primary reason or substantial motivation for users of the SaaS Offering + to access and/or use the SaaS Offering ("Prohibited SaaS Offering"); (v) + circumvent the limitations on use of Elastic Software provided to You in + Object Code format that are imposed or preserved by any License Key, or (vi) + alter or remove any Marks and Notices in the Elastic Software. If You have any + question as to whether a specific SaaS Offering constitutes a Prohibited SaaS + Offering, or are interested in obtaining Elastic's permission to engage in + commercial or non-commercial distribution of the Elastic Software, please + contact elastic_license@elastic.co. + + 1.3 Third Party Open Source Software. The Commercial Software may contain or + be provided with third party open source libraries, components, utilities and + other open source software (collectively, "Open Source Software"), which Open + Source Software may have applicable license terms as identified on a website + designated by Elastic. Notwithstanding anything to the contrary herein, use of + the Open Source Software shall be subject to the license terms and conditions + applicable to such Open Source Software, to the extent required by the + applicable licensor (which terms shall not restrict the license rights granted + to You hereunder, but may contain additional rights). To the extent any + condition of this Agreement conflicts with any license to the Open Source + Software, the Open Source Software license will govern with respect to such + Open Source Software only. Elastic may also separately provide you with + certain open source software that is licensed by Elastic. Your use of such + Elastic open source software will not be governed by this Agreement, but by + the applicable open source license terms. + +2. COMMERCIAL SOFTWARE SOURCE CODE + + 2.1 Limited License. Subject to the terms and conditions of Section 2.2 of + this Agreement, Elastic hereby grants to You, AT NO CHARGE and for so long as + you are not in breach of any provision of this Agreement, a limited, + non-exclusive, non-transferable, fully paid up royalty free right and license + to the Commercial Software in Source Code format, without the right to grant + or authorize sublicenses, to prepare Derivative Works of the Commercial + Software, provided You (i) do not hack the licensing mechanism, or otherwise + circumvent the intended limitations on the use of Elastic Software to enable + features other than Basic Features and Functions or those features You are + entitled to as part of a Subscription, and (ii) use the resulting object code + only for reasonable testing purposes. + + 2.2 Restrictions. Nothing in Section 2.1 grants You the right to (i) use the + Commercial Software Source Code other than in accordance with Section 2.1 + above, (ii) use a Derivative Work of the Commercial Software outside of a + Non-production Environment, in any production capacity, on a temporary or + permanent basis, or (iii) transfer, sell, rent, lease, distribute, sublicense, + loan or otherwise make available the Commercial Software Source Code, in whole + or in part, to any third party. Notwithstanding the foregoing, You may + maintain a copy of the repository in which the Source Code of the Commercial + Software resides and that copy may be publicly accessible, provided that you + include this Agreement with Your copy of the repository. + +3. TERMINATION + + 3.1 Termination. This Agreement will automatically terminate, whether or not + You receive notice of such Termination from Elastic, if You breach any of its + provisions. + + 3.2 Post Termination. Upon any termination of this Agreement, for any reason, + You shall promptly cease the use of the Elastic Software in Object Code format + and cease use of the Commercial Software in Source Code format. For the + avoidance of doubt, termination of this Agreement will not affect Your right + to use Elastic Software, in either Object Code or Source Code formats, made + available under the Apache License Version 2.0. + + 3.3 Survival. Sections 1.2, 2.2. 3.3, 4 and 5 shall survive any termination or + expiration of this Agreement. + +4. DISCLAIMER OF WARRANTIES AND LIMITATION OF LIABILITY + + 4.1 Disclaimer of Warranties. TO THE MAXIMUM EXTENT PERMITTED UNDER APPLICABLE + LAW, THE ELASTIC SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, + AND ELASTIC AND ITS LICENSORS MAKE NO WARRANTIES WHETHER EXPRESSED, IMPLIED OR + STATUTORY REGARDING OR RELATING TO THE ELASTIC SOFTWARE. TO THE MAXIMUM EXTENT + PERMITTED UNDER APPLICABLE LAW, ELASTIC AND ITS LICENSORS SPECIFICALLY + DISCLAIM ALL IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NON-INFRINGEMENT WITH RESPECT TO THE ELASTIC SOFTWARE, AND WITH + RESPECT TO THE USE OF THE FOREGOING. FURTHER, ELASTIC DOES NOT WARRANT RESULTS + OF USE OR THAT THE ELASTIC SOFTWARE WILL BE ERROR FREE OR THAT THE USE OF THE + ELASTIC SOFTWARE WILL BE UNINTERRUPTED. + + 4.2 Limitation of Liability. IN NO EVENT SHALL ELASTIC OR ITS LICENSORS BE + LIABLE TO YOU OR ANY THIRD PARTY FOR ANY DIRECT OR INDIRECT DAMAGES, + INCLUDING, WITHOUT LIMITATION, FOR ANY LOSS OF PROFITS, LOSS OF USE, BUSINESS + INTERRUPTION, LOSS OF DATA, COST OF SUBSTITUTE GOODS OR SERVICES, OR FOR ANY + SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES OF ANY KIND, IN CONNECTION WITH + OR ARISING OUT OF THE USE OR INABILITY TO USE THE ELASTIC SOFTWARE, OR THE + PERFORMANCE OF OR FAILURE TO PERFORM THIS AGREEMENT, WHETHER ALLEGED AS A + BREACH OF CONTRACT OR TORTIOUS CONDUCT, INCLUDING NEGLIGENCE, EVEN IF ELASTIC + HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +5. MISCELLANEOUS + + This Agreement completely and exclusively states the entire agreement of the + parties regarding the subject matter herein, and it supersedes, and its terms + govern, all prior proposals, agreements, or other communications between the + parties, oral or written, regarding such subject matter. This Agreement may be + modified by Elastic from time to time, and any such modifications will be + effective upon the "Posted Date" set forth at the top of the modified + Agreement. If any provision hereof is held unenforceable, this Agreement will + continue without said provision and be interpreted to reflect the original + intent of the parties. This Agreement and any non-contractual obligation + arising out of or in connection with it, is governed exclusively by Dutch law. + This Agreement shall not be governed by the 1980 UN Convention on Contracts + for the International Sale of Goods. All disputes arising out of or in + connection with this Agreement, including its existence and validity, shall be + resolved by the courts with jurisdiction in Amsterdam, The Netherlands, except + where mandatory law provides for the courts at another location in The + Netherlands to have jurisdiction. The parties hereby irrevocably waive any and + all claims and defenses either might otherwise have in any such action or + proceeding in any of such courts based upon any alleged lack of personal + jurisdiction, improper venue, forum non conveniens or any similar claim or + defense. A breach or threatened breach, by You of Section 2 may cause + irreparable harm for which damages at law may not provide adequate relief, and + therefore Elastic shall be entitled to seek injunctive relief without being + required to post a bond. You may not assign this Agreement (including by + operation of law in connection with a merger or acquisition), in whole or in + part to any third party without the prior written consent of Elastic, which + may be withheld or granted by Elastic in its sole and absolute discretion. + Any assignment in violation of the preceding sentence is void. Notices to + Elastic may also be sent to legal@elastic.co. + +6. DEFINITIONS + + The following terms have the meanings ascribed: + + 6.1 "Affiliate" means, with respect to a party, any entity that controls, is + controlled by, or which is under common control with, such party, where + "control" means ownership of at least fifty percent (50%) of the outstanding + voting shares of the entity, or the contractual right to establish policy for, + and manage the operations of, the entity. + + 6.2 "Basic Features and Functions" means those features and functions of the + Elastic Software that are eligible for use under a Basic license, as set forth + at https://www.elastic.co/subscriptions, as may be modified by Elastic from + time to time. + + 6.3 "Commercial Software" means the Elastic Software Source Code in any file + containing a header stating the contents are subject to the Elastic License or + which is contained in the repository folder labeled "x-pack", unless a LICENSE + file present in the directory subtree declares a different license. + + 6.4 "Derivative Work of the Commercial Software" means, for purposes of this + Agreement, any modification(s) or enhancement(s) to the Commercial Software, + which represent, as a whole, an original work of authorship. + + 6.5 "License" means a limited, non-exclusive, non-transferable, fully paid up, + royalty free, right and license, without the right to grant or authorize + sublicenses, solely for Your internal business operations to (i) install and + use the applicable Features and Functions of the Elastic Software in Object + Code, and (ii) permit Contractors and Your Affiliates to use the Elastic + software as set forth in (i) above, provided that such use by Contractors must + be solely for Your benefit and/or the benefit of Your Affiliates, and You + shall be responsible for all acts and omissions of such Contractors and + Affiliates in connection with their use of the Elastic software that are + contrary to the terms and conditions of this Agreement. + + 6.6 "License Key" means a sequence of bytes, including but not limited to a + JSON blob, that is used to enable certain features and functions of the + Elastic Software. + + 6.7 "Marks and Notices" means all Elastic trademarks, trade names, logos and + notices present on the Documentation as originally provided by Elastic. + + 6.8 "Non-production Environment" means an environment for development, testing + or quality assurance, where software is not used for production purposes. + + 6.9 "Object Code" means any form resulting from mechanical transformation or + translation of Source Code form, including but not limited to compiled object + code, generated documentation, and conversions to other media types. + + 6.10 "Source Code" means the preferred form of computer software for making + modifications, including but not limited to software source code, + documentation source, and configuration files. + + 6.11 "Subscription" means the right to receive Support Services and a License + to the Commercial Software. diff --git a/testing/environments/args.yml b/testing/environments/args.yml index 88227ce9f2eb..e36f8ea3a23f 100644 --- a/testing/environments/args.yml +++ b/testing/environments/args.yml @@ -7,4 +7,4 @@ services: args: DOWNLOAD_URL: https://snapshots.elastic.co/downloads ELASTIC_VERSION: 7.0.0-alpha1-SNAPSHOT - CACHE_BUST: 20180208 + CACHE_BUST: 20180424 diff --git a/testing/environments/docker/elasticsearch/Dockerfile b/testing/environments/docker/elasticsearch/Dockerfile index 2920021d9ede..8b1c1e348e8f 100644 --- a/testing/environments/docker/elasticsearch/Dockerfile +++ b/testing/environments/docker/elasticsearch/Dockerfile @@ -1,5 +1,4 @@ # Copy of https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/Dockerfile -#FROM docker.elastic.co/elasticsearch/elasticsearch-alpine-base:latest FROM centos:7 MAINTAINER Elastic Docker Team @@ -7,7 +6,7 @@ ARG ELASTIC_VERSION ARG DOWNLOAD_URL ARG ES_JAVA_OPTS ARG CACHE_BUST=1 -ARG XPACK +ARG IMAGE_FLAVOR=x-pack ENV ELASTIC_CONTAINER true ENV PATH /usr/share/elasticsearch/bin:$PATH @@ -19,30 +18,29 @@ RUN groupadd -g 1000 elasticsearch && adduser -u 1000 -g 1000 -d /usr/share/elas WORKDIR /usr/share/elasticsearch -# Download/extract defined ES version. busybox tar can't strip leading dir. -RUN curl -L -o elasticsearch-${ELASTIC_VERSION}.tar.gz ${DOWNLOAD_URL}/elasticsearch/elasticsearch-${ELASTIC_VERSION}.tar.gz?c=${CACHE_BUST} && \ - EXPECTED_SHA=$(wget -O - ${DOWNLOAD_URL}/elasticsearch/elasticsearch-${ELASTIC_VERSION}.tar.gz.sha512 | awk '{print $1}') && \ - test $EXPECTED_SHA == $(sha512sum elasticsearch-${ELASTIC_VERSION}.tar.gz | awk '{print $1}') && \ - tar zxf elasticsearch-${ELASTIC_VERSION}.tar.gz && \ +# Download/extract the defined ES version. +COPY download.sh /download.sh +RUN /download.sh $DOWNLOAD_URL $ELASTIC_VERSION $CACHE_BUST && rm /download.sh + +RUN tar zxf elasticsearch-${ELASTIC_VERSION}.tar.gz && \ chown -R elasticsearch:elasticsearch elasticsearch-${ELASTIC_VERSION} && \ mv elasticsearch-${ELASTIC_VERSION}/* . && \ rmdir elasticsearch-${ELASTIC_VERSION} && \ rm elasticsearch-${ELASTIC_VERSION}.tar.gz -RUN set -ex && for esdirs in config data logs; do \ +RUN set -e && for esdirs in config data logs; do \ mkdir -p "$esdirs"; \ chown -R elasticsearch:elasticsearch "$esdirs"; \ done USER elasticsearch -# Install xpack -RUN if [ ${XPACK} = "1" ]; then elasticsearch-plugin install --batch ${DOWNLOAD_URL}/packs/x-pack/x-pack-${ELASTIC_VERSION}.zip?c=${CACHE_BUST}; fi -RUN elasticsearch-plugin install --batch ${DOWNLOAD_URL}/elasticsearch-plugins/ingest-user-agent/ingest-user-agent-${ELASTIC_VERSION}.zip?c=${CACHE_BUST} -RUN elasticsearch-plugin install --batch ${DOWNLOAD_URL}/elasticsearch-plugins/ingest-geoip/ingest-geoip-${ELASTIC_VERSION}.zip?c=${CACHE_BUST} +# Install plugins. +RUN elasticsearch-plugin install --batch ${DOWNLOAD_URL}/elasticsearch-plugins/ingest-user-agent/ingest-user-agent-${ELASTIC_VERSION}.zip +RUN elasticsearch-plugin install --batch ${DOWNLOAD_URL}/elasticsearch-plugins/ingest-geoip/ingest-geoip-${ELASTIC_VERSION}.zip # Set bootstrap password (for when security is used) -RUN if [ ${XPACK} = "1" ]; then elasticsearch-keystore create; echo "changeme" | elasticsearch-keystore add -x 'bootstrap.password'; fi +RUN if [ "${IMAGE_FLAVOR}" = "x-pack" ]; then elasticsearch-keystore create; echo "changeme" | elasticsearch-keystore add -x 'bootstrap.password'; fi COPY config/elasticsearch.yml config/ COPY config/log4j2.properties config/ @@ -52,6 +50,9 @@ USER root RUN chown elasticsearch:elasticsearch config/elasticsearch.yml config/log4j2.properties bin/es-docker && \ chmod 0750 bin/es-docker +# Enable a trial license for testing ML and Alerting. +RUN if [ "${IMAGE_FLAVOR}" = "x-pack" ]; then echo "xpack.license.self_generated.type: trial" >> config/elasticsearch.yml; fi + USER elasticsearch CMD ["/bin/bash", "bin/es-docker"] diff --git a/testing/environments/docker/elasticsearch/download.sh b/testing/environments/docker/elasticsearch/download.sh new file mode 100755 index 000000000000..2610741ac263 --- /dev/null +++ b/testing/environments/docker/elasticsearch/download.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -eo pipefail + +if [ -z ${DOWNLOAD_URL+x} ]; then echo "DOWNLOAD_URL is unset"; exit 1; fi +if [ -z ${ELASTIC_VERSION+x} ]; then echo "ELASTIC_VERSION is unset"; exit 1; fi +if [ -z ${IMAGE_FLAVOR+x} ]; then echo "IMAGE_FLAVOR is unset"; exit 1; fi + +url=${DOWNLOAD_URL}/elasticsearch/elasticsearch-oss/elasticsearch-oss-${ELASTIC_VERSION}.tar.gz +if [ "${IMAGE_FLAVOR}" = "x-pack" ]; then + url=${DOWNLOAD_URL}/elasticsearch/elasticsearch-${ELASTIC_VERSION}.tar.gz +fi + +# Download. +curl -s -L -o elasticsearch-${ELASTIC_VERSION}.tar.gz $url + +# Validate SHA512. +expected_sha=$(curl -s -L $url.sha512 | awk '{print $1}') +observed_sha=$(sha512sum elasticsearch-${ELASTIC_VERSION}.tar.gz | awk '{print $1}') +test "${expected_sha}" == "${observed_sha}" diff --git a/testing/environments/docker/kibana/Dockerfile b/testing/environments/docker/kibana/Dockerfile index cf8ce1823f74..416bae46361a 100644 --- a/testing/environments/docker/kibana/Dockerfile +++ b/testing/environments/docker/kibana/Dockerfile @@ -3,12 +3,11 @@ FROM centos:7 LABEL maintainer "Elastic Docker Team " EXPOSE 5601 - ### Beats specific args #### -ARG DOWNLOAD_URL=https://snapshots.elastic.co/downloads -ARG ELASTIC_VERSION=7.0.0-alpha1-SNAPSHOT +ARG DOWNLOAD_URL +ARG ELASTIC_VERSION ARG CACHE_BUST=1 -ARG XPACK=1 +ARG IMAGE_FLAVOR=x-pack # Healthcheck create by beats team RUN yum install update -y epel-release && yum install -y jq @@ -19,7 +18,10 @@ HEALTHCHECK --interval=1s --retries=600 CMD curl -f http://localhost:5601/api/st RUN yum update -y && yum install -y fontconfig freetype && yum clean all WORKDIR /usr/share/kibana -RUN curl -Ls ${DOWNLOAD_URL}/kibana/kibana-${ELASTIC_VERSION}-linux-x86_64.tar.gz?c=${CACHE_BUST} | tar --strip-components=1 -zxf - && \ +COPY download.sh /download.sh +RUN /download.sh $DOWNLOAD_URL $ELASTIC_VERSION $CACHE_BUST && rm /download.sh +RUN tar --strip-components=1 -zxf kibana-${ELASTIC_VERSION}-linux-x86_64.tar.gz && \ + rm kibana-${ELASTIC_VERSION}-linux-x86_64.tar.gz && \ ln -s /usr/share/kibana /opt/kibana ENV ELASTIC_CONTAINER true @@ -32,9 +34,6 @@ COPY config/kibana-x-pack.yml /usr/share/kibana/config/kibana.yml # variables and translate them to Kibana CLI options. COPY bin/kibana-docker /usr/local/bin/ -# Add a self-signed SSL certificate for use in examples. -#COPY ssl/kibana.example.org.* /usr/share/kibana/config/ - # Provide a non-root user to run the process. RUN groupadd --gid 1000 kibana && \ useradd --uid 1000 --gid 1000 \ @@ -42,8 +41,5 @@ RUN groupadd --gid 1000 kibana && \ kibana USER kibana -# Beats specific check for XPACK to have both variables in one -RUN if [ ${XPACK} = "1" ]; then NODE_OPTIONS="--max-old-space-size=4096" bin/kibana-plugin install ${DOWNLOAD_URL}/kibana-plugins/x-pack/x-pack-${ELASTIC_VERSION}.zip?c=${CACHE_BUST}; fi - CMD ["/bin/bash", "/usr/local/bin/kibana-docker"] diff --git a/testing/environments/docker/kibana/download.sh b/testing/environments/docker/kibana/download.sh new file mode 100755 index 000000000000..5e8a423df2cb --- /dev/null +++ b/testing/environments/docker/kibana/download.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -eo pipefail + +if [ -z ${DOWNLOAD_URL+x} ]; then echo "DOWNLOAD_URL is unset"; exit 1; fi +if [ -z ${ELASTIC_VERSION+x} ]; then echo "ELASTIC_VERSION is unset"; exit 1; fi +if [ -z ${IMAGE_FLAVOR+x} ]; then echo "IMAGE_FLAVOR is unset"; exit 1; fi + +url=${DOWNLOAD_URL}/kibana/kibana-oss/kibana-oss-${ELASTIC_VERSION}-linux-x86_64.tar.gz +if [ "${IMAGE_FLAVOR}" = "x-pack" ]; then + url=${DOWNLOAD_URL}/kibana/kibana-${ELASTIC_VERSION}-linux-x86_64.tar.gz +fi + +# Download. +curl -s -L -o kibana-${ELASTIC_VERSION}-linux-x86_64.tar.gz $url + +# Validate SHA512. +expected_sha=$(curl -s -L $url.sha512 | awk '{print $1}') +observed_sha=$(sha512sum kibana-${ELASTIC_VERSION}-linux-x86_64.tar.gz | awk '{print $1}') +test "${expected_sha}" == "${observed_sha}" diff --git a/testing/environments/docker/logstash/Dockerfile b/testing/environments/docker/logstash/Dockerfile index eaa88ed413b2..8a55dda583f3 100644 --- a/testing/environments/docker/logstash/Dockerfile +++ b/testing/environments/docker/logstash/Dockerfile @@ -1,31 +1,52 @@ -FROM java:8-jre - -RUN apt-get update && \ - apt-get install -y netcat +FROM centos:7 +LABEL maintainer "Elastic Docker Team " +# Beats variables. ARG DOWNLOAD_URL ARG ELASTIC_VERSION ARG CACHE_BUST=1 +ARG IMAGE_FLAVOR=x-pack + +# Install Java and the "which" command, which is needed by Logstash's shell +# scripts. +RUN yum update -y && yum install -y java-1.8.0-openjdk-devel which && \ + yum clean all + +# Provide a non-root user to run the process. +RUN groupadd --gid 1000 logstash && \ + adduser --uid 1000 --gid 1000 \ + --home-dir /usr/share/logstash --no-create-home \ + logstash + +# Add Logstash itself. +COPY download.sh /download.sh +RUN /download.sh $DOWNLOAD_URL $ELASTIC_VERSION $CACHE_BUST && rm /download.sh +RUN tar zxf logstash-${ELASTIC_VERSION}.tar.gz -C /usr/share && \ + mv /usr/share/logstash-${ELASTIC_VERSION} /usr/share/logstash && \ + chown --recursive logstash:logstash /usr/share/logstash/ && \ + ln -s /usr/share/logstash /opt/logstash + +WORKDIR /usr/share/logstash + +ENV ELASTIC_CONTAINER true +ENV PATH=/usr/share/logstash/bin:$PATH + +# Provide a minimal configuration, so that simple invocations will provide +# a good experience. +ADD config/pipelines.yml config/pipelines.yml +ADD config/logstash-${IMAGE_FLAVOR}.yml config/logstash.yml +ADD config/log4j2.properties config/ +ADD pipeline/default.conf pipeline/logstash.conf +ADD pki /etc/pki +RUN chown --recursive logstash:logstash config/ pipeline/ + +# Ensure Logstash gets a UTF-8 locale by default. +ENV LANG='en_US.UTF-8' LC_ALL='en_US.UTF-8' + +HEALTHCHECK --interval=1s --retries=600 CMD curl -f http://localhost:9600/_node/stats -ENV URL ${DOWNLOAD_URL}/logstash/logstash-${ELASTIC_VERSION}.tar.gz -ENV PATH $PATH:/opt/logstash-${ELASTIC_VERSION}/bin - -# As all snapshot builds have the same url, the image is cached. The date at then can be used to invalidate the image -RUN set -x && \ - cd /opt && \ - wget -qO logstash.tar.gz $URL?${CACHE_BUST} && \ - tar xzf logstash.tar.gz - +EXPOSE 5044 5055 9600 -COPY logstash.conf.tmpl /logstash.conf.tmpl COPY docker-entrypoint.sh /entrypoint.sh - -COPY pki /etc/pki - -HEALTHCHECK --interval=1s --retries=600 CMD nc -z localhost 5044 - ENTRYPOINT ["/entrypoint.sh"] - -EXPOSE 5044 5055 9600 - -CMD logstash -f /logstash.conf --log.level=debug --config.debug --http.host=0.0.0.0 +CMD logstash diff --git a/testing/environments/docker/logstash/config/log4j2.properties b/testing/environments/docker/logstash/config/log4j2.properties new file mode 100644 index 000000000000..36bc45143eda --- /dev/null +++ b/testing/environments/docker/logstash/config/log4j2.properties @@ -0,0 +1,16 @@ +status = error +name = LogstashPropertiesConfig + +appender.console.type = Console +appender.console.name = plain_console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n + +appender.json_console.type = Console +appender.json_console.name = json_console +appender.json_console.layout.type = JSONLayout +appender.json_console.layout.compact = true +appender.json_console.layout.eventEol = true + +rootLogger.level = ${sys:ls.log.level} +rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console diff --git a/testing/environments/docker/logstash/config/logstash-oss.yml b/testing/environments/docker/logstash/config/logstash-oss.yml new file mode 100644 index 000000000000..342d19af83f9 --- /dev/null +++ b/testing/environments/docker/logstash/config/logstash-oss.yml @@ -0,0 +1 @@ +http.host: "0.0.0.0" diff --git a/testing/environments/docker/logstash/config/logstash-x-pack.yml b/testing/environments/docker/logstash/config/logstash-x-pack.yml new file mode 100644 index 000000000000..7c25c459f312 --- /dev/null +++ b/testing/environments/docker/logstash/config/logstash-x-pack.yml @@ -0,0 +1,4 @@ +http.host: "0.0.0.0" +xpack.monitoring.elasticsearch.url: http://elasticsearch:9200 +xpack.monitoring.elasticsearch.username: logstash_system +xpack.monitoring.elasticsearch.password: changeme diff --git a/testing/environments/docker/logstash/config/pipelines.yml b/testing/environments/docker/logstash/config/pipelines.yml new file mode 100644 index 000000000000..aed22ce7301a --- /dev/null +++ b/testing/environments/docker/logstash/config/pipelines.yml @@ -0,0 +1,6 @@ +# This file is where you define your pipelines. You can define multiple. +# For more information on multiple pipelines, see the documentation: +# https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html + +- pipeline.id: main + path.config: "/usr/share/logstash/pipeline" diff --git a/testing/environments/docker/logstash/docker-entrypoint.sh b/testing/environments/docker/logstash/docker-entrypoint.sh index 020c3c9a88be..71fa83de3148 100755 --- a/testing/environments/docker/logstash/docker-entrypoint.sh +++ b/testing/environments/docker/logstash/docker-entrypoint.sh @@ -15,7 +15,6 @@ readParams() { # Use default ports if not specified. : ${ES_PORT:=9200} - : ${REDIS_PORT:=6379} } es_url() { @@ -58,13 +57,8 @@ waitForElasticsearch() { exit 1 } -updateConfigFile() { - sed -e "s/hosts.*/hosts => [\"$ES_HOST:$ES_PORT\"]/" /logstash.conf.tmpl > /logstash.conf -} - # Main readParams -updateConfigFile waitForElasticsearch exec "$@" diff --git a/testing/environments/docker/logstash/download.sh b/testing/environments/docker/logstash/download.sh new file mode 100755 index 000000000000..868183bd254e --- /dev/null +++ b/testing/environments/docker/logstash/download.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -eo pipefail + +if [ -z ${DOWNLOAD_URL+x} ]; then echo "DOWNLOAD_URL is unset"; exit 1; fi +if [ -z ${ELASTIC_VERSION+x} ]; then echo "ELASTIC_VERSION is unset"; exit 1; fi +if [ -z ${IMAGE_FLAVOR+x} ]; then echo "IMAGE_FLAVOR is unset"; exit 1; fi + +url=${DOWNLOAD_URL}/logstash/logstash-oss/logstash-oss-${ELASTIC_VERSION}.tar.gz +if [ "${IMAGE_FLAVOR}" = "x-pack" ]; then + url=${DOWNLOAD_URL}/logstash/logstash-${ELASTIC_VERSION}.tar.gz +fi + +# Download. +curl -s -L -o logstash-${ELASTIC_VERSION}.tar.gz $url + +# Validate SHA512. +expected_sha=$(curl -s -L $url.sha512 | awk '{print $1}') +observed_sha=$(sha512sum logstash-${ELASTIC_VERSION}.tar.gz | awk '{print $1}') +test "${expected_sha}" == "${observed_sha}" diff --git a/testing/environments/docker/logstash/logstash.conf.tmpl b/testing/environments/docker/logstash/logstash.conf.tmpl deleted file mode 100644 index 26ae7751c892..000000000000 --- a/testing/environments/docker/logstash/logstash.conf.tmpl +++ /dev/null @@ -1,27 +0,0 @@ -input { - beats { - port => 5044 - ssl => false - } - - beats { - port => 5055 - ssl => true - ssl_certificate => "/etc/pki/tls/certs/logstash.crt" - ssl_key => "/etc/pki/tls/private/logstash.key" - } -} - - -output { - elasticsearch { - hosts => [] - #user => "beats" - #password => "testing" - index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" - document_type => "%{[@metadata][type]}" - } - - # Used for easier debugging - #stdout { codec => rubydebug { metadata => true } } -} diff --git a/testing/environments/snapshot-noxpack.yml b/testing/environments/snapshot-noxpack.yml index 3160a5388e24..cd62dcd32160 100644 --- a/testing/environments/snapshot-noxpack.yml +++ b/testing/environments/snapshot-noxpack.yml @@ -10,7 +10,7 @@ services: context: ./docker/elasticsearch dockerfile: Dockerfile args: - XPACK: 0 + IMAGE_FLAVOR: oss environment: - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - "network.host=" @@ -24,6 +24,8 @@ services: build: context: ./docker/logstash dockerfile: Dockerfile + args: + IMAGE_FLAVOR: oss environment: - ES_HOST=elasticsearch @@ -35,4 +37,4 @@ services: context: ./docker/kibana dockerfile: Dockerfile args: - XPACK: 0 + IMAGE_FLAVOR: oss diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 1874182d65d0..f01bb080da21 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -9,8 +9,6 @@ services: build: context: ./docker/elasticsearch dockerfile: Dockerfile - args: - XPACK: 1 environment: - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - "network.host=" @@ -35,5 +33,3 @@ services: build: context: ./docker/kibana dockerfile: Dockerfile - args: - XPACK: 1 diff --git a/x-pack/README.md b/x-pack/README.md new file mode 100644 index 000000000000..20fbdc310088 --- /dev/null +++ b/x-pack/README.md @@ -0,0 +1,5 @@ +# Elastic License Functionality + +This tree contains files subject to the Elastic License. The files subject to +the Elastic License are grouped in this directory to help clearly separate these +from files licensed under the Apache License 2.0.