Merge remote-tracking branch 'origin/master' into store-http-headers

.ci/build-docker-images.groovy

@@ -0,0 +1,90 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'linux && immutable' }
+  environment {
+    REPO = 'beats'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    DOCKER_REGISTRY = 'docker.elastic.co'
+    DOCKER_REGISTRY_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    GOPATH = "${env.WORKSPACE}"
+    HOME = "${env.WORKSPACE}"
+    JOB_GCS_BUCKET = credentials('gcs-bucket')
+    NOTIFY_TO = credentials('notify-to')
+    PATH = "${env.GOPATH}/bin:${env.PATH}"
+    PIPELINE_LOG_LEVEL='INFO'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
+    quietPeriod(10)
+  }
+  triggers {
+    cron 'H H(0-5) * * 1-5'
+  }
+  parameters {
+    booleanParam(name: "RELEASE_TEST_IMAGES", defaultValue: "true", description: "If it's needed to build & push Beats' test images")
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        dir("${BASE_DIR}"){
+          git("https://github.com/elastic/${REPO}.git")
+        }
+        script {
+          dir("${BASE_DIR}"){
+            env.GO_VERSION = readFile(".go-version").trim()
+          }
+        }
+      }
+    }
+    stage('Install dependencies') {
+      when {
+        expression { return params.RELEASE_TEST_IMAGES }
+      }
+      steps {
+        sh(label: 'Install virtualenv', script: 'pip install --user virtualenv')
+      }
+    }
+    stage('Release Beats Test Docker images'){
+      options {
+        warnError('Release Beats Docker images failed')
+      }
+      when {
+        expression { return params.RELEASE_TEST_IMAGES }
+      }
+      steps {
+        dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
+        retry(3){
+          sh(label: 'Build ', script: ".ci/scripts/build-beats-integrations-test-images.sh '${GO_VERSION}' '${HOME}/${BASE_DIR}/metricbeat'")
+        }
+      }
+    }
+    stage('Release X-Pack Beats Test Docker images'){
+      options {
+        warnError('Release X-Pack Beats Docker images failed')
+      }
+      when {
+        expression { return params.RELEASE_TEST_IMAGES }
+      }
+      steps {
+        dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
+        retry(3){
+          sh(label: 'Build ', script: ".ci/scripts/build-beats-integrations-test-images.sh '${GO_VERSION}' '${HOME}/${BASE_DIR}/x-pack/metricbeat'")
+        }
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult()
+    }
+  }
+}

.ci/jobs/beats-windows-mbp.yml

@@ -0,0 +1,44 @@
+---
+- job:
+    name: Beats/beats-windows-mbp
+    display-name: Beats Pipeline for Windows 
+    description: Jenkins pipeline for the Beats project running on windows agents.
+    view: Beats
+    project-type: multibranch
+    script-path: .ci/windows.groovy
+    scm:
+      - github:
+          branch-discovery: no-pr
+          discover-pr-forks-strategy: merge-current
+          discover-pr-forks-trust: permission
+          discover-pr-origin: merge-current          
+          discover-tags: false
+          head-filter-regex: 'master'
+          notification-context: 'beats-ci'
+          repo: beats
+          repo-owner: elastic
+          credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+          ssh-checkout:
+            credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+          build-strategies:
+            - tags:
+                ignore-tags-older-than: -1
+                ignore-tags-newer-than: -1
+            - regular-branches: true
+            - change-request:
+                ignore-target-only-changes: false
+          clean:
+            after: true
+            before: true
+          prune: true
+          shallow-clone: true
+          depth: 4
+          do-not-fetch-tags: true
+          submodule:
+            disable: false
+            recursive: true
+            parent-credentials: true
+            timeout: 100
+          timeout: '15'
+          use-author: true
+          wipe-workspace: 'True'

.ci/jobs/build-it-docker-images.yml

@@ -0,0 +1,22 @@
+---
+- job:
+    name: Beats/build-it-docker-images
+    display-name: ITs Docker images
+    description: Job to pre-build docker images used in integration tests.
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+      script-path: .ci/build-docker-images.groovy
+      scm:
+        - git:
+            url: [email protected]:elastic/beats.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
+            branches:
+              - 'master'
+    triggers:
+      - timed: 'H H(0-5) * * 1-5'

.ci/jobs/defaults.yml

@@ -0,0 +1,20 @@
+
+---
+
+##### GLOBAL METADATA
+
+- meta:
+    cluster: beats-ci
+
+##### JOB DEFAULTS
+
+- job:
+    logrotate:
+      numToKeep: 20
+    node: linux
+    concurrent: true
+    publishers:
+      - email:
+          recipients: [email protected]
+    periodic-folder-trigger: 1w
+    prune-dead-branches: true

.ci/scripts/build-beats-integrations-test-images.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -exo pipefail
+
+#
+# Install the given go version using the gimme script.
+#
+# Parameters:
+#   - GO_VERSION - that's the version which will be installed and enabled.
+#   - METRICBEAT_DIR - that's the location of the metricbeat directory.
+#
+
+readonly GO_VERSION="${1?Please define the Go version to be used}"
+readonly METRICBEAT_DIR="${2?Please define the location of the Metricbeat directory}"
+
+function build_test_images() {
+    local metricbeatDir="${1}"
+
+    cd "${metricbeatDir}"
+    mage compose:buildSupportedVersions
+}
+
+function install_go() {
+    local goVersion="${1}"
+
+    # Install Go using the same travis approach
+    echo "Installing ${goVersion} with gimme."
+    eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=${goVersion} bash)"
+}
+
+function install_mage() {
+    local metricbeatDir="${1}"
+
+    cd "${metricbeatDir}"
+    make mage
+}
+
+function push_test_images() {
+    local metricbeatDir="${1}"
+
+    cd "${metricbeatDir}"
+    mage compose:pushSupportedVersions
+}
+
+function main() {
+    install_go "${GO_VERSION}"
+    install_mage "${METRICBEAT_DIR}"
+
+    build_test_images "${METRICBEAT_DIR}"
+    push_test_images "${METRICBEAT_DIR}"
+}
+
+main "$@"