[CI] retry if environmental issues

Jenkinsfile

@@ -2,6 +2,13 @@
 
 @Library('apm@current') _
 
+import groovy.transform.Field
+
+/**
+ This is required to store any environmental issues to retry if so
+*/
+@Field def environmentalIssues = [:]
+
 pipeline {
   agent { label 'ubuntu-18 && immutable' }
   environment {
@@ -160,6 +167,9 @@ VERSION=${env.VERSION}-SNAPSHOT""")
                           analyzeFlakey: !isTag(), flakyReportIdx: "reporter-beats-beats-${getIdSuffix()}")
       }
     }
+    always {
+      notifyEnvironmentalIssues()
+    }
   }
 }
 
@@ -465,12 +475,31 @@ def e2e(Map args = [:]) {
   }
 }
 
+/**
+* This method is a wrapper to run the runCommand method and retry if there are 
+* environmental issues. Therefore it passes the arguments to the runCommand.
+* For further details regarding the arguments please refers to the runCommand method.
+*/
+def target(Map args = [:]) {
+  try {
+    runCommand(args)
+  } catch (err) {
+    if(environmentalIssues?.get(args.id, false)) {
+      // Retry if environmental issues
+      sleep randomNumber(min: 10, max: 30)
+      runCommand(args)
+    } else {
+      error("Error '${err.toString()}'")
+    }
+  }
+}
+
 /**
 * This method runs the given command supporting two kind of scenarios:
 *  - make -C <folder> then the dir(location) is not required, aka by disaling isMage: false
 *  - mage then the dir(location) is required, aka by enabling isMage: true.
 */
-def target(Map args = [:]) {
+def runCommand(Map args = [:]) {
   def command = args.command
   def context = args.context
   def directory = args.get('directory', '')
@@ -523,7 +552,8 @@ def withBeatsEnv(Map args = [:], Closure body) {
   def archive = args.get('archive', true)
   def withModule = args.get('withModule', false)
   def directory = args.get('directory', '')
-
+  def environmentalIssue = true
+  def uploadGeneratedFiles = false // Skip the uploading of generated files by default.
   def goRoot, path, magefile, pythonEnv, testResults, artifacts, gox_flags, userProfile
 
   if(isUnix()) {
@@ -555,60 +585,97 @@ def withBeatsEnv(Map args = [:], Closure body) {
     gox_flags = '-arch 386'
   }
 
-  deleteDir()
-  unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
-  // NOTE: This is required to run after the unstash
-  def module = withModule ? getCommonModuleInTheChangeSet(directory) : ''
-  withEnv([
-    "DOCKER_PULL=0",
-    "GOPATH=${env.WORKSPACE}",
-    "GOROOT=${goRoot}",
-    "GOX_FLAGS=${gox_flags}",
-    "HOME=${env.WORKSPACE}",
-    "MAGEFILE_CACHE=${magefile}",
-    "MODULE=${module}",
-    "PATH=${path}",
-    "PYTHON_ENV=${pythonEnv}",
-    "RACE_DETECTOR=true",
-    "TEST_COVERAGE=true",
-    "TEST_TAGS=${env.TEST_TAGS},oracle",
-    "OLD_USERPROFILE=${env.USERPROFILE}",
-    "USERPROFILE=${userProfile}"
-  ]) {
-    if(isDockerInstalled()) {
-      dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
-    }
-    dir("${env.BASE_DIR}") {
-      installTools(args)
-      // Skip to upload the generated files by default.
-      def upload = false
-      try {
+  try {
+    deleteDir()
+    unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+    // NOTE: This is required to run after the unstash
+    def module = withModule ? getCommonModuleInTheChangeSet(directory) : ''
+    withEnv([
+      "DOCKER_PULL=0",
+      "GOPATH=${env.WORKSPACE}",
+      "GOROOT=${goRoot}",
+      "GOX_FLAGS=${gox_flags}",
+      "HOME=${env.WORKSPACE}",
+      "MAGEFILE_CACHE=${magefile}",
+      "MODULE=${module}",
+      "PATH=${path}",
+      "PYTHON_ENV=${pythonEnv}",
+      "RACE_DETECTOR=true",
+      "TEST_COVERAGE=true",
+      "TEST_TAGS=${env.TEST_TAGS},oracle",
+      "OLD_USERPROFILE=${env.USERPROFILE}",
+      "USERPROFILE=${userProfile}"
+    ]) {
+      if(isDockerInstalled()) {
+        dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+      }
+      dir("${env.BASE_DIR}") {
+        installTools(args)
+        // Pre-requisites to configure the environment were ok.
+        environmentalIssue = false
         // Add more stability when dependencies are not accessible temporarily
         // See https://github.com/elastic/beats/issues/21609
         // retry/try/catch approach reports errors, let's avoid it to keep the
         // notifications cleaner.
         if (cmd(label: 'Download modules to local cache', script: 'go mod download', returnStatus: true) > 0) {
           cmd(label: 'Download modules to local cache - retry', script: 'go mod download', returnStatus: true)
         }
-        body()
-      } catch(err) {
         // Upload the generated files ONLY if the step failed. This will avoid any overhead with Google Storage
-        upload = true
-        error("Error '${err.toString()}'")
-      } finally {
-        if (archive) {
-          archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload)
-        }
-        // Tear down the setup for the permamnent workers.
-        catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
-          fixPermissions("${WORKSPACE}")
-          deleteDir()
-        }
+        uploadGeneratedFiles = true
+        body()
+        // Skip the generated files by default.
+        uploadGeneratedFiles = false
+      }
+    }
+  } finally {
+    // If there are environmental issues then let's avoid the archiving of none files.
+    if (archive && !environmentalIssue) {
+      dir("${env.BASE_DIR}") {
+        archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: uploadGeneratedFiles)
       }
     }
+    // Tear down the setup for the permanent workers.
+    catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
+      fixPermissions("${WORKSPACE}")
+      deleteDir()
+    }
+    analyseEnvironmentalIssues(id: args.id, environmentalIssue: environmentalIssue)
+  }
+}
+
+/**
+* This method analyse if the existing stage failed with some environmental issues.
+* So far the analysis is just purely based on a boolean that detects if the installation of
+* the required tools for building and testing in the agent were successfully installed or
+* some unexpected pre-building/testing errors, such as deleting the workspace.
+* We can use this method in the future to analyse the build logs for searching specific patterns
+* such as, not able to access the docker registry, or failed when pulling some docker images.
+*/
+def analyseEnvironmentalIssues(args) {
+  environmentalIssues[args.id] = args.environmentalIssue
+}
+
+/**
+* This method runs only as a post build action to notify when there are environmental issues.
+* Archive the file with the environmental issues details in case there are any
+* reported environmental issues.
+* TODO: Create JSON file to upload those details to elasticsearch
+* This won't be needed as soon as we use the APM transactions
+*/
+def notifyEnvironmentalIssues(Map args = [:]) {
+  stage('Notify environmental issues'){
+    def content = ''
+    environmentalIssues?.findAll { k, v -> return v }.each { k, v ->
+      content += "${k} failed with some environmental issues (${v})\n"
+    }
+    if (content?.trim()) {
+      writeFile file: 'environmental.issues.txt', text: content
+      archiveArtifacts artifacts: 'environmental.issues.txt'
+    }
   }
 }
 
+
 /**
 * This method fixes the filesystem permissions after the build has happenend. The reason is to
 * ensure any non-ephemeral workers don't have any leftovers that could cause some environmental