feat: add a mage command to package system tests (elastic#27295)

* feat: add a mage command to package system tests

* chore: delegate prints to the caller

* chore: do not keep writing tar file in memory

* chore: update comment

Co-authored-by: Jaime Soriano Pastor <[email protected]>

* chore: handle error while traversing the filesystem

Co-authored-by: Jaime Soriano Pastor <[email protected]>

Jenkinsfile

@@ -668,7 +668,7 @@ def withBeatsEnv(Map args = [:], Closure body) {
           error("Error '${err.toString()}'")
         } finally {
           if (archive) {
-            archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload)
+            archiveTestOutput(directory: directory, testResults: testResults, artifacts: artifacts, id: args.id, upload: upload)
           }
           tearDown()
         }
@@ -766,6 +766,8 @@ def getCommonModuleInTheChangeSet(String directory) {
 * to bypass some issues when working with big repositories.
 */
 def archiveTestOutput(Map args = [:]) {
+  def directory = args.get('directory', '')
+
   catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
     if (isUnix()) {
       fixPermissions("${WORKSPACE}")
@@ -789,13 +791,20 @@ def archiveTestOutput(Map args = [:]) {
     }
     if (args.upload) {
       catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
-        def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim()
-        log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball")
-        if (folder.trim()) {
-          // TODO: nodeOS() should support ARM
-          def os_suffix = isArm() ? 'linux' : nodeOS()
-          def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix
-          tarAndUploadArtifacts(file: "${name}.tgz", location: folder)
+        withMageEnv(version: "${env.GO_VERSION}"){
+          dir(directory){
+            cmd(label: "Archive system tests files", script: 'mage packageSystemTests')
+          }
+        }
+        def fileName = 'build/system-tests-*.tar.gz' // see dev-tools/mage/target/common/package.go#PackageSystemTests method
+        dir("${BASE_DIR}"){
+          cmd(label: "List files to upload", script: "ls -l ${BASE_DIR}/${fileName}")
+          googleStorageUploadExt(
+            bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
+            credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
+            pattern: "${BASE_DIR}/${fileName}",
+            sharedPublicly: true
+          )
         }
       }
     }

dev-tools/mage/common.go

@@ -353,6 +353,73 @@ func unzip(sourceFile, destinationDir string) error {
 	return nil
 }
 
+// Tar compress a directory using tar + gzip algorithms
+func Tar(src string, targetFile string) error {
+	fmt.Printf(">> creating TAR file from directory: %s, target: %s\n", src, targetFile)
+
+	f, err := os.Create(targetFile)
+	if err != nil {
+		return fmt.Errorf("error creating tar file: %w", err)
+	}
+	defer f.Close()
+
+	// tar > gzip > file
+	zr := gzip.NewWriter(f)
+	tw := tar.NewWriter(zr)
+
+	// walk through every file in the folder
+	filepath.Walk(src, func(file string, fi os.FileInfo, errFn error) error {
+		if errFn != nil {
+			return fmt.Errorf("error traversing the file system: %w", errFn)
+		}
+
+		// if a symlink, skip file
+		if fi.Mode().Type() == os.ModeSymlink {
+			fmt.Printf(">> skipping symlink: %s\n", file)
+			return nil
+		}
+
+		// generate tar header
+		header, err := tar.FileInfoHeader(fi, file)
+		if err != nil {
+			return fmt.Errorf("error getting file info header: %w", err)
+		}
+
+		// must provide real name
+		// (see https://golang.org/src/archive/tar/common.go?#L626)
+		header.Name = filepath.ToSlash(file)
+
+		// write header
+		if err := tw.WriteHeader(header); err != nil {
+			return fmt.Errorf("error writing header: %w", err)
+		}
+
+		// if not a dir, write file content
+		if !fi.IsDir() {
+			data, err := os.Open(file)
+			if err != nil {
+				return fmt.Errorf("error opening file: %w", err)
+			}
+			defer data.Close()
+			if _, err := io.Copy(tw, data); err != nil {
+				return fmt.Errorf("error compressing file: %w", err)
+			}
+		}
+		return nil
+	})
+
+	// produce tar
+	if err := tw.Close(); err != nil {
+		return fmt.Errorf("error closing tar file: %w", err)
+	}
+	// produce gzip
+	if err := zr.Close(); err != nil {
+		return fmt.Errorf("error closing gzip file: %w", err)
+	}
+
+	return nil
+}
+
 func untar(sourceFile, destinationDir string) error {
 	file, err := os.Open(sourceFile)
 	if err != nil {

dev-tools/mage/target/common/package.go

@@ -0,0 +1,78 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package common
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	devtools "github.com/elastic/beats/v7/dev-tools/mage"
+)
+
+// PackageSystemTests packages the python system tests results
+func PackageSystemTests() error {
+	excludeds := []string{".ci", ".git", ".github", "vendor", "dev-tools"}
+
+	// include run as it's the directory we want to compress
+	systemTestsDir := filepath.Join("build", "system-tests", "run")
+	files, err := devtools.FindFilesRecursive(func(path string, _ os.FileInfo) bool {
+		base := filepath.Base(path)
+		for _, excluded := range excludeds {
+			if strings.HasPrefix(base, excluded) {
+				return false
+			}
+		}
+
+		return strings.HasPrefix(path, systemTestsDir)
+	})
+	if err != nil {
+		return err
+	}
+
+	if len(files) == 0 {
+		fmt.Printf(">> there are no system test files under %s", systemTestsDir)
+		return nil
+	}
+
+	// create a plain directory layout for all beats
+	beat := devtools.MustExpand("{{ repo.SubDir }}")
+	beat = strings.ReplaceAll(beat, string(os.PathSeparator), "-")
+
+	targetFile := devtools.MustExpand("{{ elastic_beats_dir }}/build/system-tests-" + beat + ".tar.gz")
+	parent := filepath.Dir(targetFile)
+	if !fileExists(parent) {
+		fmt.Printf(">> creating parent dir: %s", parent)
+		os.Mkdir(parent, 0750)
+	}
+
+	err = devtools.Tar(systemTestsDir, targetFile)
+	if err != nil {
+		fmt.Printf(">> %s", err)
+		return err
+	}
+
+	return nil
+}
+
+// fileExists returns true if the specified file exists.
+func fileExists(file string) bool {
+	_, err := os.Stat(file)
+	return !os.IsNotExist(err)
+}