Packaging for elastic agent

README.md

@@ -61,6 +61,7 @@ by running `mage -l` from the root of the repository. The most commonly used com
 * `mage build` to build the data shipper binary.
 * `mage check` to check license files and dependencies.
 * `mage lint` to lint the source code using [golangci-lint](https://golangci-lint.run/).
+* `mage package` to produce release artifacts.
 * `go test ./...` to run all tests.
 
 Run mage commands with the `-v` flag for more detailed output, for example `mage -v check`.

elastic-agent-shipper.spec.yml

@@ -0,0 +1,18 @@
+version: 2
+outputs:
+  - name: elasticsearch
+    description: Elasticsearch
+    platforms: &platforms
+      - linux/amd64
+      - linux/arm64
+      - darwin/amd64
+      - darwin/arm64
+      - windows/amd64
+      - container/amd64
+      - container/arm64
+    command: {}
+  - name: logstash
+    description: Logstash
+    platforms: *platforms
+    command: {}
+

magefile.go

@@ -8,8 +8,14 @@
 package main
 
 import (
+	"archive/tar"
+	"archive/zip"
+	"compress/gzip"
 	"context"
+	"crypto/sha512"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -29,7 +35,10 @@ import (
 )
 
 const (
-	GoreleaserRepo = "github.com/goreleaser/[email protected]"
+	GoreleaserRepo   = "github.com/goreleaser/[email protected]"
+	platformsEnvName = "PLATFORMS"
+	specFileName     = devtools.ProjectName + ".spec.yml"
+	configFileName   = devtools.ProjectName + ".yml"
 )
 
 // Aliases are shortcuts to long target names.
@@ -52,7 +61,7 @@ func (Build) All() {
 }
 
 // Clean removes the build directory.
-func (Build) Clean(test string) {
+func (Build) Clean() {
 	os.RemoveAll("build") // nolint:errcheck //not required
 }
 
@@ -95,7 +104,7 @@ func (Build) Binary() error {
 		}
 	}
 
-	platform := os.Getenv("PLATFORMS")
+	platform := os.Getenv(platformsEnvName)
 	if platform != "" && devtools.PlatformFiles[platform] == nil {
 		return errors.Errorf("Platform %s not recognized, only supported options: all, darwin, linux, windows, darwin/amd64, darwin/arm64, linux/386, linux/amd64, linux/arm64, windows/386, windows/amd64", platform)
 	}
@@ -177,11 +186,7 @@ func CheckBinaries(platform string, version string) error {
 		return errors.New("No selected platform files found")
 	}
 	for _, platform := range selectedPlatformFiles {
-		var execName = devtools.ProjectName
-		if strings.Contains(platform, "windows") {
-			execName += ".exe"
-		}
-		binary := filepath.Join(path, fmt.Sprintf("%s-%s-%s", devtools.ProjectName, version, platform), execName)
+		binary := binaryName(path, version, platform)
 		if _, err := os.Stat(binary); err != nil {
 			return errors.Wrap(err, "Build: binary check failed")
 		}
@@ -215,3 +220,227 @@ func Notice() error {
 	// See https://github.com/golang/go/issues/43994#issuecomment-770053099
 	return gotool.Mod.Tidy()
 }
+
+// PACKAGE
+
+// Package runs all the checks including licence, notice, gomod, git changes, followed by binary build.
+func Package() {
+	// these are not allowed in parallel
+	mg.SerialDeps(
+		Build.Binary,
+	)
+
+	version, err := fullVersion()
+	if err != nil {
+		panic(err)
+	}
+
+	platforms := devtools.PlatformFiles[os.Getenv(platformsEnvName)]
+	// current by default
+	if platforms == nil {
+		platforms = devtools.PlatformFiles[runtime.GOOS+"/"+runtime.GOARCH]
+	}
+
+	archivePath := filepath.Join("build", "distributions")
+	if err := os.MkdirAll(archivePath, 0755); err != nil {
+		panic(err)
+	}
+
+	for _, platform := range platforms {
+		isWindows := strings.Contains(platform, "windows")
+
+		// grab binary
+		binaryPath := binaryFilePath(version, platform)
+
+		// prepare package
+		var err error
+		var archiveFileName string
+		targetDir := fmt.Sprintf("%s-%s", devtools.ProjectName, platform)
+		if isWindows {
+			archiveFileName = fmt.Sprintf("%s-%s.zip", devtools.ProjectName, platform)
+			err = prepareZipArchive(
+				archivePath,
+				targetDir,
+				archiveFileName,
+				map[string]string{
+					specFileName:       specFileName,
+					configFileName:     configFileName,
+					execName(platform): binaryPath,
+				},
+			)
+		} else {
+			archiveFileName = fmt.Sprintf("%s-%s.tar.gz", devtools.ProjectName, platform)
+			err = prepareTarArchive(
+				archivePath,
+				targetDir,
+				archiveFileName,
+				map[string]string{
+					specFileName:       specFileName,
+					configFileName:     configFileName,
+					execName(platform): binaryPath,
+				},
+			)
+		}
+
+		if err != nil {
+			panic(err)
+		}
+
+		// generate sha sum
+		archiveFullPath := filepath.Join(archivePath, archiveFileName)
+		archive, err := os.Open(archiveFullPath)
+		if err != nil {
+			panic(errors.Wrapf(err, "failed opening archive %q", archiveFileName))
+		}
+
+		h := sha512.New()
+		if _, err := io.Copy(h, archive); err != nil {
+			panic(errors.Wrapf(err, "failed computing hash for archive %q", archiveFileName))
+		}
+
+		shaFile := archiveFullPath + ".sha512"
+		content := fmt.Sprintf("%x  %s\n", h.Sum(nil), archiveFileName)
+		if err := ioutil.WriteFile(shaFile, []byte(content), 0644); err != nil {
+			panic(errors.Wrapf(err, "failed writing hash for archive %q", archiveFileName))
+		}
+
+		archive.Close()
+	}
+}
+
+func prepareZipArchive(path, targetDir, archiveFileName string, files map[string]string) error {
+	archiveFullName := filepath.Join(path, archiveFileName)
+	archiveFile, err := os.Create(archiveFullName)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create %q", archiveFullName)
+	}
+	defer archiveFile.Close()
+
+	zipWriter := zip.NewWriter(archiveFile)
+	defer zipWriter.Close()
+
+	addFile := func(fileName, filePath string, zipWriter *zip.Writer) error {
+		file, err := os.Open(filePath)
+		if err != nil {
+			return errors.Wrap(err, "failed opening a file")
+		}
+		defer file.Close()
+
+		stat, err := file.Stat()
+		if err != nil {
+			return errors.Wrap(err, "failed retrieving stat info for a file")
+		}
+
+		header, err := zip.FileInfoHeader(stat)
+		header.Method = zip.Deflate
+		header.Name = filepath.Join(targetDir, fileName)
+
+		hWriter, err := zipWriter.CreateHeader(header)
+		if err != nil {
+			return errors.Wrap(err, "failed writing zip header")
+		}
+
+		if _, err := io.Copy(hWriter, file); err != nil {
+			return errors.Wrap(err, "failed adding a file into an archive")
+		}
+		return nil
+	}
+
+	for fileName, filePath := range files {
+		if err := addFile(fileName, filePath, zipWriter); err != nil {
+			return errors.Wrapf(err, "failed adding file %q into zip archive", filePath)
+		}
+	}
+
+	return nil
+}
+
+func prepareTarArchive(path, targetDir, archiveFileName string, files map[string]string) error {
+	archiveFullName := filepath.Join(path, archiveFileName)
+	archiveFile, err := os.Create(archiveFullName)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create %q", archiveFullName)
+	}
+	defer archiveFile.Close()
+
+	gzipWriter := gzip.NewWriter(archiveFile)
+	defer gzipWriter.Close()
+
+	tarWriter := tar.NewWriter(gzipWriter)
+	defer tarWriter.Close()
+
+	addFile := func(fileName, filePath string, tarWriter *tar.Writer) error {
+		file, err := os.Open(filePath)
+		if err != nil {
+			return errors.Wrap(err, "failed opening a file")
+		}
+		defer file.Close()
+
+		stat, err := file.Stat()
+		if err != nil {
+			return errors.Wrap(err, "failed retrieving stat info for a file")
+		}
+
+		header := &tar.Header{
+			Name:    filepath.Join(targetDir, fileName),
+			Size:    stat.Size(),
+			Mode:    int64(stat.Mode()),
+			ModTime: stat.ModTime(),
+		}
+
+		if err := tarWriter.WriteHeader(header); err != nil {
+			return errors.Wrap(err, "failed writing tar header")
+		}
+
+		if _, err := io.Copy(tarWriter, file); err != nil {
+			return errors.Wrap(err, "failed adding a file into an archive")
+		}
+
+		return nil
+	}
+
+	for fileName, filePath := range files {
+		if err := addFile(fileName, filePath, tarWriter); err != nil {
+			return errors.Wrapf(err, "failed adding file %q into tar archive", filePath)
+		}
+	}
+
+	return nil
+}
+
+func fullVersion() (string, error) {
+	version := tools.DefaultBeatVersion
+	if versionQualifier, versionQualified := os.LookupEnv("VERSION_QUALIFIER"); versionQualified {
+		version += fmt.Sprintf("-%s", versionQualifier)
+	}
+
+	if snapshotEnv := os.Getenv("SNAPSHOT"); snapshotEnv != "" {
+		isSnapshot, err := strconv.ParseBool(snapshotEnv)
+		if err != nil {
+			return "", err
+		}
+		if isSnapshot {
+			version += fmt.Sprintf("-%s", "SNAPSHOT")
+		}
+	}
+
+	return version, nil
+}
+
+func binaryFilePath(version, platform string) string {
+	path := filepath.Join("build", "binaries")
+	return binaryName(path, version, platform)
+}
+
+func binaryName(path, version, platform string) string {
+	return filepath.Join(path, fmt.Sprintf("%s-%s-%s", devtools.ProjectName, version, platform), execName(platform))
+}
+
+func execName(platform string) string {
+	var execName = devtools.ProjectName
+	if strings.Contains(platform, "windows") {
+		execName += ".exe"
+	}
+
+	return execName
+}