[Agent] Fix installers

dev-tools/mage/crossbuild.go

@@ -268,6 +268,7 @@ func (b GolangCrossBuilder) Build() error {
 		"--rm",
 		"--env", "MAGEFILE_VERBOSE="+verbose,
 		"--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""),
+		"--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot),
 		"-v", repoInfo.RootDir+":"+mountPoint,
 		"-w", workDir,
 		image,

dev-tools/packaging/packages.yml

@@ -54,10 +54,10 @@ shared:
       /etc/init.d/{{.BeatServiceName}}:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/init.sh.tmpl'
         mode: 0755
-      /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
 
@@ -97,10 +97,10 @@ shared:
         source: 'agent.yml'
         mode: 0600
         config: true
-      /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
 
@@ -131,10 +131,10 @@ shared:
     <<: *common
     files:
       <<: *agent_binary_files
-      'data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+      'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
         source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      'data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+      'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
         source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
 
@@ -149,10 +149,10 @@ shared:
       uninstall-service-{{.BeatName}}.ps1:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/uninstall-service.ps1.tmpl'
         mode: 0755
-      'data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
+      'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
         source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
         mode: 0644
-      'data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
+      'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
         source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
         mode: 0644
 

go.sum

@@ -107,8 +107,8 @@ github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 h1:7rj9qZ63knnVo2Z
 github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20/go.mod h1:cI59GRkC2FRaFYtgbYEqMlgnnfvAwXzjojyZKXwklNg=
 github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 h1:WFwa9pqou0Nb4DdfBOyaBTH0GqLE74Qwdf61E7ITHwQ=
 github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43/go.mod h1:tJPYQG4mnMeUtQvQKNkbsFrnmZOg59Qnf8CcctFv5v4=
-github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
 github.com/antlr/antlr4 v0.0.0-20200225173536-225249fdaef5 h1:nkZ9axP+MvUFCu8JRN/MCY+DmTfs6lY7hE0QnJbxSdI=
 github.com/antlr/antlr4 v0.0.0-20200225173536-225249fdaef5/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
@@ -730,8 +730,8 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191021144547-ec77196f6094/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=

x-pack/agent/CHANGELOG.asciidoc

@@ -13,6 +13,7 @@
 ==== Bugfixes
 
 - Fixed tests on windows {pull}16922[16922]
+- Fixed installers for SNAPSHOTs and windows {pull}17077[17077]
 
 ==== New features
 

x-pack/agent/_meta/agent.yml

@@ -4,7 +4,7 @@
 outputs:
   default:
     type: elasticsearch
-    hosts: [127.0.0.1:9200, 127.0.0.1:9300]
+    hosts: [127.0.0.1:9200]
     username: elastic
     password: changeme
 

x-pack/agent/_meta/common.p2.yml

@@ -4,7 +4,7 @@
 outputs:
   default:
     type: elasticsearch
-    hosts: [127.0.0.1:9200, 127.0.0.1:9300]
+    hosts: [127.0.0.1:9200]
     username: elastic
     password: changeme
 

x-pack/agent/_meta/common.reference.p2.yml

@@ -4,7 +4,7 @@
 outputs:
   default:
     type: elasticsearch
-    hosts: [127.0.0.1:9200, 127.0.0.1:9300]
+    hosts: [127.0.0.1:9200]
     username: elastic
     password: changeme
 

x-pack/agent/agent.reference.yml

@@ -9,7 +9,7 @@
 outputs:
   default:
     type: elasticsearch
-    hosts: [127.0.0.1:9200, 127.0.0.1:9300]
+    hosts: [127.0.0.1:9200]
     username: elastic
     password: changeme
 

x-pack/agent/agent.yml

@@ -9,7 +9,7 @@
 outputs:
   default:
     type: elasticsearch
-    hosts: [127.0.0.1:9200, 127.0.0.1:9300]
+    hosts: [127.0.0.1:9200]
     username: elastic
     password: changeme
 

x-pack/agent/magefile.go

@@ -36,6 +36,7 @@ const (
 	goLicenserRepo = "github.com/elastic/go-licenser"
 	buildDir       = "build"
 	metaDir        = "_meta"
+	snapshotEnv    = "SNAPSHOT"
 )
 
 // Aliases for commands required by master makefile
@@ -101,6 +102,7 @@ func (Build) GenerateConfig() error {
 func GolangCrossBuildOSS() error {
 	params := devtools.DefaultGolangCrossBuildArgs()
 	params.InputFiles = []string{"cmd/agent/agent.go"}
+	params.LDFlags = flagsSet()
 	return devtools.GolangCrossBuild(params)
 }
 
@@ -110,6 +112,7 @@ func GolangCrossBuild() error {
 	params := devtools.DefaultGolangCrossBuildArgs()
 	params.InputFiles = []string{"cmd/agent/agent.go"}
 	params.OutputDir = "build/golang-crossbuild"
+	params.LDFlags = flagsSet()
 	if err := devtools.GolangCrossBuild(params); err != nil {
 		return err
 	}
@@ -210,7 +213,7 @@ func (Check) License() error {
 	)
 }
 
-// Changes run git status --porcelain and return an error if we have changes or uncommited files.
+// Changes run git status --porcelain and return an error if we have changes or uncommitted files.
 func (Check) Changes() error {
 	out, err := sh.Output("git", "status", "--porcelain")
 	if err != nil {
@@ -354,14 +357,19 @@ func commitID() string {
 }
 
 func flags() string {
+	return strings.Join(flagsSet(), " ")
+}
+
+func flagsSet() []string {
 	ts := time.Now().Format(time.RFC3339)
 	commitID := commitID()
+	isSnapshot, _ := os.LookupEnv(snapshotEnv)
 
-	return fmt.Sprintf(
-		`-X "github.com/elastic/beats/v7/x-pack/agent/pkg/release.buildTime=%s" -X "github.com/elastic/beats/v7/x-pack/agent/pkg/release.commit=%s"`,
-		ts,
-		commitID,
-	)
+	return []string{
+		fmt.Sprintf(`-X "github.com/elastic/beats/v7/x-pack/agent/pkg/release.buildTime=%s"`, ts),
+		fmt.Sprintf(`-X "github.com/elastic/beats/v7/x-pack/agent/pkg/release.commit=%s"`, commitID),
+		fmt.Sprintf(` -X "github.com/elastic/beats/v7/x-pack/agent/pkg/release.snapshot=%s"`, isSnapshot),
+	}
 }
 
 // Update is an alias for executing fields, dashboards, config, includes.

x-pack/agent/pkg/agent/operation/operator_handlers.go

@@ -11,6 +11,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/agent/pkg/artifact"
 	"github.com/elastic/beats/v7/x-pack/agent/pkg/core/plugin/app"
+	"github.com/elastic/beats/v7/x-pack/agent/pkg/release"
 )
 
 type handleFunc func(step configrequest.Step) error
@@ -66,7 +67,12 @@ func getProgramFromStepWithTags(step configrequest.Step, artifactConfig *artifac
 		return nil, nil, err
 	}
 
-	p := app.NewDescriptor(step.Process, step.Version, artifactConfig, tags)
+	version := step.Version
+	if release.Snapshot() {
+		version = fmt.Sprintf("%s-SNAPSHOT", version)
+	}
+
+	p := app.NewDescriptor(step.Process, version, artifactConfig, tags)
 	return p, config, nil
 }
 

x-pack/agent/pkg/artifact/install/installer.go

@@ -39,5 +39,6 @@ func NewInstaller(config *artifact.Config) (Installer, error) {
 	if runtime.GOOS == "windows" {
 		return zip.NewInstaller(config)
 	}
+
 	return tar.NewInstaller(config)
 }

x-pack/agent/pkg/artifact/install/tar/tar_installer.go

@@ -54,6 +54,7 @@ func unpack(r io.Reader, dir string) error {
 	}
 
 	tr := tar.NewReader(zr)
+	var rootDir string
 
 	for {
 		f, err := tr.Next()
@@ -70,6 +71,11 @@ func unpack(r io.Reader, dir string) error {
 		rel := filepath.FromSlash(f.Name)
 		abs := filepath.Join(dir, rel)
 
+		// find the root dir
+		if currentDir := filepath.Dir(abs); rootDir == "" || len(filepath.Dir(rootDir)) > len(currentDir) {
+			rootDir = currentDir
+		}
+
 		fi := f.FileInfo()
 		mode := fi.Mode()
 		switch {

x-pack/agent/pkg/artifact/install/zip/zip_installer.go

@@ -5,11 +5,11 @@
 package zip
 
 import (
+	"archive/zip"
 	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"strings"
 
 	"github.com/elastic/beats/v7/x-pack/agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/agent/pkg/artifact"
@@ -35,37 +35,72 @@ func NewInstaller(config *artifact.Config) (*Installer, error) {
 // Install performs installation of program in a specific version.
 // It expects package to be already downloaded.
 func (i *Installer) Install(programName, version, installDir string) error {
-	if err := i.unzip(programName, version, installDir); err != nil {
+	artifactPath, err := artifact.GetArtifactPath(programName, version, i.config.OS(), i.config.Arch(), i.config.TargetDirectory)
+	if err != nil {
 		return err
 	}
 
-	oldPath := filepath.Join(installDir, fmt.Sprintf("%s-%s-windows", programName, version))
-	newPath := filepath.Join(installDir, strings.Title(programName))
-	if err := os.Rename(oldPath, newPath); err != nil {
-		return errors.New(err, errors.TypeFilesystem, errors.M(errors.MetaKeyPath, newPath))
+	if err := i.unzip(artifactPath, programName, version); err != nil {
+		return err
 	}
 
-	return i.runInstall(programName, installDir)
-}
-
-func (i *Installer) unzip(programName, version, installPath string) error {
-	artifactPath, err := artifact.GetArtifactPath(programName, version, i.config.OS(), i.config.Arch(), i.config.TargetDirectory)
+	rootDir, err := i.getRootDir(artifactPath)
 	if err != nil {
 		return err
 	}
 
+	// if root directory is not the same as desired directory rename
+	// e.g contains `-windows-` or  `-SNAPSHOT-`
+	if rootDir != installDir {
+		if err := os.Rename(rootDir, installDir); err != nil {
+			return errors.New(err, errors.TypeFilesystem, errors.M(errors.MetaKeyPath, installDir))
+		}
+	}
+
+	return i.runInstall(programName, version, installDir)
+}
+
+func (i *Installer) unzip(artifactPath, programName, version string) error {
 	if _, err := os.Stat(artifactPath); err != nil {
 		return errors.New(fmt.Sprintf("artifact for '%s' version '%s' could not be found at '%s'", programName, version, artifactPath), errors.TypeFilesystem, errors.M(errors.MetaKeyPath, artifactPath))
 	}
 
-	powershellArg := fmt.Sprintf("Expand-Archive -Path \"%s\" -DestinationPath \"%s\"", artifactPath, installPath)
+	powershellArg := fmt.Sprintf("Expand-Archive -LiteralPath \"%s\" -DestinationPath \"%s\"", artifactPath, i.config.InstallPath)
 	installCmd := exec.Command("powershell", "-command", powershellArg)
 	return installCmd.Run()
 }
 
-func (i *Installer) runInstall(programName, installPath string) error {
+func (i *Installer) runInstall(programName, version, installPath string) error {
 	powershellCmd := fmt.Sprintf(powershellCmdTemplate, installPath, programName)
-
 	installCmd := exec.Command("powershell", "-command", powershellCmd)
+
 	return installCmd.Run()
 }
+
+// retrieves root directory from zip archive
+func (i *Installer) getRootDir(zipPath string) (dir string, err error) {
+	defer func() {
+		if dir != "" {
+			dir = filepath.Join(i.config.InstallPath, dir)
+		}
+	}()
+
+	zipReader, err := zip.OpenReader(zipPath)
+	if err != nil {
+		return "", err
+	}
+	defer zipReader.Close()
+
+	var rootDir string
+	for _, f := range zipReader.File {
+		if filepath.Base(f.Name) == filepath.Dir(f.Name) {
+			return f.Name, nil
+		}
+
+		if currentDir := filepath.Dir(f.Name); rootDir == "" || len(currentDir) < len(rootDir) {
+			rootDir = currentDir
+		}
+	}
+
+	return rootDir, nil
+}

x-pack/agent/pkg/basecmd/version/cmd.go

@@ -19,10 +19,15 @@ func NewCommandWithArgs(streams *cli.IOStreams) *cobra.Command {
 		Use:   "version",
 		Short: "Display the version of the agent.",
 		Run: func(_ *cobra.Command, _ []string) {
+			version := release.Version()
+			if release.Snapshot() {
+				version = version + "-SNAPSHOT"
+			}
+
 			fmt.Fprintf(
 				streams.Out,
 				"Agent version is %s (build: %s at %s)\n",
-				release.Version(),
+				version,
 				release.Commit(),
 				release.BuildTime(),
 			)

x-pack/agent/pkg/release/version.go

@@ -4,7 +4,10 @@
 
 package release
 
-import "time"
+import (
+	"strconv"
+	"time"
+)
 
 // version is the current version of the agent.
 var version = "8.0.0"
@@ -18,7 +21,10 @@ var buildTime = "<unknown>"
 // qualifier returns the version qualifier like alpha1.
 var qualifier = ""
 
-// Commit returns the current build hash or unkown if it was not injected in the build process.
+// snapshot is a flag marking build as a snapshot.
+var snapshot = ""
+
+// Commit returns the current build hash or unknown if it was not injected in the build process.
 func Commit() string {
 	return commit
 }
@@ -39,3 +45,9 @@ func Version() string {
 	}
 	return version + "-" + qualifier
 }
+
+// Snapshot returns true if binary was built as snapshot.
+func Snapshot() bool {
+	val, err := strconv.ParseBool(snapshot)
+	return err == nil && val
+}