Osquerybeat: Change osquerybeat packaging for windows, replace .MSI f…

…ile with extracted osqueryd.exe during build process (#29801) (#29823)

(cherry picked from commit e61a365)

Co-authored-by: Aleksandr Maus <[email protected]>

x-pack/osquerybeat/beater/install.go

@@ -34,9 +34,7 @@ func installOsqueryWithDir(ctx context.Context, dir string) error {
 	fn := distro.OsquerydDistroFilename()
 	var installFunc func(context.Context, string, string, bool) error
 
-	if runtime.GOOS == "windows" {
-		installFunc = install.InstallFromMSI
-	} else if runtime.GOOS == "darwin" {
+	if runtime.GOOS == "darwin" {
 		installFunc = install.InstallFromPkg
 	}
 

x-pack/osquerybeat/internal/distro/distro.go

@@ -57,19 +57,27 @@ func GetDataInstallDir(osarch OSArch) string {
 	return filepath.Join(DataInstallDir, osarch.OS, osarch.Arch)
 }
 
-func OsquerydFilename() string {
-	if runtime.GOOS == "windows" {
+func OsquerydFilenameForOS(os string) string {
+	if os == "windows" {
 		return osqueryDName + ".exe"
 	}
 	return osqueryDName
 }
 
+func OsquerydFilename() string {
+	return OsquerydFilenameForOS(runtime.GOOS)
+}
+
 func OsquerydDarwinApp() string {
 	return osqueryDarwinApp
 }
 
+func OsquerydPathForOS(os, dir string) string {
+	return filepath.Join(dir, OsquerydFilenameForOS(os))
+}
+
 func OsquerydPath(dir string) string {
-	return filepath.Join(dir, OsquerydFilename())
+	return OsquerydPathForOS(runtime.GOOS, dir)
 }
 
 func OsquerydDarwinDistroPath() string {
@@ -87,7 +95,7 @@ func OsquerydDistroFilename() string {
 func OsquerydDistroPlatformFilename(platform string) string {
 	switch platform {
 	case "windows":
-		return osqueryName + "-" + osqueryVersion + osqueryMSIExt
+		return OsquerydFilenameForOS(platform)
 	case "darwin":
 		return osqueryName + "-" + osqueryVersion + osqueryPkgExt
 	}

x-pack/osquerybeat/magefile.go

@@ -8,15 +8,20 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/magefile/mage/mg"
 
 	devtools "github.com/elastic/beats/v7/dev-tools/mage"
+	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/command"
+	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/distro"
 	osquerybeat "github.com/elastic/beats/v7/x-pack/osquerybeat/scripts/mage"
 
 	// mage:import
@@ -77,10 +82,103 @@ func Clean() error {
 	return devtools.Clean(paths)
 }
 
+func extractFromMSI() error {
+	if os.Getenv("GOOS") != "windows" {
+		return nil
+	}
+
+	ctx := context.Background()
+
+	execCommand := func(name string, args ...string) error {
+		ps := strings.Join(append([]string{name}, args...), " ")
+		fmt.Println(ps)
+		output, err := command.Execute(ctx, name, args...)
+		if err != nil {
+			fmt.Println(ps, ", failed: ", err)
+			return err
+		}
+		fmt.Print(output)
+		return err
+	}
+
+	// Install msitools
+	err := execCommand("apt", "update")
+	if err != nil {
+		return err
+	}
+
+	err = execCommand("apt", "install", "-y", "msitools")
+	if err != nil {
+		return err
+	}
+
+	osArchs := osquerybeat.OSArchs(devtools.Platforms)
+
+	for _, osarch := range osArchs {
+		if osarch.OS != "windows" {
+			continue
+		}
+		spec, err := distro.GetSpec(osarch)
+		if err != nil {
+			if errors.Is(err, distro.ErrUnsupportedOS) {
+				continue
+			} else {
+				return err
+			}
+		}
+		dip := distro.GetDataInstallDir(osarch)
+		msiFile := spec.DistroFilepath(dip)
+
+		// MSI extract
+		err = execCommand("msiextract", "--directory", dip, msiFile)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("copy osqueryd.exe from MSI")
+		dp := distro.OsquerydPathForOS(osarch.OS, dip)
+		err = devtools.Copy(filepath.Join(dip, "osquery", "osqueryd", "osqueryd.exe"), dp)
+		if err != nil {
+			fmt.Println("copy osqueryd.exe from MSI failed: ", err)
+			return err
+		}
+		// Chmod set to the same as other executables in the final package
+		if err = os.Chmod(dp, 0755); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // GolangCrossBuild build the Beat binary inside of the golang-builder.
 // Do not use directly, use crossBuild instead.
 func GolangCrossBuild() error {
-	return devtools.GolangCrossBuild(devtools.DefaultGolangCrossBuildArgs())
+	// This is to fix a defect in the field where msiexec fails to extract the osqueryd.exe
+	// from bundled osquery.msi, with error code 1603
+	// https://docs.microsoft.com/en-us/troubleshoot/windows-server/application-management/msi-installation-error-1603
+	// SDH: https://github.com/elastic/sdh-beats/issues/1575
+	// Currently we can't reproduce this is issue, but here we can eliminate the need for calling msiexec
+	// if extract the osqueryd.exe binary during the build.
+	//
+	// The builder docker images are Debian so we need to install msitools for
+	// linux in order to extract the osqueryd.exe from MSI during build process.	// Install MSI tools in order to extract file from MSI
+	// Ideally we would want these to be a part of the build docker image,
+	// but doing this here for now due to limited time before 7.16.2
+	//
+	// The cross build is currently called for two binaries osquerybeat and osqquery-extension
+	// Only install msitools and extract osqueryd.exe during osquerybeat build on windows
+	args := devtools.DefaultGolangCrossBuildArgs()
+
+	// Install msitools only
+	if !strings.HasPrefix(args.Name, "osquery-extension-") {
+		// Install msitools in the container and extract osqueryd.exe from MSI
+		if err := extractFromMSI(); err != nil {
+			return err
+		}
+	}
+
+	return devtools.GolangCrossBuild(args)
 }
 
 // BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon).

x-pack/osquerybeat/scripts/mage/distro.go

@@ -24,20 +24,20 @@ import (
 
 // FetchOsqueryDistros fetches Osquery official distros as a part of the build
 func FetchOsqueryDistros() error {
-	osArchs := osArchs(devtools.Platforms)
+	osArchs := OSArchs(devtools.Platforms)
 	log.Printf("Fetch Osquery distros for %v", osArchs)
 
 	for _, osarch := range osArchs {
 		spec, err := distro.GetSpec(osarch)
 		if err != nil {
 			if errors.Is(err, distro.ErrUnsupportedOS) {
-				log.Printf("The build spec %v is not supported, continue", spec)
+				log.Printf("The build spec %v is not supported, continue\n", spec)
 				continue
 			} else {
 				return err
 			}
 		}
-		log.Print("Found spec:", spec)
+		log.Println("Found spec:", spec)
 
 		fetched, err := checkCacheAndFetch(osarch, spec)
 		if err != nil {
@@ -66,7 +66,7 @@ func FetchOsqueryDistros() error {
 	return nil
 }
 
-func osArchs(platforms devtools.BuildPlatformList) []distro.OSArch {
+func OSArchs(platforms devtools.BuildPlatformList) []distro.OSArch {
 	mp := make(map[distro.OSArch]struct{})
 
 	for _, platform := range platforms {

x-pack/osquerybeat/scripts/mage/package.go

@@ -23,7 +23,7 @@ func CustomizePackaging() {
 		// TODO: this could be moved to dev-tools/packaging/packages.yml for the next release
 		var mode os.FileMode = 0644
 		// If distFile is osqueryd binary then it should be executable
-		if distFile == distro.OsquerydFilename() {
+		if distFile == distro.OsquerydFilenameForOS(args.OS) {
 			mode = 0750
 		}
 		arch := defaultArch