Logs refactor - migrating to zap (#281)

* removing logrus by zap

Co-authored-by: edznux-dd

* Replace all the log.* to l := log.Logger(ctx); l.*

* small refactor / moving stuff / cleaner output

* Uncommited changes

* missing file

* refactoring logs

* propagating context

* refactoring

* refactoring

* fix unit tests

* fix linter

* fix linter

* fix linter

* removing debug container

* Replace all the log.* to l := log.Logger(ctx); l.*

* small refactor / moving stuff / cleaner output

* Uncommited changes

* missing file

* refactoring logs

* propagating context

* refactoring

* refactoring

* fix unit tests

* fix linter

* fix linter

* fix linter

* removing debug container

---------

Co-authored-by: Edouard Schweisguth <[email protected]>

cmd/kubehound/config.go

@@ -28,7 +28,7 @@ var (
 			if err != nil {
 				return fmt.Errorf("get config: %w", err)
 			}
-
+			l := log.Logger(cobraCmd.Context())
 			yamlData, err := yaml.Marshal(&khCfg)
 
 			if err != nil {
@@ -46,7 +46,7 @@ var (
 					return fmt.Errorf("writing to file: %w", err)
 				}
 
-				log.I.Infof("Configuration saved to %s\n", configPath)
+				l.Info("Configuration saved", log.String("path", configPath))
 
 				return nil
 			}

cmd/kubehound/dumper.go

@@ -47,8 +47,8 @@ var (
 			if err != nil {
 				return fmt.Errorf("create temporary directory: %w", err)
 			}
-
-			log.I.Debugf("Temporary directory created: %s", tmpDir)
+			l := log.Trace(cobraCmd.Context())
+			l.Info("Temporary directory created", log.String("path", tmpDir))
 			viper.Set(config.CollectorFileDirectory, tmpDir)
 
 			// Passing the Kubehound config from viper
@@ -63,7 +63,7 @@ var (
 			}
 			// Running the ingestion on KHaaS
 			if cobraCmd.Flags().Lookup("khaas-server").Value.String() != "" {
-				return core.CoreClientGRPCIngest(khCfg.Ingestor, khCfg.Dynamic.ClusterName, khCfg.Dynamic.RunID.String())
+				return core.CoreClientGRPCIngest(cobraCmd.Context(), khCfg.Ingestor, khCfg.Dynamic.ClusterName, khCfg.Dynamic.RunID.String())
 			}
 
 			return err

cmd/kubehound/ingest.go

@@ -66,10 +66,10 @@ var (
 			}
 
 			if isIngestRemoteDefault() {
-				return core.CoreClientGRPCRehydrateLatest(khCfg.Ingestor)
+				return core.CoreClientGRPCRehydrateLatest(cobraCmd.Context(), khCfg.Ingestor)
 			}
 
-			return core.CoreClientGRPCIngest(khCfg.Ingestor, khCfg.Dynamic.ClusterName, runID)
+			return core.CoreClientGRPCIngest(cobraCmd.Context(), khCfg.Ingestor, khCfg.Dynamic.ClusterName, runID)
 		},
 	}
 )

cmd/kubehound/main.go

@@ -1,13 +1,18 @@
 package main
 
 import (
+	"errors"
+
+	"github.com/DataDog/KubeHound/pkg/cmd"
 	"github.com/DataDog/KubeHound/pkg/telemetry/log"
 	"github.com/DataDog/KubeHound/pkg/telemetry/tag"
 )
 
 func main() {
 	tag.SetupBaseTags()
-	if err := rootCmd.Execute(); err != nil {
-		log.I.Fatal(err.Error())
+	err := rootCmd.Execute()
+	err = errors.Join(err, cmd.CloseKubehoundConfig(rootCmd.Context()))
+	if err != nil {
+		log.Logger(rootCmd.Context()).Fatal(err.Error())
 	}
 }

cmd/kubehound/root.go

@@ -24,6 +24,7 @@ var (
 			return cmd.InitializeKubehoundConfig(cobraCmd.Context(), cfgFile, true, false)
 		},
 		RunE: func(cobraCmd *cobra.Command, args []string) error {
+			l := log.Logger(cobraCmd.Context())
 			// auto spawning the backend stack
 			if !skipBackend {
 				// Forcing the embed docker config to be loaded
@@ -41,7 +42,7 @@ var (
 						return err
 					}
 				} else {
-					log.I.Info("Backend stack is already running")
+					l.Info("Backend stack is already running")
 				}
 			}
 
@@ -61,17 +62,14 @@ var (
 				return err
 			}
 
-			log.I.Warn("KubeHound as finished ingesting and building the graph successfully.")
-			log.I.Warn("Please visit the UI to view the graph by clicking the link below:")
-			log.I.Warn("http://localhost:8888")
+			l.Warn("KubeHound as finished ingesting and building the graph successfully.")
+			l.Warn("Please visit the UI to view the graph by clicking the link below:")
+			l.Warn("http://localhost:8888")
 			// Yes, we should change that :D
-			log.I.Warn("Password being 'admin'")
+			l.Warn("Default password being 'admin'")
 
 			return nil
 		},
-		PersistentPostRunE: func(cobraCmd *cobra.Command, args []string) error {
-			return cmd.CloseKubehoundConfig()
-		},
 		SilenceUsage:  true,
 		SilenceErrors: true,
 	}

cmd/kubehound/server.go

@@ -26,9 +26,6 @@ var (
 
 			return core.CoreGrpcApi(cobraCmd.Context(), khCfg)
 		},
-		PersistentPostRunE: func(cobraCmd *cobra.Command, args []string) error {
-			return cmd.CloseKubehoundConfig()
-		},
 	}
 )
 

cmd/kubehound/version.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/DataDog/KubeHound/pkg/config"
 	"github.com/spf13/cobra"
@@ -15,6 +16,9 @@ var (
 		Run: func(cobraCmd *cobra.Command, args []string) {
 			fmt.Printf("kubehound version: %s (%s/%s)", config.BuildVersion, config.BuildArch, config.BuildOs) //nolint:forbidigo
 		},
+		PersistentPostRun: func(cobraCmd *cobra.Command, args []string) {
+			os.Exit(0)
+		},
 	}
 )
 

deployments/kubehound/kubehound.env.tpl

@@ -6,6 +6,7 @@ KH_INGESTOR_API_ENDPOINT=0.0.0.0:9000
 KH_INGESTOR_TEMP_DIR=/tmp/kubehound
 KH_INGESTOR_MAX_ARCHIVE_SIZE=2147483648 # 2GB
 KH_INGESTOR_ARCHIVE_NAME=archive.tar.gz
+KH_LOG_FORMAT=dd
 # AWS Bucket configuration
 KH_INGESTOR_REGION=us-east-1
 KH_INGESTOR_BUCKET_URL="" # s3://<your_bucket>

go.mod

@@ -16,11 +16,13 @@ require (
 	github.com/docker/docker v26.1.4+incompatible
 	github.com/go-playground/validator/v10 v10.21.0
 	github.com/hashicorp/go-multierror v1.1.1
+	github.com/pkg/errors v0.9.1
 	github.com/spf13/afero v1.11.0
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
 	go.mongodb.org/mongo-driver v1.15.0
 	go.uber.org/ratelimit v0.3.1
+	go.uber.org/zap v1.27.0
 	gocloud.dev v0.37.0
 	golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8
 	google.golang.org/grpc v1.64.1
@@ -194,7 +196,6 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect

go.sum

@@ -914,8 +914,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
-google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/DataDog/dd-trace-go.v1 v1.64.1 h1:HN/zoIV8FvrLKA1ZBkbyo4E1MnPh9hPc2Q0C/ojom3I=

pkg/backend/containers.go

@@ -71,7 +71,8 @@ func BuildUp(ctx context.Context, noCache bool) error {
 }
 
 func (b *Backend) buildUp(ctx context.Context, noCache bool) error {
-	log.I.Infof("Building the kubehound stack")
+	l := log.Logger(ctx)
+	l.Info("Building the kubehound stack")
 	err := b.composeService.Build(ctx, b.project, api.BuildOptions{
 		NoCache: noCache,
 		Pull:    true,
@@ -88,7 +89,8 @@ func Up(ctx context.Context) error {
 }
 
 func (b *Backend) up(ctx context.Context) error {
-	log.I.Infof("Spawning the kubehound stack")
+	l := log.Logger(ctx)
+	l.Info("Spawning the kubehound stack")
 
 	err := b.composeService.Up(ctx, b.project, api.UpOptions{
 		Create: api.CreateOptions{
@@ -117,7 +119,8 @@ func Down(ctx context.Context) error {
 }
 
 func (b *Backend) down(ctx context.Context) error {
-	log.I.Info("Tearing down the kubehound stack")
+	l := log.Logger(ctx)
+	l.Info("Tearing down the kubehound stack")
 
 	err := b.composeService.Remove(ctx, b.project.Name, api.RemoveOptions{
 		Stop:    true,
@@ -175,10 +178,11 @@ func Wipe(ctx context.Context) error {
 
 func (b *Backend) wipe(ctx context.Context) error {
 	var err error
-	log.I.Infof("Wiping the persisted backend data")
+	l := log.Logger(ctx)
+	l.Info("Wiping the persisted backend data")
 
 	for _, volumeID := range b.project.VolumeNames() {
-		log.I.Infof("Deleting volume %s", volumeID)
+		l.Info("Deleting volume", log.String("volume", volumeID))
 		err = errors.Join(err, b.dockerCli.Client().VolumeRemove(ctx, volumeID, true))
 	}
 

pkg/backend/project.go

@@ -27,13 +27,13 @@ var (
 func loadProject(ctx context.Context, composeFilePaths []string, profiles []string) (*types.Project, error) {
 	var project *types.Project
 	var err error
-
+	l := log.Logger(ctx)
 	switch {
 	case len(composeFilePaths) != 0 && len(composeFilePaths[0]) != 0:
-		log.I.Infof("Loading backend from file %s", composeFilePaths)
+		l.Info("Loading backend from file", log.Strings("path", composeFilePaths))
 		project, err = loadComposeConfig(ctx, composeFilePaths, profiles)
 	default:
-		log.I.Infof("Loading backend from default embedded")
+		l.Info("Loading backend from default embedded")
 		project, err = loadEmbeddedConfig(ctx, profiles)
 	}
 
@@ -108,9 +108,9 @@ func loadEmbeddedConfig(ctx context.Context, profiles []string) (*types.Project,
 	return loader.LoadWithContext(ctx, opts, loader.WithProfiles(profiles))
 }
 
-func loadEmbeddedDockerCompose(_ context.Context, filepath string, dockerComposeFileData map[interface{}]interface{}) (map[interface{}]interface{}, error) {
+func loadEmbeddedDockerCompose(ctx context.Context, filepath string, dockerComposeFileData map[interface{}]interface{}) (map[interface{}]interface{}, error) {
+	l := log.Logger(ctx)
 	var localYaml map[interface{}]interface{}
-
 	localData, err := embedconfigdocker.F.ReadFile(filepath)
 	if err != nil {
 		return nil, fmt.Errorf("reading embed config: %w", err)
@@ -124,7 +124,7 @@ func loadEmbeddedDockerCompose(_ context.Context, filepath string, dockerCompose
 		// For local version (when the version is "dirty", using latest to have a working binary)
 		// For any branch outside of main, using latest image as the current tag will cover (including the commit sha in the tag)
 		if strings.HasSuffix(config.BuildBranch, "dirty") || config.BuildBranch != "main" {
-			log.I.Warnf("Loading the kubehound images with tag latest - dev branch detected")
+			l.Warn("Loading the kubehound images with tag latest - dev branch detected")
 			version["VersionTag"] = "latest"
 		}
 

pkg/cmd/config.go

@@ -8,7 +8,6 @@ import (
 	"github.com/DataDog/KubeHound/pkg/telemetry"
 	"github.com/DataDog/KubeHound/pkg/telemetry/log"
 	"github.com/DataDog/KubeHound/pkg/telemetry/tag"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 )
 
@@ -24,32 +23,33 @@ func GetConfig() (*config.KubehoundConfig, error) {
 }
 
 func InitializeKubehoundConfig(ctx context.Context, configPath string, generateRunID bool, inline bool) error {
+	l := log.Logger(ctx)
 	// We define a unique run id this so we can measure run by run in addition of version per version.
 	// Useful when rerunning the same binary (same version) on different dataset or with different databases...
 	// In the case of KHaaS, the runID is taken from the GRPC request argument
 	if generateRunID {
 		viper.Set(config.DynamicRunID, config.NewRunID())
 	}
 
-	khCfg := config.NewKubehoundConfig(configPath, inline)
+	khCfg := config.NewKubehoundConfig(ctx, configPath, inline)
 	// Activate debug mode if needed
 	if khCfg.Debug {
-		log.I.Info("Debug mode activated")
-		log.I.Logger.SetLevel(logrus.DebugLevel)
+		l.Info("Debug mode activated")
 	}
 
 	InitTags(ctx, khCfg)
-	InitTelemetry(khCfg)
+	InitTelemetry(ctx, khCfg)
 
 	return nil
 
 }
 
-func InitTelemetry(khCfg *config.KubehoundConfig) {
-	log.I.Info("Initializing application telemetry")
-	err := telemetry.Initialize(khCfg)
+func InitTelemetry(ctx context.Context, khCfg *config.KubehoundConfig) {
+	l := log.Logger(ctx)
+	l.Info("Initializing application telemetry")
+	err := telemetry.Initialize(ctx, khCfg)
 	if err != nil {
-		log.I.Warnf("failed telemetry initialization: %v", err)
+		l.Warn("failed telemetry initialization", log.ErrorField(err))
 	}
 }
 
@@ -64,23 +64,23 @@ func InitTags(ctx context.Context, khCfg *config.KubehoundConfig) {
 		tag.AppendBaseTags(tag.RunID(khCfg.Dynamic.RunID.String()))
 
 		// Set the run ID as a global log tag
-		log.AddGlobalTags(map[string]string{
-			tag.RunIdTag: khCfg.Dynamic.RunID.String(),
-		})
+		// log.AddGlobalTags(map[string]string{
+		// 	tag.RunIdTag: khCfg.Dynamic.RunID.String(),
+		// })
 	}
 
 	// Update the logger behaviour from configuration
-	log.SetDD(khCfg.Telemetry.Enabled)
-	log.AddGlobalTags(khCfg.Telemetry.Tags)
+	// log.SetDD(khCfg.Telemetry.Enabled)
+	// log.AddGlobalTags(khCfg.Telemetry.Tags)
 }
 
-func CloseKubehoundConfig() error {
+func CloseKubehoundConfig(ctx context.Context) error {
 	khCfg, err := GetConfig()
 	if err != nil {
 		return err
 	}
 
-	telemetry.Shutdown(khCfg.Telemetry.Enabled)
+	telemetry.Shutdown(ctx, khCfg.Telemetry.Enabled)
 
 	return nil
 }

pkg/cmd/dump.go

@@ -38,10 +38,10 @@ func InitLocalDumpCmd(cmd *cobra.Command) {
 
 func InitRemoteDumpCmd(cmd *cobra.Command) {
 	cmd.Flags().String("bucket-url", "", "Bucket to use to push k8s resources (e.g.: s3://<your_bucket>)")
-	viper.BindPFlag(config.CollectorFileBlobBucket, cmd.Flags().Lookup("bucket-url")) //nolint: errcheck
+	viper.BindPFlag(config.IngestorBlobBucketURL, cmd.Flags().Lookup("bucket-url")) //nolint: errcheck
 
 	cmd.Flags().String("region", "", "Region to retrieve the configuration (only for s3) (e.g.: us-east-1)")
-	viper.BindPFlag(config.CollectorFileBlobRegion, cmd.Flags().Lookup("region")) //nolint: errcheck
+	viper.BindPFlag(config.IngestorBlobRegion, cmd.Flags().Lookup("region")) //nolint: errcheck
 }
 
 func InitLocalIngestCmd(cmd *cobra.Command) {

pkg/cmd/util.go

@@ -1,16 +1,18 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 
 	"strings"
 
 	"github.com/DataDog/KubeHound/pkg/telemetry/log"
 )
 
-func AskForConfirmation() (bool, error) {
-	var response string
+func AskForConfirmation(ctx context.Context) (bool, error) {
+	l := log.Logger(ctx)
 
+	var response string
 	_, err := fmt.Scanln(&response)
 	if err != nil && err.Error() != "unexpected newline" {
 		return false, fmt.Errorf("scanln: %w", err)
@@ -22,8 +24,8 @@ func AskForConfirmation() (bool, error) {
 	case "n", "no":
 		return false, nil
 	default:
-		log.I.Error("Please type (y)es or (n)o and then press enter:")
+		l.Info("Please type (y)es or (n)o and then press enter:")
 
-		return AskForConfirmation()
+		return AskForConfirmation(ctx)
 	}
 }