feat: add prometheus monitoring to hasura actions

cmd/actions/actionscmd.go

@@ -12,6 +12,7 @@ import (
 	"github.com/forbole/juno/v2/node/builder"
 	nodeconfig "github.com/forbole/juno/v2/node/config"
 	"github.com/forbole/juno/v2/node/remote"
+
 	"github.com/spf13/cobra"
 
 	"github.com/forbole/bdjuno/v2/cmd/actions/handlers"
@@ -20,10 +21,12 @@ import (
 )
 
 const (
-	flagGRPC   = "grpc"
-	flagRPC    = "rpc"
-	flagSecure = "secure"
-	flagPort   = "port"
+	flagGRPC            = "grpc"
+	flagRPC             = "rpc"
+	flagSecure          = "secure"
+	flagPort            = "port"
+	flagPortPrometheus  = "prometheus-port"
+	flagEablePrometheus = "enable-prometheus"
 )
 
 var (
@@ -47,6 +50,8 @@ func NewActionsCmd(parseCfg *parse.Config) *cobra.Command {
 			gRPC, _ := cmd.Flags().GetString(flagGRPC)
 			secure, _ := cmd.Flags().GetBool(flagSecure)
 			port, _ := cmd.Flags().GetUint(flagPort)
+			prometheusPort, _ := cmd.Flags().GetUint(flagPortPrometheus)
+			enablePrometheus, _ := cmd.Flags().GetBool(flagEablePrometheus)
 
 			log.Info().Str(flagRPC, rpc).Str(flagGRPC, gRPC).Bool(flagSecure, secure).
 				Msg("Listening to incoming Hasura actions requests....")
@@ -106,6 +111,11 @@ func NewActionsCmd(parseCfg *parse.Config) *cobra.Command {
 			waitGroup.Add(1)
 			go worker.Start(port)
 
+			// Start Prometheus
+			if enablePrometheus {
+				go actionstypes.StartPrometheus(prometheusPort)
+			}
+
 			// Block main process (signal capture will call WaitGroup's Done)
 			waitGroup.Wait()
 			return nil
@@ -116,6 +126,8 @@ func NewActionsCmd(parseCfg *parse.Config) *cobra.Command {
 	cmd.Flags().String(flagGRPC, "http://127.0.0.1:9090", "GRPC listen address. Port required")
 	cmd.Flags().Bool(flagSecure, false, "Activate secure connections")
 	cmd.Flags().Uint(flagPort, 3000, "Port to be used to expose the service")
+	cmd.Flags().Uint(flagPortPrometheus, 3001, "Port to be used to run hasura prometheus monitoring")
+	cmd.Flags().Bool(flagEablePrometheus, false, "Enable prometheus monitoring")
 
 	return cmd
 }

cmd/actions/handlers/account_balance.go

@@ -4,9 +4,14 @@ import (
 	"fmt"
 
 	actionstypes "github.com/forbole/bdjuno/v2/cmd/actions/types"
+	"github.com/rs/zerolog/log"
 )
 
 func AccountBalanceHandler(ctx *actionstypes.Context, payload *actionstypes.Payload) (interface{}, error) {
+	log.Debug().Str("address", payload.GetAddress()).
+		Int64("height", payload.Input.Height).
+		Msg("executing account balance action")
+
 	height, err := ctx.GetHeight(payload)
 	if err != nil {
 		return nil, err

cmd/actions/handlers/delegation.go

@@ -4,11 +4,16 @@ import (
 	"fmt"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/rs/zerolog/log"
 
 	actionstypes "github.com/forbole/bdjuno/v2/cmd/actions/types"
 )
 
 func DelegationHandler(ctx *actionstypes.Context, payload *actionstypes.Payload) (interface{}, error) {
+	log.Debug().Str("action", "delegations").
+		Str("address", payload.GetAddress()).
+		Msg("executing delegations action")
+
 	height, err := ctx.GetHeight(payload)
 	if err != nil {
 		return nil, err

cmd/actions/handlers/delegation_total.go

@@ -4,11 +4,16 @@ import (
 	"fmt"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/rs/zerolog/log"
 
 	actionstypes "github.com/forbole/bdjuno/v2/cmd/actions/types"
 )
 
 func TotalDelegationAmountHandler(ctx *actionstypes.Context, payload *actionstypes.Payload) (interface{}, error) {
+	log.Debug().Str("address", payload.GetAddress()).
+		Int64("height", payload.Input.Height).
+		Msg("executing total delegation amount action")
+
 	height, err := ctx.GetHeight(payload)
 	if err != nil {
 		return nil, err

cmd/actions/handlers/delegator_reward.go

@@ -4,9 +4,14 @@ import (
 	"fmt"
 
 	actionstypes "github.com/forbole/bdjuno/v2/cmd/actions/types"
+	"github.com/rs/zerolog/log"
 )
 
 func DelegationRewardHandler(ctx *actionstypes.Context, payload *actionstypes.Payload) (interface{}, error) {
+	log.Debug().Str("address", payload.GetAddress()).
+		Int64("height", payload.Input.Height).
+		Msg("executing delegation rewards action")
+
 	height, err := ctx.GetHeight(payload)
 	if err != nil {
 		return nil, err

cmd/actions/handlers/delegator_withdraw_address.go

@@ -4,9 +4,13 @@ import (
 	"fmt"
 
 	actionstypes "github.com/forbole/bdjuno/v2/cmd/actions/types"
+	"github.com/rs/zerolog/log"
 )
 
 func DelegatorWithdrawAddressHandler(ctx *actionstypes.Context, payload *actionstypes.Payload) (interface{}, error) {
+	log.Debug().Str("address", payload.GetAddress()).
+		Msg("executing delegator withdraw address action")
+
 	// Get latest node height
 	height, err := ctx.GetHeight(nil)
 	if err != nil {

cmd/actions/handlers/redelegation.go

@@ -4,11 +4,16 @@ import (
 	"fmt"
 
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
+	"github.com/rs/zerolog/log"
 
 	actionstypes "github.com/forbole/bdjuno/v2/cmd/actions/types"
 )
 
 func RedelegationHandler(ctx *actionstypes.Context, payload *actionstypes.Payload) (interface{}, error) {
+	log.Debug().Str("address", payload.GetAddress()).
+		Int64("height", payload.Input.Height).
+		Msg("executing redelegations action")
+
 	height, err := ctx.GetHeight(payload)
 	if err != nil {
 		return nil, err

cmd/actions/handlers/unbonding_delegation_total.go

@@ -5,9 +5,14 @@ import (
 	"math/big"
 
 	actionstypes "github.com/forbole/bdjuno/v2/cmd/actions/types"
+	"github.com/rs/zerolog/log"
 )
 
 func UnbondingDelegationsTotal(ctx *actionstypes.Context, payload *actionstypes.Payload) (interface{}, error) {
+	log.Debug().Str("address", payload.GetAddress()).
+		Int64("height", payload.Input.Height).
+		Msg("executing unbonding delegation total action")
+
 	height, err := ctx.GetHeight(payload)
 	if err != nil {
 		return nil, err
@@ -16,7 +21,7 @@ func UnbondingDelegationsTotal(ctx *actionstypes.Context, payload *actionstypes.
 	// Get all unbonding delegations for given delegator address
 	unbondingDelegations, err := ctx.Sources.StakingSource.GetUnbondingDelegations(height, payload.GetAddress(), nil)
 	if err != nil {
-		return nil, fmt.Errorf("error while getting delegator delegations: %s", err)
+		return nil, fmt.Errorf("error while getting delegator unbonding delegations: %s", err)
 	}
 
 	// Get the bond denom type

cmd/actions/handlers/unbonding_delegations.go

@@ -4,9 +4,14 @@ import (
 	"fmt"
 
 	actionstypes "github.com/forbole/bdjuno/v2/cmd/actions/types"
+	"github.com/rs/zerolog/log"
 )
 
 func UnbondingDelegationsHandler(ctx *actionstypes.Context, payload *actionstypes.Payload) (interface{}, error) {
+	log.Debug().Str("address", payload.GetAddress()).
+		Int64("height", payload.Input.Height).
+		Msg("executing unbonding delegations action")
+
 	height, err := ctx.GetHeight(payload)
 	if err != nil {
 		return nil, err
@@ -15,7 +20,7 @@ func UnbondingDelegationsHandler(ctx *actionstypes.Context, payload *actionstype
 	// Get all unbonding delegations for given delegator address
 	unbondingDelegations, err := ctx.Sources.StakingSource.GetUnbondingDelegations(height, payload.GetAddress(), payload.GetPagination())
 	if err != nil {
-		return nil, fmt.Errorf("error while getting delegator delegations: %s", err)
+		return nil, fmt.Errorf("error while getting delegator unbonding delegations: %s", err)
 	}
 
 	unbondingDelegationsList := make([]actionstypes.UnbondingDelegation, len(unbondingDelegations.UnbondingResponses))

cmd/actions/handlers/validator_commission.go

@@ -4,9 +4,14 @@ import (
 	"fmt"
 
 	actionstypes "github.com/forbole/bdjuno/v2/cmd/actions/types"
+	"github.com/rs/zerolog/log"
 )
 
 func ValidatorCommissionAmountHandler(ctx *actionstypes.Context, payload *actionstypes.Payload) (interface{}, error) {
+	log.Debug().Str("address", payload.GetAddress()).
+		Int64("height", payload.Input.Height).
+		Msg("executing validator commission action")
+
 	// Get latest node height
 	height, err := ctx.GetHeight(nil)
 	if err != nil {

cmd/actions/handlers/validator_delegation.go

@@ -4,11 +4,16 @@ import (
 	"fmt"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/rs/zerolog/log"
 
 	actionstypes "github.com/forbole/bdjuno/v2/cmd/actions/types"
 )
 
 func ValidatorDelegation(ctx *actionstypes.Context, payload *actionstypes.Payload) (interface{}, error) {
+	log.Debug().Str("address", payload.GetAddress()).
+		Int64("height", payload.Input.Height).
+		Msg("executing validator delegation action")
+
 	// Get latest node height
 	height, err := ctx.GetHeight(payload)
 	if err != nil {

cmd/actions/handlers/validator_redelegations_from.go

@@ -4,11 +4,16 @@ import (
 	"fmt"
 
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
+	"github.com/rs/zerolog/log"
 
 	actionstypes "github.com/forbole/bdjuno/v2/cmd/actions/types"
 )
 
 func ValidatorRedelegationsFromHandler(ctx *actionstypes.Context, payload *actionstypes.Payload) (interface{}, error) {
+	log.Debug().Str("address", payload.GetAddress()).
+		Int64("height", payload.Input.Height).
+		Msg("executing validator redelegation action")
+
 	height, err := ctx.GetHeight(payload)
 	if err != nil {
 		return nil, err

cmd/actions/handlers/validator_unbonding_delegations.go

@@ -4,9 +4,14 @@ import (
 	"fmt"
 
 	actionstypes "github.com/forbole/bdjuno/v2/cmd/actions/types"
+	"github.com/rs/zerolog/log"
 )
 
 func ValidatorUnbondingDelegationsHandler(ctx *actionstypes.Context, payload *actionstypes.Payload) (interface{}, error) {
+	log.Debug().Str("address", payload.GetAddress()).
+		Int64("height", payload.Input.Height).
+		Msg("executing validator unbonding delegations action")
+
 	// Get latest node height
 	height, err := ctx.GetHeight(payload)
 	if err != nil {

cmd/actions/logging/prometheus.go

@@ -0,0 +1,45 @@
+package logging
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// ActionResponseTime represents the Telemetry counter used to classify each executed action by response time
+var ActionResponseTime = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Name:    "bdjuno_action_response_time",
+		Help:    "Time it has taken to execute an action",
+		Buckets: []float64{0.5, 1, 2, 3, 4, 5},
+	}, []string{"path", "time"})
+
+// ActionCounter represents the Telemetry counter used to track the total number of actions executed
+var ActionCounter = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "bdjuno_actions_total_count",
+		Help: "Total number of actions executed.",
+	}, []string{"path", "http_status_code"})
+
+// ActionErrorCounter represents the Telemetry counter used to track the number of action's errors emitted
+var ActionErrorCounter = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "bdjuno_actions_error_count",
+		Help: "Total number of errors emitted.",
+	}, []string{"path", "http_status_code"},
+)
+
+func init() {
+	err := prometheus.Register(ActionResponseTime)
+	if err != nil {
+		panic(err)
+	}
+
+	err = prometheus.Register(ActionCounter)
+	if err != nil {
+		panic(err)
+	}
+
+	err = prometheus.Register(ActionErrorCounter)
+	if err != nil {
+		panic(err)
+	}
+}

cmd/actions/logging/utils.go

@@ -0,0 +1,20 @@
+package logging
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+)
+
+func SuccessCounter(path string) {
+	ActionCounter.WithLabelValues(path, fmt.Sprintf("%d", http.StatusOK)).Inc()
+}
+
+func ErrorCounter(path string) {
+	ActionErrorCounter.WithLabelValues(path, fmt.Sprintf("%d", http.StatusInternalServerError)).Inc()
+}
+
+func ReponseTimeBuckets(path string, start time.Time) {
+	ActionResponseTime.WithLabelValues(path, fmt.Sprintf("%v", time.Since(start).Seconds())).
+		Observe(time.Since(start).Seconds())
+}