Add more logging & metrics (#207)

babylonlabs-io · Jan 31, 2025 · 2f31947 · 2f31947
1 parent c4d42a1
commit 2f31947
Show file tree

Hide file tree

Showing 7 changed files with 38 additions and 13 deletions.
diff --git a/cmd/staking-api-service/main.go b/cmd/staking-api-service/main.go
@@ -83,6 +83,7 @@ func main() {
 
 	dbClients, err := dbclients.New(ctx, cfg)
 	if err != nil {
+		metrics.RecordServiceCrash("database")
 		log.Fatal().Err(err).Msg("error while setting up staking db clients")
 	}
 
@@ -126,6 +127,7 @@ func main() {
 
 	apiServer, err := api.New(ctx, cfg, services)
 	if err != nil {
+		metrics.RecordServiceCrash("api")
 		log.Fatal().Err(err).Msg("error while setting up staking api service")
 	}
 	if err = apiServer.Start(); err != nil {

diff --git a/internal/shared/api/server.go b/internal/shared/api/server.go
@@ -27,7 +27,7 @@ func New(
 
 	logLevel, err := zerolog.ParseLevel(cfg.Server.LogLevel)
 	if err != nil {
-		log.Fatal().Err(err).Msg("error while parsing log level")
+		return nil, fmt.Errorf("error while parsing log level: %w", err)
 	}
 	zerolog.SetGlobalLevel(logLevel)
 
@@ -46,12 +46,12 @@ func New(
 	}
 
 	if err != nil {
-		log.Fatal().Err(err).Msg("error while setting up handlers")
+		return nil, fmt.Errorf("error while setting up handlers: %w", err)
 	}
 
 	handlers, err := handlers.New(ctx, cfg, services)
 	if err != nil {
-		log.Fatal().Err(err).Msg("error while setting up handlers")
+		return nil, fmt.Errorf("error while setting up handlers: %w", err)
 	}
 
 	server := &Server{

diff --git a/internal/shared/db/client/db_client.go b/internal/shared/db/client/db_client.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/babylonlabs-io/staking-api-service/internal/shared/config"
+	"github.com/babylonlabs-io/staking-api-service/internal/shared/observability/metrics"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
@@ -26,6 +27,7 @@ func NewMongoClient(ctx context.Context, cfg *config.DbConfig) (*mongo.Client, e
 func (db *Database) Ping(ctx context.Context) error {
 	err := db.Client.Ping(ctx, nil)
 	if err != nil {
+		metrics.RecordDbError("ping")
 		return err
 	}
 	return nil

diff --git a/internal/shared/db/client/pk_address_mapping.go b/internal/shared/db/client/pk_address_mapping.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	dbmodel "github.com/babylonlabs-io/staking-api-service/internal/shared/db/model"
+	"github.com/babylonlabs-io/staking-api-service/internal/shared/observability/metrics"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
 )
@@ -20,6 +21,7 @@ func (db *Database) InsertPkAddressMappings(
 	}
 	_, err := client.InsertOne(ctx, addressMapping)
 	if err != nil && !mongo.IsDuplicateKeyError(err) {
+		metrics.RecordDbError("insert_pk_address_mappings")
 		return err
 	}
 	return nil
@@ -34,10 +36,12 @@ func (db *Database) FindPkMappingsByTaprootAddress(
 	addressMapping := []*dbmodel.PkAddressMapping{}
 	cursor, err := client.Find(ctx, filter)
 	if err != nil {
+		metrics.RecordDbError("find_pk_mappings_by_taproot_address")
 		return nil, err
 	}
 	defer cursor.Close(ctx)
 	if err = cursor.All(ctx, &addressMapping); err != nil {
+		metrics.RecordDbError("find_pk_mappings_by_taproot_address")
 		return nil, err
 	}
 	return addressMapping, nil
@@ -57,10 +61,12 @@ func (db *Database) FindPkMappingsByNativeSegwitAddress(
 	addressMapping := []*dbmodel.PkAddressMapping{}
 	cursor, err := client.Find(ctx, filter)
 	if err != nil {
+		metrics.RecordDbError("find_pk_mappings_by_native_segwit_address")
 		return nil, err
 	}
 	defer cursor.Close(ctx)
 	if err = cursor.All(ctx, &addressMapping); err != nil {
+		metrics.RecordDbError("find_pk_mappings_by_native_segwit_address")
 		return nil, err
 	}
 	return addressMapping, nil

diff --git a/internal/shared/db/client/unprocessable_message.go b/internal/shared/db/client/unprocessable_message.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	dbmodel "github.com/babylonlabs-io/staking-api-service/internal/shared/db/model"
+	"github.com/babylonlabs-io/staking-api-service/internal/shared/observability/metrics"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
@@ -13,10 +14,10 @@ func (db *Database) SaveUnprocessableMessage(ctx context.Context, messageBody, r
 
 	_, err := unprocessableMsgClient.InsertOne(ctx, dbmodel.NewUnprocessableMessageDocument(messageBody, receipt))
 	if err != nil {
-		return err
+		metrics.RecordDbError("save_unprocessable_message")
 	}
 
-	return nil
+	return err
 }
 
 func (db *Database) FindUnprocessableMessages(ctx context.Context) ([]dbmodel.UnprocessableMessageDocument, error) {
@@ -26,12 +27,14 @@ func (db *Database) FindUnprocessableMessages(ctx context.Context) ([]dbmodel.Un
 
 	cursor, err := client.Find(ctx, filter, &options)
 	if err != nil {
+		metrics.RecordDbError("find_unprocessable_messages")
 		return nil, err
 	}
 	defer cursor.Close(ctx)
 
 	var unprocessableMessages []dbmodel.UnprocessableMessageDocument
 	if err = cursor.All(ctx, &unprocessableMessages); err != nil {
+		metrics.RecordDbError("find_unprocessable_messages")
 		return nil, err
 	}
 
@@ -42,5 +45,8 @@ func (db *Database) DeleteUnprocessableMessage(ctx context.Context, Receipt inte
 	unprocessableMsgClient := db.Client.Database(db.DbName).Collection(dbmodel.V1UnprocessableMsgCollection)
 	filter := bson.M{"receipt": Receipt}
 	_, err := unprocessableMsgClient.DeleteOne(ctx, filter)
+	if err != nil {
+		metrics.RecordDbError("delete_unprocessable_message")
+	}
 	return err
 }
diff --git a/internal/shared/db/clients/db_clients.go b/internal/shared/db/clients/db_clients.go
@@ -3,12 +3,12 @@ package dbclients
 import (
 	"context"
 
+	"fmt"
 	indexerdbclient "github.com/babylonlabs-io/staking-api-service/internal/indexer/db/client"
 	"github.com/babylonlabs-io/staking-api-service/internal/shared/config"
 	dbclient "github.com/babylonlabs-io/staking-api-service/internal/shared/db/client"
 	v1dbclient "github.com/babylonlabs-io/staking-api-service/internal/v1/db/client"
 	v2dbclient "github.com/babylonlabs-io/staking-api-service/internal/v2/db/client"
-	"github.com/rs/zerolog/log"
 	"go.mongodb.org/mongo-driver/mongo"
 )
 
@@ -34,24 +34,21 @@ func New(ctx context.Context, cfg *config.Config) (*DbClients, error) {
 
 	v1dbClient, err := v1dbclient.New(ctx, stakingMongoClient, cfg.StakingDb)
 	if err != nil {
-		log.Ctx(ctx).Fatal().Err(err).Msg("error while creating v1 db client")
-		return nil, err
+		return nil, fmt.Errorf("error while creating v1 db client: %w", err)
 	}
 	v2dbClient, err := v2dbclient.New(ctx, stakingMongoClient, cfg.StakingDb)
 	if err != nil {
-		log.Ctx(ctx).Fatal().Err(err).Msg("error while creating v2 db client")
-		return nil, err
+		return nil, fmt.Errorf("error while creating v2 db client: %w", err)
 	}
 
 	indexerMongoClient, err := dbclient.NewMongoClient(ctx, cfg.IndexerDb)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("error while creating indexer mongo client: %w", err)
 	}
 
 	indexerDbClient, err := indexerdbclient.New(ctx, indexerMongoClient, cfg.IndexerDb)
 	if err != nil {
-		log.Ctx(ctx).Fatal().Err(err).Msg("error while creating indexer db client")
-		return nil, err
+		return nil, fmt.Errorf("error while creating indexer db client: %w", err)
 	}
 
 	dbClients := DbClients{

diff --git a/internal/shared/observability/metrics/metrics.go b/internal/shared/observability/metrics/metrics.go
@@ -35,6 +35,7 @@ var (
 	httpResponseWriteFailureCounter  *prometheus.CounterVec
 	clientRequestDurationHistogram   *prometheus.HistogramVec
 	serviceCrashCounter              *prometheus.CounterVec
+	dbErrorsCounter                  *prometheus.CounterVec
 )
 
 // Init initializes the metrics package.
@@ -132,6 +133,13 @@ func registerMetrics() {
 		},
 		[]string{"type"},
 	)
+	dbErrorsCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "db_errors",
+			Help: "",
+		},
+		[]string{"method"},
+	)
 
 	prometheus.MustRegister(
 		httpRequestDurationHistogram,
@@ -202,3 +210,7 @@ func StartClientRequestDurationTimer(baseUrl, method, path string) func(statusCo
 func RecordServiceCrash(service string) {
 	serviceCrashCounter.WithLabelValues(service).Inc()
 }
+
+func RecordDbError(method string) {
+	dbErrorsCounter.WithLabelValues(method).Inc()
+}