metrics: collect and report Go runtime.metrics (#4041)

* add metrics.RuntimeMetrics
* hook up EnableRuntimeMetrics as new config variable
* set EnableRuntimeMetrics: true in netgoal-generated configs
* add EnableRuntimeMetrics to bootstrappedScenario and custom recipes
* check that all defaultRuntimeMetrics are supported by current version of go
* add TestSanitizePrometheusName
* add partitiontest to new tests
* use algod_go_ prefix for prometheus runtime metrics

config/localTemplate.go

@@ -41,7 +41,7 @@ type Local struct {
 	// Version tracks the current version of the defaults so we can migrate old -> new
 	// This is specifically important whenever we decide to change the default value
 	// for an existing parameter. This field tag must be updated any time we add a new version.
-	Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21"`
+	Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22"`
 
 	// environmental (may be overridden)
 	// When enabled, stores blocks indefinitally, otherwise, only the most recents blocks
@@ -251,6 +251,9 @@ type Local struct {
 	// the algod api will be exposed to untrusted individuals
 	EnableProfiler bool `version[0]:"false"`
 
+	// EnableRuntimeMetrics exposes Go runtime metrics in /metrics and via node_exporter.
+	EnableRuntimeMetrics bool `version[22]:"false"`
+
 	// TelemetryToLog records messages to node.log that are normally sent to remote event monitoring
 	TelemetryToLog bool `version[5]:"true"`
 

config/local_defaults.go

@@ -20,7 +20,7 @@
 package config
 
 var defaultLocal = Local{
-	Version:                                    21,
+	Version:                                    22,
 	AccountUpdatesStatsInterval:                5000000000,
 	AccountsRebuildSynchronousMode:             1,
 	AgreementIncomingBundlesQueueLength:        7,
@@ -68,6 +68,7 @@ var defaultLocal = Local{
 	EnableProcessBlockStats:                    false,
 	EnableProfiler:                             false,
 	EnableRequestLogger:                        false,
+	EnableRuntimeMetrics:                       false,
 	EnableTopAccountsReporting:                 false,
 	EnableVerbosedTransactionSyncLogging:       false,
 	EndpointAddress:                            "127.0.0.1:0",

daemon/algod/server.go

@@ -203,6 +203,10 @@ func (s *Server) Start() {
 
 	cfg := s.node.Config()
 
+	if cfg.EnableRuntimeMetrics {
+		metrics.DefaultRegistry().Register(metrics.NewRuntimeMetrics())
+	}
+
 	if cfg.EnableMetricReporting {
 		if err := s.metricCollector.Start(context.Background()); err != nil {
 			// log this error

installer/config.json.example

@@ -1,5 +1,5 @@
 {
-    "Version": 21,
+    "Version": 22,
     "AccountUpdatesStatsInterval": 5000000000,
     "AccountsRebuildSynchronousMode": 1,
     "AgreementIncomingBundlesQueueLength": 7,
@@ -47,6 +47,7 @@
     "EnableProcessBlockStats": false,
     "EnableProfiler": false,
     "EnableRequestLogger": false,
+    "EnableRuntimeMetrics": false,
     "EnableTopAccountsReporting": false,
     "EnableVerbosedTransactionSyncLogging": false,
     "EndpointAddress": "127.0.0.1:0",

netdeploy/networkTemplate.go

@@ -246,6 +246,7 @@ func createConfigFile(node remote.NodeConfigGoal, configFile string, numNodes in
 	cfg.EndpointAddress = "127.0.0.1:0"
 	cfg.DNSBootstrapID = ""
 	cfg.EnableProfiler = true
+	cfg.EnableRuntimeMetrics = true
 	if relaysCount == 0 {
 		cfg.DisableNetworking = true
 	}

test/testdata/configs/config-v22.json

@@ -0,0 +1,105 @@
+{
+    "Version": 22,
+    "AccountUpdatesStatsInterval": 5000000000,
+    "AccountsRebuildSynchronousMode": 1,
+    "AgreementIncomingBundlesQueueLength": 7,
+    "AgreementIncomingProposalsQueueLength": 25,
+    "AgreementIncomingVotesQueueLength": 10000,
+    "AnnounceParticipationKey": true,
+    "Archival": false,
+    "BaseLoggerDebugLevel": 4,
+    "BlockServiceCustomFallbackEndpoints": "",
+    "BroadcastConnectionsLimit": -1,
+    "CadaverSizeTarget": 1073741824,
+    "CatchpointFileHistoryLength": 365,
+    "CatchpointInterval": 10000,
+    "CatchpointTracking": 0,
+    "CatchupBlockDownloadRetryAttempts": 1000,
+    "CatchupBlockValidateMode": 0,
+    "CatchupFailurePeerRefreshRate": 10,
+    "CatchupGossipBlockFetchTimeoutSec": 4,
+    "CatchupHTTPBlockFetchTimeoutSec": 4,
+    "CatchupLedgerDownloadRetryAttempts": 50,
+    "CatchupParallelBlocks": 16,
+    "ConnectionsRateLimitingCount": 60,
+    "ConnectionsRateLimitingWindowSeconds": 1,
+    "DNSBootstrapID": "<network>.algorand.network",
+    "DNSSecurityFlags": 1,
+    "DeadlockDetection": 0,
+    "DeadlockDetectionThreshold": 30,
+    "DisableLocalhostConnectionRateLimit": true,
+    "DisableNetworking": false,
+    "DisableOutgoingConnectionThrottling": false,
+    "EnableAccountUpdatesStats": false,
+    "EnableAgreementReporting": false,
+    "EnableAgreementTimeMetrics": false,
+    "EnableAssembleStats": false,
+    "EnableBlockService": false,
+    "EnableBlockServiceFallbackToArchiver": true,
+    "EnableCatchupFromArchiveServers": false,
+    "EnableDeveloperAPI": false,
+    "EnableGossipBlockService": true,
+    "EnableIncomingMessageFilter": false,
+    "EnableLedgerService": false,
+    "EnableMetricReporting": false,
+    "EnableOutgoingNetworkMessageFiltering": true,
+    "EnablePingHandler": true,
+    "EnableProcessBlockStats": false,
+    "EnableProfiler": false,
+    "EnableRequestLogger": false,
+    "EnableRuntimeMetrics": false,
+    "EnableTopAccountsReporting": false,
+    "EnableVerbosedTransactionSyncLogging": false,
+    "EndpointAddress": "127.0.0.1:0",
+    "FallbackDNSResolverAddress": "",
+    "ForceFetchTransactions": false,
+    "ForceRelayMessages": false,
+    "GossipFanout": 4,
+    "IncomingConnectionsLimit": 800,
+    "IncomingMessageFilterBucketCount": 5,
+    "IncomingMessageFilterBucketSize": 512,
+    "IsIndexerActive": false,
+    "LedgerSynchronousMode": 2,
+    "LogArchiveMaxAge": "",
+    "LogArchiveName": "node.archive.log",
+    "LogSizeLimit": 1073741824,
+    "MaxAPIResourcesPerAccount": 100000,
+    "MaxCatchpointDownloadDuration": 7200000000000,
+    "MaxConnectionsPerIP": 30,
+    "MinCatchpointFileDownloadBytesPerSecond": 20480,
+    "NetAddress": "",
+    "NetworkMessageTraceServer": "",
+    "NetworkProtocolVersion": "",
+    "NodeExporterListenAddress": ":9100",
+    "NodeExporterPath": "./node_exporter",
+    "OptimizeAccountsDatabaseOnStartup": false,
+    "OutgoingMessageFilterBucketCount": 3,
+    "OutgoingMessageFilterBucketSize": 128,
+    "ParticipationKeysRefreshInterval": 60000000000,
+    "PeerConnectionsUpdateInterval": 3600,
+    "PeerPingPeriodSeconds": 0,
+    "PriorityPeers": {},
+    "ProposalAssemblyTime": 250000000,
+    "PublicAddress": "",
+    "ReconnectTime": 60000000000,
+    "ReservedFDs": 256,
+    "RestConnectionsHardLimit": 2048,
+    "RestConnectionsSoftLimit": 1024,
+    "RestReadTimeoutSeconds": 15,
+    "RestWriteTimeoutSeconds": 120,
+    "RunHosted": false,
+    "SuggestedFeeBlockHistory": 3,
+    "SuggestedFeeSlidingWindowSize": 50,
+    "TLSCertFile": "",
+    "TLSKeyFile": "",
+    "TelemetryToLog": true,
+    "TransactionSyncDataExchangeRate": 0,
+    "TransactionSyncSignificantMessageThreshold": 0,
+    "TxPoolExponentialIncreaseFactor": 2,
+    "TxPoolSize": 15000,
+    "TxSyncIntervalSeconds": 60,
+    "TxSyncServeResponseSize": 1000000,
+    "TxSyncTimeoutSeconds": 30,
+    "UseXForwardedForAddressField": "",
+    "VerifiedTranscationsCacheSize": 30000
+}