Node Version Heartbeat (#15700)

* rough implementation of node heartbeat

* consuming services engine to simplify tick + start/close handling

* fixing duration^2 bug

* fixing health tests

* fixing testscripts test; using eng logger; using time.seconds const

* minor refactoring

* NodeHeartbeat --> Heartbeat

* removing close from Heartbeat

core/services/chainlink/application.go

@@ -7,6 +7,7 @@ import (
 	"math/big"
 	"net/http"
 	"sync"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -20,10 +21,12 @@ import (
 	"go.uber.org/multierr"
 	"go.uber.org/zap/zapcore"
 
+	"github.com/smartcontractkit/chainlink-common/pkg/beholder"
 	"github.com/smartcontractkit/chainlink-common/pkg/custmsg"
 	"github.com/smartcontractkit/chainlink-common/pkg/loop"
 	commonservices "github.com/smartcontractkit/chainlink-common/pkg/services"
 	"github.com/smartcontractkit/chainlink-common/pkg/sqlutil"
+	"github.com/smartcontractkit/chainlink-common/pkg/timeutil"
 	"github.com/smartcontractkit/chainlink-common/pkg/utils"
 	"github.com/smartcontractkit/chainlink-common/pkg/utils/jsonserializable"
 	"github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox"
@@ -80,6 +83,8 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/plugins"
 )
 
+const HeartbeatPeriod = time.Second
+
 // Application implements the common functions used in the core node.
 type Application interface {
 	Start(ctx context.Context) error
@@ -192,13 +197,71 @@ type ApplicationOpts struct {
 	NewOracleFactoryFn         standardcapabilities.NewOracleFactoryFn
 }
 
+type Heartbeat struct {
+	commonservices.Service
+	eng *commonservices.Engine
+
+	beat time.Duration
+}
+
+func NewHeartbeat(lggr logger.Logger) Heartbeat {
+	h := Heartbeat{
+		beat: HeartbeatPeriod,
+	}
+	h.Service, h.eng = commonservices.Config{
+		Name:  "Heartbeat",
+		Start: h.start,
+	}.NewServiceEngine(lggr)
+	return h
+}
+
+func (h *Heartbeat) start(_ context.Context) error {
+	// Setup beholder resources
+	gauge, err := beholder.GetMeter().Int64Gauge("heartbeat")
+	if err != nil {
+		return err
+	}
+	count, err := beholder.GetMeter().Int64Gauge("heartbeat_count")
+	if err != nil {
+		return err
+	}
+
+	cme := custmsg.NewLabeler()
+
+	// Define tick functions
+	beatFn := func(ctx context.Context) {
+		// TODO allow override of tracer provider into engine for beholder
+		_, innerSpan := beholder.GetTracer().Start(ctx, "heartbeat.beat")
+		defer innerSpan.End()
+
+		gauge.Record(ctx, 1)
+		count.Record(ctx, 1)
+
+		err = cme.Emit(ctx, "heartbeat")
+		if err != nil {
+			h.eng.Errorw("heartbeat emit failed", "err", err)
+		}
+	}
+
+	h.eng.GoTick(timeutil.NewTicker(h.getBeat), beatFn)
+	return nil
+}
+
+func (h *Heartbeat) getBeat() time.Duration {
+	return h.beat
+}
+
 // NewApplication initializes a new store if one is not already
 // present at the configured root directory (default: ~/.chainlink),
 // the logger at the same directory and returns the Application to
 // be used by the node.
 // TODO: Inject more dependencies here to save booting up useless stuff in tests
 func NewApplication(opts ApplicationOpts) (Application, error) {
 	var srvcs []services.ServiceCtx
+
+	heartbeat := NewHeartbeat(opts.Logger)
+	srvcs = append(srvcs, &heartbeat)
+
 	auditLogger := opts.AuditLogger
 	cfg := opts.Config
 	relayerChainInterops := opts.RelayerChainInteroperators

core/services/chainlink/config_telemetry.go

@@ -1,6 +1,7 @@
 package chainlink
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/smartcontractkit/chainlink/v2/core/config/toml"
@@ -42,9 +43,13 @@ func (b *telemetryConfig) OtelExporterGRPCEndpoint() string {
 //
 // These can be overridden by the TOML if the user so chooses
 func (b *telemetryConfig) ResourceAttributes() map[string]string {
+	sha, ver := static.Short()
+
 	defaults := map[string]string{
-		"service.name":    "chainlink",
-		"service.version": static.Version,
+		"service.name":         "chainlink",
+		"service.version":      static.Version,
+		"service.sha":          static.Sha,
+		"service.shortversion": fmt.Sprintf("%s@%s", ver, sha),
 	}
 
 	for k, v := range b.s.ResourceAttributes {

core/services/chainlink/config_telemetry_test.go

@@ -97,17 +97,21 @@ func TestTelemetryConfig_ResourceAttributes(t *testing.T) {
 			"DefaultAttributes",
 			toml.Telemetry{ResourceAttributes: nil},
 			map[string]string{
-				"service.name":    "chainlink",
-				"service.version": static.Version,
+				"service.name":         "chainlink",
+				"service.sha":          "unset",
+				"service.shortversion": "unset@unset",
+				"service.version":      static.Version,
 			},
 		},
 		{
 			"CustomAttributes",
 			toml.Telemetry{ResourceAttributes: map[string]string{"custom.key": "custom.value"}},
 			map[string]string{
-				"service.name":    "chainlink",
-				"service.version": static.Version,
-				"custom.key":      "custom.value",
+				"service.name":         "chainlink",
+				"service.sha":          "unset",
+				"service.shortversion": "unset@unset",
+				"service.version":      static.Version,
+				"custom.key":           "custom.value",
 			},
 		},
 	}

core/web/testdata/body/health.html

@@ -93,6 +93,9 @@
 <details open>
     <summary title="HeadReporter" class="noexpand"><span class="passing">HeadReporter</span></summary>
 </details>
+<details open>
+    <summary title="Heartbeat" class="noexpand"><span class="passing">Heartbeat</span></summary>
+</details>
 <details open>
     <summary title="JobSpawner" class="noexpand"><span class="passing">JobSpawner</span></summary>
 </details>

core/web/testdata/body/health.json

@@ -153,6 +153,15 @@
         "output": ""
       }
     },
+    {
+      "type": "checks",
+      "id": "Heartbeat",
+      "attributes": {
+        "name": "Heartbeat",
+        "status": "passing",
+        "output": ""
+      }
+    },
     {
       "type": "checks",
       "id": "JobSpawner",

core/web/testdata/body/health.txt

@@ -16,6 +16,7 @@ ok EVM.0.Txm.Confirmer
 ok EVM.0.Txm.Finalizer
 ok EVM.0.Txm.WrappedEvmEstimator
 ok HeadReporter
+ok Heartbeat
 ok JobSpawner
 ok Mailbox.Monitor
 ok Mercury.WSRPCPool

testdata/scripts/health/default.txtar

@@ -32,6 +32,7 @@ HTTPPort = $PORT
 
 -- out.txt --
 ok HeadReporter
+ok Heartbeat
 ok JobSpawner
 ok Mailbox.Monitor
 ok Mercury.WSRPCPool
@@ -55,6 +56,15 @@ ok WorkflowDBStore
         "output": ""
       }
     },
+    {
+      "type": "checks",
+      "id": "Heartbeat",
+      "attributes": {
+        "name": "Heartbeat",
+        "status": "passing",
+        "output": ""
+      }
+    },
     {
       "type": "checks",
       "id": "JobSpawner",

testdata/scripts/health/multi-chain.txtar

@@ -86,6 +86,7 @@ ok EVM.1.Txm.Confirmer
 ok EVM.1.Txm.Finalizer
 ok EVM.1.Txm.WrappedEvmEstimator
 ok HeadReporter
+ok Heartbeat
 ok JobSpawner
 ok Mailbox.Monitor
 ok Mercury.WSRPCPool
@@ -263,6 +264,15 @@ ok WorkflowDBStore
         "output": ""
       }
     },
+    {
+      "type": "checks",
+      "id": "Heartbeat",
+      "attributes": {
+        "name": "Heartbeat",
+        "status": "passing",
+        "output": ""
+      }
+    },
     {
       "type": "checks",
       "id": "JobSpawner",