feat: stuck keys

docker/validators/stuck_keys.yaml

@@ -0,0 +1,4 @@
+# this list of stuck validators keys is used to ignore some validators
+# in metrics calculation
+keys:
+  - "0x000000000000"

src/common/config/env.validation.ts

@@ -180,6 +180,12 @@ export class EnvironmentVariables {
   @IsString()
   public VALIDATOR_REGISTRY_LIDO_SOURCE_SQLITE_CACHE_PATH = './docker/validators/lido_mainnet.db';
 
+  /**
+   * Path to file with list of validators that are stuck and should be excluded from the monitoring metrics
+   */
+  @IsString()
+  public VALIDATOR_STUCK_KEYS_FILE_PATH = './docker/validators/stuck_keys.yaml';
+
   /**
    * Distance (down) from Blockchain Sync Participation average after which we think that our sync participation is bad
    * For example:

src/common/prometheus/prometheus.service.ts

@@ -89,6 +89,7 @@ export enum PrometheusValStatus {
   Slashed = 'slashed',
   WithdrawalPending = 'withdrawal_pending',
   WithdrawalDone = 'withdrawal_done',
+  Stuck = 'stuck',
 }
 
 enum TaskStatus {

src/common/validators-registry/registry.service.ts

@@ -1,25 +1,36 @@
+import { readFile } from 'fs/promises';
+
 import { LOGGER_PROVIDER } from '@lido-nestjs/logger';
 import { Inject, Injectable, LoggerService } from '@nestjs/common';
+import { load } from 'js-yaml';
 
+import { ConfigService } from 'common/config';
 import { PrometheusService, TrackTask } from 'common/prometheus';
 
 import { REGISTRY_SOURCE, RegistrySource, RegistrySourceKeyWithOperatorName } from './registry-source.interface';
 
+interface StuckValidatorsFileContent {
+  keys: string[];
+}
+
 @Injectable()
 export class RegistryService {
   constructor(
     @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService,
     @Inject(REGISTRY_SOURCE) public readonly source: RegistrySource,
     protected readonly prometheus: PrometheusService,
+    protected readonly config: ConfigService,
   ) {}
 
   protected lastTimestamp = 0;
 
   protected operators = [];
+  protected stuckKeys = [];
 
   @TrackTask('update-validators')
   public async updateKeysRegistry(timestamp: number): Promise<void> {
-    await this.source.update();
+    const tasks = await Promise.all([this.source.update(), this.readStuckKeysFile()]);
+    this.stuckKeys = tasks[1];
     await this.updateTimestamp();
     if (timestamp > this.lastTimestamp) {
       const lastUpdateTime = new Date(this.lastTimestamp * 1000).toISOString();
@@ -39,6 +50,24 @@ export class RegistryService {
     return this.operators;
   }
 
+  public getStuckKeys() {
+    return this.stuckKeys;
+  }
+
+  /**
+   * Returns keys of validators that are stuck and will not be monitored
+   * */
+  protected async readStuckKeysFile(): Promise<string[]> {
+    try {
+      const fileContent = await readFile(this.config.get('VALIDATOR_STUCK_KEYS_FILE_PATH'), 'utf-8');
+      const data = <StuckValidatorsFileContent>load(fileContent);
+      return data.keys;
+    } catch (e) {
+      this.logger.error(`Error while reading stuck validators file: ${e.message}`);
+      return [];
+    }
+  }
+
   /**
    * Updates timestamp of last source validators registry update
    */

src/duty/duty.metrics.ts

@@ -2,7 +2,6 @@ import { LOGGER_PROVIDER } from '@lido-nestjs/logger';
 import { Inject, Injectable, LoggerService } from '@nestjs/common';
 
 import { ConfigService } from 'common/config';
-import { ConsensusProviderService } from 'common/eth-providers';
 import { Epoch } from 'common/eth-providers/consensus-provider/types';
 import { allSettled } from 'common/functions/allSettled';
 import { PrometheusService, TrackTask } from 'common/prometheus';
@@ -21,7 +20,6 @@ export class DutyMetrics {
     @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService,
     protected readonly config: ConfigService,
     protected readonly prometheus: PrometheusService,
-    protected readonly clClient: ConsensusProviderService,
 
     protected readonly stateMetrics: StateMetrics,
     protected readonly attestationMetrics: AttestationMetrics,

src/duty/state/state.metrics.ts

@@ -100,6 +100,15 @@ export class StateMetrics {
       },
       (item) => item.withdrawn,
     );
+    setUserOperatorsMetric(
+      this.prometheus.userValidators,
+      data,
+      this.operators,
+      {
+        status: PrometheusValStatus.Stuck,
+      },
+      (item) => item.stuck,
+    );
   }
 
   private async userValidatorsStats() {
@@ -140,6 +149,13 @@ export class StateMetrics {
       },
       result.withdrawn,
     );
+    this.prometheus.validators.set(
+      {
+        owner: Owner.USER,
+        status: PrometheusValStatus.Stuck,
+      },
+      result.stuck,
+    );
   }
 
   private async otherValidatorsStats() {

src/duty/state/state.service.ts

@@ -39,6 +39,7 @@ export class StateService {
     this.logger.log('Processing all validators state');
     let activeValidatorsCount = 0;
     let activeValidatorsEffectiveBalance = 0n;
+    const stuckKeys = this.registry.getStuckKeys();
     const pipeline = chain([
       readStream,
       parser(),
@@ -61,6 +62,7 @@ export class StateService {
             val_status: state.status,
             val_balance: BigInt(state.balance),
             val_effective_balance: BigInt(state.validator.effective_balance),
+            val_stuck: stuckKeys.includes(state.validator.pubkey),
           });
           if ([ValStatus.ActiveOngoing, ValStatus.ActiveExiting, ValStatus.ActiveSlashed].includes(state.status)) {
             activeValidatorsCount++;

src/duty/summary/summary.service.ts

@@ -19,6 +19,7 @@ export interface ValidatorDutySummary {
   val_status?: ValStatus;
   val_balance?: bigint;
   val_balance_withdrawn?: bigint;
+  val_stuck?: boolean;
   val_effective_balance?: bigint;
   ///
   is_proposer?: boolean;