Merge branch 'main' into lemmih/mpool-push-untrusted

CHANGELOG.md

@@ -48,6 +48,9 @@
 - [#4511](https://github.com/ChainSafe/forest/pull/4511) Add support for the
   `Filecoin.EthMaxPriorityFeePerGas` RPC method.
 
+- [#4474](https://github.com/ChainSafe/forest/pull/4474) Add new subcommand
+  `forest-cli healthcheck ready`.
+
 - [#4547](https://github.com/ChainSafe/forest/pull/4547) Add support for the
   `Filecoin.MpoolPushUntrusted` RPC method.
 

documentation/src/healthcheck.md

@@ -53,8 +53,9 @@ accept traffic. In our case, we require:
 - The node is in sync with the network
 - The current epoch of the node is not too far behind the network
 - The RPC server is running
+- The Ethereum mapping is up to date
 
-If any of these conditions are not met, the nod is **not** ready to serve
+If any of these conditions are not met, the node is **not** ready to serve
 requests.
 
 Sample _ready_ response:
@@ -63,7 +64,8 @@ Sample _ready_ response:
 ❯ curl "http://127.0.0.1:2346/readyz?verbose"
 [+] sync complete
 [+] epoch up to date
-[+] rpc server running⏎
+[+] rpc server running
+[+] eth mapping up to date⏎
 ```
 
 Sample _not ready_ response:
@@ -72,7 +74,8 @@ Sample _not ready_ response:
 ❯ curl "http://127.0.0.1:2346/readyz?verbose"
 [!] sync incomplete
 [!] epoch outdated
-[+] rpc server running⏎
+[+] rpc server running
+[!] no eth mapping⏎
 ```
 
 ### `/healthz`

scripts/tests/api_compare/.env

@@ -5,6 +5,7 @@ FIL_PROOFS_PARAMETER_CACHE=/var/tmp/filecoin-proof-parameters
 LOTUS_RPC_PORT=1234
 FOREST_RPC_PORT=2345
 FOREST_OFFLINE_RPC_PORT=3456
+FOREST_HEALTHZ_RPC_PORT=2346
 CHAIN=calibnet
 
 # This is a pre-generated miner generated from Lotus

scripts/tests/api_compare/docker-compose.yml

@@ -52,6 +52,7 @@ services:
 
         forest --chain ${CHAIN} --encrypt-keystore false --no-gc \
           --rpc-address 0.0.0.0:${FOREST_RPC_PORT} \
+          --healthcheck-address 0.0.0.0:${FOREST_HEALTHZ_RPC_PORT} \
           --height=-50 \
           --import-snapshot $(ls /data/*.car.zst | tail -n 1)
     healthcheck:
@@ -60,6 +61,26 @@ services:
       timeout: 10m
       retries: 3
       start_period: 10m
+  forest-rpc-ready:
+    depends_on:
+      forest:
+        condition: service_healthy
+    build:
+      context: ../../../.
+      dockerfile: ${FOREST_DOCKERFILE_OVERRIDE:-Dockerfile}
+    volumes:
+      - node-data:/data
+    networks:
+      - api-tests
+    entrypoint: [ "/bin/bash", "-c" ]
+    user: 0:0
+    command:
+      - |
+        set -euo pipefail
+        
+        export FULLNODE_API_INFO="$(cat /data/forest-token):/dns/forest/tcp/${FOREST_RPC_PORT}/http"
+        echo "Waiting till Forest is ready"
+        forest-cli healthcheck ready --healthcheck-port ${FOREST_HEALTHZ_RPC_PORT} --wait
   forest-wallet-import:
     depends_on:
       forest:
@@ -165,6 +186,8 @@ services:
         condition: service_healthy
       forest-wallet-import:
         condition: service_completed_successfully
+      forest-rpc-ready:
+        condition: service_completed_successfully
     build:
       context: ../../../.
       dockerfile: ${FOREST_DOCKERFILE_OVERRIDE:-Dockerfile}

scripts/tests/calibnet_eth_mapping_check.sh

@@ -44,6 +44,11 @@ for ((i=0; i<=NUM_TIPSETS; i++)); do
   EPOCH=$((EPOCH - 1))
 done
 
+echo "Done"
+
+echo "Waiting eth mapping to be up to date"
+$FOREST_CLI_PATH healthcheck ready --wait
+
 ERROR=0
 echo "Testing Ethereum mapping"
 

src/chain/store/chain_store.rs

@@ -380,6 +380,10 @@ where
         Ok((lbts, *next_ts.parent_state()))
     }
 
+    pub fn settings(&self) -> Arc<dyn SettingsStore + Sync + Send> {
+        self.settings.clone()
+    }
+
     /// Filter [`SignedMessage`]'s to keep only the most recent ones, then write corresponding entries to the Ethereum mapping.
     pub fn process_signed_messages(&self, messages: &[(SignedMessage, u64)]) -> anyhow::Result<()>
     where

src/cli/main.rs

@@ -47,6 +47,7 @@ where
                 Subcommand::Snapshot(cmd) => cmd.run(client).await,
                 Subcommand::Attach { .. } => bail!("the `attach` subcommand has been removed. Please raise an issue if this breaks a workflow for you"),
                 Subcommand::Shutdown(cmd) => cmd.run(client).await,
+                Subcommand::Healthcheck(cmd) => cmd.run(client).await,
             }
         })
 }

src/cli/subcommands/healthcheck_cmd.rs

@@ -0,0 +1,72 @@
+// Copyright 2019-2024 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+use std::{
+    io::{stdout, Write},
+    time::Duration,
+};
+
+use crate::health::DEFAULT_HEALTHCHECK_PORT;
+use crate::rpc;
+use clap::Subcommand;
+use http::StatusCode;
+use ticker::Ticker;
+
+#[derive(Debug, Subcommand)]
+pub enum HealthcheckCommand {
+    /// Display ready status
+    Ready {
+        /// Don't exit until node is ready
+        #[arg(long)]
+        wait: bool,
+        /// Healthcheck port
+        #[arg(long, default_value_t=DEFAULT_HEALTHCHECK_PORT)]
+        healthcheck_port: u16,
+    },
+}
+
+impl HealthcheckCommand {
+    pub async fn run(self, client: rpc::Client) -> anyhow::Result<()> {
+        match self {
+            Self::Ready {
+                wait,
+                healthcheck_port,
+            } => {
+                let ticker = Ticker::new(0.., Duration::from_secs(1));
+                let mut stdout = stdout();
+
+                let url = format!(
+                    "http://{}:{}/readyz?verbose",
+                    client.base_url().host_str().unwrap_or("localhost"),
+                    healthcheck_port,
+                );
+
+                for _ in ticker {
+                    let response = reqwest::get(&url).await?;
+                    let status = response.status();
+                    let text = response.text().await?;
+
+                    println!("{}", text);
+
+                    if !wait {
+                        break;
+                    }
+                    if status == StatusCode::OK {
+                        println!("Done!");
+                        break;
+                    }
+
+                    for _ in 0..(text.matches('\n').count() + 1) {
+                        write!(
+                            stdout,
+                            "\r{}{}",
+                            anes::MoveCursorUp(1),
+                            anes::ClearLine::All,
+                        )?;
+                    }
+                }
+                Ok(())
+            }
+        }
+    }
+}

src/cli/subcommands/mod.rs

@@ -9,6 +9,7 @@
 mod auth_cmd;
 mod chain_cmd;
 mod config_cmd;
+mod healthcheck_cmd;
 mod info_cmd;
 mod mpool_cmd;
 mod net_cmd;
@@ -29,9 +30,9 @@ use tracing::error;
 
 pub(super) use self::{
     auth_cmd::AuthCommands, chain_cmd::ChainCommands, config_cmd::ConfigCommands,
-    mpool_cmd::MpoolCommands, net_cmd::NetCommands, send_cmd::SendCommand,
-    shutdown_cmd::ShutdownCommand, snapshot_cmd::SnapshotCommands, state_cmd::StateCommands,
-    sync_cmd::SyncCommands,
+    healthcheck_cmd::HealthcheckCommand, mpool_cmd::MpoolCommands, net_cmd::NetCommands,
+    send_cmd::SendCommand, shutdown_cmd::ShutdownCommand, snapshot_cmd::SnapshotCommands,
+    state_cmd::StateCommands, sync_cmd::SyncCommands,
 };
 use crate::cli::subcommands::info_cmd::InfoCommand;
 
@@ -94,6 +95,10 @@ pub enum Subcommand {
 
     /// Shutdown Forest
     Shutdown(ShutdownCommand),
+
+    /// Print healthcheck info
+    #[command(subcommand)]
+    Healthcheck(HealthcheckCommand),
 }
 
 /// Format a vector to a prettified string

src/cli_shared/mod.rs

@@ -5,6 +5,8 @@ pub mod cli;
 pub mod logger;
 
 use crate::cli_shared::cli::{find_config_path, Config, ConfigPath};
+use crate::db::db_engine::db_root;
+use crate::db::CAR_DB_DIR_NAME;
 use crate::networks::NetworkChain;
 use crate::utils::io::read_toml;
 use std::path::PathBuf;
@@ -19,6 +21,14 @@ pub fn chain_path(config: &Config) -> PathBuf {
     PathBuf::from(&config.client.data_dir).join(config.chain.to_string())
 }
 
+/// Gets car db path
+pub fn car_db_path(config: &Config) -> anyhow::Result<PathBuf> {
+    let chain_data_path = chain_path(config);
+    let db_root_dir = db_root(&chain_data_path)?;
+    let forest_car_db_dir = db_root_dir.join(CAR_DB_DIR_NAME);
+    Ok(forest_car_db_dir)
+}
+
 pub fn read_config(
     config_path_opt: Option<&PathBuf>,
     chain_opt: Option<NetworkChain>,

src/daemon/db_util.rs

@@ -168,15 +168,20 @@ where
 {
     let mut delegated_messages = vec![];
 
-    info!("Populating column EthMappings");
+    let hygge = state_manager.chain_config().epoch(Height::Hygge);
+    tracing::info!(
+        "Populating column EthMappings from range: [{}, {}]",
+        hygge,
+        head_ts.epoch()
+    );
 
     for ts in head_ts
         .clone()
         .chain(&state_manager.chain_store().blockstore())
     {
         // Hygge is the start of Ethereum support in the FVM (through the FEVM actor).
         // Before this height, no notion of an Ethereum-like API existed.
-        if ts.epoch() < state_manager.chain_config().epoch(Height::Hygge) {
+        if ts.epoch() < hygge {
             break;
         }
         delegated_messages.append(

src/daemon/mod.rs

@@ -9,7 +9,7 @@ use crate::auth::{create_token, generate_priv_key, ADMIN, JWT_IDENTIFIER};
 use crate::blocks::Tipset;
 use crate::chain::ChainStore;
 use crate::chain_sync::ChainMuxer;
-use crate::cli_shared::snapshot;
+use crate::cli_shared::{car_db_path, snapshot};
 use crate::cli_shared::{
     chain_path,
     cli::{CliOpts, Config},
@@ -20,7 +20,7 @@ use crate::daemon::db_util::{
 };
 use crate::db::car::ManyCar;
 use crate::db::db_engine::{db_root, open_db};
-use crate::db::{ttl::EthMappingCollector, MarkAndSweep};
+use crate::db::{ttl::EthMappingCollector, MarkAndSweep, MemoryDB, SettingsExt, CAR_DB_DIR_NAME};
 use crate::genesis::{get_network_name_from_genesis, read_genesis_header};
 use crate::key_management::{
     KeyStore, KeyStoreConfig, ENCRYPTED_KEYSTORE_NAME, FOREST_KEYSTORE_PHRASE_ENV,
@@ -43,6 +43,7 @@ use bundle::load_actor_bundles;
 use dialoguer::console::Term;
 use dialoguer::theme::ColorfulTheme;
 use futures::{select, Future, FutureExt};
+use fvm_ipld_blockstore::Blockstore;
 use once_cell::sync::Lazy;
 use raw_sync_2::events::{Event, EventInit as _, EventState};
 use shared_memory::ShmemConf;
@@ -181,7 +182,7 @@ pub(super) async fn start(
     let db_root_dir = db_root(&chain_data_path)?;
     let db_writer = Arc::new(open_db(db_root_dir.clone(), config.db_config().clone())?);
     let db = Arc::new(ManyCar::new(db_writer.clone()));
-    let forest_car_db_dir = db_root_dir.join("car_db");
+    let forest_car_db_dir = db_root_dir.join(CAR_DB_DIR_NAME);
     load_all_forest_cars(&db, &forest_car_db_dir)?;
 
     if config.client.load_actors && !opts.stateless {
@@ -368,6 +369,7 @@ pub(super) async fn start(
             genesis_timestamp: genesis_header.timestamp,
             sync_state: sync_state.clone(),
             peer_manager,
+            settings_store: chain_store.settings(),
         };
 
         let listener =
@@ -453,8 +455,6 @@ pub(super) async fn start(
             state_manager
                 .chain_store()
                 .set_heaviest_tipset(Arc::new(ts.clone()))?;
-
-            populate_eth_mappings(&state_manager, &ts)?;
         }
     }
 
@@ -483,6 +483,17 @@ pub(super) async fn start(
         return Ok(());
     }
 
+    // Populate task
+    if !opts.stateless && !chain_config.is_devnet() {
+        let state_manager = Arc::clone(&state_manager);
+        services.spawn(async move {
+            if let Err(err) = init_ethereum_mapping(state_manager, &config) {
+                tracing::warn!("Init Ethereum mapping failed: {}", err)
+            }
+            Ok(())
+        });
+    }
+
     if !opts.stateless {
         ensure_params_downloaded().await?;
     }
@@ -758,3 +769,32 @@ fn display_chain_logo(chain: &NetworkChain) {
         info!("\n{logo}");
     }
 }
+
+fn init_ethereum_mapping<DB: Blockstore>(
+    state_manager: Arc<StateManager<DB>>,
+    config: &Config,
+) -> anyhow::Result<()> {
+    match state_manager
+        .chain_store()
+        .settings()
+        .eth_mapping_up_to_date()?
+    {
+        Some(false) | None => {
+            let car_db_path = car_db_path(config)?;
+            let db: Arc<ManyCar<MemoryDB>> = Arc::default();
+            load_all_forest_cars(&db, &car_db_path)?;
+            let ts = db.heaviest_tipset()?;
+
+            populate_eth_mappings(&state_manager, &ts)?;
+
+            state_manager
+                .chain_store()
+                .settings()
+                .set_eth_mapping_up_to_date()
+        }
+        Some(true) => {
+            tracing::info!("Ethereum mapping up to date");
+            Ok(())
+        }
+    }
+}