fix(hermes-lib): Make download fully sync, and simplify re-order queuing

.config/dictionaries/project.dic

@@ -116,6 +116,7 @@ logcall
 lookaside
 maindbname
 mapref
+Mbits
 mdlint
 mdns
 memeq
@@ -223,6 +224,7 @@ unixfs
 unlinkat
 untar
 upnp
+ureq
 utimensat
 vitss
 vkey

hermes/Cargo.toml

@@ -164,4 +164,8 @@ minicbor = {version = "0.24.2", features = ["alloc", "derive", "half"]}
 brotli = "6.0.0"
 c509-certificate = { git = "https://github.com/input-output-hk/catalyst-voices.git", package = "c509-certificate", branch = "fix/c509-cleanup"}
 num-traits = "0.2.19"
-logcall = "0.1.9"
+logcall = "0.1.9"
+ureq = {version = "2.10.1", features=["native-certs"]}
+http = "1.1.0"
+hickory-resolver = { version = "0.24.1", features = ["dns-over-rustls"] }
+moka = { version = "0.12.8", features = ["sync"]}

hermes/crates/cardano-chain-follower/Cargo.toml

@@ -52,6 +52,11 @@ num-traits.workspace = true
 logcall.workspace = true
 tracing-log.workspace = true
 tar.workspace = true
+ureq.workspace = true
+http.workspace = true
+crossbeam-channel.workspace = true
+hickory-resolver.workspace = true
+moka.workspace = true
 
 [dev-dependencies]
 hex.workspace = true

hermes/crates/cardano-chain-follower/examples/follow_chains.rs

@@ -43,6 +43,8 @@ fn process_argument() -> (Vec<Network>, ArgMatches) {
                 .action(ArgAction::SetTrue),
             arg!(--"largest-metadata" "Dump The largest transaction metadata we find (as we find it).")
                 .action(ArgAction::SetTrue),
+            arg!(--"high-speed-sync" "Tune the download to get faster sync speeds (~800 Mbits/s).")
+                .action(ArgAction::SetTrue),
         ])
         .get_matches();
 

hermes/crates/cardano-chain-follower/src/mithril_snapshot_sync.rs

@@ -99,13 +99,7 @@ async fn get_snapshot_by_id(
 /// Create a client, should never fail, but return None if it does, because we can't
 /// continue.
 fn create_client(cfg: &MithrilSnapshotConfig) -> Option<(Client, Arc<MithrilTurboDownloader>)> {
-    let downloader = match MithrilTurboDownloader::new(cfg.clone()) {
-        Ok(downloader) => Arc::new(downloader),
-        Err(err) => {
-            error!(chain = cfg.chain.to_string(), "Unexpected Error [{}]: Unable to create Snapshot Downloader. Mithril Snapshots can not update.", err);
-            return None;
-        },
-    };
+    let downloader = Arc::new(MithrilTurboDownloader::new(cfg.clone()));
 
     // This can't fail, because we already tested it works. But just in case...
     let client = match mithril_client::ClientBuilder::aggregator(