feat(p2p): Hermes Peer2Peer Communications with IPFS/LibP2P

.config/dictionaries/project.dic

@@ -79,6 +79,8 @@ highwater
 hmod
 ideascale
 idents
+ipfs
+ipld
 IFMT
 Intellij
 ioerr
@@ -92,12 +94,15 @@ lcov
 Leshiy
 libsqlite
 libtest
+libipld
+libp2p
 linkat
 lintfix
 localizable
 lookaside
 maindbname
 mdlint
+mdns
 miniprotocol
 miniprotocols
 mithril
@@ -106,6 +111,7 @@ mkcron
 mkdelay
 mkdirat
 moderations
+Multiaddr
 multiera
 nanos
 netkey
@@ -150,6 +156,7 @@ rustdoc
 rustdocflags
 rustflags
 rustfmt
+rustyline
 saibatizoku
 sandboxed
 scanorder
@@ -177,7 +184,9 @@ Traceback
 txns
 typenum
 unfinalized
+unixfs
 unlinkat
+upnp
 utimensat
 vitss
 voteplan

Earthfile

@@ -1,7 +1,7 @@
 VERSION 0.8
 
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/mdlint:v3.1.7 AS mdlint-ci
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/cspell:v3.1.7 AS cspell-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/mdlint:v3.1.8 AS mdlint-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/cspell:v3.1.8 AS cspell-ci
 
 FROM debian:stable-slim
 

docs/Earthfile

@@ -1,6 +1,6 @@
 VERSION 0.8
 
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/docs:v3.1.7 AS docs-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/docs:v3.1.8 AS docs-ci
 
 IMPORT .. AS repo
 IMPORT ../hermes AS hermes
@@ -40,4 +40,4 @@ local:
     COPY +docs/ /usr/share/nginx/html
 
     # This is a local only image, we do not publish it.
-    SAVE IMAGE hermes-docs:latest
+    SAVE IMAGE hermes-docs:latest

hermes/Cargo.toml

@@ -3,6 +3,7 @@ resolver = "2"
 members = [
     "bin",
     "crates/cardano-chain-follower",
+    "crates/hermes-ipfs",
 ]
 
 [workspace.package]
@@ -100,3 +101,6 @@ blake2b_simd = "1.0.2"
 sha2 = "0.10"
 ed25519-dalek = "2.1.1"
 x509-cert = "0.2.5"
+libipld = "0.16.0"
+rust-ipfs = "0.11.19"
+rustyline-async = "0.4.2"

hermes/Earthfile

@@ -1,6 +1,6 @@
 VERSION 0.8
 
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.1.7 AS rust-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.1.8 AS rust-ci
 # Use when debugging cat-ci locally.
 # IMPORT ../../catalyst-ci/earthly/rust AS rust-ci
 

hermes/crates/cardano-chain-follower/testbed/overhead_benchmark/Earthfile

@@ -1,6 +1,6 @@
 VERSION 0.8
 
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/mithril_snapshot:v3.1.7 AS mithril-snapshot-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/mithril_snapshot:v3.1.8 AS mithril-snapshot-ci
 IMPORT ../../../.. AS hermes
 
 build:

hermes/crates/cbork/Earthfile

@@ -1,6 +1,6 @@
 VERSION 0.8
 
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.1.7 AS rust-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.1.8 AS rust-ci
 # Use when debugging cat-ci locally.
 # IMPORT ../../catalyst-ci/earthly/rust AS rust-ci
 
@@ -47,4 +47,4 @@ build:
 # Test which runs check with all supported host tooling.  Needs qemu or rosetta to run.
 # Only used to validate tooling is working across host toolsets.
 build-all-hosts:    
-    BUILD --platform=linux/amd64 --platform=linux/arm64 +build-hosted
+    BUILD --platform=linux/amd64 --platform=linux/arm64 +build-hosted

hermes/crates/cbork/deny.toml

@@ -30,7 +30,7 @@ deny = [
     { crate = "openssl-sys", use-instead = "rustls" },
     "libssh2-sys",
     # { crate = "cmake", use-instead = "cc" },
-    { crate = "windows", reason = "bloated and unnecessary", use-instead = "ideally inline bindings, practically, windows-sys" },
+    # { crate = "windows", reason = "bloated and unnecessary", use-instead = "ideally inline bindings, practically, windows-sys" },
 ]
 skip = [
     # { crate = "[email protected]", reason = "https://github.com/seanmonstar/reqwest/pull/2130 should be in the next version" },
@@ -72,6 +72,7 @@ allow = [
     "CC0-1.0",
     "ISC",
     "Unicode-3.0",
+    "MPL-2.0",
 ]
 exceptions = [
     #{ allow = ["Zlib"], crate = "tinyvec" },
@@ -89,6 +90,11 @@ crate = "hdf5-src"
 expression = "MIT"
 license-files = [{ path = "../LICENSE-MIT", hash = 0x001c7e6c }]
 
+[[licenses.clarify]]
+crate = "ring"
+expression = "MIT"
+license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }]
+
 # SPDX considers OpenSSL to encompass both the OpenSSL and SSLeay licenses
 # https://spdx.org/licenses/OpenSSL.html
 # ISC - Both BoringSSL and ring use this for their new files

hermes/crates/hermes-ipfs/.gitignore

@@ -0,0 +1,11 @@
+### Rust ###
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+target/
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb

hermes/crates/hermes-ipfs/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "hermes-ipfs"
+edition.workspace = true
+version.workspace = true
+authors.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow.workspace = true
+libipld.workspace = true
+rust-ipfs.workspace = true
+
+[dev-dependencies]
+# Dependencies used by examples
+clap = { workspace = true, features = ["derive"] }
+tokio.workspace = true
+tracing-subscriber.workspace = true
+rustyline-async.workspace = true

hermes/crates/hermes-ipfs/examples/add-file-with-pinning.rs

@@ -0,0 +1,65 @@
+//! Hermes IPFS File Publishing and Pinning
+
+use hermes_ipfs::{Cid, HermesIpfs};
+
+/// Print helper
+async fn print_cid_pinned(hermes_ipfs: &HermesIpfs, cid: &Cid) -> anyhow::Result<()> {
+    let is_pinned = hermes_ipfs.is_pinned(cid).await?;
+    println!("* Is CID pinned?: {is_pinned:?}");
+    Ok(())
+}
+
+/// Example application.
+#[tokio::main]
+#[allow(clippy::println_empty_string)]
+async fn main() -> anyhow::Result<()> {
+    let hermes_ipfs = HermesIpfs::start().await?;
+    println!("***************************************");
+    println!("* Hermes IPFS node has started.");
+    println!("***************************************");
+    println!("");
+    println!("***************************************");
+    println!("* Adding file to IPFS:");
+    println!("");
+    let ipfs_file = b"This is a demo file that is stored in IPFS.".to_vec();
+    let ipfs_path = hermes_ipfs.add_ipfs_file(ipfs_file.into()).await?;
+    println!("* IPFS file published at {ipfs_path}");
+    let cid = ipfs_path.root().cid().ok_or(anyhow::anyhow!(
+        "ERROR! Could not extract CID from IPFS path."
+    ))?;
+    println!("* CID: {cid}");
+    println!("* CID Version: {:?}", cid.version());
+    print_cid_pinned(&hermes_ipfs, cid).await?;
+    println!("***************************************");
+    println!("");
+    println!("***************************************");
+    println!("* CID Pinning:");
+    println!("");
+    println!("* Removing pin.");
+    hermes_ipfs.remove_pin(cid).await?;
+    print_cid_pinned(&hermes_ipfs, cid).await?;
+    println!("");
+    println!("* Re-pinning CID:.");
+    hermes_ipfs.insert_pin(cid).await?;
+    print_cid_pinned(&hermes_ipfs, cid).await?;
+    println!("***************************************");
+    println!("");
+    println!("***************************************");
+    println!("* Get file from IPFS:");
+    println!("");
+    println!("* Retrieving from {ipfs_path}");
+    let get_file_bytes = hermes_ipfs.get_ipfs_file(ipfs_path.into()).await?;
+    println!("* Got file, {} bytes:", get_file_bytes.len());
+    let get_file = String::from_utf8(get_file_bytes)?;
+    println!("* FILE CONTENTS:");
+    println!("");
+    println!("{get_file}");
+    println!("");
+    println!("***************************************");
+    println!("");
+    hermes_ipfs.stop().await;
+    println!("***************************************");
+    println!("* Hermes IPFS node has stopped.");
+    println!("***************************************");
+    Ok(())
+}

hermes/crates/hermes-ipfs/examples/provide-content-dht.rs

@@ -0,0 +1,113 @@
+//! Use Hermes IPFS to distribute content using DHT
+#![allow(clippy::println_empty_string)]
+
+use hermes_ipfs::{HermesIpfs, IpfsPath};
+
+/// Connect Node A, upload file and provide CID by adding to DHT
+async fn connect_node_a_upload_and_provide(
+    file_content: Vec<u8>,
+) -> anyhow::Result<(HermesIpfs, IpfsPath)> {
+    let hermes_ipfs = HermesIpfs::start().await?;
+    println!("***************************************");
+    println!("* Hermes IPFS node A has started.");
+    println!("");
+    let peer_info = hermes_ipfs.identity(None).await?;
+    let peer_id_a = peer_info.peer_id;
+    let addresses = hermes_ipfs.listening_addresses().await?;
+    println!("* Peer ID: {peer_id_a}");
+    for addr in addresses {
+        println!("    * {addr}");
+    }
+    println!("***************************************");
+    println!("");
+    println!("***************************************");
+    println!("* Adding file to IPFS:");
+    println!("");
+    let ipfs_path = hermes_ipfs.add_ipfs_file(file_content.into()).await?;
+    println!("* IPFS file published at {ipfs_path}");
+    let cid = ipfs_path.root().cid().ok_or(anyhow::anyhow!(
+        "ERROR! Could not extract CID from IPFS path."
+    ))?;
+    println!("* CID: {cid}");
+    println!("* CID Version: {:?}", cid.version());
+    println!("***************************************");
+    println!("");
+    println!("***************************************");
+    println!("* Providing content to DHT:");
+    println!("");
+    println!("* Providing {cid} as peer {peer_id_a}");
+    println!("***************************************");
+    println!("");
+    Ok((hermes_ipfs, ipfs_path))
+}
+
+/// Connect Node A, upload file and provide CID by adding to DHT
+async fn connect_node_b_to_node_a(node_a: &HermesIpfs) -> anyhow::Result<HermesIpfs> {
+    let hermes_ipfs_b = HermesIpfs::start().await?;
+    println!("***************************************");
+    println!("* Hermes IPFS node B has started.");
+    println!("");
+    let peer_info = hermes_ipfs_b.identity(None).await?;
+    let peer_id_b = peer_info.peer_id;
+    // node_b.connect(peer_id_a).await?;
+    println!("* Peer ID: {peer_id_b}");
+    println!("* Listening addresses:");
+    let addresses = hermes_ipfs_b.listening_addresses().await?;
+    for addr in addresses {
+        println!("    * {addr}");
+    }
+    println!("***************************************");
+    println!("");
+    println!("***************************************");
+    println!("* Connecting Node B to Node A:");
+    println!("");
+    println!("* Adding peer listening addresses from Node A:");
+    let node_a_addresses = node_a.listening_addresses().await?;
+    let peer_a = node_a.identity(None).await?.peer_id;
+    for addr in node_a_addresses {
+        hermes_ipfs_b.add_peer(peer_a, addr.clone()).await?;
+        println!("    * {addr} - CONNECTED");
+    }
+    println!("***************************************");
+    println!("");
+    Ok(hermes_ipfs_b)
+}
+
+/// Example application.
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    // File to be uploaded
+    let ipfs_file = b"DEMO FILE DISTRIBUTED WITH IPFS".to_vec();
+    // Start Node A, publish file, and make node provider for CID
+    let (hermes_ipfs_a, ipfs_path) = connect_node_a_upload_and_provide(ipfs_file.clone()).await?;
+    // Start Node B, add listening addresses from Node A, and
+    // connect to Node A's peer ID.
+    let hermes_ipfs_b = connect_node_b_to_node_a(&hermes_ipfs_a).await?;
+
+    println!("***************************************");
+    println!("* Get content from IPFS path {ipfs_path}");
+    println!("");
+    // For illustration, the `ipfs_path` can be obtained from a known CID
+    let ipfs_path_string = format!("{ipfs_path}");
+
+    // Fetch the content from the `ipfs_path`.
+    let fetched_bytes = hermes_ipfs_b
+        .get_ipfs_file(ipfs_path_string.parse()?)
+        .await?;
+    assert_eq!(ipfs_file, fetched_bytes);
+    let fetched_file = String::from_utf8(fetched_bytes)?;
+    println!("* Fetched: {fetched_file:?}");
+    println!("***************************************");
+    println!("");
+    // Stop the nodes and exit.
+    hermes_ipfs_a.stop().await;
+    println!("***************************************");
+    println!("* Hermes IPFS node A has stopped.");
+    println!("***************************************");
+    hermes_ipfs_b.stop().await;
+    println!("");
+    println!("***************************************");
+    println!("* Hermes IPFS node B has stopped.");
+    println!("***************************************");
+    Ok(())
+}