feat: progress bar for iroh add command (#368)

This introduces a progress bar for the `iroh add` command. It touches the following layers: * the various `add_dir`, `add_file` functions in `iroh::resolver::unixfs_builder` now return a stream of `AddEvents` * these are exposed in the `iroh-api` where they are mockable. The `iroh-api` also provides a higher level method to resolve a `AddEvent` stream to a CID * `iroh::size` introduces a way to calculate total size of each file on the filesystem * The `iroh add` command in the `iroh` crate then calculates the total size with a spinner, and then uses this to show a progress bar. Note that `indicatif`, the progress bar library, allows a lot of tweaking of what we see, both color, characters and text. I made a selection to get started. Suggestions for tweaks of what the bar looks like are welcome. I want to also include current file information in the message part of the progress bar, but getting that information requires more rework of the `unixfs_builder` and I think we should punt on that for the time being.
n0-computer · Oct 20, 2022 · bff3560 · bff3560
1 parent 5dbd907
commit bff3560
Show file tree

Hide file tree

Showing 17 changed files with 327 additions and 103 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -5,7 +5,7 @@ parameters:
     type: string
     default: "nightly-2022-06-23"
 
-orbs: 
+orbs:
   aws-cli: circleci/[email protected]
 
 executors:
@@ -172,7 +172,7 @@ commands:
           key: sccache-cache-stable-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ epoch }}
           paths:
             - "~/.cache/sccache"
-    
+
   push-to-s3:
     parameters:
       version:
@@ -191,11 +191,11 @@ commands:
       - run:
           name: push to s3
           command: |
-              source $BASH_ENV
-              aws s3 cp <<parameters.path>>iroh-gateway s3://vorc/iroh-gateway-<<parameters.os>>-<<parameters.arch>>-${CIRCLE_SHA1::7} --no-progress
-              aws s3 cp <<parameters.path>>iroh-p2p s3://vorc/iroh-p2p-<<parameters.os>>-<<parameters.arch>>-${CIRCLE_SHA1::7} --no-progress
-              aws s3 cp <<parameters.path>>iroh-store s3://vorc/iroh-store-<<parameters.os>>-<<parameters.arch>>-${CIRCLE_SHA1::7} --no-progress
-              aws s3 cp <<parameters.path>>iroh s3://vorc/iroh-<<parameters.os>>-<<parameters.arch>>-${CIRCLE_SHA1::7} --no-progress
+            source $BASH_ENV
+            aws s3 cp <<parameters.path>>iroh-gateway s3://vorc/iroh-gateway-<<parameters.os>>-<<parameters.arch>>-${CIRCLE_SHA1::7} --no-progress
+            aws s3 cp <<parameters.path>>iroh-p2p s3://vorc/iroh-p2p-<<parameters.os>>-<<parameters.arch>>-${CIRCLE_SHA1::7} --no-progress
+            aws s3 cp <<parameters.path>>iroh-store s3://vorc/iroh-store-<<parameters.os>>-<<parameters.arch>>-${CIRCLE_SHA1::7} --no-progress
+            aws s3 cp <<parameters.path>>iroh s3://vorc/iroh-<<parameters.os>>-<<parameters.arch>>-${CIRCLE_SHA1::7} --no-progress
 
   push-to-s3-latest:
     parameters:
@@ -212,11 +212,11 @@ commands:
       - run:
           name: push to s3 latest
           command: |
-              source $BASH_ENV
-              aws s3 cp <<parameters.path>>iroh-gateway s3://vorc/iroh-gateway-<<parameters.os>>-<<parameters.arch>>-latest --no-progress
-              aws s3 cp <<parameters.path>>iroh-p2p s3://vorc/iroh-p2p-<<parameters.os>>-<<parameters.arch>>-latest --no-progress
-              aws s3 cp <<parameters.path>>iroh-store s3://vorc/iroh-store-<<parameters.os>>-<<parameters.arch>>-latest --no-progress
-              aws s3 cp <<parameters.path>>iroh s3://vorc/iroh-<<parameters.os>>-<<parameters.arch>>-latest --no-progress
+            source $BASH_ENV
+            aws s3 cp <<parameters.path>>iroh-gateway s3://vorc/iroh-gateway-<<parameters.os>>-<<parameters.arch>>-latest --no-progress
+            aws s3 cp <<parameters.path>>iroh-p2p s3://vorc/iroh-p2p-<<parameters.os>>-<<parameters.arch>>-latest --no-progress
+            aws s3 cp <<parameters.path>>iroh-store s3://vorc/iroh-store-<<parameters.os>>-<<parameters.arch>>-latest --no-progress
+            aws s3 cp <<parameters.path>>iroh s3://vorc/iroh-<<parameters.os>>-<<parameters.arch>>-latest --no-progress
 
 jobs:
   cargo_fetch:
@@ -331,7 +331,7 @@ jobs:
           name: Run cargo clippy (default features)
           command: cargo clippy --all --all-targets -- -D warnings
       - save-sccache-cache
-  
+
   audit:
     executor: docker-executor
     environment:
@@ -380,7 +380,7 @@ jobs:
           os: linux
           arch: amd64
       - save-sccache-cache
-  
+
   build_release_aarch64:
     executor: arm-executor
     steps:
@@ -454,7 +454,6 @@ jobs:
           arch: aarch64
       - save-sccache-cache
 
-
 workflows:
   version: 2.1
 

diff --git a/iroh-api/src/api.rs b/iroh-api/src/api.rs
@@ -5,7 +5,7 @@ use crate::config::{Config, CONFIG_FILE_NAME, ENV_PREFIX};
 #[cfg(feature = "testing")]
 use crate::p2p::MockP2p;
 use crate::p2p::{ClientP2p, P2p};
-use crate::{Cid, IpfsPath};
+use crate::{AddEvent, IpfsPath};
 use anyhow::Result;
 use futures::future::{BoxFuture, LocalBoxFuture};
 use futures::stream::LocalBoxStream;
@@ -48,9 +48,21 @@ pub trait Api {
         ipfs_path: &IpfsPath,
     ) -> LocalBoxStream<'_, Result<(RelativePathBuf, OutType)>>;
 
-    fn add_file<'a>(&'a self, path: &'a Path, wrap: bool) -> LocalBoxFuture<'_, Result<Cid>>;
-    fn add_dir<'a>(&'a self, path: &'a Path, wrap: bool) -> LocalBoxFuture<'_, Result<Cid>>;
-    fn add_symlink<'a>(&'a self, path: &'a Path, wrap: bool) -> LocalBoxFuture<'_, Result<Cid>>;
+    fn add_file(
+        &self,
+        path: &Path,
+        wrap: bool,
+    ) -> LocalBoxFuture<'_, Result<LocalBoxStream<'static, Result<AddEvent>>>>;
+    fn add_dir(
+        &self,
+        path: &Path,
+        wrap: bool,
+    ) -> LocalBoxFuture<'_, Result<LocalBoxStream<'static, Result<AddEvent>>>>;
+    fn add_symlink(
+        &self,
+        path: &Path,
+        wrap: bool,
+    ) -> LocalBoxFuture<'_, Result<LocalBoxStream<'static, Result<AddEvent>>>>;
 
     fn check(&self) -> BoxFuture<'_, StatusTable>;
     fn watch(&self) -> LocalBoxFuture<'static, LocalBoxStream<'static, StatusTable>>;
@@ -131,32 +143,53 @@ impl Api for Iroh {
         .boxed_local()
     }
 
-    fn add_file<'a>(&'a self, path: &'a Path, wrap: bool) -> LocalBoxFuture<'_, Result<Cid>> {
+    fn add_file(
+        &self,
+        path: &Path,
+        wrap: bool,
+    ) -> LocalBoxFuture<'_, Result<LocalBoxStream<'static, Result<AddEvent>>>> {
+        let providing_client = iroh_resolver::unixfs_builder::StoreAndProvideClient {
+            client: self.client.clone(),
+        };
+        let path = path.to_path_buf();
         async move {
-            let providing_client = iroh_resolver::unixfs_builder::StoreAndProvideClient {
-                client: Box::new(&self.client),
-            };
-            unixfs_builder::add_file(Some(&providing_client), path, wrap).await
+            unixfs_builder::add_file(Some(providing_client), &path, wrap)
+                .await
+                .map(|s| s.boxed_local())
         }
         .boxed_local()
     }
 
-    fn add_dir<'a>(&'a self, path: &'a Path, wrap: bool) -> LocalBoxFuture<'_, Result<Cid>> {
+    fn add_dir(
+        &self,
+        path: &Path,
+        wrap: bool,
+    ) -> LocalBoxFuture<'_, Result<LocalBoxStream<'static, Result<AddEvent>>>> {
+        let providing_client = iroh_resolver::unixfs_builder::StoreAndProvideClient {
+            client: self.client.clone(),
+        };
+        let path = path.to_path_buf();
         async move {
-            let providing_client = iroh_resolver::unixfs_builder::StoreAndProvideClient {
-                client: Box::new(&self.client),
-            };
-            unixfs_builder::add_dir(Some(&providing_client), path, wrap).await
+            unixfs_builder::add_dir(Some(providing_client), &path, wrap)
+                .await
+                .map(|s| s.boxed_local())
         }
         .boxed_local()
     }
 
-    fn add_symlink<'a>(&'a self, path: &'a Path, wrap: bool) -> LocalBoxFuture<'_, Result<Cid>> {
+    fn add_symlink(
+        &self,
+        path: &Path,
+        wrap: bool,
+    ) -> LocalBoxFuture<'_, Result<LocalBoxStream<'static, Result<AddEvent>>>> {
+        let providing_client = iroh_resolver::unixfs_builder::StoreAndProvideClient {
+            client: self.client.clone(),
+        };
+        let path = path.to_path_buf();
         async move {
-            let providing_client = iroh_resolver::unixfs_builder::StoreAndProvideClient {
-                client: Box::new(&self.client),
-            };
-            unixfs_builder::add_symlink(Some(&providing_client), path, wrap).await
+            unixfs_builder::add_symlink(Some(providing_client), &path, wrap)
+                .await
+                .map(|s| s.boxed_local())
         }
         .boxed_local()
     }

diff --git a/iroh-api/src/api_ext.rs b/iroh-api/src/api_ext.rs
@@ -1,10 +1,12 @@
 use std::path::{Path, PathBuf};
 
-use crate::{Api, Cid, IpfsPath, OutType};
-use anyhow::{anyhow, Result};
+use crate::{AddEvent, Api, Cid, IpfsPath, OutType};
+use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
+use futures::stream::LocalBoxStream;
 use futures::Stream;
 use futures::StreamExt;
+use futures::TryStreamExt;
 use relative_path::RelativePathBuf;
 
 #[async_trait(?Send)]
@@ -30,7 +32,11 @@ pub trait ApiExt: Api {
         Ok(root_path)
     }
 
-    async fn add(&self, path: &Path, wrap: bool) -> Result<Cid> {
+    async fn add_stream(
+        &self,
+        path: &Path,
+        wrap: bool,
+    ) -> Result<LocalBoxStream<'static, Result<AddEvent>>> {
         if path.is_dir() {
             self.add_dir(path, wrap).await
         } else if path.is_symlink() {
@@ -41,6 +47,21 @@ pub trait ApiExt: Api {
             anyhow::bail!("can only add files or directories")
         }
     }
+
+    async fn add(&self, path: &Path, wrap: bool) -> Result<Cid> {
+        let add_events = self.add_stream(path, wrap).await?;
+
+        add_events
+            .try_fold(None, |acc, add_event| async move {
+                Ok(if let AddEvent::Done(cid) = add_event {
+                    Some(cid)
+                } else {
+                    acc
+                })
+            })
+            .await?
+            .context("No cid found")
+    }
 }
 
 impl<T> ApiExt for T where T: Api {}

diff --git a/iroh-api/src/lib.rs b/iroh-api/src/lib.rs
@@ -14,6 +14,7 @@ pub use crate::p2p::{Lookup, PeerIdOrAddr};
 pub use bytes::Bytes;
 pub use cid::Cid;
 pub use iroh_resolver::resolver::Path as IpfsPath;
+pub use iroh_resolver::unixfs_builder::AddEvent;
 pub use iroh_rpc_client::{ServiceStatus, StatusRow, StatusTable};
 pub use libp2p::gossipsub::MessageId;
 pub use libp2p::{Multiaddr, PeerId};
diff --git a/iroh-resolver/benches/unixfs.rs b/iroh-resolver/benches/unixfs.rs
@@ -1,4 +1,6 @@
+use anyhow::Context;
 use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use futures::TryStreamExt;
 use iroh_metrics::config::Config as MetricsConfig;
 use iroh_rpc_client::Client;
 use iroh_rpc_client::Config as RpcClientConfig;
@@ -51,10 +53,30 @@ pub fn add_benchmark(c: &mut Criterion) {
                 });
                 let rpc_ref = &rpc;
                 b.to_async(&executor).iter(|| async move {
-                    let c = iroh_resolver::unixfs_builder::add_file(Some(rpc_ref), path, false)
+                    let stream =
+                        iroh_resolver::unixfs_builder::add_file(Some(rpc_ref), path, false)
+                            .await
+                            .unwrap();
+                    // we have to consume the stream here, otherwise we are
+                    // not actually benchmarking anything
+                    // TODO(faassen) rewrite the benchmark in terms of the iroh-api which
+                    // can consume the stream for you
+                    let cid = stream
+                        .try_fold(None, |acc, add_event| async move {
+                            Ok(
+                                if let iroh_resolver::unixfs_builder::AddEvent::Done(cid) =
+                                    add_event
+                                {
+                                    Some(cid)
+                                } else {
+                                    acc
+                                },
+                            )
+                        })
                         .await
-                        .unwrap();
-                    black_box(c)
+                        .unwrap()
+                        .context("No cid found");
+                    black_box(cid)
                 });
             },
         );

diff --git a/iroh-resolver/src/resolver.rs b/iroh-resolver/src/resolver.rs
@@ -62,6 +62,14 @@ impl Block {
         &self.links
     }
 
+    pub fn raw_data_size(&self) -> Option<u64> {
+        let codec = Codec::try_from(self.cid.codec()).unwrap();
+        match codec {
+            Codec::Raw => Some(self.data.len() as u64),
+            _ => None,
+        }
+    }
+
     /// Validate the block. Will return an error if the hash or the links are wrong.
     pub fn validate(&self) -> Result<()> {
         // check that the cid is supported