From 8668c68acf84b04bccc9328fa37b26b5bb10b61a Mon Sep 17 00:00:00 2001 From: Caio Date: Fri, 31 Jul 2020 08:34:20 -0300 Subject: [PATCH] Split Entry into two different enums to remove unnecessary expects --- http/src/v0/root_files.rs | 84 ++++++++---------- unixfs/examples/get.rs | 61 ++++++------- unixfs/src/walk.rs | 178 ++++++++++++++++++++------------------ 3 files changed, 157 insertions(+), 166 deletions(-) diff --git a/http/src/v0/root_files.rs b/http/src/v0/root_files.rs index 8f8a4d1d4..8d7baf54a 100644 --- a/http/src/v0/root_files.rs +++ b/http/src/v0/root_files.rs @@ -6,7 +6,7 @@ use async_stream::try_stream; use bytes::Bytes; use cid::{Cid, Codec}; use futures::stream::TryStream; -use ipfs::unixfs::ll::walk::{self, ContinuedWalk, Walker}; +use ipfs::unixfs::ll::walk::{self, ContinuedWalk, Entry, MetadataEntry, Walker}; use ipfs::unixfs::{ll::file::FileReadFailed, TraversalFailed}; use ipfs::Block; use ipfs::{Ipfs, IpfsTypes}; @@ -159,75 +159,63 @@ fn walk( visit = match walker.continue_walk(&data, &mut cache)? { ContinuedWalk::File(segment, item) => { - let total_size = item.as_entry() - .total_file_size() - .expect("files do have total_size"); - - if segment.is_first() { - let path = item.as_entry().path(); - let metadata = item - .as_entry() - .metadata() - .expect("files must have metadata"); - - for mut bytes in tar_helper.apply_file(path, metadata, total_size)?.iter_mut() { - if let Some(bytes) = bytes.take() { - yield bytes; + if let Entry::Metadata(MetadataEntry::File(.., p, md, size)) = item.as_entry() { + if segment.is_first() { + for mut bytes in tar_helper.apply_file(p, md, size)?.iter_mut() { + if let Some(bytes) = bytes.take() { + yield bytes; + } } } - } - // even if the largest of files can have 256 kB blocks and about the same - // amount of content, try to consume it in small parts not to grow the buffers - // too much. + // even if the largest of files can have 256 kB blocks and about the same + // amount of content, try to consume it in small parts not to grow the buffers + // too much. - let mut n = 0usize; - let slice = segment.as_ref(); - let total = slice.len(); + let mut n = 0usize; + let slice = segment.as_ref(); + let total = slice.len(); - while n < total { - let next = tar_helper.buffer_file_contents(&slice[n..]); - n += next.len(); - yield next; - } + while n < total { + let next = tar_helper.buffer_file_contents(&slice[n..]); + n += next.len(); + yield next; + } - if segment.is_last() { - if let Some(zeroes) = tar_helper.pad(total_size) { - yield zeroes; + if segment.is_last() { + if let Some(zeroes) = tar_helper.pad(size) { + yield zeroes; + } } } - item.into_inner() }, ContinuedWalk::Directory(item) => { - - // only first instances of directories will have the metadata - if let Some(metadata) = item.as_entry().metadata() { - let path = item.as_entry().path(); - + if let Entry::Metadata(metadata_entry) = item.as_entry() { + let metadata = metadata_entry.metadata(); + let path = metadata_entry.path(); for mut bytes in tar_helper.apply_directory(path, metadata)?.iter_mut() { if let Some(bytes) = bytes.take() { yield bytes; } } } - item.into_inner() }, ContinuedWalk::Symlink(bytes, item) => { - - // converting a symlink is the most tricky part - let path = item.as_entry().path(); - let target = std::str::from_utf8(bytes).map_err(|_| GetError::NonUtf8Symlink)?; - let target = Path::new(target); - let metadata = item.as_entry().metadata().expect("symlink must have metadata"); - - for mut bytes in tar_helper.apply_symlink(path, target, metadata)?.iter_mut() { - if let Some(bytes) = bytes.take() { - yield bytes; + if let Entry::Metadata(metadata_entry) = item.as_entry() { + // converting a symlink is the most tricky part + let path = metadata_entry.path(); + let target = std::str::from_utf8(bytes).map_err(|_| GetError::NonUtf8Symlink)?; + let target = Path::new(target); + let metadata = metadata_entry.metadata(); + + for mut bytes in tar_helper.apply_symlink(path, target, metadata)?.iter_mut() { + if let Some(bytes) = bytes.take() { + yield bytes; + } } } - item.into_inner() }, }; diff --git a/unixfs/examples/get.rs b/unixfs/examples/get.rs index 95e3bfdee..35a5e4c16 100644 --- a/unixfs/examples/get.rs +++ b/unixfs/examples/get.rs @@ -52,7 +52,7 @@ fn main() { } fn walk(blocks: ShardedBlockStore, start: &Cid) -> Result<(), Error> { - use ipfs_unixfs::walk::{ContinuedWalk, Walker}; + use ipfs_unixfs::walk::{ContinuedWalk, Entry, MetadataEntry, Walker}; let mut buf = Vec::new(); let mut cache = None; @@ -72,56 +72,45 @@ fn walk(blocks: ShardedBlockStore, start: &Cid) -> Result<(), Error> { // items. visit = match walker.continue_walk(&buf, &mut cache)? { ContinuedWalk::File(segment, item) => { - let entry = item.as_entry(); - let total_size = entry.total_file_size().expect("all files have total size"); - // metadata is picked up from the root file and carried until the last block - let metadata = entry.metadata().expect("all files have metadata"); - - if segment.is_first() { - // this is set on the root block, no actual bytes are present for multiblock - // files + if let Entry::Metadata(MetadataEntry::File(.., path, md, size)) = item.as_entry() { + if segment.is_first() { + // this is set on the root block, no actual bytes are present for multiblock + // files + } + + if segment.is_last() { + let mode = md.mode().unwrap_or(0o0644) & 0o7777; + let (seconds, _) = md.mtime().unwrap_or((0, 0)); + println!("f {:o} {:>12} {:>16} {:?}", mode, seconds, size, path); + } } - - if segment.is_last() { - let path = entry.path(); - let mode = metadata.mode().unwrap_or(0o0644) & 0o7777; - let (seconds, _) = metadata.mtime().unwrap_or((0, 0)); - - println!("f {:o} {:>12} {:>16} {:?}", mode, seconds, total_size, path); - } - - // continue the walk item.into_inner() } ContinuedWalk::Directory(item) => { // presense of metadata can be used to determine if this is the first apperiance of // a directory by looking at the metadata: sibling hamt shard buckets do not have // metadata. - if let Some(metadata) = item.as_entry().metadata() { + if let Entry::Metadata(metadata_entry) = item.as_entry() { + let metadata = metadata_entry.metadata(); let path = item.as_entry().path(); - let mode = metadata.mode().unwrap_or(0o0755) & 0o7777; let (seconds, _) = metadata.mtime().unwrap_or((0, 0)); - println!("d {:o} {:>12} {:>16} {:?}", mode, seconds, "-", path); } - item.into_inner() } ContinuedWalk::Symlink(bytes, item) => { - let entry = item.as_entry(); - let metadata = entry.metadata().expect("symlink must have metadata"); - - let path = entry.path(); - let target = Path::new(std::str::from_utf8(bytes).unwrap()); - let mode = metadata.mode().unwrap_or(0o0755) & 0o7777; - let (seconds, _) = metadata.mtime().unwrap_or((0, 0)); - - println!( - "s {:o} {:>12} {:>16} {:?} -> {:?}", - mode, seconds, "-", path, target - ); - + if let Entry::Metadata(metadata_entry) = item.as_entry() { + let metadata = metadata_entry.metadata(); + let path = metadata_entry.path(); + let target = Path::new(std::str::from_utf8(bytes).unwrap()); + let mode = metadata.mode().unwrap_or(0o0755) & 0o7777; + let (seconds, _) = metadata.mtime().unwrap_or((0, 0)); + println!( + "s {:o} {:>12} {:>16} {:?} -> {:?}", + mode, seconds, "-", path, target + ); + } item.into_inner() } }; diff --git a/unixfs/src/walk.rs b/unixfs/src/walk.rs index 03d306c69..903ebbdeb 100644 --- a/unixfs/src/walk.rs +++ b/unixfs/src/walk.rs @@ -10,6 +10,43 @@ use std::convert::TryFrom; use std::fmt; use std::path::{Path, PathBuf}; +/// Representation of the current item of Walker or the last observed item with medatada. +#[derive(Debug)] +pub enum MetadataEntry<'a> { + /// Current item is a non-root plain Directory or a HAMTShard directory. + Directory(&'a Cid, &'a Path, &'a Metadata), + /// Current item is possibly a root file with a path, metadata, and a total file size. + File(&'a Cid, &'a Path, &'a Metadata, u64), + /// Current item is the root directory (HAMTShard or plain Directory). + RootDirectory(&'a Cid, &'a Path, &'a Metadata), + /// Current item is possibly a root symlink. + Symlink(&'a Cid, &'a Path, &'a Metadata), +} + +impl MetadataEntry<'_> { + /// Returns the metadata for the latest entry. It exists for initial directory entries, files, + /// and symlinks but not for continued HamtShards. + pub fn metadata(&self) -> &'_ Metadata { + match self { + Self::Directory(_, _, m) + | Self::File(_, _, m, ..) + | Self::RootDirectory(_, _, m) + | Self::Symlink(_, _, m) => m, + } + } + + /// Returns the path for the latest entry. This is created from a UTF-8 string and, as such, is always + /// representable on all supported platforms. + pub fn path(&self) -> &'_ Path { + match self { + MetadataEntry::Directory(_, p, ..) + | MetadataEntry::File(_, p, ..) + | MetadataEntry::RootDirectory(_, p, ..) + | MetadataEntry::Symlink(_, p, ..) => p, + } + } +} + /// `Walker` helps with walking a UnixFS tree, including all of the content and files. It is created with /// `Walker::new` and walked over each block with `Walker::continue_block`. Use /// `Walker::pending_links` to obtain the next [`Cid`] to be loaded and the prefetchable links. @@ -185,7 +222,6 @@ impl Walker { // replacing this with try_fold takes as many lines as the R: Try cannot be // deduced without specifying the Error - for link in links { pending.push(link?); } @@ -254,20 +290,20 @@ impl Walker { UnixFsType::Raw | UnixFsType::File => { let (bytes, file_size, metadata, step) = IdleFileVisit::default().start_from_parsed(flat, cache)?; - let (cid, name, depth) = next.expect("validated at new and earlier in this method"); - let file_continues = step.is_some(); - - let ie = match current { - Some(mut ie) => { - ie.as_file(cid, &name, depth, metadata, step, file_size); + let (cid, name, depth) = next.expect("validated at new and earlier in this method"); + let file_continues = step.is_some(); + + let ie = match current { + Some(mut ie) => { + ie.as_file(cid, &name, depth, metadata, step, file_size); ie } _ => InnerEntry::new_root_file(cid, metadata, &name, step, file_size, depth), }; - + let next = pending.pop(); let segment = FileSegment::first(bytes, !file_continues); - + let state = if file_continues || next.is_some() { State::Unfinished(Self { current: Some(ie), @@ -355,12 +391,12 @@ impl fmt::Debug for InnerKind { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { use InnerKind::*; match self { - RootDirectory(cid) => write!(fmt, "RootDirectory({})", cid), - BucketAtRoot(cid) => write!(fmt, "BucketAtRoot({})", cid), - RootBucket(cid) => write!(fmt, "RootBucket({})", cid), Bucket(cid) => write!(fmt, "Bucket({})", cid), + BucketAtRoot(cid) => write!(fmt, "BucketAtRoot({})", cid), Directory(cid) => write!(fmt, "Directory({})", cid), File(cid, _, sz) => write!(fmt, "File({}, _, {})", cid, sz), + RootBucket(cid) => write!(fmt, "RootBucket({})", cid), + RootDirectory(cid) => write!(fmt, "RootDirectory({})", cid), Symlink(cid) => write!(fmt, "Symlink({})", cid), } } @@ -369,63 +405,34 @@ impl fmt::Debug for InnerKind { /// Representation of the current item of Walker or the last observed item. #[derive(Debug)] pub enum Entry<'a> { - /// Current item is the root directory (HAMTShard or plain Directory). - RootDirectory(&'a Cid, &'a Path, &'a Metadata), /// Current item is a continuation of a HAMTShard directory. Only the root HAMTShard will have /// file metadata. Bucket(&'a Cid, &'a Path), - /// Current item is a non-root plain Directory or a HAMTShard directory. - Directory(&'a Cid, &'a Path, &'a Metadata), - /// Current item is possibly a root file with a path, metadata, and a total file size. - File(&'a Cid, &'a Path, &'a Metadata, u64), - /// Current item is possibly a root symlink. - Symlink(&'a Cid, &'a Path, &'a Metadata), + /// All items that have metadata. + Metadata(MetadataEntry<'a>), } impl<'a> Entry<'a> { - /// Returns the path for the latest entry. This is created from a UTF-8 string and, as such, is always - /// representable on all supported platforms. - pub fn path(&self) -> &'a Path { - use Entry::*; - match self { - RootDirectory(_, p, _) - | Bucket(_, p) - | Directory(_, p, _) - | File(_, p, _, _) - | Symlink(_, p, _) => p, - } - } - - /// Returns the metadata for the latest entry. It exists for initial directory entries, files, - /// and symlinks but not for continued HamtShards. - pub fn metadata(&self) -> Option<&'a Metadata> { - use Entry::*; - match self { - Bucket(_, _) => None, - RootDirectory(_, _, m) | Directory(_, _, m) | File(_, _, m, _) | Symlink(_, _, m) => { - Some(m) - } - } - } - - /// Returns the total size of the file this entry represents, or `None` if not a file. - pub fn total_file_size(&self) -> Option { - use Entry::*; + /// Returns the Cid for the latest entry. + pub fn cid(&self) -> &Cid { match self { - File(_, _, _, sz) => Some(*sz), - _ => None, + Self::Bucket(cid, ..) => cid, + Self::Metadata(MetadataEntry::Directory(cid, ..)) + | Self::Metadata(MetadataEntry::File(cid, ..)) + | Self::Metadata(MetadataEntry::RootDirectory(cid, ..)) + | Self::Metadata(MetadataEntry::Symlink(cid, ..)) => cid, } } - /// Returns the Cid for the latest entry. - pub fn cid(&self) -> &Cid { - use Entry::*; + /// Returns the path for the latest entry. This is created from a UTF-8 string and, as such, is always + /// representable on all supported platforms. + pub fn path(&self) -> &'a Path { match self { - RootDirectory(cid, _, _) - | Bucket(cid, _) - | Directory(cid, _, _) - | File(cid, _, _, _) - | Symlink(cid, _, _) => cid, + Self::Bucket(_, p) => p, + Self::Metadata(MetadataEntry::Directory(_, p, ..)) + | Self::Metadata(MetadataEntry::File(_, p, ..)) + | Self::Metadata(MetadataEntry::RootDirectory(_, p, ..)) + | Self::Metadata(MetadataEntry::Symlink(_, p, ..)) => p, } } } @@ -483,16 +490,23 @@ impl InnerEntry { } pub fn as_entry(&self) -> Entry<'_> { - use InnerKind::*; match &self.kind { - RootDirectory(cid) | BucketAtRoot(cid) => { - Entry::RootDirectory(cid, &self.path, &self.metadata) + InnerKind::Bucket(cid) => Entry::Bucket(cid, &self.path), + InnerKind::Directory(cid) => { + Entry::Metadata(MetadataEntry::Directory(cid, &self.path, &self.metadata)) + } + InnerKind::File(cid, _, sz) => { + Entry::Metadata(MetadataEntry::File(cid, &self.path, &self.metadata, *sz)) + } + InnerKind::RootBucket(cid) => { + Entry::Metadata(MetadataEntry::Directory(cid, &self.path, &self.metadata)) + } + InnerKind::RootDirectory(cid) | InnerKind::BucketAtRoot(cid) => Entry::Metadata( + MetadataEntry::RootDirectory(cid, &self.path, &self.metadata), + ), + InnerKind::Symlink(cid) => { + Entry::Metadata(MetadataEntry::Symlink(cid, &self.path, &self.metadata)) } - RootBucket(cid) => Entry::Directory(cid, &self.path, &self.metadata), - Bucket(cid) => Entry::Bucket(cid, &self.path), - Directory(cid) => Entry::Directory(cid, &self.path, &self.metadata), - File(cid, _, sz) => Entry::File(cid, &self.path, &self.metadata, *sz), - Symlink(cid) => Entry::Symlink(cid, &self.path, &self.metadata), } } @@ -609,12 +623,12 @@ impl InnerEntry { fn as_symlink(&mut self, cid: Cid, name: &str, depth: usize, metadata: Metadata) { use InnerKind::*; match self.kind { - RootDirectory(_) + Bucket(_) | BucketAtRoot(_) - | RootBucket(_) - | Bucket(_) | Directory(_) | File(_, None, _) + | RootBucket(_) + | RootDirectory(_) | Symlink(_) => { self.kind = Symlink(cid); self.set_path(name, depth); @@ -774,13 +788,14 @@ pub enum Error { UnsupportedHAMTShard(ShardError), } -impl From> for Error { - fn from(e: ParsingFailed<'_>) -> Self { - use ParsingFailed::*; +impl From for Error { + fn from(e: FileReadFailed) -> Self { + use FileReadFailed::*; match e { - InvalidDagPb(e) => Error::DagPbParsingFailed(e), - InvalidUnixFs(e, _) => Error::UnixFsParsingFailed(e), - NoData(_) => Error::EmptyDagPbNode, + File(e) => Error::File(e), + UnexpectedType(ut) => Error::UnexpectedType(ut), + Read(_) => unreachable!("FileVisit does not parse any blocks"), + InvalidCid(l) => Error::InvalidCid(l), } } } @@ -791,14 +806,13 @@ impl From for Error { } } -impl From for Error { - fn from(e: FileReadFailed) -> Self { - use FileReadFailed::*; +impl From> for Error { + fn from(e: ParsingFailed<'_>) -> Self { + use ParsingFailed::*; match e { - File(e) => Error::File(e), - UnexpectedType(ut) => Error::UnexpectedType(ut), - Read(_) => unreachable!("FileVisit does not parse any blocks"), - InvalidCid(l) => Error::InvalidCid(l), + InvalidDagPb(e) => Error::DagPbParsingFailed(e), + InvalidUnixFs(e, _) => Error::UnixFsParsingFailed(e), + NoData(_) => Error::EmptyDagPbNode, } } }