Skip to content

Commit

Permalink
feat: Refactor unixfs
Browse files Browse the repository at this point in the history
  • Loading branch information
@ppodolsky
ppodolsky committed Dec 15, 2022
1 parent 312ad77 commit d79bda0
Show file tree
Hide file tree
Showing 15 changed files with 276 additions and 337 deletions.
5 changes: 2 additions & 3 deletions iroh-api/src/api.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ use iroh_rpc_client::StatusTable;
use iroh_unixfs::{
builder::Entry as UnixfsEntry,
content_loader::{FullLoader, FullLoaderConfig},
ResponseClip,
};
use iroh_util::{iroh_config_path, make_config};
#[cfg(feature = "testing")]
Expand Down Expand Up @@ -140,13 +139,13 @@ impl Api {
if out.is_dir() {
yield (relative_path, OutType::Dir);
} else if out.is_symlink() {
let mut reader = out.pretty(resolver.clone(), Default::default(), ResponseClip::NoClip)?;
let mut reader = out.pretty(resolver.clone(), Default::default(), None)?;
let mut target = String::new();
reader.read_to_string(&mut target).await?;
let target = PathBuf::from(target);
yield (relative_path, OutType::Symlink(target));
} else {
let reader = out.pretty(resolver.clone(), Default::default(), ResponseClip::NoClip)?;
let reader = out.pretty(resolver.clone(), Default::default(), None)?;
yield (relative_path, OutType::Reader(Box::new(reader)));
}
}
Expand Down
10 changes: 3 additions & 7 deletions iroh-gateway/src/client.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use iroh_metrics::{
};
use iroh_resolver::dns_resolver::Config;
use iroh_resolver::resolver::{CidOrDomain, Metadata, Out, OutPrettyReader, OutType, Resolver};
use iroh_unixfs::{content_loader::ContentLoader, ResponseClip, Source};
use iroh_unixfs::{content_loader::ContentLoader, Source};
use mime::Mime;
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncSeekExt, AsyncWrite};
use tokio_util::io::ReaderStream;
Expand Down Expand Up @@ -120,15 +120,11 @@ impl<T: ContentLoader + std::marker::Unpin> Client<T> {
let body = FileResult::Directory(path_metadata);
Ok((body, metadata))
} else {
let mut clip = 0;
if let Some(range) = &range {
clip = range.end as usize;
}
let reader = path_metadata
.pretty(
self.resolver.clone(),
OutMetrics { start: start_time },
ResponseClip::from(clip),
range.as_ref().map(|range| range.end as usize),
)
.map_err(|e| e.to_string())?;

Expand Down Expand Up @@ -194,7 +190,7 @@ impl<T: ContentLoader + std::marker::Unpin> Client<T> {
let reader = res.pretty(
self.resolver.clone(),
OutMetrics { start: start_time },
ResponseClip::NoClip,
None,
);
match reader {
Ok(mut reader) => {
Expand Down
2 changes: 1 addition & 1 deletion iroh-gateway/src/core.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -617,7 +617,7 @@ mod tests {

assert_eq!(http::StatusCode::MOVED_PERMANENTLY, res.status());
assert_eq!(
format!("http://{}.ipfs.ipfs.io/world.txt", test_setup.root_cid,),
format!("http://{}.ipfs.ipfs.io/world.txt", test_setup.root_cid),
res.headers().get("Location").unwrap().to_str().unwrap(),
);

Expand Down
41 changes: 25 additions & 16 deletions iroh-gateway/src/handlers.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ pub async fn handler<T: ContentLoader + Unpin>(
.retrieve_path_metadata(req.resolved_path)
.await
.map_err(|e| GatewayError::new(StatusCode::INTERNAL_SERVER_ERROR, &e))?;
add_content_length_header(&mut response_headers, path_metadata.metadata().clone());
add_content_length_header(&mut response_headers, path_metadata.metadata().size);
Ok(GatewayResponse::empty(response_headers))
}
Method::GET => {
Expand Down Expand Up @@ -556,9 +556,9 @@ async fn serve_raw<T: ContentLoader + Unpin>(
if let Some(res) = etag_check(&headers, &req.cid, &req.format) {
return Ok(res);
}
add_cache_control_headers(&mut headers, metadata.clone());
add_ipfs_roots_headers(&mut headers, metadata.clone());
add_content_length_header(&mut headers, metadata.clone());
add_cache_control_headers(&mut headers, &metadata);
add_ipfs_roots_headers(&mut headers, &metadata);
add_content_length_header(&mut headers, metadata.size);

if let Some(mut capped_range) = range {
if let Some(size) = metadata.size {
Expand Down Expand Up @@ -606,14 +606,14 @@ async fn serve_car<T: ContentLoader + Unpin>(

set_content_disposition_headers(&mut headers, &file_name, DISPOSITION_ATTACHMENT);

add_cache_control_headers(&mut headers, metadata.clone());
add_content_length_header(&mut headers, metadata.clone());
add_cache_control_headers(&mut headers, &metadata);
add_content_length_header(&mut headers, metadata.size);
let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone())));
set_etag_headers(&mut headers, etag);
if let Some(res) = etag_check(&headers, &req.cid, &req.format) {
return Ok(res);
}
add_ipfs_roots_headers(&mut headers, metadata);
add_ipfs_roots_headers(&mut headers, &metadata);
Ok(GatewayResponse::new(StatusCode::OK, body, headers))
}
FileResult::Directory(_) => Err(GatewayError::new(
Expand Down Expand Up @@ -674,7 +674,7 @@ async fn serve_fs<T: ContentLoader + Unpin>(
.get_file(req.resolved_path.clone(), start_time, range.clone())
.await
.map_err(|e| GatewayError::new(StatusCode::INTERNAL_SERVER_ERROR, &e))?;
add_ipfs_roots_headers(&mut headers, metadata.clone());
add_ipfs_roots_headers(&mut headers, &metadata);
match body {
FileResult::Directory(res) => {
let dir_list: anyhow::Result<Vec<_>> = res
Expand All @@ -701,8 +701,14 @@ async fn serve_fs<T: ContentLoader + Unpin>(
Some(_) => {
// todo(arqu): error on no size
// todo(arqu): add lazy seeking
add_cache_control_headers(&mut headers, metadata.clone());
add_content_length_header(&mut headers, metadata.clone());
add_cache_control_headers(&mut headers, &metadata);
add_content_length_header(
&mut headers,
range
.as_ref()
.map(|range| range.end - range.start)
.or(metadata.size),
);
set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone())));
if let Some(res) = etag_check(&headers, &req.cid, &req.format) {
return Ok(res);
Expand All @@ -722,12 +728,12 @@ async fn serve_fs<T: ContentLoader + Unpin>(
let content_sniffed_mime = body.get_mime();
add_content_type_headers(&mut headers, &name, content_sniffed_mime);
}
if let Some(mut capped_range) = range {
if let Some(mut range) = range {
if let Some(size) = metadata.size {
capped_range.end = std::cmp::min(capped_range.end, size);
range.end = std::cmp::min(range.end, size);
}
add_etag_range(&mut headers, capped_range.clone());
add_content_range_headers(&mut headers, capped_range, metadata.size);
add_etag_range(&mut headers, range.clone());
add_content_range_headers(&mut headers, range, metadata.size);
Ok(GatewayResponse::new(
StatusCode::PARTIAL_CONTENT,
body,
Expand All @@ -746,8 +752,11 @@ async fn serve_fs<T: ContentLoader + Unpin>(
FileResult::Raw(body) => {
// todo(arqu): error on no size
// todo(arqu): add lazy seeking
add_cache_control_headers(&mut headers, metadata.clone());
add_content_length_header(&mut headers, metadata.clone());
add_cache_control_headers(&mut headers, &metadata);
add_content_length_header(
&mut headers,
range.map(|range| range.end - range.start).or(metadata.size),
);
set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone())));
if let Some(res) = etag_check(&headers, &req.cid, &req.format) {
return Ok(res);
Expand Down
17 changes: 10 additions & 7 deletions iroh-gateway/src/headers.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,14 @@ pub fn parse_range_header(range: &HeaderValue) -> Option<Range<u64>> {
if start >= end || end == 0 {
return None;
}
Some(Range { start, end })
Some(Range {
start,
end: end + 1,
})
}

#[tracing::instrument()]
pub fn add_cache_control_headers(headers: &mut HeaderMap, metadata: Metadata) {
pub fn add_cache_control_headers(headers: &mut HeaderMap, metadata: &Metadata) {
if metadata.path.typ() == PathType::Ipns {
let lmdt: OffsetDateTime = time::SystemTime::now().into();
// TODO: better last modified headers based on actual dns ttls
Expand All @@ -121,19 +124,19 @@ pub fn add_cache_control_headers(headers: &mut HeaderMap, metadata: Metadata) {
}

#[tracing::instrument()]
pub fn add_content_length_header(headers: &mut HeaderMap, metadata: Metadata) {
if let Some(size) = metadata.size {
pub fn add_content_length_header(headers: &mut HeaderMap, content_length: Option<u64>) {
if let Some(content_length) = content_length {
headers.insert(
CONTENT_LENGTH,
HeaderValue::from_str(&size.to_string()).unwrap(),
HeaderValue::from_str(&content_length.to_string()).unwrap(),
);
}
}

#[tracing::instrument()]
pub fn add_ipfs_roots_headers(headers: &mut HeaderMap, metadata: Metadata) {
pub fn add_ipfs_roots_headers(headers: &mut HeaderMap, metadata: &Metadata) {
let mut roots = "".to_string();
for rcid in metadata.resolved_path {
for rcid in &metadata.resolved_path {
write!(roots, "{},", rcid).unwrap();
}
roots.pop();
Expand Down
Loading

0 comments on commit d79bda0

Please sign in to comment.