gateway: spec compliance part 2 (#350)

* head request support

* align with spec & dir etags

* CR

iroh-gateway/Cargo.toml

@@ -60,6 +60,7 @@ http-body = "0.4.5"
 mime_classifier = "0.0.1"
 mime = "0.3"
 phf = { version = "0.11", features = ["macros"] }
+once_cell = "1.15.0"
 
 [dev-dependencies]
 axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messages on handlers

iroh-gateway/src/bad_bits.rs

@@ -210,7 +210,7 @@ mod tests {
             .unwrap();
         let client = hyper::Client::new();
         let res = client.get(uri).await.unwrap();
-        assert_eq!(StatusCode::FORBIDDEN, res.status());
+        assert_eq!(StatusCode::GONE, res.status());
 
         let uri = hyper::Uri::builder()
             .scheme("http")
@@ -220,7 +220,7 @@ mod tests {
             .unwrap();
         let client = hyper::Client::new();
         let res = client.get(uri).await.unwrap();
-        assert_eq!(StatusCode::FORBIDDEN, res.status());
+        assert_eq!(StatusCode::GONE, res.status());
 
         let uri = hyper::Uri::builder()
             .scheme("http")
@@ -231,7 +231,7 @@ mod tests {
         let client = hyper::Client::new();
         let res = client.get(uri).await.unwrap();
         let status = res.status();
-        assert_eq!(StatusCode::FORBIDDEN, status);
+        assert_eq!(StatusCode::GONE, status);
 
         let uri = hyper::Uri::builder()
             .scheme("http")
@@ -242,7 +242,7 @@ mod tests {
         let client = hyper::Client::new();
         let res = client.get(uri).await.unwrap();
         let status = res.status();
-        assert_eq!(StatusCode::FORBIDDEN, status);
+        assert_eq!(StatusCode::GONE, status);
 
         let uri = hyper::Uri::builder()
             .scheme("http")
@@ -252,7 +252,7 @@ mod tests {
             .unwrap();
         let client = hyper::Client::new();
         let res = client.get(uri).await.unwrap();
-        assert!(res.status() != StatusCode::FORBIDDEN);
+        assert!(res.status() != StatusCode::GONE);
 
         let uri = hyper::Uri::builder()
             .scheme("http")
@@ -262,7 +262,7 @@ mod tests {
             .unwrap();
         let client = hyper::Client::new();
         let res = client.get(uri).await.unwrap();
-        assert!(res.status() != StatusCode::FORBIDDEN);
+        assert!(res.status() != StatusCode::GONE);
 
         let uri = hyper::Uri::builder()
             .scheme("http")
@@ -272,7 +272,7 @@ mod tests {
             .unwrap();
         let client = hyper::Client::new();
         let res = client.get(uri).await.unwrap();
-        assert!(res.status() != StatusCode::FORBIDDEN);
+        assert!(res.status() != StatusCode::GONE);
 
         core_task.abort();
         core_task.await.unwrap_err();

iroh-gateway/src/config.rs

@@ -102,8 +102,6 @@ fn default_headers() -> HeaderMap {
         .into_iter()
         .collect::<AccessControlAllowHeaders>(),
     );
-    // todo(arqu): remove these once propperly implmented
-    headers.insert(ACCEPT_RANGES, VALUE_NONE.clone());
     headers
 }
 
@@ -182,7 +180,7 @@ mod tests {
     #[test]
     fn test_default_headers() {
         let headers = default_headers();
-        assert_eq!(headers.len(), 4);
+        assert_eq!(headers.len(), 3);
         let h = headers.get(&ACCESS_CONTROL_ALLOW_ORIGIN).unwrap();
         assert_eq!(h, "*");
     }
@@ -242,7 +240,6 @@ mod tests {
                 "if-none-match, accept, cache-control, range, service-worker",
             ),
         );
-        expect.insert("accept-ranges".to_string(), Value::new(None, "none"));
         let got = collect_headers(&default_headers()).unwrap();
         assert_eq!(expect, got);
     }

iroh-gateway/src/error.rs

@@ -4,11 +4,15 @@ use std::{
 };
 
 use axum::{
+    body::BoxBody,
     http::StatusCode,
     response::{IntoResponse, Response},
 };
+use http::{HeaderMap, HeaderValue};
 use serde_json::json;
 
+use crate::constants::HEADER_X_TRACE_ID;
+
 #[derive(Debug)]
 pub struct GatewayError {
     pub status_code: StatusCode,
@@ -28,16 +32,30 @@ impl GatewayError {
 
 impl IntoResponse for GatewayError {
     fn into_response(self) -> Response {
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            &HEADER_X_TRACE_ID,
+            HeaderValue::from_str(&self.trace_id).unwrap(),
+        );
         match self.method {
-            Some(http::Method::HEAD) => (self.status_code).into_response(),
+            Some(http::Method::HEAD) => {
+                let mut rb = Response::builder().status(self.status_code);
+                let rh = rb.headers_mut().unwrap();
+                rh.extend(headers);
+                rb.body(BoxBody::default()).unwrap()
+            }
             _ => {
                 let body = axum::Json(json!({
                     "code": self.status_code.as_u16(),
                     "success": false,
                     "message": self.message,
                     "trace_id": self.trace_id,
                 }));
-                (self.status_code, body).into_response()
+                let mut res = body.into_response();
+                res.headers_mut().extend(headers);
+                let status = res.status_mut();
+                *status = self.status_code;
+                res
             }
         }
     }

iroh-gateway/src/handlers.rs

@@ -4,6 +4,7 @@ use axum::{
     error_handling::HandleErrorLayer,
     extract::{Extension, Path, Query},
     http::{header::*, Request as HttpRequest, StatusCode},
+    middleware,
     response::IntoResponse,
     routing::get,
     BoxError, Router,
@@ -71,6 +72,7 @@ pub fn get_app_routes<T: ContentLoader + std::marker::Unpin>(state: &Arc<State<T
             ServiceBuilder::new()
                 // Handle errors from middleware
                 .layer(Extension(Arc::clone(state)))
+                .layer(middleware::from_fn(request_middleware))
                 .layer(CompressionLayer::new())
                 .layer(HandleErrorLayer::new(middleware_error_handler::<T>))
                 .load_shed()
@@ -149,11 +151,7 @@ pub async fn get_handler<T: ContentLoader + std::marker::Unpin>(
     unsuported_header_check(&request_headers, &state)?;
 
     if check_bad_bits(&state, cid, cpath).await {
-        return Err(error(
-            StatusCode::FORBIDDEN,
-            "CID is in the denylist",
-            &state,
-        ));
+        return Err(error(StatusCode::GONE, "CID is in the denylist", &state));
     }
 
     let full_content_path = format!("/{}/{}{}", scheme, cid, cpath);
@@ -169,11 +167,7 @@ pub async fn get_handler<T: ContentLoader + std::marker::Unpin>(
     }
 
     if check_bad_bits(&state, resolved_cid.to_string().as_str(), cpath).await {
-        return Err(error(
-            StatusCode::FORBIDDEN,
-            "CID is in the denylist",
-            &state,
-        ));
+        return Err(error(StatusCode::GONE, "CID is in the denylist", &state));
     }
 
     // parse query params
@@ -391,10 +385,10 @@ fn etag_check<T: ContentLoader>(
             .to_str()
             .unwrap();
         if !inm.is_empty() {
-            // todo(arqu): handle dir etags
             let cid_etag = get_etag(resolved_cid, Some(format.clone()));
+            let dir_etag = get_dir_etag(resolved_cid);
 
-            if etag_matches(inm, &cid_etag) {
+            if etag_matches(inm, &cid_etag) || etag_matches(inm, &dir_etag) {
                 return Some(GatewayResponse::not_modified());
             }
         }
@@ -431,8 +425,12 @@ async fn serve_raw<T: ContentLoader + std::marker::Unpin>(
 
             set_content_disposition_headers(&mut headers, &file_name, DISPOSITION_ATTACHMENT);
             set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone())));
+            if let Some(res) = etag_check(&headers, &req.cid, &req.format, &state) {
+                return Ok(res);
+            }
             add_cache_control_headers(&mut headers, metadata.clone());
             add_ipfs_roots_headers(&mut headers, metadata.clone());
+            add_content_length_header(&mut headers, metadata.clone());
 
             if let Some(mut capped_range) = range {
                 if let Some(size) = metadata.size {
@@ -478,9 +476,12 @@ async fn serve_car<T: ContentLoader + std::marker::Unpin>(
             set_content_disposition_headers(&mut headers, &file_name, DISPOSITION_ATTACHMENT);
 
             add_cache_control_headers(&mut headers, metadata.clone());
+            add_content_length_header(&mut headers, metadata.clone());
             let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone())));
             set_etag_headers(&mut headers, etag);
-            // todo(arqu): handle if none match
+            if let Some(res) = etag_check(&headers, &req.cid, &req.format, &state) {
+                return Ok(res);
+            }
             add_ipfs_roots_headers(&mut headers, metadata);
             response(StatusCode::OK, body, headers)
         }
@@ -516,8 +517,9 @@ async fn serve_car_recursive<T: ContentLoader + std::marker::Unpin>(
     // add_cache_control_headers(&mut headers, metadata.clone());
     let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone())));
     set_etag_headers(&mut headers, etag);
-
-    // todo(arqu): handle if none match
+    if let Some(res) = etag_check(&headers, &req.cid, &req.format, &state) {
+        return Ok(res);
+    }
     // add_ipfs_roots_headers(&mut headers, metadata);
     response(StatusCode::OK, body, headers)
 }
@@ -573,7 +575,11 @@ async fn serve_fs<T: ContentLoader + std::marker::Unpin>(
                     // todo(arqu): error on no size
                     // todo(arqu): add lazy seeking
                     add_cache_control_headers(&mut headers, metadata.clone());
+                    add_content_length_header(&mut headers, metadata.clone());
                     set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone())));
+                    if let Some(res) = etag_check(&headers, &req.cid, &req.format, &state) {
+                        return Ok(res);
+                    }
                     let name = add_content_disposition_headers(
                         &mut headers,
                         &req.query_file_name,
@@ -612,7 +618,11 @@ async fn serve_fs<T: ContentLoader + std::marker::Unpin>(
             // todo(arqu): error on no size
             // todo(arqu): add lazy seeking
             add_cache_control_headers(&mut headers, metadata.clone());
+            add_content_length_header(&mut headers, metadata.clone());
             set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone())));
+            if let Some(res) = etag_check(&headers, &req.cid, &req.format, &state) {
+                return Ok(res);
+            }
             let name = add_content_disposition_headers(
                 &mut headers,
                 &req.query_file_name,
@@ -649,16 +659,18 @@ async fn serve_fs_dir<T: ContentLoader + std::marker::Unpin>(
                 req.resolved_path,
                 req.query_params.to_query_string()
             );
-            return Ok(GatewayResponse::redirect(&redirect_path));
+            return Ok(GatewayResponse::redirect_permanently(&redirect_path));
         }
         let mut new_req = req.clone();
         new_req.resolved_path.push("index.html");
         return serve_fs(&new_req, state, headers, http_req, start_time).await;
     }
 
     headers.insert(CONTENT_TYPE, HeaderValue::from_str("text/html").unwrap());
-    // todo(arqu): set etag
-    // set_etag_headers(&mut headers, metadata.dir_hash.clone());
+    set_etag_headers(&mut headers, get_dir_etag(&req.cid));
+    if let Some(res) = etag_check(&headers, &req.cid, &req.format, &state) {
+        return Ok(res);
+    }
 
     let mut template_data: Map<String, Json> = Map::new();
     let mut root_path = req.resolved_path.clone();
@@ -760,6 +772,20 @@ fn error<T: ContentLoader>(
     }
 }
 
+// #[tracing::instrument()]
+pub async fn request_middleware<B>(
+    request: axum::http::Request<B>,
+    next: axum::middleware::Next<B>,
+) -> axum::response::Response {
+    let method = request.method().clone();
+    let mut r = next.run(request).await;
+    if method == Method::HEAD {
+        let b = r.body_mut();
+        *b = http_body::combinators::UnsyncBoxBody::default();
+    }
+    r
+}
+
 #[tracing::instrument()]
 pub async fn middleware_error_handler<T: ContentLoader>(
     method: Method,
@@ -773,12 +799,12 @@ pub async fn middleware_error_handler<T: ContentLoader>(
     }
 
     if err.is::<tower::timeout::error::Elapsed>() {
-        return error(StatusCode::REQUEST_TIMEOUT, "request timed out", &state);
+        return error(StatusCode::GATEWAY_TIMEOUT, "request timed out", &state);
     }
 
     if err.is::<tower::load_shed::error::Overloaded>() {
         return error(
-            StatusCode::SERVICE_UNAVAILABLE,
+            StatusCode::TOO_MANY_REQUESTS,
             "service is overloaded, try again later",
             &state,
         );

iroh-gateway/src/headers.rs

@@ -3,6 +3,8 @@ use ::time::OffsetDateTime;
 use axum::http::header::*;
 use iroh_resolver::resolver::{CidOrDomain, Metadata, PathType};
 use mime::Mime;
+use once_cell::sync::Lazy;
+use sha2::Digest;
 use std::{fmt::Write, ops::Range, time};
 
 #[tracing::instrument()]
@@ -118,6 +120,16 @@ pub fn add_cache_control_headers(headers: &mut HeaderMap, metadata: Metadata) {
     }
 }
 
+#[tracing::instrument()]
+pub fn add_content_length_header(headers: &mut HeaderMap, metadata: Metadata) {
+    if let Some(size) = metadata.size {
+        headers.insert(
+            CONTENT_LENGTH,
+            HeaderValue::from_str(&size.to_string()).unwrap(),
+        );
+    }
+}
+
 #[tracing::instrument()]
 pub fn add_ipfs_roots_headers(headers: &mut HeaderMap, metadata: Metadata) {
     let mut roots = "".to_string();
@@ -163,6 +175,19 @@ pub fn get_etag(cid: &CidOrDomain, response_format: Option<ResponseFormat>) -> S
     }
 }
 
+#[tracing::instrument()]
+pub fn get_dir_etag(cid: &CidOrDomain) -> String {
+    match cid {
+        CidOrDomain::Cid(cid) => {
+            format!("\"Dir-{}-CID-{}\"", *VERSION_TEMPLATE_HASH, cid)
+        }
+        CidOrDomain::Domain(_) => {
+            // TODO:
+            String::new()
+        }
+    }
+}
+
 #[tracing::instrument()]
 pub fn etag_matches(inm: &str, cid_etag: &str) -> bool {
     let mut buf = inm.trim();
@@ -227,6 +252,22 @@ pub fn get_filename(content_path: &str) -> String {
         .unwrap_or_default()
 }
 
+pub fn version_and_template_hash() -> String {
+    let v = format!(
+        "{}-{}-{}-{}",
+        env!("CARGO_PKG_NAME"),
+        env!("CARGO_PKG_VERSION"),
+        crate::templates::DIR_LIST_TEMPLATE,
+        crate::templates::NOT_FOUND_TEMPLATE,
+    );
+    let mut hasher = sha2::Sha256::new();
+    hasher.update(v.as_bytes());
+    let hash = hasher.finalize();
+    hex::encode(hash)
+}
+
+pub(crate) static VERSION_TEMPLATE_HASH: Lazy<String> = Lazy::new(version_and_template_hash);
+
 #[cfg(test)]
 mod tests {
     use cid::Cid;