From 402b79fae330a397e9989e8215bf5ece7456ed07 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Tue, 10 May 2022 11:41:58 +0200 Subject: [PATCH 01/31] extend --- iroh-gateway/Cargo.toml | 1 + iroh-gateway/src/client.rs | 3 +- iroh-gateway/src/constants.rs | 6 +- iroh-gateway/src/core.rs | 289 +++++++++++++++++++++++++++++----- iroh-gateway/src/response.rs | 80 +++++++--- 5 files changed, 314 insertions(+), 65 deletions(-) diff --git a/iroh-gateway/Cargo.toml b/iroh-gateway/Cargo.toml index 6af9a25fca..b6508d131e 100644 --- a/iroh-gateway/Cargo.toml +++ b/iroh-gateway/Cargo.toml @@ -24,6 +24,7 @@ git-version = "0.3.5" rand = "0.8.5" tracing-opentelemetry = "0.17.2" opentelemetry = { version = "0.17.0", features = ["rt-tokio"] } +chrono = "0.4.19" [dev-dependencies] axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messages on handlers diff --git a/iroh-gateway/src/client.rs b/iroh-gateway/src/client.rs index b6b585c87c..e0f39406f1 100644 --- a/iroh-gateway/src/client.rs +++ b/iroh-gateway/src/client.rs @@ -3,6 +3,7 @@ use crate::metrics::*; use crate::response::ResponseFormat; use axum::body::Body; +use cid::Cid; use metrics::{counter, gauge, histogram, increment_counter}; use rand::{prelude::StdRng, Rng, SeedableRng}; use std::{fs::File, io::Read, path::Path, time::Duration}; @@ -86,7 +87,7 @@ impl Client { #[derive(Debug, Clone)] pub struct Request { pub format: ResponseFormat, - pub cid: String, + pub cid: Cid, pub full_content_path: String, pub query_file_name: String, pub content_path: String, diff --git a/iroh-gateway/src/constants.rs b/iroh-gateway/src/constants.rs index 1096fc1e4c..1b99b32c1a 100644 --- a/iroh-gateway/src/constants.rs +++ b/iroh-gateway/src/constants.rs @@ -2,6 +2,8 @@ pub const HEADER_X_IPFS_PATH: &str = "X-Ipfs-Path"; pub const HEADER_X_CONTENT_TYPE_OPTIONS: &str = "X-Content-Type-Options"; pub const HEADER_X_TRACE_ID: &str = "X-Trace-Id"; +pub const HEADER_X_IPFS_GATEWAY_PREFIX: &str = "X-Ipfs-Gateway-Prefix"; +pub const HEADER_SERVICE_WORKER: &str = "Service-Worker"; // Common Header Values pub const VALUE_XCTO_NOSNIFF: &str = "nosniff"; @@ -14,4 +16,6 @@ pub const DISPOSITION_INLINE: &str = "inline"; pub const CONTENT_TYPE_IPLD_RAW: &str = "application/vnd.ipld.raw"; pub const CONTENT_TYPE_IPLD_CAR: &str = "application/vnd.ipld.car; version=1"; pub const CONTENT_TYPE_OCTET_STREAM: &str = "application/octet-stream"; -pub const CONTENT_TYPE_HTML: &str = "text/html"; + +// Values +pub const VAL_IMMUTABLE_MAX_AGE: &str = "public, max-age=31536000, immutable"; diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index d29dc53c57..624a83231c 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -3,14 +3,20 @@ use axum::{ error_handling::HandleErrorLayer, extract::{Extension, Path, Query}, http::{header::*, StatusCode}, - response::IntoResponse, + response::{IntoResponse, Redirect}, routing::get, BoxError, Router, }; +use chrono::prelude::Utc; use cid::Cid; use metrics::increment_counter; use serde::Deserialize; -use std::{borrow::Cow, collections::HashMap, sync::Arc, time::Duration}; +use std::{ + borrow::Cow, + collections::HashMap, + sync::Arc, + time::{self, Duration}, +}; use tower::ServiceBuilder; use crate::{ @@ -39,6 +45,32 @@ pub struct GetParams { download: Option, } +impl GetParams { + pub fn to_query_string(&self) -> String { + let mut query = String::new(); + if let Some(format) = &self.format { + query.push_str(&format!("format={}", format)); + } + if let Some(filename) = &self.filename { + if !query.is_empty() { + query.push('&'); + } + query.push_str(&format!("filename={}", filename)); + } + if let Some(download) = &self.download { + if !query.is_empty() { + query.push('&'); + } + query.push_str(&format!("download={}", download)); + } + if query.is_empty() { + "".to_string() + } else { + format!("?{}", query) + } + } +} + impl Core { pub fn new(config: Config) -> Self { Self { @@ -48,9 +80,12 @@ impl Core { } pub async fn serve(self) { + // todo(arqu): ?uri=... https://github.com/ipfs/go-ipfs/pull/7802 let app = Router::new() .route("/ipfs/:cid", get(get_ipfs)) .route("/ipfs/:cid/*cpath", get(get_ipfs)) + .route("/ipfs/ipfs/:cid", get(redundant_ipfs)) + .route("/ipfs/ipfs/:cid/*cpath", get(redundant_ipfs)) .layer(Extension(Arc::clone(&self.config))) .layer(Extension(Arc::clone(&self.client))) .layer( @@ -73,18 +108,49 @@ impl Core { } } +#[tracing::instrument()] +async fn redundant_ipfs( + Path(params): Path>, + Query(query_params): Query, +) -> impl IntoResponse { + let cid = params.get("cid").unwrap(); + let cpath = "".to_string(); + let cpath = params.get("cpath").unwrap_or(&cpath); + let redirect_path: String = if cpath.is_empty() { + format!("/ipfs/{}", cid) + } else { + format!("/ipfs/{}/{}", cid, cpath) + }; + let redirect_path = format!("{}{}", redirect_path, query_params.to_query_string()); + Redirect::to(&redirect_path).into_response() +} + #[tracing::instrument()] async fn get_ipfs( Extension(config): Extension>, Extension(client): Extension>, Path(params): Path>, Query(query_params): Query, + request_headers: HeaderMap, ) -> Result { increment_counter!(METRICS_CNT_REQUESTS_TOTAL); - let start_time = std::time::Instant::now(); + let start_time = time::Instant::now(); // parse path params let cid_param = params.get("cid").unwrap(); let cid = Cid::try_from(cid_param.clone()); + let cpath = "".to_string(); + let cpath = params.get("cpath").unwrap_or(&cpath); + + if request_headers.contains_key(HEADER_SERVICE_WORKER) { + let sw = request_headers.get(HEADER_SERVICE_WORKER).unwrap(); + if sw.to_str().unwrap() == "script" && cpath.is_empty() { + return error(StatusCode::BAD_REQUEST, "Service Worker not supported"); + } + } + if request_headers.contains_key(HEADER_X_IPFS_GATEWAY_PREFIX) { + return error(StatusCode::BAD_REQUEST, "Unsupported HTTP header"); + } + let cid = match cid { Ok(cid) => cid, Err(_) => { @@ -92,34 +158,47 @@ async fn get_ipfs( return error(StatusCode::BAD_REQUEST, "invalid cid"); } }; - - let cpath = "".to_string(); - let cpath = params.get("cpath").unwrap_or(&cpath); let full_content_path = format!("/ipfs/{}{}", cid, cpath); + + // todo(arqu): actually plug in a resolver + let resolved_cid = resolve_cid(&cid).await.unwrap(); + // parse query params - let format = if let Some(format) = query_params.format { - match ResponseFormat::try_from(format.as_str()) { - Ok(format) => format, - Err(err) => { - return error(StatusCode::BAD_REQUEST, &err); - } + let format = match get_response_format(&request_headers, query_params.format) { + Ok(format) => format, + Err(err) => { + return error(StatusCode::BAD_REQUEST, &err); } - } else { - ResponseFormat::Fs }; + let query_file_name = query_params.filename.unwrap_or_default(); let download = query_params.download.unwrap_or_default(); - // init headers let mut headers = HashMap::new(); + + if request_headers.contains_key("If-None-Match") { + // todo(arqu): handle dir etags + let cid_etag = get_etag(&resolved_cid, Some(format.clone())); + let inm = request_headers + .get("If-None-Match") + .unwrap() + .to_str() + .unwrap(); + if etag_matches(inm, &cid_etag) { + return response(StatusCode::NOT_MODIFIED, body::BoxBody::default(), headers); + } + } + + // init headers format.write_headers(&mut headers); - headers.insert(HEADER_X_IPFS_PATH.to_string(), full_content_path.clone()); add_user_headers(&mut headers, config.headers.clone()); + headers.insert(HEADER_X_IPFS_PATH.to_string(), full_content_path.clone()); + // todo(arqu): add X-Ipfs-Roots // handle request and fetch data let req = Request { format, - cid: cid.to_string(), + cid, full_content_path, query_file_name, content_path: cpath.to_string(), @@ -128,9 +207,115 @@ async fn get_ipfs( match req.format { ResponseFormat::Raw => serve_raw(&req, *client, headers, start_time).await, ResponseFormat::Car => serve_car(&req, *client, headers, start_time).await, - ResponseFormat::Html => serve_html(&req, *client, headers, start_time).await, - ResponseFormat::Fs => serve_fs(&req, *client, headers, start_time).await, + ResponseFormat::Fs(_) => serve_fs(&req, *client, headers, start_time).await, + } +} + +// todo(arqu): flesh out resolving +#[tracing::instrument()] +async fn resolve_cid(cid: &Cid) -> Result { + Ok(*cid) +} + +#[tracing::instrument()] +fn get_response_format( + request_headers: &HeaderMap, + query_format: Option, +) -> Result { + let format = if let Some(format) = query_format { + if format.is_empty() { + match ResponseFormat::try_from_headers(request_headers) { + Ok(format) => format, + Err(_) => { + return Err("invalid format".to_string()); + } + } + } else { + match ResponseFormat::try_from(format.as_str()) { + Ok(format) => format, + Err(_) => { + match ResponseFormat::try_from_headers(request_headers) { + Ok(format) => format, + Err(_) => { + return Err("invalid format".to_string()); + } + }; + return Err("invalid format".to_string()); + } + } + } + } else { + match ResponseFormat::try_from_headers(request_headers) { + Ok(format) => format, + Err(_) => { + return Err("invalid format".to_string()); + } + } + }; + Ok(format) +} + +#[tracing::instrument()] +fn get_etag(cid: &Cid, response_format: Option) -> String { + let mut suffix = "".to_string(); + if let Some(fmt) = response_format { + suffix = format!(".{}", fmt.get_extenstion()); + } + format!("\"{}{}\"", cid, suffix) +} + +#[tracing::instrument()] +fn etag_matches(inm: &str, cid_etag: &str) -> bool { + let mut buf = inm.trim(); + loop { + if buf.is_empty() { + break; + } + if buf.starts_with(',') { + buf = &buf[1..]; + continue; + } + if buf.starts_with('*') { + return true; + } + let (etag, remain) = scan_etag(buf); + if etag.is_empty() { + break; + } + if etag_weak_match(etag, cid_etag) { + return true; + } + buf = remain; } + false +} + +#[tracing::instrument()] +fn scan_etag(buf: &str) -> (&str, &str) { + let s = buf.trim(); + let mut start = 0; + if s.starts_with("W/") { + start = 2; + } + if s.len() - start < 2 || s.chars().nth(start) != Some('"') { + return ("", ""); + } + for i in start + 1..s.len() { + let c = s.as_bytes().get(i).unwrap(); + if *c == 0x21 || (0x23..0x7E).contains(c) || *c >= 0x80 { + continue; + } + if *c == b'"' { + return (&s[..i + 1], &s[i + 1..]); + } + return ("", ""); + } + ("", "") +} + +#[tracing::instrument()] +fn etag_weak_match(etag: &str, cid_etag: &str) -> bool { + etag.trim_start_matches("W/") == cid_etag.trim_start_matches("W/") } #[tracing::instrument()] @@ -154,6 +339,8 @@ async fn serve_raw( format!("{}.bin", req.cid).as_str(), DISPOSITION_ATTACHMENT, ); + set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); + add_cache_control_headers(&mut headers, req.full_content_path.to_string()); response(StatusCode::OK, body::boxed(body), headers.clone()) } @@ -178,25 +365,10 @@ async fn serve_car( format!("{}.car", req.cid).as_str(), DISPOSITION_ATTACHMENT, ); - response(StatusCode::OK, body::boxed(body), headers.clone()) -} - -#[tracing::instrument()] -async fn serve_html( - req: &Request, - client: Client, - headers: HashMap, - start_time: std::time::Instant, -) -> Result { - let body = client - .get_file_simulated(req.full_content_path.as_str(), start_time) - .await; - let body = match body { - Ok(b) => b, - Err(e) => { - return error(StatusCode::INTERNAL_SERVER_ERROR, &e); - } - }; + // todo(arqu): this should be root cid + let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone()))); + set_etag_headers(&mut headers, etag); + // todo(arqu): check if etag matches for root cid response(StatusCode::OK, body::boxed(body), headers.clone()) } @@ -222,6 +394,8 @@ async fn serve_fs( &req.content_path, req.download, ); + set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); + add_cache_control_headers(&mut headers, req.full_content_path.to_string()); add_content_type_headers(&mut headers, &name); response(StatusCode::OK, body::boxed(body), headers.clone()) } @@ -272,6 +446,24 @@ fn set_content_disposition_headers( ); } +#[tracing::instrument()] +fn add_cache_control_headers(headers: &mut HashMap, content_path: String) { + if true { + // todo(arqu): work out if cpath is mutable + // now we just treat everything as mutable + // should also utilize the cache flag on config + headers.insert(LAST_MODIFIED.to_string(), Utc::now().to_string()); + } else { + headers.insert(LAST_MODIFIED.to_string(), 0.to_string()); + headers.insert(CACHE_CONTROL.to_string(), VAL_IMMUTABLE_MAX_AGE.to_string()); + } +} + +#[tracing::instrument()] +fn set_etag_headers(headers: &mut HashMap, etag: String) { + headers.insert(ETAG.to_string(), etag); +} + #[tracing::instrument()] fn get_filename(content_path: &str) -> String { content_path @@ -438,4 +630,25 @@ mod tests { assert_eq!(get_filename("QmSomeCid"), "QmSomeCid"); assert_eq!(get_filename(""), ""); } + + #[test] + fn etag_test() { + let any_etag = "*"; + let etag = get_etag( + &Cid::try_from("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy").unwrap(), + Some(ResponseFormat::Raw), + ); + let wetag = format!("W/{}", etag); + let other_etag = get_etag( + &Cid::try_from("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4aaaaa").unwrap(), + Some(ResponseFormat::Raw), + ); + let other_wetag = format!("W/{}", other_etag); + let long_etag = format!("{},{}", other_etag, wetag); + + assert!(etag_matches(any_etag, &etag)); + assert!(etag_matches(&etag, &wetag)); + assert!(etag_matches(&long_etag, &etag)); + assert!(!etag_matches(&etag, &other_wetag)); + } } diff --git a/iroh-gateway/src/response.rs b/iroh-gateway/src/response.rs index d3b7591447..1eb7f5d0db 100644 --- a/iroh-gateway/src/response.rs +++ b/iroh-gateway/src/response.rs @@ -12,10 +12,9 @@ pub const ERR_UNSUPPORTED_FORMAT: &str = "unsuported format"; #[derive(Debug, Clone, PartialEq)] pub enum ResponseFormat { - Html, Raw, Car, - Fs, + Fs(String), } impl std::convert::TryFrom<&str> for ResponseFormat { @@ -23,11 +22,16 @@ impl std::convert::TryFrom<&str> for ResponseFormat { fn try_from(s: &str) -> Result { match s.to_lowercase().as_str() { - "html" => Ok(ResponseFormat::Html), - "raw" => Ok(ResponseFormat::Raw), - "car" => Ok(ResponseFormat::Car), - "fs" | "" => Ok(ResponseFormat::Fs), - _ => Err(format!("{}: {}", ERR_UNSUPPORTED_FORMAT, s)), + "application/vnd.ipld.raw" | "raw" => Ok(ResponseFormat::Raw), + "application/vnd.ipld.car" | "car" => Ok(ResponseFormat::Car), + "fs" | "" => Ok(ResponseFormat::Fs(String::new())), + rf => { + if rf.starts_with("application/vnd.ipld.") { + Ok(ResponseFormat::Fs(rf.to_string())) + } else { + Err(format!("{}: {}", ERR_UNSUPPORTED_FORMAT, rf)) + } + } } } } @@ -48,11 +52,13 @@ impl ResponseFormat { HEADER_X_CONTENT_TYPE_OPTIONS.to_string(), VALUE_XCTO_NOSNIFF.to_string(), ); + headers.insert(ACCEPT_RANGES.to_string(), "none".to_string()); + headers.insert( + CACHE_CONTROL.to_string(), + "no-cache, no-transform".to_string(), + ); } - ResponseFormat::Html => { - headers.insert(CONTENT_TYPE.to_string(), CONTENT_TYPE_HTML.to_string()); - } - ResponseFormat::Fs => { + ResponseFormat::Fs(_) => { headers.insert( CONTENT_TYPE.to_string(), CONTENT_TYPE_OCTET_STREAM.to_string(), @@ -60,6 +66,41 @@ impl ResponseFormat { } } } + + pub fn get_extenstion(&self) -> String { + match self { + ResponseFormat::Raw => "bin".to_string(), + ResponseFormat::Car => "car".to_string(), + ResponseFormat::Fs(s) => { + if s.is_empty() { + String::new() + } else { + s.split('.').last().unwrap().to_string() + } + } + } + } + + pub fn try_from_headers(headers: &HeaderMap) -> Result { + if headers.contains_key("Accept") { + let h_values = headers.get("Accept").unwrap().to_str().unwrap(); + let h_values = h_values.split(',').collect::>(); + for h_value in h_values { + let h_value = h_value.trim(); + if h_value.starts_with("application/vnd.ipld.") { + // if valid media type use it, otherwise return error + // todo(arqu): add support for better media type detection + if h_value != "application/vnd.ipld.raw" + && h_value != "application/vnd.ipld.car" + { + return Err(format!("{}: {}", ERR_UNSUPPORTED_FORMAT, h_value)); + } + return ResponseFormat::try_from(h_value); + } + } + } + Ok(ResponseFormat::Fs(String::new())) + } } #[derive(Debug)] @@ -97,12 +138,10 @@ mod tests { assert_eq!(rf, Ok(ResponseFormat::Raw)); let rf = ResponseFormat::try_from("car"); assert_eq!(rf, Ok(ResponseFormat::Car)); - let rf = ResponseFormat::try_from("html"); - assert_eq!(rf, Ok(ResponseFormat::Html)); let rf = ResponseFormat::try_from("fs"); - assert_eq!(rf, Ok(ResponseFormat::Fs)); + assert_eq!(rf, Ok(ResponseFormat::Fs(String::new()))); let rf = ResponseFormat::try_from(""); - assert_eq!(rf, Ok(ResponseFormat::Fs)); + assert_eq!(rf, Ok(ResponseFormat::Fs(String::new()))); let rf = ResponseFormat::try_from("RaW"); assert_eq!(rf, Ok(ResponseFormat::Raw)); @@ -131,7 +170,7 @@ mod tests { let rf = ResponseFormat::try_from("car").unwrap(); let mut headers = HashMap::new(); rf.write_headers(&mut headers); - assert_eq!(headers.len(), 2); + assert_eq!(headers.len(), 4); assert_eq!( headers.get(&CONTENT_TYPE.to_string()).unwrap(), &CONTENT_TYPE_IPLD_CAR.to_string() @@ -143,15 +182,6 @@ mod tests { &VALUE_XCTO_NOSNIFF.to_string() ); - let rf = ResponseFormat::try_from("html").unwrap(); - let mut headers = HashMap::new(); - rf.write_headers(&mut headers); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_TYPE.to_string()).unwrap(), - &CONTENT_TYPE_HTML.to_string() - ); - let rf = ResponseFormat::try_from("fs").unwrap(); let mut headers = HashMap::new(); rf.write_headers(&mut headers); From 3744073da48d5f260ed98479178f586779a1d6bc Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Thu, 12 May 2022 14:19:06 +0200 Subject: [PATCH 02/31] refactor --- iroh-gateway/src/core.rs | 317 +---------------------------------- iroh-gateway/src/headers.rs | 283 +++++++++++++++++++++++++++++++ iroh-gateway/src/lib.rs | 1 + iroh-gateway/src/response.rs | 38 +++++ 4 files changed, 324 insertions(+), 315 deletions(-) create mode 100644 iroh-gateway/src/headers.rs diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index 624a83231c..bfd2234c33 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -7,7 +7,6 @@ use axum::{ routing::get, BoxError, Router, }; -use chrono::prelude::Utc; use cid::Cid; use metrics::increment_counter; use serde::Deserialize; @@ -24,8 +23,9 @@ use crate::{ config::Config, constants::*, error::GatewayError, + headers::*, metrics::{get_current_trace_id, METRICS_CNT_REQUESTS_TOTAL}, - response::{GatewayResponse, ResponseFormat}, + response::{get_response_format, GatewayResponse, ResponseFormat}, }; #[derive(Debug)] @@ -217,107 +217,6 @@ async fn resolve_cid(cid: &Cid) -> Result { Ok(*cid) } -#[tracing::instrument()] -fn get_response_format( - request_headers: &HeaderMap, - query_format: Option, -) -> Result { - let format = if let Some(format) = query_format { - if format.is_empty() { - match ResponseFormat::try_from_headers(request_headers) { - Ok(format) => format, - Err(_) => { - return Err("invalid format".to_string()); - } - } - } else { - match ResponseFormat::try_from(format.as_str()) { - Ok(format) => format, - Err(_) => { - match ResponseFormat::try_from_headers(request_headers) { - Ok(format) => format, - Err(_) => { - return Err("invalid format".to_string()); - } - }; - return Err("invalid format".to_string()); - } - } - } - } else { - match ResponseFormat::try_from_headers(request_headers) { - Ok(format) => format, - Err(_) => { - return Err("invalid format".to_string()); - } - } - }; - Ok(format) -} - -#[tracing::instrument()] -fn get_etag(cid: &Cid, response_format: Option) -> String { - let mut suffix = "".to_string(); - if let Some(fmt) = response_format { - suffix = format!(".{}", fmt.get_extenstion()); - } - format!("\"{}{}\"", cid, suffix) -} - -#[tracing::instrument()] -fn etag_matches(inm: &str, cid_etag: &str) -> bool { - let mut buf = inm.trim(); - loop { - if buf.is_empty() { - break; - } - if buf.starts_with(',') { - buf = &buf[1..]; - continue; - } - if buf.starts_with('*') { - return true; - } - let (etag, remain) = scan_etag(buf); - if etag.is_empty() { - break; - } - if etag_weak_match(etag, cid_etag) { - return true; - } - buf = remain; - } - false -} - -#[tracing::instrument()] -fn scan_etag(buf: &str) -> (&str, &str) { - let s = buf.trim(); - let mut start = 0; - if s.starts_with("W/") { - start = 2; - } - if s.len() - start < 2 || s.chars().nth(start) != Some('"') { - return ("", ""); - } - for i in start + 1..s.len() { - let c = s.as_bytes().get(i).unwrap(); - if *c == 0x21 || (0x23..0x7E).contains(c) || *c >= 0x80 { - continue; - } - if *c == b'"' { - return (&s[..i + 1], &s[i + 1..]); - } - return ("", ""); - } - ("", "") -} - -#[tracing::instrument()] -fn etag_weak_match(etag: &str, cid_etag: &str) -> bool { - etag.trim_start_matches("W/") == cid_etag.trim_start_matches("W/") -} - #[tracing::instrument()] async fn serve_raw( req: &Request, @@ -400,80 +299,6 @@ async fn serve_fs( response(StatusCode::OK, body::boxed(body), headers.clone()) } -#[tracing::instrument()] -fn add_user_headers(headers: &mut HashMap, user_headers: HashMap) { - headers.extend(user_headers.into_iter()); -} - -#[tracing::instrument()] -fn add_content_type_headers(headers: &mut HashMap, name: &str) { - let guess = mime_guess::from_path(name); - let content_type = guess.first_or_octet_stream().to_string(); - headers.insert(CONTENT_TYPE.to_string(), content_type); -} - -#[tracing::instrument()] -fn add_content_disposition_headers( - headers: &mut HashMap, - filename: &str, - content_path: &str, - should_download: bool, -) -> String { - let mut name = get_filename(content_path); - if !filename.is_empty() { - name = filename.to_string(); - } - if !name.is_empty() { - let disposition = if should_download { - DISPOSITION_ATTACHMENT - } else { - DISPOSITION_INLINE - }; - set_content_disposition_headers(headers, &name, disposition); - } - name -} - -#[tracing::instrument()] -fn set_content_disposition_headers( - headers: &mut HashMap, - filename: &str, - disposition: &str, -) { - headers.insert( - CONTENT_DISPOSITION.to_string(), - format!("{}; filename={}", disposition, filename), - ); -} - -#[tracing::instrument()] -fn add_cache_control_headers(headers: &mut HashMap, content_path: String) { - if true { - // todo(arqu): work out if cpath is mutable - // now we just treat everything as mutable - // should also utilize the cache flag on config - headers.insert(LAST_MODIFIED.to_string(), Utc::now().to_string()); - } else { - headers.insert(LAST_MODIFIED.to_string(), 0.to_string()); - headers.insert(CACHE_CONTROL.to_string(), VAL_IMMUTABLE_MAX_AGE.to_string()); - } -} - -#[tracing::instrument()] -fn set_etag_headers(headers: &mut HashMap, etag: String) { - headers.insert(ETAG.to_string(), etag); -} - -#[tracing::instrument()] -fn get_filename(content_path: &str) -> String { - content_path - .split('/') - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) - .last() - .unwrap_or_default() -} - #[tracing::instrument()] fn response( status_code: StatusCode, @@ -514,141 +339,3 @@ async fn middleware_error_handler(error: BoxError) -> impl IntoResponse { Cow::from(format!("unhandled internal error: {}", error)), ) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn add_user_headers_test() { - let mut headers = HashMap::new(); - let user_headers = HashMap::from_iter(vec![ - (HEADER_X_IPFS_PATH.to_string(), "QmHeaderPath1".to_string()), - (HEADER_X_IPFS_PATH.to_string(), "QmHeaderPath2".to_string()), - ]); - add_user_headers(&mut headers, user_headers); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&HEADER_X_IPFS_PATH.to_string()).unwrap(), - &"QmHeaderPath2".to_string() - ); - } - - #[test] - fn add_content_type_headers_test() { - let mut headers = HashMap::new(); - let name = "test.txt"; - add_content_type_headers(&mut headers, name); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_TYPE.to_string()).unwrap(), - &"text/plain".to_string() - ); - - let mut headers = HashMap::new(); - let name = "test.RAND_EXT"; - add_content_type_headers(&mut headers, name); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_TYPE.to_string()).unwrap(), - &CONTENT_TYPE_OCTET_STREAM.to_string() - ); - } - - #[test] - fn add_content_disposition_headers_test() { - // inline - let mut headers = HashMap::new(); - let filename = "test.txt"; - let content_path = "QmSomeCid"; - let download = false; - let name = add_content_disposition_headers(&mut headers, filename, content_path, download); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_DISPOSITION.to_string()).unwrap(), - &"inline; filename=test.txt".to_string() - ); - assert_eq!(name, "test.txt"); - - // attachment - let mut headers = HashMap::new(); - let filename = "test.txt"; - let content_path = "QmSomeCid"; - let download = true; - let name = add_content_disposition_headers(&mut headers, filename, content_path, download); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_DISPOSITION.to_string()).unwrap(), - &"attachment; filename=test.txt".to_string() - ); - assert_eq!(name, "test.txt"); - - // no filename & no content path filename - let mut headers = HashMap::new(); - let filename = ""; - let content_path = "QmSomeCid"; - let download = true; - let name = add_content_disposition_headers(&mut headers, filename, content_path, download); - assert_eq!(headers.len(), 1); - assert_eq!(name, "QmSomeCid"); - - // no filename & with content path filename - let mut headers = HashMap::new(); - let filename = ""; - let content_path = "QmSomeCid/folder/test.txt"; - let download = true; - let name = add_content_disposition_headers(&mut headers, filename, content_path, download); - assert_eq!(headers.len(), 1); - assert_eq!(name, "test.txt"); - } - - #[test] - fn set_content_disposition_headers_test() { - let mut headers = HashMap::new(); - let filename = "test_inline.txt"; - set_content_disposition_headers(&mut headers, filename, DISPOSITION_INLINE); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_DISPOSITION.to_string()).unwrap(), - &"inline; filename=test_inline.txt".to_string() - ); - - let mut headers = HashMap::new(); - let filename = "test_attachment.txt"; - set_content_disposition_headers(&mut headers, filename, DISPOSITION_ATTACHMENT); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_DISPOSITION.to_string()).unwrap(), - &"attachment; filename=test_attachment.txt".to_string() - ); - } - - #[test] - fn get_filename_test() { - assert_eq!(get_filename("QmSomeCid/folder/test.txt"), "test.txt"); - assert_eq!(get_filename("QmSomeCid/folder"), "folder"); - assert_eq!(get_filename("QmSomeCid"), "QmSomeCid"); - assert_eq!(get_filename(""), ""); - } - - #[test] - fn etag_test() { - let any_etag = "*"; - let etag = get_etag( - &Cid::try_from("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy").unwrap(), - Some(ResponseFormat::Raw), - ); - let wetag = format!("W/{}", etag); - let other_etag = get_etag( - &Cid::try_from("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4aaaaa").unwrap(), - Some(ResponseFormat::Raw), - ); - let other_wetag = format!("W/{}", other_etag); - let long_etag = format!("{},{}", other_etag, wetag); - - assert!(etag_matches(any_etag, &etag)); - assert!(etag_matches(&etag, &wetag)); - assert!(etag_matches(&long_etag, &etag)); - assert!(!etag_matches(&etag, &other_wetag)); - } -} diff --git a/iroh-gateway/src/headers.rs b/iroh-gateway/src/headers.rs new file mode 100644 index 0000000000..22129c414e --- /dev/null +++ b/iroh-gateway/src/headers.rs @@ -0,0 +1,283 @@ +use crate::{constants::*, response::ResponseFormat}; +use axum::http::header::*; +use chrono::prelude::Utc; +use cid::Cid; +use std::collections::HashMap; + +#[tracing::instrument()] +pub fn add_user_headers( + headers: &mut HashMap, + user_headers: HashMap, +) { + headers.extend(user_headers.into_iter()); +} + +#[tracing::instrument()] +pub fn add_content_type_headers(headers: &mut HashMap, name: &str) { + let guess = mime_guess::from_path(name); + let content_type = guess.first_or_octet_stream().to_string(); + headers.insert(CONTENT_TYPE.to_string(), content_type); +} + +#[tracing::instrument()] +pub fn add_content_disposition_headers( + headers: &mut HashMap, + filename: &str, + content_path: &str, + should_download: bool, +) -> String { + let mut name = get_filename(content_path); + if !filename.is_empty() { + name = filename.to_string(); + } + if !name.is_empty() { + let disposition = if should_download { + DISPOSITION_ATTACHMENT + } else { + DISPOSITION_INLINE + }; + set_content_disposition_headers(headers, &name, disposition); + } + name +} + +#[tracing::instrument()] +pub fn set_content_disposition_headers( + headers: &mut HashMap, + filename: &str, + disposition: &str, +) { + headers.insert( + CONTENT_DISPOSITION.to_string(), + format!("{}; filename={}", disposition, filename), + ); +} + +#[tracing::instrument()] +pub fn add_cache_control_headers(headers: &mut HashMap, content_path: String) { + if true { + // todo(arqu): work out if cpath is mutable + // now we just treat everything as mutable + // should also utilize the cache flag on config + headers.insert(LAST_MODIFIED.to_string(), Utc::now().to_string()); + } else { + headers.insert(LAST_MODIFIED.to_string(), 0.to_string()); + headers.insert(CACHE_CONTROL.to_string(), VAL_IMMUTABLE_MAX_AGE.to_string()); + } +} + +#[tracing::instrument()] +pub fn set_etag_headers(headers: &mut HashMap, etag: String) { + headers.insert(ETAG.to_string(), etag); +} + +#[tracing::instrument()] +pub fn get_etag(cid: &Cid, response_format: Option) -> String { + let mut suffix = "".to_string(); + if let Some(fmt) = response_format { + suffix = format!(".{}", fmt.get_extenstion()); + } + format!("\"{}{}\"", cid, suffix) +} + +#[tracing::instrument()] +pub fn etag_matches(inm: &str, cid_etag: &str) -> bool { + let mut buf = inm.trim(); + loop { + if buf.is_empty() { + break; + } + if buf.starts_with(',') { + buf = &buf[1..]; + continue; + } + if buf.starts_with('*') { + return true; + } + let (etag, remain) = scan_etag(buf); + if etag.is_empty() { + break; + } + if etag_weak_match(etag, cid_etag) { + return true; + } + buf = remain; + } + false +} + +#[tracing::instrument()] +pub fn scan_etag(buf: &str) -> (&str, &str) { + let s = buf.trim(); + let mut start = 0; + if s.starts_with("W/") { + start = 2; + } + if s.len() - start < 2 || s.chars().nth(start) != Some('"') { + return ("", ""); + } + for i in start + 1..s.len() { + let c = s.as_bytes().get(i).unwrap(); + if *c == 0x21 || (0x23..0x7E).contains(c) || *c >= 0x80 { + continue; + } + if *c == b'"' { + return (&s[..i + 1], &s[i + 1..]); + } + return ("", ""); + } + ("", "") +} + +#[tracing::instrument()] +pub fn etag_weak_match(etag: &str, cid_etag: &str) -> bool { + etag.trim_start_matches("W/") == cid_etag.trim_start_matches("W/") +} + +#[tracing::instrument()] +fn get_filename(content_path: &str) -> String { + content_path + .split('/') + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .last() + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn add_user_headers_test() { + let mut headers = HashMap::new(); + let user_headers = HashMap::from_iter(vec![ + (HEADER_X_IPFS_PATH.to_string(), "QmHeaderPath1".to_string()), + (HEADER_X_IPFS_PATH.to_string(), "QmHeaderPath2".to_string()), + ]); + add_user_headers(&mut headers, user_headers); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&HEADER_X_IPFS_PATH.to_string()).unwrap(), + &"QmHeaderPath2".to_string() + ); + } + + #[test] + fn add_content_type_headers_test() { + let mut headers = HashMap::new(); + let name = "test.txt"; + add_content_type_headers(&mut headers, name); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_TYPE.to_string()).unwrap(), + &"text/plain".to_string() + ); + + let mut headers = HashMap::new(); + let name = "test.RAND_EXT"; + add_content_type_headers(&mut headers, name); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_TYPE.to_string()).unwrap(), + &CONTENT_TYPE_OCTET_STREAM.to_string() + ); + } + + #[test] + fn add_content_disposition_headers_test() { + // inline + let mut headers = HashMap::new(); + let filename = "test.txt"; + let content_path = "QmSomeCid"; + let download = false; + let name = add_content_disposition_headers(&mut headers, filename, content_path, download); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_DISPOSITION.to_string()).unwrap(), + &"inline; filename=test.txt".to_string() + ); + assert_eq!(name, "test.txt"); + + // attachment + let mut headers = HashMap::new(); + let filename = "test.txt"; + let content_path = "QmSomeCid"; + let download = true; + let name = add_content_disposition_headers(&mut headers, filename, content_path, download); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_DISPOSITION.to_string()).unwrap(), + &"attachment; filename=test.txt".to_string() + ); + assert_eq!(name, "test.txt"); + + // no filename & no content path filename + let mut headers = HashMap::new(); + let filename = ""; + let content_path = "QmSomeCid"; + let download = true; + let name = add_content_disposition_headers(&mut headers, filename, content_path, download); + assert_eq!(headers.len(), 1); + assert_eq!(name, "QmSomeCid"); + + // no filename & with content path filename + let mut headers = HashMap::new(); + let filename = ""; + let content_path = "QmSomeCid/folder/test.txt"; + let download = true; + let name = add_content_disposition_headers(&mut headers, filename, content_path, download); + assert_eq!(headers.len(), 1); + assert_eq!(name, "test.txt"); + } + + #[test] + fn set_content_disposition_headers_test() { + let mut headers = HashMap::new(); + let filename = "test_inline.txt"; + set_content_disposition_headers(&mut headers, filename, DISPOSITION_INLINE); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_DISPOSITION.to_string()).unwrap(), + &"inline; filename=test_inline.txt".to_string() + ); + + let mut headers = HashMap::new(); + let filename = "test_attachment.txt"; + set_content_disposition_headers(&mut headers, filename, DISPOSITION_ATTACHMENT); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_DISPOSITION.to_string()).unwrap(), + &"attachment; filename=test_attachment.txt".to_string() + ); + } + + #[test] + fn get_filename_test() { + assert_eq!(get_filename("QmSomeCid/folder/test.txt"), "test.txt"); + assert_eq!(get_filename("QmSomeCid/folder"), "folder"); + assert_eq!(get_filename("QmSomeCid"), "QmSomeCid"); + assert_eq!(get_filename(""), ""); + } + + #[test] + fn etag_test() { + let any_etag = "*"; + let etag = get_etag( + &Cid::try_from("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy").unwrap(), + Some(ResponseFormat::Raw), + ); + let wetag = format!("W/{}", etag); + let other_etag = get_etag( + &Cid::try_from("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4aaaaa").unwrap(), + Some(ResponseFormat::Raw), + ); + let other_wetag = format!("W/{}", other_etag); + let long_etag = format!("{},{}", other_etag, wetag); + + assert!(etag_matches(any_etag, &etag)); + assert!(etag_matches(&etag, &wetag)); + assert!(etag_matches(&long_etag, &etag)); + assert!(!etag_matches(&etag, &other_wetag)); + } +} diff --git a/iroh-gateway/src/lib.rs b/iroh-gateway/src/lib.rs index d819cbc8d0..a3e617d817 100644 --- a/iroh-gateway/src/lib.rs +++ b/iroh-gateway/src/lib.rs @@ -3,5 +3,6 @@ pub mod config; mod constants; pub mod core; mod error; +mod headers; pub mod metrics; mod response; diff --git a/iroh-gateway/src/response.rs b/iroh-gateway/src/response.rs index 1eb7f5d0db..c969f5a95d 100644 --- a/iroh-gateway/src/response.rs +++ b/iroh-gateway/src/response.rs @@ -103,6 +103,44 @@ impl ResponseFormat { } } +#[tracing::instrument()] +pub fn get_response_format( + request_headers: &HeaderMap, + query_format: Option, +) -> Result { + let format = if let Some(format) = query_format { + if format.is_empty() { + match ResponseFormat::try_from_headers(request_headers) { + Ok(format) => format, + Err(_) => { + return Err("invalid format".to_string()); + } + } + } else { + match ResponseFormat::try_from(format.as_str()) { + Ok(format) => format, + Err(_) => { + match ResponseFormat::try_from_headers(request_headers) { + Ok(format) => format, + Err(_) => { + return Err("invalid format".to_string()); + } + }; + return Err("invalid format".to_string()); + } + } + } + } else { + match ResponseFormat::try_from_headers(request_headers) { + Ok(format) => format, + Err(_) => { + return Err("invalid format".to_string()); + } + } + }; + Ok(format) +} + #[derive(Debug)] pub struct GatewayResponse { pub status_code: StatusCode, From 5f3cf4bc96956a2b1f810723674e61b2acfa1af1 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Thu, 12 May 2022 15:41:31 +0200 Subject: [PATCH 03/31] fix headers --- iroh-gateway/src/config.rs | 29 +++++------- iroh-gateway/src/constants.rs | 38 +++++++++------ iroh-gateway/src/core.rs | 21 +++++---- iroh-gateway/src/error.rs | 1 - iroh-gateway/src/headers.rs | 89 ++++++++++++++++++----------------- iroh-gateway/src/response.rs | 78 ++++++++++-------------------- 6 files changed, 120 insertions(+), 136 deletions(-) diff --git a/iroh-gateway/src/config.rs b/iroh-gateway/src/config.rs index fc42d1414d..62ad8f5e5c 100644 --- a/iroh-gateway/src/config.rs +++ b/iroh-gateway/src/config.rs @@ -1,5 +1,4 @@ -use std::collections::HashMap; - +use crate::constants::*; use axum::http::header::*; pub const DEFAULT_PORT: u16 = 9050; @@ -13,7 +12,7 @@ pub struct Config { /// flag to toggle whether the gateway enables/utilizes caching pub cache: bool, /// set of user provided headers to attach to all responses - pub headers: HashMap, //todo(arqu): convert to use axum::http::header + pub headers: HeaderMap, /// default port to listen on pub port: u16, } @@ -24,21 +23,19 @@ impl Config { writeable, fetch, cache, - headers: HashMap::new(), + headers: HeaderMap::new(), port, } } pub fn set_default_headers(&mut self) { - let mut headers = HashMap::new(); - headers.insert(ACCESS_CONTROL_ALLOW_ORIGIN.to_string(), "*".to_string()); - headers.insert(ACCESS_CONTROL_ALLOW_HEADERS.to_string(), "*".to_string()); - headers.insert(ACCESS_CONTROL_ALLOW_METHODS.to_string(), "*".to_string()); - headers.insert( - CACHE_CONTROL.to_string(), - "no-cache, no-transform".to_string(), - ); - headers.insert(ACCEPT_RANGES.to_string(), "none".to_string()); + let mut headers = HeaderMap::new(); + headers.insert(ACCESS_CONTROL_ALLOW_ORIGIN, VALUE_STAR.clone()); + headers.insert(ACCESS_CONTROL_ALLOW_HEADERS, VALUE_STAR.clone()); + headers.insert(ACCESS_CONTROL_ALLOW_METHODS, VALUE_STAR.clone()); + // todo(arqu): remove these once propperly implmented + headers.insert(CACHE_CONTROL, VALUE_NO_CACHE_NO_TRANSFORM.clone()); + headers.insert(ACCEPT_RANGES, VALUE_NONE.clone()); self.headers = headers; } } @@ -49,7 +46,7 @@ impl Default for Config { writeable: false, fetch: false, cache: false, - headers: HashMap::new(), + headers: HeaderMap::new(), port: DEFAULT_PORT, }; t.set_default_headers(); @@ -67,8 +64,8 @@ mod tests { config.set_default_headers(); assert_eq!(config.headers.len(), 5); assert_eq!( - config.headers.get(&ACCESS_CONTROL_ALLOW_ORIGIN.to_string()), - Some(&"*".to_string()) + config.headers.get(&ACCESS_CONTROL_ALLOW_ORIGIN), + Some(&VALUE_STAR) ); } diff --git a/iroh-gateway/src/constants.rs b/iroh-gateway/src/constants.rs index 1b99b32c1a..420ae63ca7 100644 --- a/iroh-gateway/src/constants.rs +++ b/iroh-gateway/src/constants.rs @@ -1,21 +1,31 @@ +use axum::http::{header::HeaderName, HeaderValue}; + // Headers -pub const HEADER_X_IPFS_PATH: &str = "X-Ipfs-Path"; -pub const HEADER_X_CONTENT_TYPE_OPTIONS: &str = "X-Content-Type-Options"; -pub const HEADER_X_TRACE_ID: &str = "X-Trace-Id"; -pub const HEADER_X_IPFS_GATEWAY_PREFIX: &str = "X-Ipfs-Gateway-Prefix"; -pub const HEADER_SERVICE_WORKER: &str = "Service-Worker"; +pub static HEADER_X_IPFS_PATH: HeaderName = HeaderName::from_static("x-ipfs-path"); +pub static HEADER_X_CONTENT_TYPE_OPTIONS: HeaderName = + HeaderName::from_static("x-content-type-options"); +pub static HEADER_X_TRACE_ID: HeaderName = HeaderName::from_static("x-trace-id"); +pub static HEADER_X_IPFS_GATEWAY_PREFIX: HeaderName = + HeaderName::from_static("x-ipfs-gateway-prefix"); +pub static HEADER_SERVICE_WORKER: HeaderName = HeaderName::from_static("service-worker"); // Common Header Values -pub const VALUE_XCTO_NOSNIFF: &str = "nosniff"; +pub static VALUE_XCTO_NOSNIFF: HeaderValue = HeaderValue::from_static("nosniff"); +pub static VALUE_STAR: HeaderValue = HeaderValue::from_static("*"); +pub static VALUE_NONE: HeaderValue = HeaderValue::from_static("none"); +pub static VALUE_NO_CACHE_NO_TRANSFORM: HeaderValue = + HeaderValue::from_static("no-cache, no-transform"); +pub static VAL_IMMUTABLE_MAX_AGE: HeaderValue = + HeaderValue::from_static("public, max-age=31536000, immutable"); // Dispositions -pub const DISPOSITION_ATTACHMENT: &str = "attachment"; -pub const DISPOSITION_INLINE: &str = "inline"; +pub static DISPOSITION_ATTACHMENT: &str = "attachment"; +pub static DISPOSITION_INLINE: &str = "inline"; // Content Types -pub const CONTENT_TYPE_IPLD_RAW: &str = "application/vnd.ipld.raw"; -pub const CONTENT_TYPE_IPLD_CAR: &str = "application/vnd.ipld.car; version=1"; -pub const CONTENT_TYPE_OCTET_STREAM: &str = "application/octet-stream"; - -// Values -pub const VAL_IMMUTABLE_MAX_AGE: &str = "public, max-age=31536000, immutable"; +pub static CONTENT_TYPE_IPLD_RAW: HeaderValue = + HeaderValue::from_static("application/vnd.ipld.raw"); +pub static CONTENT_TYPE_IPLD_CAR: HeaderValue = + HeaderValue::from_static("application/vnd.ipld.car; version=1"); +pub static CONTENT_TYPE_OCTET_STREAM: HeaderValue = + HeaderValue::from_static("application/octet-stream"); diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index bfd2234c33..4ad71c6f8a 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -141,13 +141,13 @@ async fn get_ipfs( let cpath = "".to_string(); let cpath = params.get("cpath").unwrap_or(&cpath); - if request_headers.contains_key(HEADER_SERVICE_WORKER) { - let sw = request_headers.get(HEADER_SERVICE_WORKER).unwrap(); + if request_headers.contains_key(&HEADER_SERVICE_WORKER) { + let sw = request_headers.get(&HEADER_SERVICE_WORKER).unwrap(); if sw.to_str().unwrap() == "script" && cpath.is_empty() { return error(StatusCode::BAD_REQUEST, "Service Worker not supported"); } } - if request_headers.contains_key(HEADER_X_IPFS_GATEWAY_PREFIX) { + if request_headers.contains_key(&HEADER_X_IPFS_GATEWAY_PREFIX) { return error(StatusCode::BAD_REQUEST, "Unsupported HTTP header"); } @@ -174,7 +174,7 @@ async fn get_ipfs( let query_file_name = query_params.filename.unwrap_or_default(); let download = query_params.download.unwrap_or_default(); - let mut headers = HashMap::new(); + let mut headers = HeaderMap::new(); if request_headers.contains_key("If-None-Match") { // todo(arqu): handle dir etags @@ -192,7 +192,10 @@ async fn get_ipfs( // init headers format.write_headers(&mut headers); add_user_headers(&mut headers, config.headers.clone()); - headers.insert(HEADER_X_IPFS_PATH.to_string(), full_content_path.clone()); + headers.insert( + &HEADER_X_IPFS_PATH, + HeaderValue::from_str(&full_content_path).unwrap(), + ); // todo(arqu): add X-Ipfs-Roots // handle request and fetch data @@ -221,7 +224,7 @@ async fn resolve_cid(cid: &Cid) -> Result { async fn serve_raw( req: &Request, client: Client, - mut headers: HashMap, + mut headers: HeaderMap, start_time: std::time::Instant, ) -> Result { let body = client @@ -247,7 +250,7 @@ async fn serve_raw( async fn serve_car( req: &Request, client: Client, - mut headers: HashMap, + mut headers: HeaderMap, start_time: std::time::Instant, ) -> Result { let body = client @@ -275,7 +278,7 @@ async fn serve_car( async fn serve_fs( req: &Request, client: Client, - mut headers: HashMap, + mut headers: HeaderMap, start_time: std::time::Instant, ) -> Result { let body = client @@ -303,7 +306,7 @@ async fn serve_fs( fn response( status_code: StatusCode, body: BoxBody, - headers: HashMap, + headers: HeaderMap, ) -> Result { Ok(GatewayResponse { status_code, diff --git a/iroh-gateway/src/error.rs b/iroh-gateway/src/error.rs index b320a5f407..e277333b2c 100644 --- a/iroh-gateway/src/error.rs +++ b/iroh-gateway/src/error.rs @@ -23,7 +23,6 @@ impl IntoResponse for GatewayError { "message": self.message, "trace_id": self.trace_id, })); - // todo(arqu): add headers (self.status_code, body).into_response() } } diff --git a/iroh-gateway/src/headers.rs b/iroh-gateway/src/headers.rs index 22129c414e..099e6e9752 100644 --- a/iroh-gateway/src/headers.rs +++ b/iroh-gateway/src/headers.rs @@ -2,26 +2,22 @@ use crate::{constants::*, response::ResponseFormat}; use axum::http::header::*; use chrono::prelude::Utc; use cid::Cid; -use std::collections::HashMap; #[tracing::instrument()] -pub fn add_user_headers( - headers: &mut HashMap, - user_headers: HashMap, -) { +pub fn add_user_headers(headers: &mut HeaderMap, user_headers: HeaderMap) { headers.extend(user_headers.into_iter()); } #[tracing::instrument()] -pub fn add_content_type_headers(headers: &mut HashMap, name: &str) { +pub fn add_content_type_headers(headers: &mut HeaderMap, name: &str) { let guess = mime_guess::from_path(name); let content_type = guess.first_or_octet_stream().to_string(); - headers.insert(CONTENT_TYPE.to_string(), content_type); + headers.insert(CONTENT_TYPE, HeaderValue::from_str(&content_type).unwrap()); } #[tracing::instrument()] pub fn add_content_disposition_headers( - headers: &mut HashMap, + headers: &mut HeaderMap, filename: &str, content_path: &str, should_download: bool, @@ -42,40 +38,42 @@ pub fn add_content_disposition_headers( } #[tracing::instrument()] -pub fn set_content_disposition_headers( - headers: &mut HashMap, - filename: &str, - disposition: &str, -) { +pub fn set_content_disposition_headers(headers: &mut HeaderMap, filename: &str, disposition: &str) { headers.insert( - CONTENT_DISPOSITION.to_string(), - format!("{}; filename={}", disposition, filename), + CONTENT_DISPOSITION, + HeaderValue::from_str(&format!("{}; filename={}", disposition, filename)).unwrap(), ); } #[tracing::instrument()] -pub fn add_cache_control_headers(headers: &mut HashMap, content_path: String) { +pub fn add_cache_control_headers(headers: &mut HeaderMap, content_path: String) { if true { // todo(arqu): work out if cpath is mutable // now we just treat everything as mutable // should also utilize the cache flag on config - headers.insert(LAST_MODIFIED.to_string(), Utc::now().to_string()); + headers.insert( + LAST_MODIFIED, + HeaderValue::from_str(&Utc::now().to_string()).unwrap(), + ); } else { - headers.insert(LAST_MODIFIED.to_string(), 0.to_string()); - headers.insert(CACHE_CONTROL.to_string(), VAL_IMMUTABLE_MAX_AGE.to_string()); + headers.insert(LAST_MODIFIED, HeaderValue::from_str("0").unwrap()); + headers.insert(CACHE_CONTROL, VAL_IMMUTABLE_MAX_AGE.clone()); } } #[tracing::instrument()] -pub fn set_etag_headers(headers: &mut HashMap, etag: String) { - headers.insert(ETAG.to_string(), etag); +pub fn set_etag_headers(headers: &mut HeaderMap, etag: String) { + headers.insert(ETAG, HeaderValue::from_str(&etag).unwrap()); } #[tracing::instrument()] pub fn get_etag(cid: &Cid, response_format: Option) -> String { let mut suffix = "".to_string(); if let Some(fmt) = response_format { - suffix = format!(".{}", fmt.get_extenstion()); + let ext = fmt.get_extenstion(); + if !ext.is_empty() { + suffix = format!(".{}", ext); + } } format!("\"{}{}\"", cid, suffix) } @@ -150,70 +148,75 @@ mod tests { #[test] fn add_user_headers_test() { - let mut headers = HashMap::new(); - let user_headers = HashMap::from_iter(vec![ - (HEADER_X_IPFS_PATH.to_string(), "QmHeaderPath1".to_string()), - (HEADER_X_IPFS_PATH.to_string(), "QmHeaderPath2".to_string()), - ]); + let mut headers = HeaderMap::new(); + let mut user_headers = HeaderMap::new(); + user_headers.insert( + &HEADER_X_IPFS_PATH, + HeaderValue::from_str("QmHeaderPath1").unwrap(), + ); + user_headers.insert( + &HEADER_X_IPFS_PATH, + HeaderValue::from_str("QmHeaderPath2").unwrap(), + ); add_user_headers(&mut headers, user_headers); assert_eq!(headers.len(), 1); assert_eq!( - headers.get(&HEADER_X_IPFS_PATH.to_string()).unwrap(), + headers.get(&HEADER_X_IPFS_PATH).unwrap(), &"QmHeaderPath2".to_string() ); } #[test] fn add_content_type_headers_test() { - let mut headers = HashMap::new(); + let mut headers = HeaderMap::new(); let name = "test.txt"; add_content_type_headers(&mut headers, name); assert_eq!(headers.len(), 1); assert_eq!( - headers.get(&CONTENT_TYPE.to_string()).unwrap(), + headers.get(&CONTENT_TYPE).unwrap(), &"text/plain".to_string() ); - let mut headers = HashMap::new(); + let mut headers = HeaderMap::new(); let name = "test.RAND_EXT"; add_content_type_headers(&mut headers, name); assert_eq!(headers.len(), 1); assert_eq!( - headers.get(&CONTENT_TYPE.to_string()).unwrap(), - &CONTENT_TYPE_OCTET_STREAM.to_string() + headers.get(&CONTENT_TYPE).unwrap(), + &CONTENT_TYPE_OCTET_STREAM ); } #[test] fn add_content_disposition_headers_test() { // inline - let mut headers = HashMap::new(); + let mut headers = HeaderMap::new(); let filename = "test.txt"; let content_path = "QmSomeCid"; let download = false; let name = add_content_disposition_headers(&mut headers, filename, content_path, download); assert_eq!(headers.len(), 1); assert_eq!( - headers.get(&CONTENT_DISPOSITION.to_string()).unwrap(), + headers.get(&CONTENT_DISPOSITION).unwrap(), &"inline; filename=test.txt".to_string() ); assert_eq!(name, "test.txt"); // attachment - let mut headers = HashMap::new(); + let mut headers = HeaderMap::new(); let filename = "test.txt"; let content_path = "QmSomeCid"; let download = true; let name = add_content_disposition_headers(&mut headers, filename, content_path, download); assert_eq!(headers.len(), 1); assert_eq!( - headers.get(&CONTENT_DISPOSITION.to_string()).unwrap(), + headers.get(&CONTENT_DISPOSITION).unwrap(), &"attachment; filename=test.txt".to_string() ); assert_eq!(name, "test.txt"); // no filename & no content path filename - let mut headers = HashMap::new(); + let mut headers = HeaderMap::new(); let filename = ""; let content_path = "QmSomeCid"; let download = true; @@ -222,7 +225,7 @@ mod tests { assert_eq!(name, "QmSomeCid"); // no filename & with content path filename - let mut headers = HashMap::new(); + let mut headers = HeaderMap::new(); let filename = ""; let content_path = "QmSomeCid/folder/test.txt"; let download = true; @@ -233,21 +236,21 @@ mod tests { #[test] fn set_content_disposition_headers_test() { - let mut headers = HashMap::new(); + let mut headers = HeaderMap::new(); let filename = "test_inline.txt"; set_content_disposition_headers(&mut headers, filename, DISPOSITION_INLINE); assert_eq!(headers.len(), 1); assert_eq!( - headers.get(&CONTENT_DISPOSITION.to_string()).unwrap(), + headers.get(&CONTENT_DISPOSITION).unwrap(), &"inline; filename=test_inline.txt".to_string() ); - let mut headers = HashMap::new(); + let mut headers = HeaderMap::new(); let filename = "test_attachment.txt"; set_content_disposition_headers(&mut headers, filename, DISPOSITION_ATTACHMENT); assert_eq!(headers.len(), 1); assert_eq!( - headers.get(&CONTENT_DISPOSITION.to_string()).unwrap(), + headers.get(&CONTENT_DISPOSITION).unwrap(), &"attachment; filename=test_attachment.txt".to_string() ); } diff --git a/iroh-gateway/src/response.rs b/iroh-gateway/src/response.rs index c969f5a95d..de3eb700c6 100644 --- a/iroh-gateway/src/response.rs +++ b/iroh-gateway/src/response.rs @@ -1,5 +1,3 @@ -use std::{collections::HashMap, str::FromStr}; - use axum::{ body::BoxBody, http::{header::*, HeaderValue, StatusCode}, @@ -37,32 +35,20 @@ impl std::convert::TryFrom<&str> for ResponseFormat { } impl ResponseFormat { - pub fn write_headers(&self, headers: &mut HashMap) { + pub fn write_headers(&self, headers: &mut HeaderMap) { match self { ResponseFormat::Raw => { - headers.insert(CONTENT_TYPE.to_string(), CONTENT_TYPE_IPLD_RAW.to_string()); - headers.insert( - HEADER_X_CONTENT_TYPE_OPTIONS.to_string(), - VALUE_XCTO_NOSNIFF.to_string(), - ); + headers.insert(CONTENT_TYPE, CONTENT_TYPE_IPLD_RAW.clone()); + headers.insert(&HEADER_X_CONTENT_TYPE_OPTIONS, VALUE_XCTO_NOSNIFF.clone()); } ResponseFormat::Car => { - headers.insert(CONTENT_TYPE.to_string(), CONTENT_TYPE_IPLD_CAR.to_string()); - headers.insert( - HEADER_X_CONTENT_TYPE_OPTIONS.to_string(), - VALUE_XCTO_NOSNIFF.to_string(), - ); - headers.insert(ACCEPT_RANGES.to_string(), "none".to_string()); - headers.insert( - CACHE_CONTROL.to_string(), - "no-cache, no-transform".to_string(), - ); + headers.insert(CONTENT_TYPE, CONTENT_TYPE_IPLD_CAR.clone()); + headers.insert(&HEADER_X_CONTENT_TYPE_OPTIONS, VALUE_XCTO_NOSNIFF.clone()); + headers.insert(ACCEPT_RANGES, VALUE_NONE.clone()); + headers.insert(CACHE_CONTROL, VALUE_NO_CACHE_NO_TRANSFORM.clone()); } ResponseFormat::Fs(_) => { - headers.insert( - CONTENT_TYPE.to_string(), - CONTENT_TYPE_OCTET_STREAM.to_string(), - ); + headers.insert(CONTENT_TYPE, CONTENT_TYPE_OCTET_STREAM.clone()); } } } @@ -145,22 +131,19 @@ pub fn get_response_format( pub struct GatewayResponse { pub status_code: StatusCode, pub body: BoxBody, - pub headers: HashMap, + pub headers: HeaderMap, pub trace_id: String, } impl IntoResponse for GatewayResponse { - fn into_response(self) -> Response { + fn into_response(mut self) -> Response { let mut rb = Response::builder().status(self.status_code); - let headers = rb.headers_mut().unwrap(); - for (key, value) in &self.headers { - let header_name = HeaderName::from_str(key).unwrap(); - headers.insert(header_name, HeaderValue::from_str(value).unwrap()); - } - headers.insert( - HEADER_X_TRACE_ID, + self.headers.insert( + &HEADER_X_TRACE_ID, HeaderValue::from_str(&self.trace_id).unwrap(), ); + let rh = rb.headers_mut().unwrap(); + rh.extend(self.headers); rb.body(self.body).unwrap() } } @@ -168,7 +151,6 @@ impl IntoResponse for GatewayResponse { #[cfg(test)] mod tests { use super::*; - use std::collections::HashMap; #[test] fn response_format_try_from() { @@ -191,42 +173,32 @@ mod tests { #[test] fn response_format_write_headers() { let rf = ResponseFormat::try_from("raw").unwrap(); - let mut headers = HashMap::new(); + let mut headers = HeaderMap::new(); rf.write_headers(&mut headers); assert_eq!(headers.len(), 2); + assert_eq!(headers.get(&CONTENT_TYPE).unwrap(), &CONTENT_TYPE_IPLD_RAW); assert_eq!( - headers.get(&CONTENT_TYPE.to_string()).unwrap(), - &CONTENT_TYPE_IPLD_RAW.to_string() - ); - assert_eq!( - headers - .get(&HEADER_X_CONTENT_TYPE_OPTIONS.to_string()) - .unwrap(), - &VALUE_XCTO_NOSNIFF.to_string() + headers.get(&HEADER_X_CONTENT_TYPE_OPTIONS).unwrap(), + &VALUE_XCTO_NOSNIFF ); let rf = ResponseFormat::try_from("car").unwrap(); - let mut headers = HashMap::new(); + let mut headers = HeaderMap::new(); rf.write_headers(&mut headers); assert_eq!(headers.len(), 4); + assert_eq!(headers.get(&CONTENT_TYPE).unwrap(), &CONTENT_TYPE_IPLD_CAR); assert_eq!( - headers.get(&CONTENT_TYPE.to_string()).unwrap(), - &CONTENT_TYPE_IPLD_CAR.to_string() - ); - assert_eq!( - headers - .get(&HEADER_X_CONTENT_TYPE_OPTIONS.to_string()) - .unwrap(), - &VALUE_XCTO_NOSNIFF.to_string() + headers.get(&HEADER_X_CONTENT_TYPE_OPTIONS).unwrap(), + &VALUE_XCTO_NOSNIFF ); let rf = ResponseFormat::try_from("fs").unwrap(); - let mut headers = HashMap::new(); + let mut headers = HeaderMap::new(); rf.write_headers(&mut headers); assert_eq!(headers.len(), 1); assert_eq!( - headers.get(&CONTENT_TYPE.to_string()).unwrap(), - &CONTENT_TYPE_OCTET_STREAM.to_string() + headers.get(&CONTENT_TYPE).unwrap(), + &CONTENT_TYPE_OCTET_STREAM ); } } From 1e8217b45a4dbb4099144ccf056d45a6a55e134e Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 13 May 2022 11:09:08 +0200 Subject: [PATCH 04/31] cleanup --- iroh-gateway/Cargo.toml | 3 ++- iroh-gateway/src/config.rs | 1 - iroh-gateway/src/core.rs | 28 +++++++--------------------- iroh-gateway/src/headers.rs | 6 ++++-- iroh-gateway/src/response.rs | 25 +++++++++++++------------ 5 files changed, 26 insertions(+), 37 deletions(-) diff --git a/iroh-gateway/Cargo.toml b/iroh-gateway/Cargo.toml index b6508d131e..deb7e57514 100644 --- a/iroh-gateway/Cargo.toml +++ b/iroh-gateway/Cargo.toml @@ -14,6 +14,7 @@ axum = "0.5.1" clap = { version = "3.1.8", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.78" +serde_qs = "0.9.2" tower = { version = "0.4", features = ["util", "timeout", "load-shed", "limit"] } mime_guess = "2.0.4" iroh-metrics = { path = "../iroh-metrics" } @@ -24,7 +25,7 @@ git-version = "0.3.5" rand = "0.8.5" tracing-opentelemetry = "0.17.2" opentelemetry = { version = "0.17.0", features = ["rt-tokio"] } -chrono = "0.4.19" +time = "0.3.9" [dev-dependencies] axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messages on handlers diff --git a/iroh-gateway/src/config.rs b/iroh-gateway/src/config.rs index 62ad8f5e5c..ec2896254a 100644 --- a/iroh-gateway/src/config.rs +++ b/iroh-gateway/src/config.rs @@ -1,6 +1,5 @@ use crate::constants::*; use axum::http::header::*; - pub const DEFAULT_PORT: u16 = 9050; #[derive(Debug, Clone)] diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index 4ad71c6f8a..b01be2accb 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -9,7 +9,8 @@ use axum::{ }; use cid::Cid; use metrics::increment_counter; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; +use serde_qs; use std::{ borrow::Cow, collections::HashMap, @@ -34,7 +35,7 @@ pub struct Core { client: Arc, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct GetParams { // todo(arqu): swap this for ResponseFormat /// specifies the expected format of the response @@ -47,26 +48,11 @@ pub struct GetParams { impl GetParams { pub fn to_query_string(&self) -> String { - let mut query = String::new(); - if let Some(format) = &self.format { - query.push_str(&format!("format={}", format)); - } - if let Some(filename) = &self.filename { - if !query.is_empty() { - query.push('&'); - } - query.push_str(&format!("filename={}", filename)); - } - if let Some(download) = &self.download { - if !query.is_empty() { - query.push('&'); - } - query.push_str(&format!("download={}", download)); - } - if query.is_empty() { - "".to_string() + let q = serde_qs::to_string(self).unwrap(); + if q.is_empty() { + q } else { - format!("?{}", query) + format!("?{}", q) } } } diff --git a/iroh-gateway/src/headers.rs b/iroh-gateway/src/headers.rs index 099e6e9752..5938835b69 100644 --- a/iroh-gateway/src/headers.rs +++ b/iroh-gateway/src/headers.rs @@ -1,7 +1,8 @@ use crate::{constants::*, response::ResponseFormat}; +use ::time::OffsetDateTime; use axum::http::header::*; -use chrono::prelude::Utc; use cid::Cid; +use std::time; #[tracing::instrument()] pub fn add_user_headers(headers: &mut HeaderMap, user_headers: HeaderMap) { @@ -51,9 +52,10 @@ pub fn add_cache_control_headers(headers: &mut HeaderMap, content_path: String) // todo(arqu): work out if cpath is mutable // now we just treat everything as mutable // should also utilize the cache flag on config + let lmdt: OffsetDateTime = time::SystemTime::now().into(); headers.insert( LAST_MODIFIED, - HeaderValue::from_str(&Utc::now().to_string()).unwrap(), + HeaderValue::from_str(&lmdt.to_string()).unwrap(), ); } else { headers.insert(LAST_MODIFIED, HeaderValue::from_str("0").unwrap()); diff --git a/iroh-gateway/src/response.rs b/iroh-gateway/src/response.rs index de3eb700c6..5a2d37944d 100644 --- a/iroh-gateway/src/response.rs +++ b/iroh-gateway/src/response.rs @@ -69,19 +69,20 @@ impl ResponseFormat { pub fn try_from_headers(headers: &HeaderMap) -> Result { if headers.contains_key("Accept") { - let h_values = headers.get("Accept").unwrap().to_str().unwrap(); - let h_values = h_values.split(',').collect::>(); - for h_value in h_values { - let h_value = h_value.trim(); - if h_value.starts_with("application/vnd.ipld.") { - // if valid media type use it, otherwise return error - // todo(arqu): add support for better media type detection - if h_value != "application/vnd.ipld.raw" - && h_value != "application/vnd.ipld.car" - { - return Err(format!("{}: {}", ERR_UNSUPPORTED_FORMAT, h_value)); + if let Some(h_values) = headers.get("Accept") { + let h_values = h_values.to_str().unwrap().split(','); + for h_value in h_values { + let h_value = h_value.trim(); + if h_value.starts_with("application/vnd.ipld.") { + // if valid media type use it, otherwise return error + // todo(arqu): add support for better media type detection + if h_value != "application/vnd.ipld.raw" + && h_value != "application/vnd.ipld.car" + { + return Err(format!("{}: {}", ERR_UNSUPPORTED_FORMAT, h_value)); + } + return ResponseFormat::try_from(h_value); } - return ResponseFormat::try_from(h_value); } } } From f9bc0b58a87cbdb78f1b6b669741a3b8a2887c3d Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 13 May 2022 15:40:55 +0200 Subject: [PATCH 05/31] fix: metrics subscriber config --- iroh-gateway/src/main.rs | 5 ++++- iroh-gateway/src/metrics.rs | 11 +++++++++-- iroh-metrics/Cargo.toml | 2 +- iroh-metrics/src/lib.rs | 18 ++++++++---------- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/iroh-gateway/src/main.rs b/iroh-gateway/src/main.rs index f5aa99498a..cbb8e44c7e 100644 --- a/iroh-gateway/src/main.rs +++ b/iroh-gateway/src/main.rs @@ -12,6 +12,8 @@ struct Args { fetch: bool, #[clap(short, long)] cache: bool, + #[clap(long = "no-metrics")] + no_metrics: bool, } #[tokio::main] @@ -22,7 +24,8 @@ async fn main() { config.set_default_headers(); println!("{:#?}", config); - iroh_metrics::init(metrics::metrics_config()).expect("failed to initialize metrics"); + iroh_metrics::init(metrics::metrics_config(args.no_metrics)) + .expect("failed to initialize metrics"); metrics::register_counters(); let handler = Core::new(config); diff --git a/iroh-gateway/src/metrics.rs b/iroh-gateway/src/metrics.rs index 9028f462cc..cf4d71030b 100644 --- a/iroh-gateway/src/metrics.rs +++ b/iroh-gateway/src/metrics.rs @@ -4,7 +4,7 @@ use metrics::{describe_counter, describe_gauge, describe_histogram, Unit}; use opentelemetry::trace::{TraceContextExt, TraceId}; use tracing_opentelemetry::OpenTelemetrySpanExt; -pub fn metrics_config() -> iroh_metrics::Config { +pub fn metrics_config(logger_only: bool) -> iroh_metrics::Config { // compile time configuration let service_name = env!("CARGO_PKG_NAME").to_string(); let build = git_version!().to_string(); @@ -14,7 +14,14 @@ pub fn metrics_config() -> iroh_metrics::Config { let instance_id = std::env::var("IROH_INSTANCE_ID") .unwrap_or_else(|_| names::Generator::default().next().unwrap()); let service_env = std::env::var("IROH_ENV").unwrap_or_else(|_| "dev".to_string()); - iroh_metrics::Config::new(service_name, instance_id, build, version, service_env) + iroh_metrics::Config::new( + service_name, + instance_id, + build, + version, + service_env, + logger_only, + ) } pub const METRICS_CNT_REQUESTS_TOTAL: &str = "gw_requests_total"; diff --git a/iroh-metrics/Cargo.toml b/iroh-metrics/Cargo.toml index d1fb527029..b19a7fe0bf 100644 --- a/iroh-metrics/Cargo.toml +++ b/iroh-metrics/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/n0-computer/iroh" [dependencies] tracing = "0.1.33" tracing-futures = "0.2.5" -tracing-subscriber = "0.3.9" +tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } tracing-opentelemetry = "0.17.2" opentelemetry = { version = "0.17.0", features = ["rt-tokio"] } opentelemetry-otlp = { version = "0.10.0", features = ["grpc-sys"] } diff --git a/iroh-metrics/src/lib.rs b/iroh-metrics/src/lib.rs index 837bd101e4..9d181a0c76 100644 --- a/iroh-metrics/src/lib.rs +++ b/iroh-metrics/src/lib.rs @@ -3,7 +3,7 @@ use opentelemetry::sdk::{trace, Resource}; use opentelemetry_otlp::WithExportConfig; use std::env::consts::{ARCH, OS}; use std::time::Duration; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer}; /// Initialize the tracing and metrics subsystems. pub fn init(cfg: Config) -> Result<(), Box> { @@ -27,14 +27,11 @@ pub fn init_metrics(cfg: Config) -> Result<(), Box> { /// Initialize the tracing subsystem. pub fn init_tracer(cfg: Config) -> Result<(), Box> { + let log_subscriber = fmt::layer() + .pretty() + .with_filter(EnvFilter::from_default_env()); if cfg.debug { - tracing_subscriber::fmt() - .pretty() - .with_thread_names(true) - // enable everything - .with_max_level(tracing::Level::TRACE) - // sets this to be the default, global collector for this application. - .init(); + tracing_subscriber::registry().with(log_subscriber).init(); } else { let tracer = opentelemetry_otlp::new_pipeline() .tracing() @@ -57,10 +54,10 @@ pub fn init_tracer(cfg: Config) -> Result<(), Box> { let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer); tracing_subscriber::registry() + .with(log_subscriber) .with(opentelemetry) .try_init()?; } - Ok(()) } @@ -96,9 +93,10 @@ impl Config { build: String, version: String, service_env: String, + debug: bool, ) -> Self { let debug = - std::env::var("IROH_METRICS_DEBUG").unwrap_or_else(|_| "false".to_string()) == "true"; + std::env::var("IROH_METRICS_DEBUG").unwrap_or_else(|_| debug.to_string()) == "true"; let collector_endpoint = std::env::var("IROH_METRICS_COLLECTOR_ENDPOINT") .unwrap_or_else(|_| "http://localhost:4317".to_string()); let prometheus_gateway_endpoint = std::env::var("IROH_METRICS_PROM_GATEWAY_ENDPOINT") From 2d4364079d65988e355ea0a0cf3fd770abd6ace8 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 13 May 2022 22:57:14 +0200 Subject: [PATCH 06/31] fix: protoc --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00ccc05f09..3666a6675a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,6 +111,9 @@ jobs: - name: Start sccache server run: sccache --start-server + - name: Install Protoc + uses: arduino/setup-protoc@v1 + - name: check uses: actions-rs/cargo@v1 with: From 9fa8d04010345be081a7492b4c502cbe972cb203 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 13 May 2022 23:19:15 +0200 Subject: [PATCH 07/31] fix: windows build --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3666a6675a..8dcfa505d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,6 +114,10 @@ jobs: - name: Install Protoc uses: arduino/setup-protoc@v1 + - name: configure windows + if: matirx.os == 'windows-latest' + uses: ilammy/msvc-dev-cmd@v1.4.1 + - name: check uses: actions-rs/cargo@v1 with: From a31517078a31f40f1eb66864a450888455ca5b83 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 13 May 2022 23:37:01 +0200 Subject: [PATCH 08/31] ci: pin to win-2019 --- .github/workflows/ci.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8dcfa505d8..e62274fa15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,14 +14,14 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + os: [ubuntu-latest, windows-2019, macOS-latest] rust: [nightly, stable] include: - os: ubuntu-latest sccache-path: /home/runner/.cache/sccache - os: macos-latest sccache-path: /Users/runner/Library/Caches/Mozilla.sccache - - os: windows-latest + - os: windows-2019 sccache-path: "%LOCALAPPDATA%\\sccache" env: RUST_BACKTRACE: full @@ -33,11 +33,11 @@ jobs: steps: - uses: actions/checkout@master - name: Set sccache env path - if: matrix.os != 'windows-latest' + if: matrix.os != 'windows-2019' run: | echo "SCCACHE_DIR=${{ matrix.sccache-path }}" >> $GITHUB_ENV - name: Set windows arch - if: matrix.os == 'windows-latest' + if: matrix.os == 'windows-2019' run: | echo "RELEASE_ARCH=windows" >> $GITHUB_ENV - name: Set linux arch @@ -60,16 +60,16 @@ jobs: mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache chmod 755 $HOME/.local/bin/sccache echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: Install scoop (windows-latest) - if: matrix.os == 'windows-latest' + - name: Install scoop (windows-2019) + if: matrix.os == 'windows-2019' shell: powershell run: | Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser iwr -useb get.scoop.sh -outfile 'install.ps1' .\\install.ps1 -RunAsAdmin Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH - - name: Install sccache (windows-latest) - if: matrix.os == 'windows-latest' + - name: Install sccache (windows-2019) + if: matrix.os == 'windows-2019' shell: powershell run: scoop install sccache - name: Install sccache (macos-latest) @@ -115,8 +115,8 @@ jobs: uses: arduino/setup-protoc@v1 - name: configure windows - if: matirx.os == 'windows-latest' uses: ilammy/msvc-dev-cmd@v1.4.1 + if: matirx.os == 'windows-2019' - name: check uses: actions-rs/cargo@v1 @@ -146,14 +146,14 @@ jobs: - name: build release uses: actions-rs/cargo@v1 - if: matrix.os != 'windows-latest' && matrix.rust=='stable' && github.ref_name=='main' + if: matrix.os != 'windows-2019' && matrix.rust=='stable' && github.ref_name=='main' with: command: build args: --release - name: Get current iroh-gateway version id: ig_version - if: matrix.os != 'windows-latest' && matrix.rust=='stable' && github.ref_name=='main' + if: matrix.os != 'windows-2019' && matrix.rust=='stable' && github.ref_name=='main' uses: dante-signal31/rust-app-version@v1.0.0 with: cargo_toml_folder: iroh-gateway/ @@ -165,7 +165,7 @@ jobs: sudo installer -pkg AWSCLIV2.pkg -target / - name: Set aws credentials - if: matrix.os != 'windows-latest' + if: matrix.os != 'windows-2019' run: | echo "AWS_ACCESS_KEY_ID=${{secrets.S3_ACCESS_KEY_ID}}" >> $GITHUB_ENV echo "AWS_SECRET_ACCESS_KEY=${{secrets.S3_ACCESS_KEY}}" >> $GITHUB_ENV From e24262e0520c36239f2c2ff0525de595c31e8b1a Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 13 May 2022 23:38:06 +0200 Subject: [PATCH 09/31] typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e62274fa15..992660b9dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,7 +116,7 @@ jobs: - name: configure windows uses: ilammy/msvc-dev-cmd@v1.4.1 - if: matirx.os == 'windows-2019' + if: matrix.os == 'windows-2019' - name: check uses: actions-rs/cargo@v1 From 1176d16ab7ca251a49e9ef4139976aebecb024a6 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 13 May 2022 23:56:00 +0200 Subject: [PATCH 10/31] windows... --- .github/workflows/ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 992660b9dc..4da2528190 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,16 @@ jobs: curl -L "$LINK/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache chmod 755 $HOME/.local/bin/sccache - echo "$HOME/.local/bin" >> $GITHUB_PATH + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Install LLVM and Clang # required for bindgen to work, see https://github.com/rust-lang/rust-bindgen/issues/1797 + uses: KyleMayes/install-llvm-action@32c4866ebb71e0949e8833eb49beeebed48532bd + if: matrix.os == 'windows-2019' + with: + version: "11.0" + directory: ${{ runner.temp }}/llvm + - name: Set LIBCLANG_PATH + run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV + if: matrix.os == 'windows-2019' - name: Install scoop (windows-2019) if: matrix.os == 'windows-2019' shell: powershell From 0186b3ca5f0c58d3c288de1795faf79a83651971 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Sat, 14 May 2022 00:09:48 +0200 Subject: [PATCH 11/31] clear sccache --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4da2528190..20d1828ea9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: RUSTV: ${{ matrix.rust }} SCCACHE_CACHE_SIZE: 2G - # SCCACHE_RECACHE: 1 # Uncomment this to clear cache, then comment it back out + SCCACHE_RECACHE: 1 # Uncomment this to clear cache, then comment it back out steps: - uses: actions/checkout@master - name: Set sccache env path From f015bc55055152e28f37366440b4f13532903da3 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Sat, 14 May 2022 00:18:57 +0200 Subject: [PATCH 12/31] try windows 2022 again --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20d1828ea9..6e54f7972d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,14 +14,14 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-2019, macOS-latest] + os: [ubuntu-latest, windows-latest, macOS-latest] rust: [nightly, stable] include: - os: ubuntu-latest sccache-path: /home/runner/.cache/sccache - os: macos-latest sccache-path: /Users/runner/Library/Caches/Mozilla.sccache - - os: windows-2019 + - os: windows-latest sccache-path: "%LOCALAPPDATA%\\sccache" env: RUST_BACKTRACE: full @@ -29,15 +29,15 @@ jobs: RUSTV: ${{ matrix.rust }} SCCACHE_CACHE_SIZE: 2G - SCCACHE_RECACHE: 1 # Uncomment this to clear cache, then comment it back out + # SCCACHE_RECACHE: 1 # Uncomment this to clear cache, then comment it back out steps: - uses: actions/checkout@master - name: Set sccache env path - if: matrix.os != 'windows-2019' + if: matrix.os != 'windows-latest' run: | echo "SCCACHE_DIR=${{ matrix.sccache-path }}" >> $GITHUB_ENV - name: Set windows arch - if: matrix.os == 'windows-2019' + if: matrix.os == 'windows-latest' run: | echo "RELEASE_ARCH=windows" >> $GITHUB_ENV - name: Set linux arch @@ -62,23 +62,23 @@ jobs: echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Install LLVM and Clang # required for bindgen to work, see https://github.com/rust-lang/rust-bindgen/issues/1797 uses: KyleMayes/install-llvm-action@32c4866ebb71e0949e8833eb49beeebed48532bd - if: matrix.os == 'windows-2019' + if: matrix.os == 'windows-latest' with: version: "11.0" directory: ${{ runner.temp }}/llvm - name: Set LIBCLANG_PATH run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV - if: matrix.os == 'windows-2019' - - name: Install scoop (windows-2019) - if: matrix.os == 'windows-2019' + if: matrix.os == 'windows-latest' + - name: Install scoop (windows-latest) + if: matrix.os == 'windows-latest' shell: powershell run: | Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser iwr -useb get.scoop.sh -outfile 'install.ps1' .\\install.ps1 -RunAsAdmin Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH - - name: Install sccache (windows-2019) - if: matrix.os == 'windows-2019' + - name: Install sccache (windows-latest) + if: matrix.os == 'windows-latest' shell: powershell run: scoop install sccache - name: Install sccache (macos-latest) @@ -125,7 +125,7 @@ jobs: - name: configure windows uses: ilammy/msvc-dev-cmd@v1.4.1 - if: matrix.os == 'windows-2019' + if: matrix.os == 'windows-latest' - name: check uses: actions-rs/cargo@v1 @@ -155,14 +155,14 @@ jobs: - name: build release uses: actions-rs/cargo@v1 - if: matrix.os != 'windows-2019' && matrix.rust=='stable' && github.ref_name=='main' + if: matrix.os != 'windows-latest' && matrix.rust=='stable' && github.ref_name=='main' with: command: build args: --release - name: Get current iroh-gateway version id: ig_version - if: matrix.os != 'windows-2019' && matrix.rust=='stable' && github.ref_name=='main' + if: matrix.os != 'windows-latest' && matrix.rust=='stable' && github.ref_name=='main' uses: dante-signal31/rust-app-version@v1.0.0 with: cargo_toml_folder: iroh-gateway/ @@ -174,7 +174,7 @@ jobs: sudo installer -pkg AWSCLIV2.pkg -target / - name: Set aws credentials - if: matrix.os != 'windows-2019' + if: matrix.os != 'windows-latest' run: | echo "AWS_ACCESS_KEY_ID=${{secrets.S3_ACCESS_KEY_ID}}" >> $GITHUB_ENV echo "AWS_SECRET_ACCESS_KEY=${{secrets.S3_ACCESS_KEY}}" >> $GITHUB_ENV From d0441ee7b7586246b03cb607af64a3ba40ffc0d3 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Sat, 14 May 2022 00:42:29 +0200 Subject: [PATCH 13/31] try windows 2019 again --- .github/workflows/ci.yml | 50 ++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e54f7972d..57bcac987c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,14 +14,14 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + os: [ubuntu-latest, windows-2019, macOS-latest] rust: [nightly, stable] include: - os: ubuntu-latest sccache-path: /home/runner/.cache/sccache - os: macos-latest sccache-path: /Users/runner/Library/Caches/Mozilla.sccache - - os: windows-latest + - os: windows-2019 sccache-path: "%LOCALAPPDATA%\\sccache" env: RUST_BACKTRACE: full @@ -33,11 +33,11 @@ jobs: steps: - uses: actions/checkout@master - name: Set sccache env path - if: matrix.os != 'windows-latest' + if: matrix.os != 'windows-2019' run: | echo "SCCACHE_DIR=${{ matrix.sccache-path }}" >> $GITHUB_ENV - name: Set windows arch - if: matrix.os == 'windows-latest' + if: matrix.os == 'windows-2019' run: | echo "RELEASE_ARCH=windows" >> $GITHUB_ENV - name: Set linux arch @@ -60,25 +60,25 @@ jobs: mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache chmod 755 $HOME/.local/bin/sccache echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: Install LLVM and Clang # required for bindgen to work, see https://github.com/rust-lang/rust-bindgen/issues/1797 - uses: KyleMayes/install-llvm-action@32c4866ebb71e0949e8833eb49beeebed48532bd - if: matrix.os == 'windows-latest' - with: - version: "11.0" - directory: ${{ runner.temp }}/llvm - - name: Set LIBCLANG_PATH - run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV - if: matrix.os == 'windows-latest' - - name: Install scoop (windows-latest) - if: matrix.os == 'windows-latest' + # - name: Install LLVM and Clang # required for bindgen to work, see https://github.com/rust-lang/rust-bindgen/issues/1797 + # uses: KyleMayes/install-llvm-action@32c4866ebb71e0949e8833eb49beeebed48532bd + # if: matrix.os == 'windows-2019' + # with: + # version: "11.0" + # directory: ${{ runner.temp }}/llvm + # - name: Set LIBCLANG_PATH + # run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV + # if: matrix.os == 'windows-2019' + - name: Install scoop (windows-2019) + if: matrix.os == 'windows-2019' shell: powershell run: | Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser iwr -useb get.scoop.sh -outfile 'install.ps1' .\\install.ps1 -RunAsAdmin Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH - - name: Install sccache (windows-latest) - if: matrix.os == 'windows-latest' + - name: Install sccache (windows-2019) + if: matrix.os == 'windows-2019' shell: powershell run: scoop install sccache - name: Install sccache (macos-latest) @@ -120,12 +120,12 @@ jobs: - name: Start sccache server run: sccache --start-server - - name: Install Protoc - uses: arduino/setup-protoc@v1 + # - name: Install Protoc + # uses: arduino/setup-protoc@v1 - - name: configure windows - uses: ilammy/msvc-dev-cmd@v1.4.1 - if: matrix.os == 'windows-latest' + # - name: configure windows + # uses: ilammy/msvc-dev-cmd@v1.4.1 + # if: matrix.os == 'windows-2019' - name: check uses: actions-rs/cargo@v1 @@ -155,14 +155,14 @@ jobs: - name: build release uses: actions-rs/cargo@v1 - if: matrix.os != 'windows-latest' && matrix.rust=='stable' && github.ref_name=='main' + if: matrix.os != 'windows-2019' && matrix.rust=='stable' && github.ref_name=='main' with: command: build args: --release - name: Get current iroh-gateway version id: ig_version - if: matrix.os != 'windows-latest' && matrix.rust=='stable' && github.ref_name=='main' + if: matrix.os != 'windows-2019' && matrix.rust=='stable' && github.ref_name=='main' uses: dante-signal31/rust-app-version@v1.0.0 with: cargo_toml_folder: iroh-gateway/ @@ -174,7 +174,7 @@ jobs: sudo installer -pkg AWSCLIV2.pkg -target / - name: Set aws credentials - if: matrix.os != 'windows-latest' + if: matrix.os != 'windows-2019' run: | echo "AWS_ACCESS_KEY_ID=${{secrets.S3_ACCESS_KEY_ID}}" >> $GITHUB_ENV echo "AWS_SECRET_ACCESS_KEY=${{secrets.S3_ACCESS_KEY}}" >> $GITHUB_ENV From 8308f35b3175ec08da287effb5fe5708c184b6e2 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 09:33:33 +0200 Subject: [PATCH 14/31] try windows 2022 again --- .github/workflows/ci.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57bcac987c..5c3994e12f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,14 +14,14 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-2019, macOS-latest] + os: [ubuntu-latest, windows-latest, macOS-latest] rust: [nightly, stable] include: - os: ubuntu-latest sccache-path: /home/runner/.cache/sccache - os: macos-latest sccache-path: /Users/runner/Library/Caches/Mozilla.sccache - - os: windows-2019 + - os: windows-latest sccache-path: "%LOCALAPPDATA%\\sccache" env: RUST_BACKTRACE: full @@ -33,11 +33,11 @@ jobs: steps: - uses: actions/checkout@master - name: Set sccache env path - if: matrix.os != 'windows-2019' + if: matrix.os != 'windows-latest' run: | echo "SCCACHE_DIR=${{ matrix.sccache-path }}" >> $GITHUB_ENV - name: Set windows arch - if: matrix.os == 'windows-2019' + if: matrix.os == 'windows-latest' run: | echo "RELEASE_ARCH=windows" >> $GITHUB_ENV - name: Set linux arch @@ -62,23 +62,23 @@ jobs: echo "$HOME/.local/bin" >> $GITHUB_PATH # - name: Install LLVM and Clang # required for bindgen to work, see https://github.com/rust-lang/rust-bindgen/issues/1797 # uses: KyleMayes/install-llvm-action@32c4866ebb71e0949e8833eb49beeebed48532bd - # if: matrix.os == 'windows-2019' + # if: matrix.os == 'windows-latest' # with: # version: "11.0" # directory: ${{ runner.temp }}/llvm # - name: Set LIBCLANG_PATH # run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV - # if: matrix.os == 'windows-2019' - - name: Install scoop (windows-2019) - if: matrix.os == 'windows-2019' + # if: matrix.os == 'windows-latest' + - name: Install scoop (windows-latest) + if: matrix.os == 'windows-latest' shell: powershell run: | Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser iwr -useb get.scoop.sh -outfile 'install.ps1' .\\install.ps1 -RunAsAdmin Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH - - name: Install sccache (windows-2019) - if: matrix.os == 'windows-2019' + - name: Install sccache (windows-latest) + if: matrix.os == 'windows-latest' shell: powershell run: scoop install sccache - name: Install sccache (macos-latest) @@ -125,7 +125,7 @@ jobs: # - name: configure windows # uses: ilammy/msvc-dev-cmd@v1.4.1 - # if: matrix.os == 'windows-2019' + # if: matrix.os == 'windows-latest' - name: check uses: actions-rs/cargo@v1 @@ -155,14 +155,14 @@ jobs: - name: build release uses: actions-rs/cargo@v1 - if: matrix.os != 'windows-2019' && matrix.rust=='stable' && github.ref_name=='main' + if: matrix.os != 'windows-latest' && matrix.rust=='stable' && github.ref_name=='main' with: command: build args: --release - name: Get current iroh-gateway version id: ig_version - if: matrix.os != 'windows-2019' && matrix.rust=='stable' && github.ref_name=='main' + if: matrix.os != 'windows-latest' && matrix.rust=='stable' && github.ref_name=='main' uses: dante-signal31/rust-app-version@v1.0.0 with: cargo_toml_folder: iroh-gateway/ @@ -174,7 +174,7 @@ jobs: sudo installer -pkg AWSCLIV2.pkg -target / - name: Set aws credentials - if: matrix.os != 'windows-2019' + if: matrix.os != 'windows-latest' run: | echo "AWS_ACCESS_KEY_ID=${{secrets.S3_ACCESS_KEY_ID}}" >> $GITHUB_ENV echo "AWS_SECRET_ACCESS_KEY=${{secrets.S3_ACCESS_KEY}}" >> $GITHUB_ENV From a633f08831b47a7f730cf83248b88b55ba6c4366 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 09:57:58 +0200 Subject: [PATCH 15/31] idk --- .github/workflows/ci.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c3994e12f..6e54f7972d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,15 +60,15 @@ jobs: mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache chmod 755 $HOME/.local/bin/sccache echo "$HOME/.local/bin" >> $GITHUB_PATH - # - name: Install LLVM and Clang # required for bindgen to work, see https://github.com/rust-lang/rust-bindgen/issues/1797 - # uses: KyleMayes/install-llvm-action@32c4866ebb71e0949e8833eb49beeebed48532bd - # if: matrix.os == 'windows-latest' - # with: - # version: "11.0" - # directory: ${{ runner.temp }}/llvm - # - name: Set LIBCLANG_PATH - # run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV - # if: matrix.os == 'windows-latest' + - name: Install LLVM and Clang # required for bindgen to work, see https://github.com/rust-lang/rust-bindgen/issues/1797 + uses: KyleMayes/install-llvm-action@32c4866ebb71e0949e8833eb49beeebed48532bd + if: matrix.os == 'windows-latest' + with: + version: "11.0" + directory: ${{ runner.temp }}/llvm + - name: Set LIBCLANG_PATH + run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV + if: matrix.os == 'windows-latest' - name: Install scoop (windows-latest) if: matrix.os == 'windows-latest' shell: powershell @@ -120,12 +120,12 @@ jobs: - name: Start sccache server run: sccache --start-server - # - name: Install Protoc - # uses: arduino/setup-protoc@v1 + - name: Install Protoc + uses: arduino/setup-protoc@v1 - # - name: configure windows - # uses: ilammy/msvc-dev-cmd@v1.4.1 - # if: matrix.os == 'windows-latest' + - name: configure windows + uses: ilammy/msvc-dev-cmd@v1.4.1 + if: matrix.os == 'windows-latest' - name: check uses: actions-rs/cargo@v1 From 42340b773340868dfb99ce5656e1113b91e369ea Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 10:42:41 +0200 Subject: [PATCH 16/31] manually config cmake windows --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e54f7972d..3f0cb1ce5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,12 @@ jobs: - name: Set LIBCLANG_PATH run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV if: matrix.os == 'windows-latest' + - name: Configure CMAGE windows + if: matrix.os == 'windows.latest' + run: | + echo "CMAKE_C_COMPILER=C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.32.31326/bin/Hostx64/x64/cl.exe" >> $env:GITHUB_ENV + echo "CMAKE_CXX_COMPILER=C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.32.31326/bin/Hostx64/x64/cl.exe" >> $env:GITHUB_ENV + - name: Install scoop (windows-latest) if: matrix.os == 'windows-latest' shell: powershell From 38a412d9a4664ca80bb9fb96954dc33f5f9a786b Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 11:09:49 +0200 Subject: [PATCH 17/31] disable windows --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f0cb1ce5a..01ce60ba11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,15 +14,15 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + os: [ubuntu-latest, macOS-latest] rust: [nightly, stable] include: - os: ubuntu-latest sccache-path: /home/runner/.cache/sccache - os: macos-latest sccache-path: /Users/runner/Library/Caches/Mozilla.sccache - - os: windows-latest - sccache-path: "%LOCALAPPDATA%\\sccache" + # - os: windows-latest + # sccache-path: "%LOCALAPPDATA%\\sccache" env: RUST_BACKTRACE: full RUSTC_WRAPPER: sccache From 60bcb09e97f315bfccd0df4e8317131f54ce6c4a Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 11:23:43 +0200 Subject: [PATCH 18/31] make windows builds optional --- .github/workflows/ci.yml | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01ce60ba11..da17fc29f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,17 +12,20 @@ jobs: build_and_test: name: Build and test runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.experimental }} strategy: matrix: - os: [ubuntu-latest, macOS-latest] + os: [ubuntu-latest, windows-latest, macOS-latest] rust: [nightly, stable] + experimental: [false] include: - os: ubuntu-latest sccache-path: /home/runner/.cache/sccache - os: macos-latest sccache-path: /Users/runner/Library/Caches/Mozilla.sccache - # - os: windows-latest - # sccache-path: "%LOCALAPPDATA%\\sccache" + - os: windows-latest + sccache-path: "%LOCALAPPDATA%\\sccache" + experimental: true env: RUST_BACKTRACE: full RUSTC_WRAPPER: sccache @@ -60,21 +63,6 @@ jobs: mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache chmod 755 $HOME/.local/bin/sccache echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: Install LLVM and Clang # required for bindgen to work, see https://github.com/rust-lang/rust-bindgen/issues/1797 - uses: KyleMayes/install-llvm-action@32c4866ebb71e0949e8833eb49beeebed48532bd - if: matrix.os == 'windows-latest' - with: - version: "11.0" - directory: ${{ runner.temp }}/llvm - - name: Set LIBCLANG_PATH - run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV - if: matrix.os == 'windows-latest' - - name: Configure CMAGE windows - if: matrix.os == 'windows.latest' - run: | - echo "CMAKE_C_COMPILER=C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.32.31326/bin/Hostx64/x64/cl.exe" >> $env:GITHUB_ENV - echo "CMAKE_CXX_COMPILER=C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.32.31326/bin/Hostx64/x64/cl.exe" >> $env:GITHUB_ENV - - name: Install scoop (windows-latest) if: matrix.os == 'windows-latest' shell: powershell @@ -126,13 +114,6 @@ jobs: - name: Start sccache server run: sccache --start-server - - name: Install Protoc - uses: arduino/setup-protoc@v1 - - - name: configure windows - uses: ilammy/msvc-dev-cmd@v1.4.1 - if: matrix.os == 'windows-latest' - - name: check uses: actions-rs/cargo@v1 with: From 6afa52843cf4c4d2370983bbce880d9b7b688f2d Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 11:34:28 +0200 Subject: [PATCH 19/31] return protoc --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da17fc29f2..c864c06ca6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,6 +114,9 @@ jobs: - name: Start sccache server run: sccache --start-server + - name: Install Protoc + uses: arduino/setup-protoc@v1 + - name: check uses: actions-rs/cargo@v1 with: From 4bdf298c33866d94c105f6d00d5d1ca9d80e13ea Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 12:04:24 +0200 Subject: [PATCH 20/31] windows optional --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c864c06ca6..5d262126e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: continue-on-error: ${{ matrix.experimental }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + os: [ubuntu-latest, macOS-latest] rust: [nightly, stable] experimental: [false] include: @@ -24,6 +24,7 @@ jobs: - os: macos-latest sccache-path: /Users/runner/Library/Caches/Mozilla.sccache - os: windows-latest + rust: stable sccache-path: "%LOCALAPPDATA%\\sccache" experimental: true env: From c34358ec6c8826a289cfb3da2ea5ba3e052f6893 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 12:17:20 +0200 Subject: [PATCH 21/31] protoc --- .github/workflows/ci.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d262126e6..f9506cd53a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,9 +115,30 @@ jobs: - name: Start sccache server run: sccache --start-server - - name: Install Protoc + - name: Install Protoc windows + if: matrix.os == 'windows-latest' uses: arduino/setup-protoc@v1 + - name: Install Protoc linux + if: matrix.os == 'ubuntu-latest' + run: | + PROTOC_VERSION=3.20.0 + PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip + curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP + sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc + sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' + rm -f $PROTOC_ZIP + + - name: Install Protoc macOS + if: matrix.os == 'macOS-latest' + run: | + PROTOC_VERSION=3.20.0 + PROTOC_ZIP=protoc-$PROTOC_VERSION-osx-x86_64.zip + curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP + sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc + sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' + rm -f $PROTOC_ZIP + - name: check uses: actions-rs/cargo@v1 with: From 2f5d9a4bedff191a1411f82186bd5a3cca66c9e6 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 12:29:02 +0200 Subject: [PATCH 22/31] protoc --- .github/workflows/ci.yml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9506cd53a..47a8e8bb11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,22 +122,24 @@ jobs: - name: Install Protoc linux if: matrix.os == 'ubuntu-latest' run: | - PROTOC_VERSION=3.20.0 - PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip - curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP - sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc - sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' - rm -f $PROTOC_ZIP + apt install -y protobuf-compiler + # PROTOC_VERSION=3.20.0 + # PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip + # curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP + # sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc + # sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' + # rm -f $PROTOC_ZIP - name: Install Protoc macOS if: matrix.os == 'macOS-latest' run: | - PROTOC_VERSION=3.20.0 - PROTOC_ZIP=protoc-$PROTOC_VERSION-osx-x86_64.zip - curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP - sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc - sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' - rm -f $PROTOC_ZIP + brew install protobuf + # PROTOC_VERSION=3.20.0 + # PROTOC_ZIP=protoc-$PROTOC_VERSION-osx-x86_64.zip + # curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP + # sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc + # sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' + # rm -f $PROTOC_ZIP - name: check uses: actions-rs/cargo@v1 From 456fe0e07a9c4f28427e979907dc554686bd0151 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 12:44:14 +0200 Subject: [PATCH 23/31] sudo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47a8e8bb11..bfe0798306 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,7 +122,7 @@ jobs: - name: Install Protoc linux if: matrix.os == 'ubuntu-latest' run: | - apt install -y protobuf-compiler + sudo apt install -y protobuf-compiler # PROTOC_VERSION=3.20.0 # PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip # curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP From f94457e758e10f1ccfc84a5e3b69b3fe5708adea Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 12:55:49 +0200 Subject: [PATCH 24/31] recache --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfe0798306..7dbc3bfa02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: RUSTV: ${{ matrix.rust }} SCCACHE_CACHE_SIZE: 2G - # SCCACHE_RECACHE: 1 # Uncomment this to clear cache, then comment it back out + SCCACHE_RECACHE: 1 # Uncomment this to clear cache, then comment it back out steps: - uses: actions/checkout@master - name: Set sccache env path From d7faf33a7cb060252efd5b1bfd83f9155246bb04 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 13:00:42 +0200 Subject: [PATCH 25/31] recache --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7dbc3bfa02..bfe0798306 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: RUSTV: ${{ matrix.rust }} SCCACHE_CACHE_SIZE: 2G - SCCACHE_RECACHE: 1 # Uncomment this to clear cache, then comment it back out + # SCCACHE_RECACHE: 1 # Uncomment this to clear cache, then comment it back out steps: - uses: actions/checkout@master - name: Set sccache env path From e659450ca0eea20a018c64fb2ed879d8e535d4f3 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 13:08:28 +0200 Subject: [PATCH 26/31] playing with deps --- iroh-bitswap/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/iroh-bitswap/Cargo.toml b/iroh-bitswap/Cargo.toml index a74ad0740f..38387842a4 100644 --- a/iroh-bitswap/Cargo.toml +++ b/iroh-bitswap/Cargo.toml @@ -18,6 +18,7 @@ futures = "0.3.5" libp2p = { version = "0.44", default-features = false } multihash = "0.16.0" prost = "0.10" +prost-build = "0.10" thiserror = "1.0.20" unsigned-varint = "0.7.0" ahash = "0.7.6" From 11265b37847c21da1b04946ec1f9f523e5dae121 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 13:15:32 +0200 Subject: [PATCH 27/31] playing with ci --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfe0798306..f4cc6b01aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,6 +123,9 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt install -y protobuf-compiler + echo $PROTOC + echo $PROTOC_INCLUDE + protoc --help # PROTOC_VERSION=3.20.0 # PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip # curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP From f31e36d6e00155d967d3fc1896cec6670caefd85 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 13:20:46 +0200 Subject: [PATCH 28/31] playing with ci --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4cc6b01aa..977ec82ea2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,8 @@ jobs: echo $PROTOC echo $PROTOC_INCLUDE protoc --help + echo "PROTOC=/usr/bin/protoc" >> $GITHUB_ENV + echo "PROTOC_INCLUDE=/usr/include" >> $GITHUB_ENV # PROTOC_VERSION=3.20.0 # PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip # curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP From 965b71fdd274e6135eea4335dedd324c6b345999 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 13:38:14 +0200 Subject: [PATCH 29/31] cleanup ci --- .github/workflows/ci.yml | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 977ec82ea2..f3fe946a9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,37 +21,32 @@ jobs: include: - os: ubuntu-latest sccache-path: /home/runner/.cache/sccache + release-arch: linux - os: macos-latest sccache-path: /Users/runner/Library/Caches/Mozilla.sccache + release-arch: darwin - os: windows-latest rust: stable sccache-path: "%LOCALAPPDATA%\\sccache" + release-arch: windows experimental: true env: RUST_BACKTRACE: full RUSTC_WRAPPER: sccache RUSTV: ${{ matrix.rust }} SCCACHE_CACHE_SIZE: 2G - # SCCACHE_RECACHE: 1 # Uncomment this to clear cache, then comment it back out steps: - uses: actions/checkout@master + - name: Set sccache env path if: matrix.os != 'windows-latest' run: | echo "SCCACHE_DIR=${{ matrix.sccache-path }}" >> $GITHUB_ENV - - name: Set windows arch - if: matrix.os == 'windows-latest' - run: | - echo "RELEASE_ARCH=windows" >> $GITHUB_ENV - - name: Set linux arch - if: matrix.os == 'ubuntu-latest' - run: | - echo "RELEASE_ARCH=linux" >> $GITHUB_ENV - - name: Set macOS arch - if: matrix.os == 'macos-latest' + - name: Set build arch run: | - echo "RELEASE_ARCH=darwin" >> $GITHUB_ENV + echo "RELEASE_ARCH=${{ matrix.release-arch }}" >> $GITHUB_ENV + - name: Install sccache (ubuntu-latest) if: matrix.os == 'ubuntu-latest' env: @@ -64,6 +59,7 @@ jobs: mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache chmod 755 $HOME/.local/bin/sccache echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Install scoop (windows-latest) if: matrix.os == 'windows-latest' shell: powershell @@ -72,15 +68,18 @@ jobs: iwr -useb get.scoop.sh -outfile 'install.ps1' .\\install.ps1 -RunAsAdmin Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH + - name: Install sccache (windows-latest) if: matrix.os == 'windows-latest' shell: powershell run: scoop install sccache + - name: Install sccache (macos-latest) if: matrix.os == 'macos-latest' run: | brew update brew install sccache + - name: Install ${{ matrix.rust }} uses: actions-rs/toolchain@v1 with: @@ -123,28 +122,13 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt install -y protobuf-compiler - echo $PROTOC - echo $PROTOC_INCLUDE - protoc --help echo "PROTOC=/usr/bin/protoc" >> $GITHUB_ENV echo "PROTOC_INCLUDE=/usr/include" >> $GITHUB_ENV - # PROTOC_VERSION=3.20.0 - # PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip - # curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP - # sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc - # sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' - # rm -f $PROTOC_ZIP - name: Install Protoc macOS if: matrix.os == 'macOS-latest' run: | brew install protobuf - # PROTOC_VERSION=3.20.0 - # PROTOC_ZIP=protoc-$PROTOC_VERSION-osx-x86_64.zip - # curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP - # sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc - # sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' - # rm -f $PROTOC_ZIP - name: check uses: actions-rs/cargo@v1 From 366611defa7c68168735cb6ad3381226f07bec08 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 14:22:19 +0200 Subject: [PATCH 30/31] headers --- iroh-bitswap/Cargo.toml | 1 - iroh-gateway/Cargo.toml | 1 + iroh-gateway/src/config.rs | 40 ++++++++++++++++++++++++++++------- iroh-gateway/src/constants.rs | 1 - 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/iroh-bitswap/Cargo.toml b/iroh-bitswap/Cargo.toml index 38387842a4..a74ad0740f 100644 --- a/iroh-bitswap/Cargo.toml +++ b/iroh-bitswap/Cargo.toml @@ -18,7 +18,6 @@ futures = "0.3.5" libp2p = { version = "0.44", default-features = false } multihash = "0.16.0" prost = "0.10" -prost-build = "0.10" thiserror = "1.0.20" unsigned-varint = "0.7.0" ahash = "0.7.6" diff --git a/iroh-gateway/Cargo.toml b/iroh-gateway/Cargo.toml index deb7e57514..cdd2f9346c 100644 --- a/iroh-gateway/Cargo.toml +++ b/iroh-gateway/Cargo.toml @@ -26,6 +26,7 @@ rand = "0.8.5" tracing-opentelemetry = "0.17.2" opentelemetry = { version = "0.17.0", features = ["rt-tokio"] } time = "0.3.9" +headers = "0.3.7" [dev-dependencies] axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messages on handlers diff --git a/iroh-gateway/src/config.rs b/iroh-gateway/src/config.rs index ec2896254a..b214044ff4 100644 --- a/iroh-gateway/src/config.rs +++ b/iroh-gateway/src/config.rs @@ -1,5 +1,8 @@ use crate::constants::*; -use axum::http::header::*; +use axum::http::{header::*, Method}; +use headers::{ + AccessControlAllowHeaders, AccessControlAllowMethods, AccessControlAllowOrigin, HeaderMapExt, +}; pub const DEFAULT_PORT: u16 = 9050; #[derive(Debug, Clone)] @@ -29,9 +32,32 @@ impl Config { pub fn set_default_headers(&mut self) { let mut headers = HeaderMap::new(); - headers.insert(ACCESS_CONTROL_ALLOW_ORIGIN, VALUE_STAR.clone()); - headers.insert(ACCESS_CONTROL_ALLOW_HEADERS, VALUE_STAR.clone()); - headers.insert(ACCESS_CONTROL_ALLOW_METHODS, VALUE_STAR.clone()); + headers.typed_insert(AccessControlAllowOrigin::ANY); + headers.typed_insert( + [ + Method::GET, + Method::PUT, + Method::POST, + Method::DELETE, + Method::HEAD, + Method::OPTIONS, + ] + .into_iter() + .collect::(), + ); + headers.typed_insert(AccessControlAllowHeaders::from_iter(vec![ + CONTENT_TYPE, + CONTENT_DISPOSITION, + LAST_MODIFIED, + CACHE_CONTROL, + ACCEPT_RANGES, + ETAG, + HEADER_SERVICE_WORKER.clone(), + HEADER_X_IPFS_GATEWAY_PREFIX.clone(), + HEADER_X_TRACE_ID.clone(), + HEADER_X_CONTENT_TYPE_OPTIONS.clone(), + HEADER_X_IPFS_PATH.clone(), + ])); // todo(arqu): remove these once propperly implmented headers.insert(CACHE_CONTROL, VALUE_NO_CACHE_NO_TRANSFORM.clone()); headers.insert(ACCEPT_RANGES, VALUE_NONE.clone()); @@ -62,10 +88,8 @@ mod tests { let mut config = Config::new(false, false, false, 9050); config.set_default_headers(); assert_eq!(config.headers.len(), 5); - assert_eq!( - config.headers.get(&ACCESS_CONTROL_ALLOW_ORIGIN), - Some(&VALUE_STAR) - ); + let h = config.headers.get(&ACCESS_CONTROL_ALLOW_ORIGIN).unwrap(); + assert_eq!(h, "*"); } #[test] diff --git a/iroh-gateway/src/constants.rs b/iroh-gateway/src/constants.rs index 420ae63ca7..9a0cff76e4 100644 --- a/iroh-gateway/src/constants.rs +++ b/iroh-gateway/src/constants.rs @@ -11,7 +11,6 @@ pub static HEADER_SERVICE_WORKER: HeaderName = HeaderName::from_static("service- // Common Header Values pub static VALUE_XCTO_NOSNIFF: HeaderValue = HeaderValue::from_static("nosniff"); -pub static VALUE_STAR: HeaderValue = HeaderValue::from_static("*"); pub static VALUE_NONE: HeaderValue = HeaderValue::from_static("none"); pub static VALUE_NO_CACHE_NO_TRANSFORM: HeaderValue = HeaderValue::from_static("no-cache, no-transform"); From 4326f4e874f9744492a039fd53a1819a65e21e0e Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 16 May 2022 14:57:25 +0200 Subject: [PATCH 31/31] cleanup --- iroh-gateway/src/config.rs | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/iroh-gateway/src/config.rs b/iroh-gateway/src/config.rs index b214044ff4..869a81fab3 100644 --- a/iroh-gateway/src/config.rs +++ b/iroh-gateway/src/config.rs @@ -45,19 +45,23 @@ impl Config { .into_iter() .collect::(), ); - headers.typed_insert(AccessControlAllowHeaders::from_iter(vec![ - CONTENT_TYPE, - CONTENT_DISPOSITION, - LAST_MODIFIED, - CACHE_CONTROL, - ACCEPT_RANGES, - ETAG, - HEADER_SERVICE_WORKER.clone(), - HEADER_X_IPFS_GATEWAY_PREFIX.clone(), - HEADER_X_TRACE_ID.clone(), - HEADER_X_CONTENT_TYPE_OPTIONS.clone(), - HEADER_X_IPFS_PATH.clone(), - ])); + headers.typed_insert( + [ + CONTENT_TYPE, + CONTENT_DISPOSITION, + LAST_MODIFIED, + CACHE_CONTROL, + ACCEPT_RANGES, + ETAG, + HEADER_SERVICE_WORKER.clone(), + HEADER_X_IPFS_GATEWAY_PREFIX.clone(), + HEADER_X_TRACE_ID.clone(), + HEADER_X_CONTENT_TYPE_OPTIONS.clone(), + HEADER_X_IPFS_PATH.clone(), + ] + .into_iter() + .collect::(), + ); // todo(arqu): remove these once propperly implmented headers.insert(CACHE_CONTROL, VALUE_NO_CACHE_NO_TRANSFORM.clone()); headers.insert(ACCEPT_RANGES, VALUE_NONE.clone());