From 7098756d33af9391e32059a9fea82261214a9056 Mon Sep 17 00:00:00 2001 From: raph Date: Mon, 4 Mar 2024 02:07:47 +0100 Subject: [PATCH 01/11] Add content proxy --- src/lib.rs | 1 + src/server_config.rs | 1 + src/subcommand/server.rs | 88 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a8a86ac9b1..0a0d638cca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -51,6 +51,7 @@ use { lazy_static::lazy_static, ordinals::{DeserializeFromStr, Epoch, Height, Rarity, Sat, SatPoint}, regex::Regex, + reqwest::Url, serde::{Deserialize, Deserializer, Serialize, Serializer}, std::{ cmp::{self, Reverse}, diff --git a/src/server_config.rs b/src/server_config.rs index b22688d880..2184dff8ed 100644 --- a/src/server_config.rs +++ b/src/server_config.rs @@ -8,4 +8,5 @@ pub(crate) struct ServerConfig { pub(crate) domain: Option, pub(crate) index_sats: bool, pub(crate) json_api_enabled: bool, + pub(crate) proxy_content: Option, } diff --git a/src/subcommand/server.rs b/src/subcommand/server.rs index a4658b28ae..abe3216fd2 100644 --- a/src/subcommand/server.rs +++ b/src/subcommand/server.rs @@ -135,6 +135,8 @@ pub struct Server { pub(crate) redirect_http_to_https: bool, #[arg(long, alias = "nosync", help = "Do not update the index.")] pub(crate) no_sync: bool, + #[arg(long, help = "Proxy `/content/` to this host.")] + pub(crate) proxy_content: Option, #[arg( long, default_value = "5s", @@ -178,6 +180,7 @@ impl Server { index_sats: index.has_sat_index(), json_api_enabled: !self.disable_json_api, decompress: self.decompress, + proxy_content: self.proxy_content.clone(), }); let router = Router::new() @@ -1200,6 +1203,32 @@ impl Server { Redirect::to("https://docs.ordinals.com/bounty/") } + fn content_proxy( + inscription_id: InscriptionId, + proxy: Url, + ) -> ServerResult)>> { + let response = reqwest::blocking::Client::new() + .get(format!("{}content/{}", proxy, inscription_id)) + .send() + .unwrap(); + + if response.status() != StatusCode::OK { + return Ok(None); + } + + let mut headers = response.headers().clone(); + + headers.insert( + header::CONTENT_SECURITY_POLICY, + HeaderValue::from_str(&format!( + "default-src 'self' {proxy} 'unsafe-eval' 'unsafe-inline' data: blob:" + )) + .map_err(|err| ServerError::Internal(Error::from(err)))?, + ); + + Ok(Some((headers, response.bytes().unwrap().to_vec()))) + } + async fn content( Extension(index): Extension>, Extension(settings): Extension>, @@ -1212,9 +1241,23 @@ impl Server { return Ok(PreviewUnknownHtml.into_response()); } - let mut inscription = index - .get_inscription_by_id(inscription_id)? - .ok_or_not_found(|| format!("inscription {inscription_id}"))?; + let mut inscription = match index.get_inscription_by_id(inscription_id)? { + None => { + if let Some(proxy) = server_config.proxy_content.clone() { + return Ok( + Self::content_proxy(inscription_id, proxy)? + .ok_or_not_found(|| format!("inscription {inscription_id} content"))? + .into_response(), + ); + } else { + return Err(ServerError::NotFound(format!( + "{} not found", + inscription_id + ))); + } + } + Some(inscription) => inscription, + }; if let Some(delegate) = inscription.delegate() { inscription = index @@ -1771,6 +1814,11 @@ mod tests { self } + fn server_option(mut self, option: &str, value: &str) -> Self { + self.server_args.insert(option.into(), Some(value.into())); + self + } + fn server_flag(mut self, flag: &str) -> Self { self.server_args.insert(flag.into(), None); self @@ -5459,6 +5507,40 @@ next server.assert_response(format!("/preview/{id}"), StatusCode::OK, "foo"); } + #[test] + fn proxy() { + let server = TestServer::builder().chain(Chain::Regtest).build(); + + server.mine_blocks(1); + + let inscription = Inscription { + content_type: Some("text/html".into()), + body: Some("foo".into()), + ..Default::default() + }; + + let txid = server.bitcoin_rpc_server.broadcast_tx(TransactionTemplate { + inputs: &[(1, 0, 0, inscription.to_witness())], + ..Default::default() + }); + + server.mine_blocks(1); + + let id = InscriptionId { txid, index: 0 }; + + server.assert_response(format!("/content/{id}"), StatusCode::OK, "foo"); + + let server_with_proxy = TestServer::builder() + .chain(Chain::Regtest) + .server_option("--proxy-content", server.url.as_ref()) + .build(); + + server_with_proxy.mine_blocks(1); + + server.assert_response(format!("/content/{id}"), StatusCode::OK, "foo"); + server_with_proxy.assert_response(format!("/content/{id}"), StatusCode::OK, "foo"); + } + #[test] fn chainwork_conversion_to_integer() { assert_eq!(chainwork(&[]), 0); From 3ed9832e82205922315ae8e51a97ad4af3216738 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 4 Mar 2024 15:13:48 -0800 Subject: [PATCH 02/11] Amend --- src/server_config.rs | 2 +- src/subcommand/server.rs | 39 ++++++++++++++++++--------------------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/server_config.rs b/src/server_config.rs index 2184dff8ed..57cd3e0dd1 100644 --- a/src/server_config.rs +++ b/src/server_config.rs @@ -3,10 +3,10 @@ use super::*; #[derive(Default)] pub(crate) struct ServerConfig { pub(crate) chain: Chain, + pub(crate) content_proxy: Option, pub(crate) csp_origin: Option, pub(crate) decompress: bool, pub(crate) domain: Option, pub(crate) index_sats: bool, pub(crate) json_api_enabled: bool, - pub(crate) proxy_content: Option, } diff --git a/src/subcommand/server.rs b/src/subcommand/server.rs index abe3216fd2..9b1d984872 100644 --- a/src/subcommand/server.rs +++ b/src/subcommand/server.rs @@ -136,7 +136,7 @@ pub struct Server { #[arg(long, alias = "nosync", help = "Do not update the index.")] pub(crate) no_sync: bool, #[arg(long, help = "Proxy `/content/` to this host.")] - pub(crate) proxy_content: Option, + pub(crate) content_proxy: Option, #[arg( long, default_value = "5s", @@ -175,12 +175,12 @@ impl Server { let server_config = Arc::new(ServerConfig { chain: settings.chain(), + content_proxy: self.content_proxy.clone(), csp_origin: self.csp_origin.clone(), + decompress: self.decompress, domain: acme_domains.first().cloned(), index_sats: index.has_sat_index(), json_api_enabled: !self.disable_json_api, - decompress: self.decompress, - proxy_content: self.proxy_content.clone(), }); let router = Router::new() @@ -1205,7 +1205,7 @@ impl Server { fn content_proxy( inscription_id: InscriptionId, - proxy: Url, + proxy: &Url, ) -> ServerResult)>> { let response = reqwest::blocking::Client::new() .get(format!("{}content/{}", proxy, inscription_id)) @@ -1241,22 +1241,19 @@ impl Server { return Ok(PreviewUnknownHtml.into_response()); } - let mut inscription = match index.get_inscription_by_id(inscription_id)? { - None => { - if let Some(proxy) = server_config.proxy_content.clone() { - return Ok( - Self::content_proxy(inscription_id, proxy)? - .ok_or_not_found(|| format!("inscription {inscription_id} content"))? - .into_response(), - ); - } else { - return Err(ServerError::NotFound(format!( - "{} not found", - inscription_id - ))); - } - } - Some(inscription) => inscription, + let Some(mut inscription) = index.get_inscription_by_id(inscription_id)? else { + return if let Some(proxy) = server_config.content_proxy.as_ref() { + Ok( + Self::content_proxy(inscription_id, proxy)? + .ok_or_not_found(|| format!("inscription {inscription_id} content"))? + .into_response(), + ) + } else { + Err(ServerError::NotFound(format!( + "{} not found", + inscription_id + ))) + }; }; if let Some(delegate) = inscription.delegate() { @@ -5532,7 +5529,7 @@ next let server_with_proxy = TestServer::builder() .chain(Chain::Regtest) - .server_option("--proxy-content", server.url.as_ref()) + .server_option("--content-proxy", server.url.as_ref()) .build(); server_with_proxy.mine_blocks(1); From ea0623fdbb2ba112b72ab3c9486452a7219b828b Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 4 Mar 2024 15:15:00 -0800 Subject: [PATCH 03/11] Amend --- src/subcommand/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/subcommand/server.rs b/src/subcommand/server.rs index 9b1d984872..2c7d7645a7 100644 --- a/src/subcommand/server.rs +++ b/src/subcommand/server.rs @@ -135,7 +135,7 @@ pub struct Server { pub(crate) redirect_http_to_https: bool, #[arg(long, alias = "nosync", help = "Do not update the index.")] pub(crate) no_sync: bool, - #[arg(long, help = "Proxy `/content/` to this host.")] + #[arg(long, help = "Proxy `/content/` to .")] pub(crate) content_proxy: Option, #[arg( long, From f8c02aa0dfe0323ef5e300a13b8bb5279e08f25a Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 4 Mar 2024 15:25:53 -0800 Subject: [PATCH 04/11] Amend --- docs/src/guides/testing.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/src/guides/testing.md b/docs/src/guides/testing.md index 223fdfd33f..1595ddd8f5 100644 --- a/docs/src/guides/testing.md +++ b/docs/src/guides/testing.md @@ -113,3 +113,16 @@ Finally you will have to mine some blocks and start the server: bitcoin-cli generatetoaddress 6 ord -r server ``` + +To reference inscriptions on mainnet you may now specify a +with `--content-proxy`, like so: + +``` +ord -r server --content-proxy https://ordinals.com +``` + +Essentially this will forward the content from an inscription id it cannot find +in the local regtest to the specified proxy, allowing you to test recursive +inscriptions with already inscribed content. + + From 2d6a57aeaabda8c2136730c8d3b7207b0f2a2edb Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 4 Mar 2024 15:28:12 -0800 Subject: [PATCH 05/11] Amend --- docs/src/guides/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/guides/testing.md b/docs/src/guides/testing.md index 1595ddd8f5..e1388b63c8 100644 --- a/docs/src/guides/testing.md +++ b/docs/src/guides/testing.md @@ -121,7 +121,7 @@ with `--content-proxy`, like so: ord -r server --content-proxy https://ordinals.com ``` -Essentially this will forward the content from an inscription id it cannot find +Essentially this will forward the `/content/` from an inscription id it cannot find in the local regtest to the specified proxy, allowing you to test recursive inscriptions with already inscribed content. From 5f66fdbac78c0de39b248b8c1f226db665c45a8e Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 4 Mar 2024 15:40:27 -0800 Subject: [PATCH 06/11] Amend --- docs/src/guides/testing.md | 53 ++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/docs/src/guides/testing.md b/docs/src/guides/testing.md index e1388b63c8..5430e8bc0e 100644 --- a/docs/src/guides/testing.md +++ b/docs/src/guides/testing.md @@ -49,38 +49,46 @@ Example ------- Run bitcoind in regtest with: + ``` bitcoind -regtest -txindex ``` +Run ord server in regtest with: + +``` +ord -r server +``` + + Create a wallet in regtest with: + ``` ord -r wallet create ``` Get a regtest receive address with: + ``` ord -r wallet receive ``` Mine 101 blocks (to unlock the coinbase) with: + ``` bitcoin-cli -regtest generatetoaddress 101 ``` Inscribe in regtest with: + ``` ord -r wallet inscribe --fee-rate 1 --file ``` Mine the inscription with: -``` -bitcoin-cli -regtest generatetoaddress 1 -``` -View the inscription in the regtest explorer: ``` -ord -r server +bitcoin-cli -regtest generatetoaddress 1 ``` By default, browsers don't support compression over HTTP. To test compressed @@ -94,35 +102,42 @@ Testing Recursion When testing out [recursion](../inscriptions/recursion.md), inscribe the dependencies first (example with [p5.js](https://p5js.org)): + ``` ord -r wallet inscribe --fee-rate 1 --file p5.js ``` -This should return a `inscription_id` which you can then reference in your -recursive inscription. -ATTENTION: These ids will be different when inscribing on -mainnet or signet, so be sure to change those in your recursive inscription for -each chain. +This will return the inscription ID of the dependency which you can then +reference in your inscription. + +However, inscription IDs differ between mainnet and test chains, so you must +change the inscription IDs in your inscription to the mainnet inscription IDs of +your dependencies before making the final inscription on mainnet. Then you can inscribe your recursive inscription with: + ``` ord -r wallet inscribe --fee-rate 1 --file recursive-inscription.html ``` + Finally you will have to mine some blocks and start the server: + ``` bitcoin-cli generatetoaddress 6 -ord -r server ``` -To reference inscriptions on mainnet you may now specify a -with `--content-proxy`, like so: +### Mainnet Dependencies + +To avoid having to change dependency inscription IDs to mainnet inscription IDs, +you may utilize a content proxy when testing. `ord server` accepts a +`--content-proxy` option, which takes the URL of a another `ord server` +instance. When making a request to `/content/` when a content +proxy is set and the inscription is not found, `ord server` will forward the +request to the content proxy. This allows you to run a test `ord server` +instance with a mainnet content proxy. You can then use mainnet inscription IDs +in your test inscription, which will then return the content of the mainnet +inscriptions. ``` ord -r server --content-proxy https://ordinals.com ``` - -Essentially this will forward the `/content/` from an inscription id it cannot find -in the local regtest to the specified proxy, allowing you to test recursive -inscriptions with already inscribed content. - - From 6ba7e41bf90456687981825ebd1428263f5d0663 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 4 Mar 2024 15:41:32 -0800 Subject: [PATCH 07/11] Amend --- docs/src/guides/testing.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/src/guides/testing.md b/docs/src/guides/testing.md index 5430e8bc0e..c488edb1b3 100644 --- a/docs/src/guides/testing.md +++ b/docs/src/guides/testing.md @@ -57,20 +57,19 @@ bitcoind -regtest -txindex Run ord server in regtest with: ``` -ord -r server +ord --regtest server ``` - Create a wallet in regtest with: ``` -ord -r wallet create +ord --regtest wallet create ``` Get a regtest receive address with: ``` -ord -r wallet receive +ord --regtest wallet receive ``` Mine 101 blocks (to unlock the coinbase) with: @@ -82,7 +81,7 @@ bitcoin-cli -regtest generatetoaddress 101 Inscribe in regtest with: ``` -ord -r wallet inscribe --fee-rate 1 --file +ord --regtest wallet inscribe --fee-rate 1 --file ``` Mine the inscription with: @@ -94,7 +93,7 @@ bitcoin-cli -regtest generatetoaddress 1 By default, browsers don't support compression over HTTP. To test compressed content over HTTP, use the `--decompress` flag: ``` -ord -r server --decompress +ord --regtest server --decompress ``` Testing Recursion @@ -104,7 +103,7 @@ When testing out [recursion](../inscriptions/recursion.md), inscribe the dependencies first (example with [p5.js](https://p5js.org)): ``` -ord -r wallet inscribe --fee-rate 1 --file p5.js +ord --regtest wallet inscribe --fee-rate 1 --file p5.js ``` This will return the inscription ID of the dependency which you can then @@ -117,7 +116,7 @@ your dependencies before making the final inscription on mainnet. Then you can inscribe your recursive inscription with: ``` -ord -r wallet inscribe --fee-rate 1 --file recursive-inscription.html +ord --regtest wallet inscribe --fee-rate 1 --file recursive-inscription.html ``` Finally you will have to mine some blocks and start the server: @@ -139,5 +138,5 @@ in your test inscription, which will then return the content of the mainnet inscriptions. ``` -ord -r server --content-proxy https://ordinals.com +ord --regtest server --content-proxy https://ordinals.com ``` From 1951612c5ac170f96cdc86f775a10727b7633071 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 4 Mar 2024 15:42:14 -0800 Subject: [PATCH 08/11] Amend --- docs/src/guides/testing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/guides/testing.md b/docs/src/guides/testing.md index c488edb1b3..0d98dbce3a 100644 --- a/docs/src/guides/testing.md +++ b/docs/src/guides/testing.md @@ -48,13 +48,13 @@ Regtest doesn't require downloading the blockchain or indexing ord. Example ------- -Run bitcoind in regtest with: +Run `bitcoind` in regtest with: ``` bitcoind -regtest -txindex ``` -Run ord server in regtest with: +Run `ord server` in regtest with: ``` ord --regtest server From ef9e3f07850af2f2403a859ab4e1e978c6ff9b99 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 4 Mar 2024 15:42:48 -0800 Subject: [PATCH 09/11] Amend --- docs/src/guides/testing.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/guides/testing.md b/docs/src/guides/testing.md index 0d98dbce3a..a25e04204d 100644 --- a/docs/src/guides/testing.md +++ b/docs/src/guides/testing.md @@ -92,6 +92,7 @@ bitcoin-cli -regtest generatetoaddress 1 By default, browsers don't support compression over HTTP. To test compressed content over HTTP, use the `--decompress` flag: + ``` ord --regtest server --decompress ``` From a92cd275f71833573b5ccad8ed92be6476196da6 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 4 Mar 2024 15:48:10 -0800 Subject: [PATCH 10/11] Amend --- src/subcommand/server.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/subcommand/server.rs b/src/subcommand/server.rs index 2c7d7645a7..015812fd91 100644 --- a/src/subcommand/server.rs +++ b/src/subcommand/server.rs @@ -135,7 +135,10 @@ pub struct Server { pub(crate) redirect_http_to_https: bool, #[arg(long, alias = "nosync", help = "Do not update the index.")] pub(crate) no_sync: bool, - #[arg(long, help = "Proxy `/content/` to .")] + #[arg( + long, + help = "Proxy `/content/INSCRIPTION_ID` requests to `/content/INSCRIPTION_ID` if the inscription is not present on current chain." + )] pub(crate) content_proxy: Option, #[arg( long, From 0d4715fe7113392733a100c7e0533c4938efeba3 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 4 Mar 2024 15:59:34 -0800 Subject: [PATCH 11/11] Amend --- src/subcommand/server.rs | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/subcommand/server.rs b/src/subcommand/server.rs index 015812fd91..54adaa808e 100644 --- a/src/subcommand/server.rs +++ b/src/subcommand/server.rs @@ -1206,18 +1206,11 @@ impl Server { Redirect::to("https://docs.ordinals.com/bounty/") } - fn content_proxy( - inscription_id: InscriptionId, - proxy: &Url, - ) -> ServerResult)>> { + fn proxy_content(proxy: &Url, inscription_id: InscriptionId) -> ServerResult { let response = reqwest::blocking::Client::new() .get(format!("{}content/{}", proxy, inscription_id)) .send() - .unwrap(); - - if response.status() != StatusCode::OK { - return Ok(None); - } + .map_err(|err| anyhow!(err))?; let mut headers = response.headers().clone(); @@ -1229,7 +1222,14 @@ impl Server { .map_err(|err| ServerError::Internal(Error::from(err)))?, ); - Ok(Some((headers, response.bytes().unwrap().to_vec()))) + Ok( + ( + response.status(), + headers, + response.bytes().map_err(|err| anyhow!(err))?, + ) + .into_response(), + ) } async fn content( @@ -1246,11 +1246,7 @@ impl Server { let Some(mut inscription) = index.get_inscription_by_id(inscription_id)? else { return if let Some(proxy) = server_config.content_proxy.as_ref() { - Ok( - Self::content_proxy(inscription_id, proxy)? - .ok_or_not_found(|| format!("inscription {inscription_id} content"))? - .into_response(), - ) + Self::proxy_content(proxy, inscription_id) } else { Err(ServerError::NotFound(format!( "{} not found",