new template escape

crates/rari-doc/Cargo.toml

@@ -15,7 +15,6 @@ percent-encoding = "2"
 pest = "2"
 pest_derive = "2"
 regex = "1"
-base64 = "0.22"
 validator = { version = "0.18", features = ["derive"] }
 once_cell = "1"
 scraper = { version = "0.19", features = ["deterministic"] }

crates/rari-doc/src/docs/build.rs

@@ -22,10 +22,11 @@ use super::sections::{split_sections, BuildSection, BuildSectionType};
 use super::title::{page_title, transform_title};
 use crate::baseline::get_baseline;
 use crate::error::DocError;
+use crate::html::modifier::add_missing_ids;
 use crate::html::rewriter::post_process_html;
 use crate::html::sidebar::render_sidebar;
 use crate::specs::extract_specifications;
-use crate::templ::render::render;
+use crate::templ::render::{decode_ref, render};
 
 impl<'a> From<BuildSection<'a>> for Section {
     fn from(value: BuildSection) -> Self {
@@ -131,14 +132,17 @@ pub fn make_toc(sections: &[BuildSection], with_h3: bool) -> Vec<TocEntry> {
 }
 
 pub fn build_content<T: PageLike>(doc: &T) -> Result<PageContent, DocError> {
-    let ks_rendered_doc = if let Some(rari_env) = &doc.rari_env() {
-        Cow::Owned(render(rari_env, doc.content())?)
+    let (ks_rendered_doc, templs) = if let Some(rari_env) = &doc.rari_env() {
+        let (out, templs) = render(rari_env, doc.content())?;
+        (Cow::Owned(out), templs)
     } else {
-        Cow::Borrowed(doc.content())
+        (Cow::Borrowed(doc.content()), vec![])
     };
-    let html = render_md_to_html(&ks_rendered_doc, doc.locale())?;
+    let encoded_html = render_md_to_html(&ks_rendered_doc, doc.locale())?;
+    let html = decode_ref(&encoded_html, &templs)?;
     let post_processed_html = post_process_html(&html, doc, false)?;
-    let fragment = Html::parse_fragment(&post_processed_html);
+    let mut fragment = Html::parse_fragment(&post_processed_html);
+    add_missing_ids(&mut fragment)?;
     let (sections, summary) = split_sections(&fragment).expect("DOOM");
     let toc = make_toc(&sections, matches!(doc.page_type(), PageType::Curriculum));
     let body = sections.into_iter().map(Into::into).collect();

crates/rari-doc/src/error.rs

@@ -13,6 +13,10 @@ use crate::docs::page::PageCategory;
 
 #[derive(Debug, Error)]
 pub enum DocError {
+    #[error("Cannot parse templ index")]
+    TemplIndexParseError(#[from] std::num::ParseIntError),
+    #[error("Invalid templ index {0}")]
+    InvalidTemplIndex(usize),
     #[error("No parent")]
     NoParent(PathBuf),
     #[error(transparent)]
@@ -65,8 +69,6 @@ pub enum DocError {
     ArgError(#[from] ArgError),
     #[error("pest error: {0}")]
     PestError(String),
-    #[error("failed to decode ks: {0}")]
-    DecodeError(#[from] base64::DecodeError),
     #[error("failed to de/serialize")]
     SerializationError,
     #[error(transparent)]

crates/rari-doc/src/html/mod.rs

@@ -1,3 +1,4 @@
 pub mod links;
+pub mod modifier;
 pub mod rewriter;
 pub mod sidebar;

crates/rari-doc/src/html/modifier.rs

@@ -0,0 +1,44 @@
+use ego_tree::NodeId;
+use html5ever::{namespace_url, ns, QualName};
+use rari_md::anchor::anchorize;
+use scraper::{Html, Node, Selector};
+
+use crate::error::DocError;
+
+pub fn add_attribute(html: &mut Html, node_id: NodeId, key: &str, value: &str) {
+    if let Some(mut details) = html.tree.get_mut(node_id) {
+        if let Node::Element(ref mut el) = details.value() {
+            el.attrs.insert(
+                QualName {
+                    prefix: None,
+                    ns: ns!(),
+                    local: key.into(),
+                },
+                value.into(),
+            );
+        }
+    }
+}
+
+pub fn add_missing_ids(html: &mut Html) -> Result<(), DocError> {
+    let a_selector = Selector::parse("*[id=---update-id]").unwrap();
+    let subs = html
+        .select(&a_selector)
+        .map(|el| {
+            let text = if let Some(text) = el
+                .first_child()
+                .and_then(|child| child.value().as_text().map(|t| t.to_string()))
+            {
+                text
+            } else {
+                el.text().collect::<String>()
+            };
+            let id = anchorize(&text);
+            (el.id(), id)
+        })
+        .collect::<Vec<_>>();
+    for (el_id, id) in subs {
+        add_attribute(html, el_id, "id", &id);
+    }
+    Ok(())
+}

crates/rari-doc/src/html/sidebar.rs

@@ -2,8 +2,6 @@ use std::collections::HashMap;
 pub use std::ops::Deref;
 use std::sync::{Arc, RwLock};
 
-use ego_tree::NodeId;
-use html5ever::{namespace_url, ns, QualName};
 use once_cell::sync::Lazy;
 use rari_types::fm_types::PageType;
 use rari_types::globals::cache_content;
@@ -12,6 +10,7 @@ use scraper::{Html, Node, Selector};
 use serde::{Deserialize, Serialize};
 
 use super::links::{render_link_from_page, render_link_via_page, LinkModifier};
+use super::modifier::add_attribute;
 use super::rewriter::post_process_html;
 use crate::cached_readers::read_sidebar;
 use crate::docs::doc::Doc;
@@ -36,22 +35,7 @@ type SidebarCache = Arc<RwLock<HashMap<Locale, HashMap<String, String>>>>;
 
 static SIDEBAR_CACHE: Lazy<SidebarCache> = Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
 
-fn add_attribute(html: &mut Html, node_id: NodeId, key: &str, value: &str) {
-    if let Some(mut details) = html.tree.get_mut(node_id) {
-        if let Node::Element(ref mut el) = details.value() {
-            el.attrs.insert(
-                QualName {
-                    prefix: None,
-                    ns: ns!(),
-                    local: key.into(),
-                },
-                value.into(),
-            );
-        }
-    }
-}
-
-fn highlight_current(mut html: Html, url: &str) -> Result<String, DocError> {
+fn expand_details_to_for_current(mut html: Html, url: &str) -> Result<String, DocError> {
     let a_selector = Selector::parse(&format!("a[href=\"{url}\"]")).unwrap();
     let mut details = vec![];
     let mut parent_id = None;
@@ -111,7 +95,7 @@ pub fn render_sidebar(doc: &Doc) -> Result<Option<String>, DocError> {
         })
         .map(|ks_rendered_sidebar| {
             let fragment = Html::parse_fragment(&ks_rendered_sidebar?);
-            let pre_processed_html = highlight_current(fragment, &doc.meta.url)?;
+            let pre_processed_html = expand_details_to_for_current(fragment, &doc.meta.url)?;
             let post_processed_html = post_process_html(&pre_processed_html, doc, true)?;
             Ok::<_, DocError>(post_processed_html)
         })

crates/rari-doc/src/templ/macros/compat.rs

@@ -16,14 +16,15 @@ mod test {
     use rari_types::RariEnv;
 
     use crate::error::DocError;
-    use crate::templ::render::render;
+    use crate::templ::render::{decode_ref, render};
 
     #[test]
     fn test_compat_none() -> Result<(), DocError> {
         let env = RariEnv {
             ..Default::default()
         };
-        let out = render(&env, r#"{{ compat }}"#)?;
+        let (out, templs) = render(&env, r#"{{ compat }}"#)?;
+        let out = decode_ref(&out, &templs)?;
         assert_eq!(out, r#""#);
         Ok(())
     }
@@ -37,7 +38,8 @@ mod test {
         let exp = r#"<div class="bc-data" data-query="javascript.builtins.Array.concat" data-depth="1" data-multiple="false">
 If you're able to see this, something went wrong on this page.
 </div>"#;
-        let out = render(&env, r#"{{ compat }}"#)?;
+        let (out, templs) = render(&env, r#"{{ compat }}"#)?;
+        let out = decode_ref(&out, &templs)?;
         assert_eq!(out, exp);
         Ok(())
     }
@@ -57,7 +59,8 @@ If you're able to see this, something went wrong on this page.
 <div class="bc-data" data-query="javascript.builtins.Array.filter" data-depth="1" data-multiple="true">
 If you're able to see this, something went wrong on this page.
 </div>"#;
-        let out = render(&env, r#"{{ compat }}"#)?;
+        let (out, templs) = render(&env, r#"{{ compat }}"#)?;
+        let out = decode_ref(&out, &templs)?;
         assert_eq!(out, exp);
         Ok(())
     }

crates/rari-doc/src/templ/macros/links.rs

@@ -112,14 +112,15 @@ pub fn link_internal(
 mod test {
 
     use crate::error::DocError;
-    use crate::templ::render::render;
+    use crate::templ::render::{decode_ref, render};
 
     #[test]
     fn test_link() -> Result<(), DocError> {
         let env = rari_types::RariEnv {
             ..Default::default()
         };
-        let out = render(&env, r#"{{ link("/en-US/docs/basic") }}"#)?;
+        let (out, templs) = render(&env, r#"{{ link("/en-US/docs/basic") }}"#)?;
+        let out = decode_ref(&out, &templs)?;
         assert_eq!(out, r#"<a href="/en-US/docs/Basic">The Basic Page</a>"#);
         Ok(())
     }

crates/rari-doc/src/templ/parser.rs

@@ -111,6 +111,15 @@ pub(crate) fn parse(input: &str) -> Result<Vec<Token>, DocError> {
     Ok(tokens)
 }
 
+fn _strip_escape_residues(s: &str) -> &str {
+    let s = s.strip_prefix("&gt;").or(s.strip_prefix('>')).unwrap_or(s);
+    let s = s
+        .strip_suffix("!&lt;")
+        .or(s.strip_suffix("!<"))
+        .unwrap_or(s);
+    s
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

crates/rari-doc/src/templ/render.rs

@@ -8,12 +8,64 @@ use super::macros::invoke;
 use super::parser::{parse, Token};
 use crate::error::DocError;
 
-pub fn render(env: &RariEnv, input: &str) -> Result<String, DocError> {
+pub fn render(env: &RariEnv, input: &str) -> Result<(String, Vec<String>), DocError> {
     let tokens = parse(input)?;
     render_tokens(env, tokens, input)
 }
+fn encode_ref(index: usize, out: &mut String) -> Result<(), DocError> {
+    Ok(write!(out, "!::::{index}::::!",)?)
+}
 
-pub fn render_tokens(env: &RariEnv, tokens: Vec<Token>, input: &str) -> Result<String, DocError> {
+pub(crate) fn decode_ref(input: &str, templs: &[String]) -> Result<String, DocError> {
+    let mut decoded = String::with_capacity(input.len());
+    if !input.contains("!::::") {
+        return Ok(input.to_string());
+    }
+    let mut frags = vec![];
+    for frag in input.split("!::::") {
+        let has_ks = frag.contains("::::!");
+        for (i, sub_frag) in frag.splitn(2, "::::!").enumerate() {
+            if i == 0 && has_ks {
+                frags.push(sub_frag);
+                //decode_macro(sub_frag, &mut decoded)?;
+            } else {
+                //decoded.push_str(strip_escape_residues(sub_frag))
+                frags.push(sub_frag)
+            }
+        }
+    }
+    for i in 0..frags.len() {
+        if i % 2 == 1
+            && i < frags.len() + 1
+            && frags[i - 1].ends_with("<p>")
+            && frags[i + 1].starts_with("</p>")
+        {
+            frags[i - 1] = frags[i - 1].strip_suffix("<p>").unwrap();
+            frags[i + 1] = frags[i + 1].strip_prefix("</p>").unwrap();
+        }
+    }
+
+    for (i, frag) in frags.iter().enumerate() {
+        if i % 2 == 1 {
+            let index = frag.parse::<usize>()?;
+            if let Some(templ) = templs.get(index) {
+                decoded.push_str(templ);
+            } else {
+                return Err(DocError::InvalidTemplIndex(index));
+            };
+        } else {
+            decoded.push_str(frag)
+        }
+    }
+
+    Ok(decoded)
+}
+pub fn render_tokens(
+    env: &RariEnv,
+    tokens: Vec<Token>,
+    input: &str,
+) -> Result<(String, Vec<String>), DocError> {
+    let mut templs = vec![];
     let mut out = String::with_capacity(input.len());
     for token in tokens {
         match token {
@@ -31,7 +83,10 @@ pub fn render_tokens(env: &RariEnv, tokens: Vec<Token>, input: &str) -> Result<S
                 let span = span!(Level::ERROR, "templ", "{}", &ident);
                 let _enter = span.enter();
                 match invoke(env, &mac.ident, mac.args) {
-                    Ok(rendered) => out.push_str(&rendered),
+                    Ok(rendered) => {
+                        encode_ref(templs.len(), &mut out)?;
+                        templs.push(rendered)
+                    }
                     Err(e) if deny_warnings() => return Err(e),
                     Err(e) => {
                         error!("{e}");
@@ -41,5 +96,5 @@ pub fn render_tokens(env: &RariEnv, tokens: Vec<Token>, input: &str) -> Result<S
             }
         }
     }
-    Ok(out)
+    Ok((out, templs))
 }

crates/rari-md/src/html.rs

@@ -493,17 +493,11 @@ impl<'o> HtmlFormatter<'o> {
             if matches!(child.data.borrow().value, NodeValue::Link(_)) {
                 return true;
             }
-            if let NodeValue::HtmlInline(res) = &child.data.borrow().value {
-                return res.starts_with("<a ");
-            }
             if matches!(child.data.borrow().value, NodeValue::Paragraph) {
                 if let Some(child) = child.children().next() {
                     if matches!(child.data.borrow().value, NodeValue::Link(_)) {
                         return true;
                     }
-                    if let NodeValue::HtmlInline(res) = &child.data.borrow().value {
-                        return res.starts_with("<a ");
-                    }
                 }
             }
         }
@@ -603,10 +597,15 @@ impl<'o> HtmlFormatter<'o> {
                     self.cr()?;
                     let mut text_content = Vec::with_capacity(20);
                     Self::collect_first_child_text(node, &mut text_content);
-                    let mut id = String::from_utf8(text_content).unwrap();
-                    id = self.anchorizer.anchorize(id);
-                    write!(self.output, "<dt id=\"{}\"", id)?;
-                    if !Self::next_is_link(node) {
+                    let raw_id = String::from_utf8(text_content).unwrap();
+                    let is_templ = raw_id.starts_with("!::::");
+                    if is_templ {
+                        write!(self.output, "<dt id=\"---update-id\"")?;
+                    } else {
+                        let id = self.anchorizer.anchorize(raw_id);
+                        write!(self.output, "<dt id=\"{}\"", id)?;
+                    };
+                    if !is_templ && !Self::next_is_link(node) {
                         write!(self.output, " data-add-link")?;
                     }
                     self.render_sourcepos(node)?;

crates/rari-md/src/lib.rs

@@ -90,7 +90,7 @@ mod test {
         let out = m2h("- {{foo}}\n  - : bar", Locale::EnUs)?;
         assert_eq!(
             out,
-            "<dl>\n<dt id=\"foo\">{{foo}}</dt>\n<dd>\n<p>bar</p>\n</dd>\n</dl>\n"
+            "<dl>\n<dt id=\"foo\" data-add-link>{{foo}}</dt>\n<dd>\n<p>bar</p>\n</dd>\n</dl>\n"
         );
         Ok(())
     }