Rollup merge of #83992 - GuillaumeGomez:merge-idents, r=notriddle

Merge idents when generating source content

The idea here is to not have a span for each part of a path. Currently, for `a::b::c` we generate `<span>a</span>::<span>b</span>::<span>c</span>`, with this change, we will generate `<span>a::b::c</span>`.

A nice "side-effect" is that it reduces the size of the output HTML too. :)

cc `@notriddle`

src/librustdoc/html/highlight.rs

@@ -136,6 +136,16 @@ impl Iterator for TokenIter<'a> {
     }
 }
 
+fn get_real_ident_class(text: &str, edition: Edition) -> Class {
+    match text {
+        "ref" | "mut" => Class::RefKeyWord,
+        "self" | "Self" => Class::Self_,
+        "false" | "true" => Class::Bool,
+        _ if Symbol::intern(text).is_reserved(|| edition) => Class::KeyWord,
+        _ => Class::Ident,
+    }
+}
+
 /// Processes program tokens, classifying strings of text by highlighting
 /// category (`Class`).
 struct Classifier<'a> {
@@ -144,6 +154,8 @@ struct Classifier<'a> {
     in_macro: bool,
     in_macro_nonterminal: bool,
     edition: Edition,
+    byte_pos: u32,
+    src: &'a str,
 }
 
 impl<'a> Classifier<'a> {
@@ -155,6 +167,68 @@ impl<'a> Classifier<'a> {
             in_macro: false,
             in_macro_nonterminal: false,
             edition,
+            byte_pos: 0,
+            src,
+        }
+    }
+
+    /// Concatenate colons and idents as one when possible.
+    fn get_full_ident_path(&mut self) -> Vec<(TokenKind, usize, usize)> {
+        let start = self.byte_pos as usize;
+        let mut pos = start;
+        let mut has_ident = false;
+        let edition = self.edition;
+
+        loop {
+            let mut nb = 0;
+            while let Some((TokenKind::Colon, _)) = self.tokens.peek() {
+                self.tokens.next();
+                nb += 1;
+            }
+            // Ident path can start with "::" but if we already have content in the ident path,
+            // the "::" is mandatory.
+            if has_ident && nb == 0 {
+                return vec![(TokenKind::Ident, start, pos)];
+            } else if nb != 0 && nb != 2 {
+                if has_ident {
+                    return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)];
+                } else {
+                    return vec![(TokenKind::Colon, pos, pos + nb)];
+                }
+            }
+
+            if let Some((Class::Ident, text)) = self.tokens.peek().map(|(token, text)| {
+                if *token == TokenKind::Ident {
+                    let class = get_real_ident_class(text, edition);
+                    (class, text)
+                } else {
+                    // Doesn't matter which Class we put in here...
+                    (Class::Comment, text)
+                }
+            }) {
+                // We only "add" the colon if there is an ident behind.
+                pos += text.len() + nb;
+                has_ident = true;
+                self.tokens.next();
+            } else if nb > 0 && has_ident {
+                return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)];
+            } else if nb > 0 {
+                return vec![(TokenKind::Colon, pos, pos + nb)];
+            } else if has_ident {
+                return vec![(TokenKind::Ident, start, pos)];
+            } else {
+                return Vec::new();
+            }
+        }
+    }
+
+    /// Wraps the tokens iteration to ensure that the byte_pos is always correct.
+    fn next(&mut self) -> Option<(TokenKind, &'a str)> {
+        if let Some((kind, text)) = self.tokens.next() {
+            self.byte_pos += text.len() as u32;
+            Some((kind, text))
+        } else {
+            None
         }
     }
 
@@ -165,8 +239,25 @@ impl<'a> Classifier<'a> {
     /// token is used.
     fn highlight(mut self, sink: &mut dyn FnMut(Highlight<'a>)) {
         with_default_session_globals(|| {
-            while let Some((token, text)) = self.tokens.next() {
-                self.advance(token, text, sink);
+            loop {
+                if self
+                    .tokens
+                    .peek()
+                    .map(|t| matches!(t.0, TokenKind::Colon | TokenKind::Ident))
+                    .unwrap_or(false)
+                {
+                    let tokens = self.get_full_ident_path();
+                    for (token, start, end) in tokens {
+                        let text = &self.src[start..end];
+                        self.advance(token, text, sink);
+                        self.byte_pos += text.len() as u32;
+                    }
+                }
+                if let Some((token, text)) = self.next() {
+                    self.advance(token, text, sink);
+                } else {
+                    break;
+                }
             }
         })
     }
@@ -203,12 +294,12 @@ impl<'a> Classifier<'a> {
             },
             TokenKind::And => match lookahead {
                 Some(TokenKind::And) => {
-                    let _and = self.tokens.next();
+                    self.next();
                     sink(Highlight::Token { text: "&&", class: Some(Class::Op) });
                     return;
                 }
                 Some(TokenKind::Eq) => {
-                    let _eq = self.tokens.next();
+                    self.next();
                     sink(Highlight::Token { text: "&=", class: Some(Class::Op) });
                     return;
                 }
@@ -260,7 +351,7 @@ impl<'a> Classifier<'a> {
                 match lookahead {
                     // Case 1: #![inner_attribute]
                     Some(TokenKind::Bang) => {
-                        let _not = self.tokens.next().unwrap();
+                        self.next();
                         if let Some(TokenKind::OpenBracket) = self.peek() {
                             self.in_attribute = true;
                             sink(Highlight::EnterSpan { class: Class::Attribute });
@@ -304,19 +395,17 @@ impl<'a> Classifier<'a> {
                 sink(Highlight::Token { text, class: None });
                 return;
             }
-            TokenKind::Ident => match text {
-                "ref" | "mut" => Class::RefKeyWord,
-                "self" | "Self" => Class::Self_,
-                "false" | "true" => Class::Bool,
-                "Option" | "Result" => Class::PreludeTy,
-                "Some" | "None" | "Ok" | "Err" => Class::PreludeVal,
-                // Keywords are also included in the identifier set.
-                _ if Symbol::intern(text).is_reserved(|| self.edition) => Class::KeyWord,
-                _ if self.in_macro_nonterminal => {
-                    self.in_macro_nonterminal = false;
-                    Class::MacroNonTerminal
-                }
-                _ => Class::Ident,
+            TokenKind::Ident => match get_real_ident_class(text, self.edition) {
+                Class::Ident => match text {
+                    "Option" | "Result" => Class::PreludeTy,
+                    "Some" | "None" | "Ok" | "Err" => Class::PreludeVal,
+                    _ if self.in_macro_nonterminal => {
+                        self.in_macro_nonterminal = false;
+                        Class::MacroNonTerminal
+                    }
+                    _ => Class::Ident,
+                },
+                c => c,
             },
             TokenKind::RawIdent => Class::Ident,
             TokenKind::Lifetime { .. } => Class::Lifetime,

src/librustdoc/html/highlight/fixtures/sample.html

@@ -10,6 +10,8 @@
 </style>
 <pre><code><span class="attribute">#![<span class="ident">crate_type</span> <span class="op">=</span> <span class="string">&quot;lib&quot;</span>]</span>
 
+<span class="kw">use</span> <span class="ident">std::path</span>::{<span class="ident">Path</span>, <span class="ident">PathBuf</span>};
+
 <span class="attribute">#[<span class="ident">cfg</span>(<span class="ident">target_os</span> <span class="op">=</span> <span class="string">&quot;linux&quot;</span>)]</span>
 <span class="kw">fn</span> <span class="ident">main</span>() {
     <span class="kw">let</span> <span class="ident">foo</span> <span class="op">=</span> <span class="bool-val">true</span> <span class="op">&amp;&amp;</span> <span class="bool-val">false</span> <span class="op">|</span><span class="op">|</span> <span class="bool-val">true</span>;
@@ -19,6 +21,14 @@
     <span class="kw">let</span> <span class="kw">_</span> <span class="op">=</span> <span class="kw-2">*</span><span class="ident">foo</span>;
     <span class="macro">mac!</span>(<span class="ident">foo</span>, <span class="kw-2">&amp;</span><span class="kw-2">mut</span> <span class="ident">bar</span>);
     <span class="macro">assert!</span>(<span class="self">self</span>.<span class="ident">length</span> <span class="op">&lt;</span> <span class="ident">N</span> <span class="op">&amp;&amp;</span> <span class="ident">index</span> <span class="op">&lt;</span><span class="op">=</span> <span class="self">self</span>.<span class="ident">length</span>);
+    <span class="ident">::std::env::var</span>(<span class="string">&quot;gateau&quot;</span>).<span class="ident">is_ok</span>();
+    <span class="attribute">#[<span class="ident">rustfmt::skip</span>]</span>
+    <span class="kw">let</span> <span class="ident">s</span>:<span class="ident">std</span><span class="ident">::path::PathBuf</span> <span class="op">=</span> <span class="ident">std::path::PathBuf::new</span>();
+    <span class="kw">let</span> <span class="kw-2">mut</span> <span class="ident">s</span> <span class="op">=</span> <span class="ident">String::new</span>();
+
+    <span class="kw">match</span> <span class="kw-2">&amp;</span><span class="ident">s</span> {
+        <span class="kw-2">ref</span> <span class="kw-2">mut</span> <span class="ident">x</span> <span class="op">=</span><span class="op">&gt;</span> {}
+    }
 }
 
 <span class="macro">macro_rules!</span> <span class="ident">bar</span> {

src/librustdoc/html/highlight/fixtures/sample.rs

@@ -1,5 +1,7 @@
 #![crate_type = "lib"]
 
+use std::path::{Path, PathBuf};
+
 #[cfg(target_os = "linux")]
 fn main() {
     let foo = true && false || true;
@@ -9,6 +11,14 @@ fn main() {
     let _ = *foo;
     mac!(foo, &mut bar);
     assert!(self.length < N && index <= self.length);
+    ::std::env::var("gateau").is_ok();
+    #[rustfmt::skip]
+    let s:std::path::PathBuf = std::path::PathBuf::new();
+    let mut s = String::new();
+
+    match &s {
+        ref mut x => {}
+    }
 }
 
 macro_rules! bar {