Append TokenTree with ToTokens in proc_macro::quote!

rust-lang · Jan 4, 2025 · 5d0a6c5 · 5d0a6c5
1 parent 2a2e87b
commit 5d0a6c5
Show file tree

Hide file tree

Showing 20 changed files with 117 additions and 68 deletions.
diff --git a/library/proc_macro/src/quote.rs b/library/proc_macro/src/quote.rs
@@ -4,7 +4,9 @@
 //! This quasiquoter uses macros 2.0 hygiene to reliably access
 //! items from `proc_macro`, to build a `proc_macro::TokenStream`.
 
-use crate::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
+use crate::{
+    Delimiter, Group, Ident, Literal, Punct, Spacing, Span, ToTokens, TokenStream, TokenTree,
+};
 
 macro_rules! minimal_quote_tt {
     (($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, minimal_quote!($($t)*)) };
@@ -50,7 +52,7 @@ macro_rules! minimal_quote {
     () => { TokenStream::new() };
     ($($t:tt)*) => {
         [
-            $(TokenStream::from(minimal_quote_ts!($t)),)*
+            $(ToTokens::into_token_stream(minimal_quote_ts!($t)),)*
         ].iter().cloned().collect::<TokenStream>()
     };
 }
@@ -66,48 +68,56 @@ pub fn quote(stream: TokenStream) -> TokenStream {
     }
     let proc_macro_crate = minimal_quote!(crate);
     let mut after_dollar = false;
-    let tokens = stream
-        .into_iter()
-        .filter_map(|tree| {
-            if after_dollar {
-                after_dollar = false;
-                match tree {
-                    TokenTree::Ident(_) => {
-                        return Some(minimal_quote!(Into::<crate::TokenStream>::into(
-                        Clone::clone(&(@ tree))),));
-                    }
-                    TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
-                    _ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
-                }
-            } else if let TokenTree::Punct(ref tt) = tree {
-                if tt.as_char() == '$' {
-                    after_dollar = true;
-                    return None;
+
+    let mut tokens = crate::TokenStream::new();
+    for tree in stream {
+        if after_dollar {
+            after_dollar = false;
+            match tree {
+                TokenTree::Ident(_) => {
+                    minimal_quote!(crate::ToTokens::to_tokens(&(@ tree), &mut ts);)
+                        .to_tokens(&mut tokens);
+                    continue;
                 }
+                TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
+                _ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
             }
+        } else if let TokenTree::Punct(ref tt) = tree {
+            if tt.as_char() == '$' {
+                after_dollar = true;
+                continue;
+            }
+        }
 
-            Some(minimal_quote!(crate::TokenStream::from((@ match tree {
-                TokenTree::Punct(tt) => minimal_quote!(crate::TokenTree::Punct(crate::Punct::new(
+        match tree {
+            TokenTree::Punct(tt) => {
+                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(
                     (@ TokenTree::from(Literal::character(tt.as_char()))),
                     (@ match tt.spacing() {
                         Spacing::Alone => minimal_quote!(crate::Spacing::Alone),
                         Spacing::Joint => minimal_quote!(crate::Spacing::Joint),
                     }),
-                ))),
-                TokenTree::Group(tt) => minimal_quote!(crate::TokenTree::Group(crate::Group::new(
+                )), &mut ts);)
+            }
+            TokenTree::Group(tt) => {
+                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Group(crate::Group::new(
                     (@ match tt.delimiter() {
                         Delimiter::Parenthesis => minimal_quote!(crate::Delimiter::Parenthesis),
                         Delimiter::Brace => minimal_quote!(crate::Delimiter::Brace),
                         Delimiter::Bracket => minimal_quote!(crate::Delimiter::Bracket),
                         Delimiter::None => minimal_quote!(crate::Delimiter::None),
                     }),
                     (@ quote(tt.stream())),
-                ))),
-                TokenTree::Ident(tt) => minimal_quote!(crate::TokenTree::Ident(crate::Ident::new(
+                )), &mut ts);)
+            }
+            TokenTree::Ident(tt) => {
+                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Ident(crate::Ident::new(
                     (@ TokenTree::from(Literal::string(&tt.to_string()))),
                     (@ quote_span(proc_macro_crate.clone(), tt.span())),
-                ))),
-                TokenTree::Literal(tt) => minimal_quote!(crate::TokenTree::Literal({
+                )), &mut ts);)
+            }
+            TokenTree::Literal(tt) => {
+                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Literal({
                     let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string())))
                         .parse::<crate::TokenStream>()
                         .unwrap()
@@ -120,16 +130,22 @@ pub fn quote(stream: TokenStream) -> TokenStream {
                     } else {
                         unreachable!()
                     }
-                }))
-            })),))
-        })
-        .collect::<TokenStream>();
-
+                }), &mut ts);)
+            }
+        }
+        .to_tokens(&mut tokens);
+    }
     if after_dollar {
         panic!("unexpected trailing `$` in `quote!`");
     }
 
-    minimal_quote!([(@ tokens)].iter().cloned().collect::<crate::TokenStream>())
+    minimal_quote! {
+        {
+            let mut ts = crate::TokenStream::new();
+            (@ tokens)
+            ts
+        }
+    }
 }
 
 /// Quote a `Span` into a `TokenStream`.

diff --git a/tests/ui/deriving/auxiliary/another-proc-macro.rs b/tests/ui/deriving/auxiliary/another-proc-macro.rs
@@ -1,4 +1,5 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 

diff --git a/tests/ui/hygiene/auxiliary/opaque-hygiene.rs b/tests/ui/hygiene/auxiliary/opaque-hygiene.rs
@@ -1,4 +1,5 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 use proc_macro::{TokenStream, quote};

diff --git a/tests/ui/macros/auxiliary/hello_macro.rs b/tests/ui/macros/auxiliary/hello_macro.rs
@@ -1,4 +1,5 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 

diff --git a/tests/ui/macros/auxiliary/issue-100199.rs b/tests/ui/macros/auxiliary/issue-100199.rs
@@ -1,8 +1,9 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 
-use proc_macro::{quote, Ident, Span, TokenStream, TokenTree};
+use proc_macro::{Ident, Span, TokenStream, TokenTree, quote};
 
 #[proc_macro_attribute]
 pub fn struct_with_bound(_: TokenStream, _: TokenStream) -> TokenStream {

diff --git a/tests/ui/macros/auxiliary/proc_macro_def.rs b/tests/ui/macros/auxiliary/proc_macro_def.rs
@@ -1,4 +1,5 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 

diff --git a/tests/ui/macros/auxiliary/proc_macro_sequence.rs b/tests/ui/macros/auxiliary/proc_macro_sequence.rs
@@ -1,8 +1,8 @@
-#![feature(proc_macro_span, proc_macro_quote)]
+#![feature(proc_macro_span, proc_macro_quote, proc_macro_totokens)]
 
 extern crate proc_macro;
 
-use proc_macro::{quote, Span, TokenStream, TokenTree};
+use proc_macro::{Span, TokenStream, TokenTree, quote};
 
 // This macro generates a macro with the same macro definition as `manual_foo` in
 // `same-sequence-span.rs` but with the same span for all sequences.

diff --git a/tests/ui/proc-macro/auxiliary/cond_plugin.rs b/tests/ui/proc-macro/auxiliary/cond_plugin.rs
@@ -1,4 +1,5 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 

diff --git a/tests/ui/proc-macro/auxiliary/count_compound_ops.rs b/tests/ui/proc-macro/auxiliary/count_compound_ops.rs
@@ -1,8 +1,9 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 
-use proc_macro::{TokenStream, TokenTree, Spacing, Literal, quote};
+use proc_macro::{Literal, Spacing, TokenStream, TokenTree, quote};
 
 #[proc_macro]
 pub fn count_compound_ops(input: TokenStream) -> TokenStream {

diff --git a/tests/ui/proc-macro/auxiliary/double.rs b/tests/ui/proc-macro/auxiliary/double.rs
@@ -1,4 +1,5 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 use proc_macro::*;

diff --git a/tests/ui/proc-macro/auxiliary/generate-dollar-ident.rs b/tests/ui/proc-macro/auxiliary/generate-dollar-ident.rs
@@ -1,4 +1,5 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 use proc_macro::*;

diff --git a/tests/ui/proc-macro/auxiliary/hygiene_example_codegen.rs b/tests/ui/proc-macro/auxiliary/hygiene_example_codegen.rs
@@ -1,4 +1,5 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro as proc_macro_renamed; // This does not break `quote!`
 

diff --git a/tests/ui/proc-macro/auxiliary/mixed-site-span.rs b/tests/ui/proc-macro/auxiliary/mixed-site-span.rs
@@ -1,4 +1,5 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 use proc_macro::*;
@@ -13,10 +14,8 @@ pub fn proc_macro_rules(input: TokenStream) -> TokenStream {
         let local_use = id("local_use");
         let mut single_quote = Punct::new('\'', Spacing::Joint);
         single_quote.set_span(Span::mixed_site());
-        let label_use: TokenStream = [
-            TokenTree::from(single_quote),
-            id("label_use"),
-        ].iter().cloned().collect();
+        let label_use: TokenStream =
+            [TokenTree::from(single_quote), id("label_use")].iter().cloned().collect();
         quote!(
             struct $item_def;
             let $local_def = 0;

diff --git a/tests/ui/proc-macro/auxiliary/nonterminal-recollect-attr.rs b/tests/ui/proc-macro/auxiliary/nonterminal-recollect-attr.rs
@@ -1,4 +1,5 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 use proc_macro::{TokenStream, quote};

diff --git a/tests/ui/proc-macro/auxiliary/resolved-located-at.rs b/tests/ui/proc-macro/auxiliary/resolved-located-at.rs
@@ -1,6 +1,7 @@
 #![feature(proc_macro_def_site)]
 #![feature(proc_macro_diagnostic)]
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 use proc_macro::*;

diff --git a/tests/ui/proc-macro/auxiliary/span-from-proc-macro.rs b/tests/ui/proc-macro/auxiliary/span-from-proc-macro.rs
@@ -1,17 +1,18 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 #![feature(proc_macro_internals)] // FIXME - this shouldn't be necessary
 
-extern crate proc_macro;
 extern crate custom_quote;
+extern crate proc_macro;
 
-use proc_macro::{quote, TokenStream};
+use proc_macro::{TokenStream, quote};
 
 macro_rules! expand_to_quote {
     () => {
         quote! {
             let bang_error: bool = 25;
         }
-    }
+    };
 }
 
 #[proc_macro]

diff --git a/tests/ui/proc-macro/quote-debug.stdout b/tests/ui/proc-macro/quote-debug.stdout
@@ -19,25 +19,29 @@ extern crate std;
 extern crate proc_macro;
 
 fn main() {
-    [crate::TokenStream::from(crate::TokenTree::Ident(crate::Ident::new("let",
-                                    crate::Span::recover_proc_macro_span(0)))),
-                        crate::TokenStream::from(crate::TokenTree::Ident(crate::Ident::new("hello",
-                                    crate::Span::recover_proc_macro_span(1)))),
-                        crate::TokenStream::from(crate::TokenTree::Punct(crate::Punct::new('=',
-                                    crate::Spacing::Alone))),
-                        crate::TokenStream::from(crate::TokenTree::Literal({
-                                    let mut iter =
-                                        "\"world\"".parse::<crate::TokenStream>().unwrap().into_iter();
-                                    if let (Some(crate::TokenTree::Literal(mut lit)), None) =
-                                                (iter.next(), iter.next()) {
-                                            lit.set_span(crate::Span::recover_proc_macro_span(2));
-                                            lit
-                                        } else {
-                                           ::core::panicking::panic("internal error: entered unreachable code")
-                                       }
-                                })),
-                        crate::TokenStream::from(crate::TokenTree::Punct(crate::Punct::new(';',
-                                    crate::Spacing::Alone)))].iter().cloned().collect::<crate::TokenStream>()
+    {
+        let mut ts = crate::TokenStream::new();
+        crate::ToTokens::to_tokens(&crate::TokenTree::Ident(crate::Ident::new("let",
+                        crate::Span::recover_proc_macro_span(0))), &mut ts);
+        crate::ToTokens::to_tokens(&crate::TokenTree::Ident(crate::Ident::new("hello",
+                        crate::Span::recover_proc_macro_span(1))), &mut ts);
+        crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new('=',
+                        crate::Spacing::Alone)), &mut ts);
+        crate::ToTokens::to_tokens(&crate::TokenTree::Literal({
+                        let mut iter =
+                            "\"world\"".parse::<crate::TokenStream>().unwrap().into_iter();
+                        if let (Some(crate::TokenTree::Literal(mut lit)), None) =
+                                    (iter.next(), iter.next()) {
+                                lit.set_span(crate::Span::recover_proc_macro_span(2));
+                                lit
+                            } else {
+                               ::core::panicking::panic("internal error: entered unreachable code")
+                           }
+                    }), &mut ts);
+        crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(';',
+                        crate::Spacing::Alone)), &mut ts);
+        ts
+    }
 }
 const _: () =
     {

diff --git a/tests/ui/proc-macro/quote-interpolation.rs b/tests/ui/proc-macro/quote-interpolation.rs
@@ -0,0 +1,16 @@
+//@ check-pass
+
+#![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::*;
+
+fn main() {
+    let x = Ident::new("foo", Span::call_site());
+    let _ = quote! {
+        let $x = 199;
+    };
+}
diff --git a/tests/ui/proc-macro/span-from-proc-macro.stderr b/tests/ui/proc-macro/span-from-proc-macro.stderr
@@ -1,5 +1,5 @@
 error[E0412]: cannot find type `MissingType` in this scope
-  --> $DIR/auxiliary/span-from-proc-macro.rs:33:20
+  --> $DIR/auxiliary/span-from-proc-macro.rs:34:20
    |
 LL | pub fn error_from_attribute(_args: TokenStream, _input: TokenStream) -> TokenStream {
    | ----------------------------------------------------------------------------------- in this expansion of `#[error_from_attribute]`
@@ -13,7 +13,7 @@ LL | #[error_from_attribute]
    | ----------------------- in this procedural macro expansion
 
 error[E0412]: cannot find type `OtherMissingType` in this scope
-  --> $DIR/auxiliary/span-from-proc-macro.rs:42:21
+  --> $DIR/auxiliary/span-from-proc-macro.rs:43:21
    |
 LL | pub fn error_from_derive(_input: TokenStream) -> TokenStream {
    | ------------------------------------------------------------ in this expansion of `#[derive(ErrorFromDerive)]`
@@ -27,7 +27,7 @@ LL | #[derive(ErrorFromDerive)]
    |          --------------- in this derive macro expansion
 
 error[E0425]: cannot find value `my_ident` in this scope
-  --> $DIR/auxiliary/span-from-proc-macro.rs:25:9
+  --> $DIR/auxiliary/span-from-proc-macro.rs:26:9
    |
 LL | pub fn other_error_from_bang(_input: TokenStream) -> TokenStream {
    | ---------------------------------------------------------------- in this expansion of `other_error_from_bang!`
@@ -41,7 +41,7 @@ LL |     other_error_from_bang!();
    |     ------------------------ in this macro invocation
 
 error[E0308]: mismatched types
-  --> $DIR/auxiliary/span-from-proc-macro.rs:12:36
+  --> $DIR/auxiliary/span-from-proc-macro.rs:13:36
    |
 LL |             let bang_error: bool = 25;
    |                             ----   ^^ expected `bool`, found integer

diff --git a/tests/ui/suggestions/auxiliary/proc-macro-type-error.rs b/tests/ui/suggestions/auxiliary/proc-macro-type-error.rs
@@ -1,8 +1,9 @@
 #![feature(proc_macro_quote)]
+#![feature(proc_macro_totokens)]
 
 extern crate proc_macro;
 
-use proc_macro::{quote, TokenStream};
+use proc_macro::{TokenStream, quote};
 
 #[proc_macro_attribute]
 pub fn hello(_: TokenStream, _: TokenStream) -> TokenStream {