Skip to content

Commit

Permalink
Append TokenTree with ToTokens in proc_macro::quote!
Browse files Browse the repository at this point in the history
  • Loading branch information
@SpriteOvO
SpriteOvO committed Jan 5, 2025
1 parent 2a2e87b commit 6404cf5
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 53 deletions.
2 changes: 1 addition & 1 deletion library/proc_macro/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,7 @@ pub mod token_stream {
/// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term.
/// To quote `$` itself, use `$$`.
#[unstable(feature = "proc_macro_quote", issue = "54722")]
#[allow_internal_unstable(proc_macro_def_site, proc_macro_internals)]
#[allow_internal_unstable(proc_macro_def_site, proc_macro_internals, proc_macro_totokens)]
#[rustc_builtin_macro]
pub macro quote($($t:tt)*) {
/* compiler built-in */
Expand Down
82 changes: 49 additions & 33 deletions library/proc_macro/src/quote.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
//! This quasiquoter uses macros 2.0 hygiene to reliably access
//! items from `proc_macro`, to build a `proc_macro::TokenStream`.
use crate::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
use crate::{
Delimiter, Group, Ident, Literal, Punct, Spacing, Span, ToTokens, TokenStream, TokenTree,
};

macro_rules! minimal_quote_tt {
(($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, minimal_quote!($($t)*)) };
Expand Down Expand Up @@ -50,7 +52,7 @@ macro_rules! minimal_quote {
() => { TokenStream::new() };
($($t:tt)*) => {
[
$(TokenStream::from(minimal_quote_ts!($t)),)*
$(ToTokens::into_token_stream(minimal_quote_ts!($t)),)*
].iter().cloned().collect::<TokenStream>()
};
}
Expand All @@ -66,48 +68,56 @@ pub fn quote(stream: TokenStream) -> TokenStream {
}
let proc_macro_crate = minimal_quote!(crate);
let mut after_dollar = false;
let tokens = stream
.into_iter()
.filter_map(|tree| {
if after_dollar {
after_dollar = false;
match tree {
TokenTree::Ident(_) => {
return Some(minimal_quote!(Into::<crate::TokenStream>::into(
Clone::clone(&(@ tree))),));
}
TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
_ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
}
} else if let TokenTree::Punct(ref tt) = tree {
if tt.as_char() == '$' {
after_dollar = true;
return None;

let mut tokens = crate::TokenStream::new();
for tree in stream {
if after_dollar {
after_dollar = false;
match tree {
TokenTree::Ident(_) => {
minimal_quote!(crate::ToTokens::to_tokens(&(@ tree), &mut ts);)
.to_tokens(&mut tokens);
continue;
}
TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
_ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
}
} else if let TokenTree::Punct(ref tt) = tree {
if tt.as_char() == '$' {
after_dollar = true;
continue;
}
}

Some(minimal_quote!(crate::TokenStream::from((@ match tree {
TokenTree::Punct(tt) => minimal_quote!(crate::TokenTree::Punct(crate::Punct::new(
match tree {
TokenTree::Punct(tt) => {
minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(
(@ TokenTree::from(Literal::character(tt.as_char()))),
(@ match tt.spacing() {
Spacing::Alone => minimal_quote!(crate::Spacing::Alone),
Spacing::Joint => minimal_quote!(crate::Spacing::Joint),
}),
))),
TokenTree::Group(tt) => minimal_quote!(crate::TokenTree::Group(crate::Group::new(
)), &mut ts);)
}
TokenTree::Group(tt) => {
minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Group(crate::Group::new(
(@ match tt.delimiter() {
Delimiter::Parenthesis => minimal_quote!(crate::Delimiter::Parenthesis),
Delimiter::Brace => minimal_quote!(crate::Delimiter::Brace),
Delimiter::Bracket => minimal_quote!(crate::Delimiter::Bracket),
Delimiter::None => minimal_quote!(crate::Delimiter::None),
}),
(@ quote(tt.stream())),
))),
TokenTree::Ident(tt) => minimal_quote!(crate::TokenTree::Ident(crate::Ident::new(
)), &mut ts);)
}
TokenTree::Ident(tt) => {
minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Ident(crate::Ident::new(
(@ TokenTree::from(Literal::string(&tt.to_string()))),
(@ quote_span(proc_macro_crate.clone(), tt.span())),
))),
TokenTree::Literal(tt) => minimal_quote!(crate::TokenTree::Literal({
)), &mut ts);)
}
TokenTree::Literal(tt) => {
minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Literal({
let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string())))
.parse::<crate::TokenStream>()
.unwrap()
Expand All @@ -120,16 +130,22 @@ pub fn quote(stream: TokenStream) -> TokenStream {
} else {
unreachable!()
}
}))
})),))
})
.collect::<TokenStream>();

}), &mut ts);)
}
}
.to_tokens(&mut tokens);
}
if after_dollar {
panic!("unexpected trailing `$` in `quote!`");
}

minimal_quote!([(@ tokens)].iter().cloned().collect::<crate::TokenStream>())
minimal_quote! {
{
let mut ts = crate::TokenStream::new();
(@ tokens)
ts
}
}
}

/// Quote a `Span` into a `TokenStream`.
Expand Down
42 changes: 23 additions & 19 deletions tests/ui/proc-macro/quote-debug.stdout
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,25 +19,29 @@ extern crate std;
extern crate proc_macro;

fn main() {
[crate::TokenStream::from(crate::TokenTree::Ident(crate::Ident::new("let",
crate::Span::recover_proc_macro_span(0)))),
crate::TokenStream::from(crate::TokenTree::Ident(crate::Ident::new("hello",
crate::Span::recover_proc_macro_span(1)))),
crate::TokenStream::from(crate::TokenTree::Punct(crate::Punct::new('=',
crate::Spacing::Alone))),
crate::TokenStream::from(crate::TokenTree::Literal({
let mut iter =
"\"world\"".parse::<crate::TokenStream>().unwrap().into_iter();
if let (Some(crate::TokenTree::Literal(mut lit)), None) =
(iter.next(), iter.next()) {
lit.set_span(crate::Span::recover_proc_macro_span(2));
lit
} else {
::core::panicking::panic("internal error: entered unreachable code")
}
})),
crate::TokenStream::from(crate::TokenTree::Punct(crate::Punct::new(';',
crate::Spacing::Alone)))].iter().cloned().collect::<crate::TokenStream>()
{
let mut ts = crate::TokenStream::new();
crate::ToTokens::to_tokens(&crate::TokenTree::Ident(crate::Ident::new("let",
crate::Span::recover_proc_macro_span(0))), &mut ts);
crate::ToTokens::to_tokens(&crate::TokenTree::Ident(crate::Ident::new("hello",
crate::Span::recover_proc_macro_span(1))), &mut ts);
crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new('=',
crate::Spacing::Alone)), &mut ts);
crate::ToTokens::to_tokens(&crate::TokenTree::Literal({
let mut iter =
"\"world\"".parse::<crate::TokenStream>().unwrap().into_iter();
if let (Some(crate::TokenTree::Literal(mut lit)), None) =
(iter.next(), iter.next()) {
lit.set_span(crate::Span::recover_proc_macro_span(2));
lit
} else {
::core::panicking::panic("internal error: entered unreachable code")
}
}), &mut ts);
crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(';',
crate::Spacing::Alone)), &mut ts);
ts
}
}
const _: () =
{
Expand Down

0 comments on commit 6404cf5

Please sign in to comment.