Skip to content

Commit

Permalink
Handle Groups with Delimiter::None (#50)
Browse files Browse the repository at this point in the history
Currently, rustc does not pass the exact original `TokenStream` to
proc-macros in several cases. This has many undesirable effects, such as
losing correct location information in error message.
See rust-lang/rust#43081 for more details

In the future, rustc will begin passing the correct `TokenStream` to
proc-macros. As a result, some tokens may be wrapped in a
`TokenTree::Group` with `Delimiter::None` (when the tokens originally
came from a `macro_rules!`) macro expansion.

I've determined that this change will cause your crate to stop working
on some inputs. This PR updates `hex-literal-impl` to be compatible with both the
old and new `TokenStream` contents.

If you have any questions, feel free to ask me. See rust-lang/rust#72622 for more details
  • Loading branch information
Aaron1011 authored Jun 1, 2020
1 parent a0e4801 commit bd10ffa
Showing 1 changed file with 23 additions and 2 deletions.
25 changes: 23 additions & 2 deletions hex-literal/hex-literal-impl/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
extern crate proc_macro;

use proc_macro::{TokenStream, TokenTree};
use proc_macro::{TokenStream, TokenTree, Delimiter};
use proc_macro_hack::proc_macro_hack;

fn is_hex_char(c: &char) -> bool {
Expand All @@ -17,8 +17,29 @@ fn is_format_char(c: &char) -> bool {
}
}


/// Strips any outer `Delimiter::None` groups from the input,
/// returning a `TokenStream` consisting of the innermost
/// non-empty-group `TokenTree`.
/// This is used to handle a proc macro being invoked
/// by a `macro_rules!` expansion.
/// See https://github.com/rust-lang/rust/issues/72545 for background
fn ignore_groups(mut input: TokenStream) -> TokenStream {
let mut tokens = input.clone().into_iter();
loop {
if let Some(TokenTree::Group(group)) = tokens.next() {
if group.delimiter() == Delimiter::None {
input = group.stream();
continue;
}
}
return input;
}
}

#[proc_macro_hack]
pub fn hex(input: TokenStream) -> TokenStream {
pub fn hex(mut input: TokenStream) -> TokenStream {
input = ignore_groups(input);
let mut ts = input.into_iter();
let input = match (ts.next(), ts.next()) {
(Some(TokenTree::Literal(literal)), None) => literal.to_string(),
Expand Down

0 comments on commit bd10ffa

Please sign in to comment.