Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove TokenStream::JointTree. #56964

Merged
merged 1 commit into from
Dec 24, 2018
Merged

Remove TokenStream::JointTree. #56964

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions src/libsyntax/parse/lexer/tokentrees.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
use print::pprust::token_to_string;
use parse::lexer::StringReader;
use parse::{token, PResult};
use tokenstream::{DelimSpan, TokenStream, TokenTree};
use tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree};
nnethercote marked this conversation as resolved.
Show resolved Hide resolved

impl<'a> StringReader<'a> {
// Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
Expand Down Expand Up @@ -178,8 +178,7 @@ impl<'a> StringReader<'a> {
let raw = self.span_src_raw;
self.real_token();
let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token);
nnethercote marked this conversation as resolved.
Show resolved Hide resolved

Ok(if is_joint { tt.joint() } else { tt.into() })
Ok(TokenStream::Tree(tt, if is_joint { Joint } else { NonJoint }))
}
}
}
Expand Down
93 changes: 42 additions & 51 deletions src/libsyntax/tokenstream.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ impl TokenTree {
}

pub fn joint(self) -> TokenStream {
TokenStream::JointTree(self)
TokenStream::Tree(self, Joint)
}

/// Returns the opening delimiter as a token tree.
Expand Down Expand Up @@ -156,15 +156,22 @@ impl TokenTree {
#[derive(Clone, Debug)]
pub enum TokenStream {
Empty,
Tree(TokenTree),
JointTree(TokenTree),
Tree(TokenTree, IsJoint),
nnethercote marked this conversation as resolved.
Show resolved Hide resolved
Stream(Lrc<Vec<TokenStream>>),
}

// `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger.
#[cfg(target_arch = "x86_64")]
static_assert!(MEM_SIZE_OF_TOKEN_STREAM: mem::size_of::<TokenStream>() == 32);

#[derive(Clone, Copy, Debug, PartialEq)]
pub enum IsJoint {
Joint,
NonJoint
}
nnethercote marked this conversation as resolved.
Show resolved Hide resolved

use self::IsJoint::*;

impl TokenStream {
/// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream`
/// separating the two arguments with a comma for diagnostic suggestions.
Expand All @@ -176,16 +183,16 @@ impl TokenStream {
while let Some((pos, ts)) = iter.next() {
if let Some((_, next)) = iter.peek() {
let sp = match (&ts, &next) {
(TokenStream::Tree(TokenTree::Token(_, token::Token::Comma)), _) |
(_, TokenStream::Tree(TokenTree::Token(_, token::Token::Comma))) => {
continue;
}
(TokenStream::Tree(TokenTree::Token(sp, _)), _) => *sp,
(TokenStream::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(),
(TokenStream::Tree(TokenTree::Token(_, token::Token::Comma), NonJoint), _) |
(_, TokenStream::Tree(TokenTree::Token(_, token::Token::Comma), NonJoint))
=> continue,
(TokenStream::Tree(TokenTree::Token(sp, _), NonJoint), _) => *sp,
(TokenStream::Tree(TokenTree::Delimited(sp, ..), NonJoint), _) =>
sp.entire(),
_ => continue,
};
let sp = sp.shrink_to_hi();
let comma = TokenStream::Tree(TokenTree::Token(sp, token::Comma));
let comma = TokenStream::Tree(TokenTree::Token(sp, token::Comma), NonJoint);
suggestion = Some((pos, comma, sp));
}
}
Expand All @@ -204,7 +211,7 @@ impl TokenStream {

impl From<TokenTree> for TokenStream {
fn from(tt: TokenTree) -> TokenStream {
TokenStream::Tree(tt)
TokenStream::Tree(tt, NonJoint)
}
}

Expand Down Expand Up @@ -232,7 +239,7 @@ impl Extend<TokenStream> for TokenStream {
vec.reserve(iter.size_hint().0);
vec
}
TokenStream::Tree(_) | TokenStream::JointTree(_) => {
TokenStream::Tree(..) => {
let mut vec = Vec::new();
vec.reserve(1 + iter.size_hint().0);
vec.push(this);
Expand Down Expand Up @@ -367,8 +374,7 @@ impl TokenStream {
/// Returns true if the token tree is a joint operation w.r.t. `proc_macro::TokenNode`.
pub fn as_tree(self) -> (TokenTree, bool /* joint? */) {
match self {
TokenStream::Tree(tree) => (tree, false),
TokenStream::JointTree(tree) => (tree, true),
TokenStream::Tree(tree, is_joint) => (tree, is_joint == Joint),
_ => unreachable!(),
}
}
Expand All @@ -379,8 +385,7 @@ impl TokenStream {
let mut i = 0;
while let Some(stream) = trees.next_as_stream() {
result.push(match stream {
TokenStream::Tree(tree) => f(i, tree).into(),
TokenStream::JointTree(tree) => f(i, tree).joint(),
TokenStream::Tree(tree, is_joint) => TokenStream::Tree(f(i, tree), is_joint),
_ => unreachable!()
});
i += 1;
Expand All @@ -393,27 +398,25 @@ impl TokenStream {
let mut result = Vec::new();
while let Some(stream) = trees.next_as_stream() {
result.push(match stream {
TokenStream::Tree(tree) => f(tree).into(),
TokenStream::JointTree(tree) => f(tree).joint(),
TokenStream::Tree(tree, is_joint) => TokenStream::Tree(f(tree), is_joint),
_ => unreachable!()
});
}
TokenStream::new(result)
}

fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> {
fn first_tree_and_joint(&self) -> Option<(TokenTree, IsJoint)> {
match self {
TokenStream::Empty => None,
TokenStream::Tree(ref tree) => Some((tree.clone(), false)),
TokenStream::JointTree(ref tree) => Some((tree.clone(), true)),
TokenStream::Tree(ref tree, is_joint) => Some((tree.clone(), *is_joint)),
TokenStream::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(),
}
}

fn last_tree_if_joint(&self) -> Option<TokenTree> {
match self {
TokenStream::Empty | TokenStream::Tree(..) => None,
TokenStream::JointTree(ref tree) => Some(tree.clone()),
TokenStream::Empty | TokenStream::Tree(_, NonJoint) => None,
TokenStream::Tree(ref tree, Joint) => Some(tree.clone()),
TokenStream::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(),
}
}
Expand All @@ -437,11 +440,7 @@ impl TokenStreamBuilder {
self.push_all_but_last_tree(&last_stream);
let glued_span = last_span.to(span);
let glued_tt = TokenTree::Token(glued_span, glued_tok);
let glued_tokenstream = if is_joint {
glued_tt.joint()
} else {
glued_tt.into()
};
let glued_tokenstream = TokenStream::Tree(glued_tt, is_joint);
self.0.push(glued_tokenstream);
self.push_all_but_first_tree(&stream);
return
Expand Down Expand Up @@ -491,8 +490,7 @@ pub struct Cursor(CursorKind);
#[derive(Clone)]
enum CursorKind {
Empty,
Tree(TokenTree, bool /* consumed? */),
JointTree(TokenTree, bool /* consumed? */),
Tree(TokenTree, IsJoint, bool /* consumed? */),
nnethercote marked this conversation as resolved.
Show resolved Hide resolved
Stream(StreamCursor),
}

Expand All @@ -514,9 +512,9 @@ impl StreamCursor {
self.index += 1;
let next = self.stream[self.index - 1].clone();
match next {
TokenStream::Tree(..) | TokenStream::JointTree(..) => return Some(next),
TokenStream::Stream(stream) => self.insert(stream),
TokenStream::Empty => {}
TokenStream::Tree(..) => return Some(next),
TokenStream::Stream(stream) => self.insert(stream),
}
} else if let Some((stream, index)) = self.stack.pop() {
self.stream = stream;
Expand All @@ -538,7 +536,7 @@ impl Iterator for Cursor {

fn next(&mut self) -> Option<TokenTree> {
self.next_as_stream().map(|stream| match stream {
TokenStream::Tree(tree) | TokenStream::JointTree(tree) => tree,
TokenStream::Tree(tree, _) => tree,
_ => unreachable!()
})
}
Expand All @@ -548,18 +546,15 @@ impl Cursor {
fn new(stream: TokenStream) -> Self {
Cursor(match stream {
TokenStream::Empty => CursorKind::Empty,
TokenStream::Tree(tree) => CursorKind::Tree(tree, false),
TokenStream::JointTree(tree) => CursorKind::JointTree(tree, false),
TokenStream::Tree(tree, is_joint) => CursorKind::Tree(tree, is_joint, false),
TokenStream::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)),
})
}

pub fn next_as_stream(&mut self) -> Option<TokenStream> {
let (stream, consumed) = match self.0 {
CursorKind::Tree(ref tree, ref mut consumed @ false) =>
(tree.clone().into(), consumed),
CursorKind::JointTree(ref tree, ref mut consumed @ false) =>
(tree.clone().joint(), consumed),
CursorKind::Tree(ref tree, ref is_joint, ref mut consumed @ false) =>
(TokenStream::Tree(tree.clone(), *is_joint), consumed),
CursorKind::Stream(ref mut cursor) => return cursor.next_as_stream(),
_ => return None,
};
Expand All @@ -572,7 +567,7 @@ impl Cursor {
match self.0 {
_ if stream.is_empty() => return,
CursorKind::Empty => *self = stream.trees(),
CursorKind::Tree(_, consumed) | CursorKind::JointTree(_, consumed) => {
CursorKind::Tree(_, _, consumed) => {
*self = TokenStream::new(vec![self.original_stream(), stream]).trees();
if consumed {
self.next();
Expand All @@ -587,8 +582,8 @@ impl Cursor {
pub fn original_stream(&self) -> TokenStream {
match self.0 {
CursorKind::Empty => TokenStream::empty(),
CursorKind::Tree(ref tree, _) => tree.clone().into(),
CursorKind::JointTree(ref tree, _) => tree.clone().joint(),
CursorKind::Tree(ref tree, ref is_joint, _) =>
TokenStream::Tree(tree.clone(), *is_joint),
CursorKind::Stream(ref cursor) => TokenStream::Stream(
cursor.stack.get(0).cloned().map(|(stream, _)| stream)
.unwrap_or_else(|| cursor.stream.clone())
Expand All @@ -600,9 +595,8 @@ impl Cursor {
fn look_ahead(streams: &[TokenStream], mut n: usize) -> Result<TokenTree, usize> {
for stream in streams {
n = match stream {
TokenStream::Tree(ref tree) | TokenStream::JointTree(ref tree)
if n == 0 => return Ok(tree.clone()),
TokenStream::Tree(..) | TokenStream::JointTree(..) => n - 1,
TokenStream::Tree(ref tree, _) if n == 0 => return Ok(tree.clone()),
TokenStream::Tree(..) => n - 1,
TokenStream::Stream(ref stream) => match look_ahead(stream, n) {
Ok(tree) => return Ok(tree),
Err(n) => n,
Expand All @@ -615,10 +609,8 @@ impl Cursor {

match self.0 {
CursorKind::Empty |
CursorKind::Tree(_, true) |
CursorKind::JointTree(_, true) => Err(n),
CursorKind::Tree(ref tree, false) |
CursorKind::JointTree(ref tree, false) => look_ahead(&[tree.clone().into()], n),
CursorKind::Tree(_, _, true) => Err(n),
CursorKind::Tree(ref tree, _, false) => look_ahead(&[tree.clone().into()], n),
CursorKind::Stream(ref cursor) => {
look_ahead(&cursor.stream[cursor.index ..], n).or_else(|mut n| {
for &(ref stream, index) in cursor.stack.iter().rev() {
Expand Down Expand Up @@ -651,8 +643,7 @@ impl From<TokenStream> for ThinTokenStream {
fn from(stream: TokenStream) -> ThinTokenStream {
ThinTokenStream(match stream {
TokenStream::Empty => None,
TokenStream::Tree(tree) => Some(Lrc::new(vec![tree.into()])),
TokenStream::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])),
TokenStream::Tree(..) => Some(Lrc::new(vec![stream])),
TokenStream::Stream(stream) => Some(stream),
})
}
Expand Down
8 changes: 2 additions & 6 deletions src/libsyntax_ext/proc_macro_server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ use syntax::ast;
use syntax::ext::base::ExtCtxt;
use syntax::parse::lexer::comments;
use syntax::parse::{self, token, ParseSess};
use syntax::tokenstream::{self, DelimSpan, TokenStream};
use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream};
use syntax_pos::hygiene::{SyntaxContext, Transparency};
use syntax_pos::symbol::{keywords, Symbol};
use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
Expand Down Expand Up @@ -297,11 +297,7 @@ impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
};

let tree = tokenstream::TokenTree::Token(span, token);
if joint {
tree.joint()
} else {
tree.into()
}
TokenStream::Tree(tree, if joint { Joint } else { NonJoint })
}
}

Expand Down