diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 877293bac54f7..b2b63d0dbb4bd 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -237,7 +237,7 @@ fn expand_mac_invoc(mac: ast::Mac, ident: Option, attrs: Vec(mac: ast::Mac, ident: Option, attrs: Vec, m: Mrk) -> Vec { +fn mark_tts(tts: &[TokenTree], m: Mrk) -> Vec { noop_fold_tts(tts, &mut Marker{mark:m, expn_id: None}) } diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 68527b0797d5b..ffc950d76dd27 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -32,6 +32,7 @@ pub mod rt { use ext::base::ExtCtxt; use parse::{self, token, classify}; use ptr::P; + use std::rc::Rc; use tokenstream::{self, TokenTree}; @@ -215,12 +216,12 @@ pub mod rt { if self.node.style == ast::AttrStyle::Inner { r.push(TokenTree::Token(self.span, token::Not)); } - r.push(TokenTree::Delimited(self.span, tokenstream::Delimited { + r.push(TokenTree::Delimited(self.span, Rc::new(tokenstream::Delimited { delim: token::Bracket, open_span: self.span, tts: self.node.value.to_tokens(cx), close_span: self.span, - })); + }))); r } } @@ -235,12 +236,12 @@ pub mod rt { impl ToTokens for () { fn to_tokens(&self, _cx: &ExtCtxt) -> Vec { - vec![TokenTree::Delimited(DUMMY_SP, tokenstream::Delimited { + vec![TokenTree::Delimited(DUMMY_SP, Rc::new(tokenstream::Delimited { delim: token::Paren, open_span: DUMMY_SP, tts: vec![], close_span: DUMMY_SP, - })] + }))] } } @@ -791,9 +792,14 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, matcher: bool) -> Vec { parser: RefCell>, @@ -262,7 +263,7 @@ pub fn compile<'cx>(cx: &'cx mut ExtCtxt, let match_lhs_tok = MatchNt(lhs_nm, token::str_to_ident("tt")); let match_rhs_tok = MatchNt(rhs_nm, token::str_to_ident("tt")); let argument_gram = vec![ - TokenTree::Sequence(DUMMY_SP, tokenstream::SequenceRepetition { + TokenTree::Sequence(DUMMY_SP, Rc::new(tokenstream::SequenceRepetition { tts: vec![ TokenTree::Token(DUMMY_SP, match_lhs_tok), TokenTree::Token(DUMMY_SP, token::FatArrow), @@ -271,14 +272,14 @@ pub fn compile<'cx>(cx: &'cx mut ExtCtxt, separator: Some(token::Semi), op: tokenstream::KleeneOp::OneOrMore, num_captures: 2, - }), + })), // to phase into semicolon-termination instead of semicolon-separation - TokenTree::Sequence(DUMMY_SP, tokenstream::SequenceRepetition { + TokenTree::Sequence(DUMMY_SP, Rc::new(tokenstream::SequenceRepetition { tts: vec![TokenTree::Token(DUMMY_SP, token::Semi)], separator: None, op: tokenstream::KleeneOp::ZeroOrMore, num_captures: 0 - }), + })), ]; // Parse the macro_rules! invocation (`none` is for no interpolations): diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 40944a9a1c2d3..7c0d10669f30e 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -79,11 +79,11 @@ pub fn new_tt_reader_with_doc_flag(sp_diag: &Handler, let mut r = TtReader { sp_diag: sp_diag, stack: vec!(TtFrame { - forest: TokenTree::Sequence(DUMMY_SP, tokenstream::SequenceRepetition { + forest: TokenTree::Sequence(DUMMY_SP, Rc::new(tokenstream::SequenceRepetition { tts: src, // doesn't matter. This merely holds the root unzipping. separator: None, op: tokenstream::KleeneOp::ZeroOrMore, num_captures: 0 - }), + })), idx: 0, dotdotdoted: false, sep: None, diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index ed6f09eed645f..ac3d643b185ca 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -28,6 +28,8 @@ use tokenstream::*; use util::small_vector::SmallVector; use util::move_map::MoveMap; +use std::rc::Rc; + pub trait Folder : Sized { // Any additions to this trait should happen in form // of a call to a public `noop_*` function that only calls @@ -222,11 +224,11 @@ pub trait Folder : Sized { noop_fold_ty_params(tps, self) } - fn fold_tt(&mut self, tt: TokenTree) -> TokenTree { + fn fold_tt(&mut self, tt: &TokenTree) -> TokenTree { noop_fold_tt(tt, self) } - fn fold_tts(&mut self, tts: Vec) -> Vec { + fn fold_tts(&mut self, tts: &[TokenTree]) -> Vec { noop_fold_tts(tts, self) } @@ -501,7 +503,7 @@ pub fn noop_fold_mac(Spanned {node, span}: Mac, fld: &mut T) -> Mac { Spanned { node: Mac_ { path: fld.fold_path(node.path), - tts: fld.fold_tts(node.tts), + tts: fld.fold_tts(&node.tts), }, span: fld.new_span(span) } @@ -528,26 +530,32 @@ pub fn noop_fold_arg(Arg {id, pat, ty}: Arg, fld: &mut T) -> Arg { } } -pub fn noop_fold_tt(tt: TokenTree, fld: &mut T) -> TokenTree { - match tt { +pub fn noop_fold_tt(tt: &TokenTree, fld: &mut T) -> TokenTree { + match *tt { TokenTree::Token(span, ref tok) => TokenTree::Token(span, fld.fold_token(tok.clone())), - TokenTree::Delimited(span, delimed) => TokenTree::Delimited(span, Delimited { - delim: delimed.delim, - open_span: delimed.open_span, - tts: fld.fold_tts(delimed.tts), - close_span: delimed.close_span, - }), - TokenTree::Sequence(span, seq) => TokenTree::Sequence(span, SequenceRepetition { - tts: fld.fold_tts(seq.tts), - separator: seq.separator.clone().map(|tok| fld.fold_token(tok)), - ..seq - }), + TokenTree::Delimited(span, ref delimed) => { + TokenTree::Delimited(span, Rc::new( + Delimited { + delim: delimed.delim, + open_span: delimed.open_span, + tts: fld.fold_tts(&delimed.tts), + close_span: delimed.close_span, + } + )) + }, + TokenTree::Sequence(span, ref seq) => + TokenTree::Sequence(span, + Rc::new(SequenceRepetition { + tts: fld.fold_tts(&seq.tts), + separator: seq.separator.clone().map(|tok| fld.fold_token(tok)), + ..**seq + })), } } -pub fn noop_fold_tts(tts: Vec, fld: &mut T) -> Vec { - tts.move_map(|tt| fld.fold_tt(tt)) +pub fn noop_fold_tts(tts: &[TokenTree], fld: &mut T) -> Vec { + tts.iter().map(|tt| fld.fold_tt(tt)).collect() } // apply ident folder if it's an ident, apply other folds to interpolated nodes @@ -605,7 +613,7 @@ pub fn noop_fold_interpolated(nt: token::Nonterminal, fld: &mut T) token::NtIdent(Box::new(Spanned::{node: fld.fold_ident(id.node), ..*id})), token::NtMeta(meta_item) => token::NtMeta(fld.fold_meta_item(meta_item)), token::NtPath(path) => token::NtPath(Box::new(fld.fold_path(*path))), - token::NtTT(tt) => token::NtTT(tt.map(|tt| fld.fold_tt(tt))), + token::NtTT(tt) => token::NtTT(P(fld.fold_tt(&tt))), token::NtArm(arm) => token::NtArm(fld.fold_arm(arm)), token::NtImplItem(arm) => token::NtImplItem(arm.map(|arm| fld.fold_impl_item(arm) diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index bbcc044d43c6b..9502bc48a3e11 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -662,6 +662,7 @@ pub fn integer_lit(s: &str, #[cfg(test)] mod tests { use super::*; + use std::rc::Rc; use syntax_pos::{Span, BytePos, Pos, NO_EXPANSION}; use codemap::Spanned; use ast::{self, PatKind}; @@ -763,7 +764,7 @@ mod tests { ) if first_delimed.delim == token::Paren && ident.name.as_str() == "a" => {}, - _ => panic!("value 3: {:?}", *first_delimed), + _ => panic!("value 3: {:?}", **first_delimed), } let tts = &second_delimed.tts[..]; match (tts.len(), tts.get(0), tts.get(1)) { @@ -774,10 +775,10 @@ mod tests { ) if second_delimed.delim == token::Paren && ident.name.as_str() == "a" => {}, - _ => panic!("value 4: {:?}", *second_delimed), + _ => panic!("value 4: {:?}", **second_delimed), } }, - _ => panic!("value 2: {:?}", *macro_delimed), + _ => panic!("value 2: {:?}", **macro_delimed), } }, _ => panic!("value: {:?}",tts), @@ -793,7 +794,7 @@ mod tests { TokenTree::Token(sp(3, 4), token::Ident(str_to_ident("a"))), TokenTree::Delimited( sp(5, 14), - tokenstream::Delimited { + Rc::new(tokenstream::Delimited { delim: token::DelimToken::Paren, open_span: sp(5, 6), tts: vec![ @@ -802,10 +803,10 @@ mod tests { TokenTree::Token(sp(10, 13), token::Ident(str_to_ident("i32"))), ], close_span: sp(13, 14), - }), + })), TokenTree::Delimited( sp(15, 21), - tokenstream::Delimited { + Rc::new(tokenstream::Delimited { delim: token::DelimToken::Brace, open_span: sp(15, 16), tts: vec![ @@ -813,7 +814,7 @@ mod tests { TokenTree::Token(sp(18, 19), token::Semi), ], close_span: sp(20, 21), - }) + })) ]; assert_eq!(tts, expected); diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index a06270bb7727a..4cf14e6229954 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -2688,12 +2688,13 @@ impl<'a> Parser<'a> { )?; let (sep, repeat) = self.parse_sep_and_kleene_op()?; let name_num = macro_parser::count_names(&seq); - return Ok(TokenTree::Sequence(mk_sp(sp.lo, seq_span.hi), SequenceRepetition { - tts: seq, - separator: sep, - op: repeat, - num_captures: name_num - })); + return Ok(TokenTree::Sequence(mk_sp(sp.lo, seq_span.hi), + Rc::new(SequenceRepetition { + tts: seq, + separator: sep, + op: repeat, + num_captures: name_num + }))); } else if self.token.is_keyword(keywords::Crate) { self.bump(); return Ok(TokenTree::Token(sp, SpecialVarNt(SpecialMacroVar::CrateMacroVar))); @@ -2848,12 +2849,12 @@ impl<'a> Parser<'a> { _ => {} } - Ok(TokenTree::Delimited(span, Delimited { + Ok(TokenTree::Delimited(span, Rc::new(Delimited { delim: delim, open_span: open_span, tts: tts, close_span: close_span, - })) + }))) }, _ => { // invariants: the current token is not a left-delimiter, diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 35377d14bab7c..f0f0a7bc580d3 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -21,6 +21,8 @@ use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; use parse::lexer; use parse::token; +use std::rc::Rc; + /// A delimited sequence of token trees #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Delimited { @@ -94,13 +96,13 @@ pub enum TokenTree { /// A single token Token(Span, token::Token), /// A delimited sequence of token trees - Delimited(Span, Delimited), + Delimited(Span, Rc), // This only makes sense in MBE macros. /// A kleene-style repetition sequence with a span // FIXME(eddyb) #12938 Use DST. - Sequence(Span, SequenceRepetition), + Sequence(Span, Rc), } impl TokenTree { @@ -149,7 +151,7 @@ impl TokenTree { Some(*cnt) }).max().unwrap_or(0); - TokenTree::Delimited(sp, Delimited { + TokenTree::Delimited(sp, Rc::new(Delimited { delim: token::Bracket, open_span: sp, tts: vec![TokenTree::Token(sp, token::Ident(token::str_to_ident("doc"))), @@ -157,7 +159,7 @@ impl TokenTree { TokenTree::Token(sp, token::Literal( token::StrRaw(token::intern(&stripped), num_of_hashes), None))], close_span: sp, - }) + })) } (&TokenTree::Delimited(_, ref delimed), _) => { if index == 0 {