Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ignore whitespace tokens when re-computing spans in save_analysis #18562

Merged
merged 1 commit into from
Nov 4, 2014
Merged

Ignore whitespace tokens when re-computing spans in save_analysis #18562

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions src/librustc/middle/save/span_utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ impl<'a> SpanUtils<'a> {
let mut toks = self.retokenise_span(span);
let mut bracket_count = 0u;
loop {
let ts = toks.next_token();
let ts = toks.real_token();
if ts.tok == token::Eof {
return self.make_sub_span(span, result)
}
Expand All @@ -115,7 +115,7 @@ impl<'a> SpanUtils<'a> {
let mut toks = self.retokenise_span(span);
let mut bracket_count = 0u;
loop {
let ts = toks.next_token();
let ts = toks.real_token();
if ts.tok == token::Eof {
return None;
}
Expand All @@ -137,13 +137,13 @@ impl<'a> SpanUtils<'a> {
// any brackets, or the last span.
pub fn sub_span_for_meth_name(&self, span: Span) -> Option<Span> {
let mut toks = self.retokenise_span(span);
let mut prev = toks.next_token();
let mut prev = toks.real_token();
let mut result = None;
let mut bracket_count = 0u;
let mut last_span = None;
while prev.tok != token::Eof {
last_span = None;
let mut next = toks.next_token();
let mut next = toks.real_token();

if (next.tok == token::OpenDelim(token::Paren) ||
next.tok == token::Lt) &&
Expand All @@ -156,7 +156,7 @@ impl<'a> SpanUtils<'a> {
next.tok == token::ModSep {
let old = prev;
prev = next;
next = toks.next_token();
next = toks.real_token();
if next.tok == token::Lt &&
old.tok.is_ident() {
result = Some(old.sp);
Expand Down Expand Up @@ -185,11 +185,11 @@ impl<'a> SpanUtils<'a> {
// brackets, or the last span.
pub fn sub_span_for_type_name(&self, span: Span) -> Option<Span> {
let mut toks = self.retokenise_span(span);
let mut prev = toks.next_token();
let mut prev = toks.real_token();
let mut result = None;
let mut bracket_count = 0u;
loop {
let next = toks.next_token();
let next = toks.real_token();

if (next.tok == token::Lt ||
next.tok == token::Colon) &&
Expand Down Expand Up @@ -234,7 +234,7 @@ impl<'a> SpanUtils<'a> {
// We keep track of how many brackets we're nested in
let mut bracket_count = 0i;
loop {
let ts = toks.next_token();
let ts = toks.real_token();
if ts.tok == token::Eof {
if bracket_count != 0 {
let loc = self.sess.codemap().lookup_char_pos(span.lo);
Expand Down Expand Up @@ -263,12 +263,12 @@ impl<'a> SpanUtils<'a> {

pub fn sub_span_before_token(&self, span: Span, tok: Token) -> Option<Span> {
let mut toks = self.retokenise_span(span);
let mut prev = toks.next_token();
let mut prev = toks.real_token();
loop {
if prev.tok == token::Eof {
return None;
}
let next = toks.next_token();
let next = toks.real_token();
if next.tok == tok {
return self.make_sub_span(span, Some(prev.sp));
}
Expand All @@ -281,15 +281,16 @@ impl<'a> SpanUtils<'a> {
keyword: keywords::Keyword) -> Option<Span> {
let mut toks = self.retokenise_span(span);
loop {
let ts = toks.next_token();
let ts = toks.real_token();
if ts.tok == token::Eof {
return None;
}
if ts.tok.is_keyword(keyword) {
let ts = toks.next_token();
let ts = toks.real_token();
if ts.tok == token::Eof {
return None
} else {
println!("found keyword: {} at {}", ts, ts.sp);
return self.make_sub_span(span, Some(ts.sp));
}
}
Expand Down
13 changes: 13 additions & 0 deletions src/libsyntax/parse/lexer/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,19 @@ pub trait Reader {
/// Report a non-fatal error with the current span.
fn err(&self, &str);
fn peek(&self) -> TokenAndSpan;
/// Get a token the parser cares about.
fn real_token(&mut self) -> TokenAndSpan {
let mut t = self.next_token();
loop {
match t.tok {
token::Whitespace | token::Comment | token::Shebang(_) => {
t = self.next_token();
},
_ => break
}
}
t
}
}

#[deriving(Clone, PartialEq, Eq, Show)]
Expand Down
20 changes: 3 additions & 17 deletions src/libsyntax/parse/parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,27 +338,13 @@ fn is_plain_ident_or_underscore(t: &token::Token) -> bool {
t.is_plain_ident() || *t == token::Underscore
}

/// Get a token the parser cares about
fn real_token(rdr: &mut Reader) -> TokenAndSpan {
let mut t = rdr.next_token();
loop {
match t.tok {
token::Whitespace | token::Comment | token::Shebang(_) => {
t = rdr.next_token();
},
_ => break
}
}
t
}

impl<'a> Parser<'a> {
pub fn new(sess: &'a ParseSess,
cfg: ast::CrateConfig,
mut rdr: Box<Reader+'a>)
-> Parser<'a>
{
let tok0 = real_token(&mut *rdr);
let tok0 = rdr.real_token();
let span = tok0.sp;
let placeholder = TokenAndSpan {
tok: token::Underscore,
Expand Down Expand Up @@ -898,7 +884,7 @@ impl<'a> Parser<'a> {
None
};
let next = if self.buffer_start == self.buffer_end {
real_token(&mut *self.reader)
self.reader.real_token()
} else {
// Avoid token copies with `replace`.
let buffer_start = self.buffer_start as uint;
Expand Down Expand Up @@ -942,7 +928,7 @@ impl<'a> Parser<'a> {
-> R {
let dist = distance as int;
while self.buffer_length() < dist {
self.buffer[self.buffer_end as uint] = real_token(&mut *self.reader);
self.buffer[self.buffer_end as uint] = self.reader.real_token();
self.buffer_end = (self.buffer_end + 1) & 3;
}
f(&self.buffer[((self.buffer_start + dist - 1) & 3) as uint].tok)
Expand Down