Skip to content

Commit

Permalink
Prepare for invisible delimiters.
Browse files Browse the repository at this point in the history 
Current places where `Interpolated` is used are going to change to
instead use invisible delimiters. This prepares for that.
- It adds invisible delimiter cases to the `can_begin_*`/`may_be_*`
  methods and the `failed_to_match_macro` that are equivalent to the
  existing `Interpolated` cases.
- It adds panics/asserts in some places where invisible delimiters
  should never occur.
- In `Parser::parse_struct_fields` it excludes an ident + invisible
  delimiter from special consideration in an error message, because
  that's quite different to an ident + paren/brace/bracket.
  • Loading branch information
@nnethercote
nnethercote committed Nov 5, 2024
1 parent 5fb9b2d commit 7b1887d
Show file tree
Hide file tree
Showing 5 changed files with 109 additions and 14 deletions.
38 changes: 36 additions & 2 deletions compiler/rustc_ast/src/token.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -598,10 +598,11 @@ impl Token {
/// **NB**: Take care when modifying this function, since it will change
/// the stable set of tokens that are allowed to match an expr nonterminal.
pub fn can_begin_expr(&self) -> bool {
use Delimiter::*;
match self.uninterpolate().kind {
Ident(name, is_raw) =>
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
OpenDelim(..) | // tuple, array or block
OpenDelim(Parenthesis | Brace | Bracket) | // tuple, array or block
Literal(..) | // literal
Not | // operator not
BinOp(Minus) | // unary minus
Expand All @@ -612,7 +613,7 @@ impl Token {
// DotDotDot is no longer supported, but we need some way to display the error
DotDot | DotDotDot | DotDotEq | // range notation
Lt | BinOp(Shl) | // associated path
PathSep | // global path
PathSep | // global path
Lifetime(..) | // labeled loop
Pound => true, // expression attributes
Interpolated(ref nt) =>
Expand All @@ -622,6 +623,12 @@ impl Token {
NtLiteral(..) |
NtPath(..)
),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Block |
MetaVarKind::Expr { .. } |
MetaVarKind::Literal |
MetaVarKind::Path
))) => true,
_ => false,
}
}
Expand Down Expand Up @@ -655,6 +662,14 @@ impl Token {
| NtPath(..)
| NtTy(..)
),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Expr { .. } |
MetaVarKind::Literal |
MetaVarKind::Meta |
MetaVarKind::Pat(_) |
MetaVarKind::Path |
MetaVarKind::Ty
))) => true,
_ => false,
}
}
Expand All @@ -675,6 +690,10 @@ impl Token {
Lt | BinOp(Shl) | // associated path
PathSep => true, // global path
Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Ty |
MetaVarKind::Path
))) => true,
// For anonymous structs or unions, which only appear in specific positions
// (type of struct fields or union fields), we don't consider them as regular types
_ => false,
Expand All @@ -687,6 +706,9 @@ impl Token {
OpenDelim(Delimiter::Brace) | Literal(..) | BinOp(Minus) => true,
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
))) => true,
_ => false,
}
}
Expand Down Expand Up @@ -743,6 +765,13 @@ impl Token {
},
_ => false,
},
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
MetaVarKind::Literal => true,
MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => {
can_begin_literal_maybe_minus
}
_ => false,
},
_ => false,
}
}
Expand All @@ -758,6 +787,11 @@ impl Token {
},
_ => false,
},
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
MetaVarKind::Literal => true,
MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal,
_ => false,
},
_ => false,
}
}
Expand Down
6 changes: 4 additions & 2 deletions compiler/rustc_expand/src/mbe/diagnostics.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use std::borrow::Cow;

use rustc_ast::token::{self, Token, TokenKind};
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::TokenStream;
use rustc_errors::{Applicability, Diag, DiagCtxtHandle, DiagMessage};
use rustc_macros::Subdiagnostic;
Expand Down Expand Up @@ -68,7 +68,9 @@ pub(super) fn failed_to_match_macro(

if let MatcherLoc::Token { token: expected_token } = &remaining_matcher
&& (matches!(expected_token.kind, TokenKind::Interpolated(_))
|| matches!(token.kind, TokenKind::Interpolated(_)))
|| matches!(token.kind, TokenKind::Interpolated(_))
|| matches!(expected_token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_)))
|| matches!(token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_))))
{
err.note("captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens");
err.note("see <https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment> for more information");
Expand Down
16 changes: 12 additions & 4 deletions compiler/rustc_parse/src/lexer/tokentrees.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,19 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> {
let mut buf = Vec::new();
loop {
match self.token.kind {
token::OpenDelim(delim) => buf.push(match self.lex_token_tree_open_delim(delim) {
Ok(val) => val,
Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)),
}),
token::OpenDelim(delim) => {
// Invisible delimiters cannot occur here because `TokenTreesReader` parses
// code directly from strings, with no macro expansion involved.
debug_assert!(!matches!(delim, Delimiter::Invisible(_)));
buf.push(match self.lex_token_tree_open_delim(delim) {
Ok(val) => val,
Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)),
})
}
token::CloseDelim(delim) => {
// Invisible delimiters cannot occur here because `TokenTreesReader` parses
// code directly from strings, with no macro expansion involved.
debug_assert!(!matches!(delim, Delimiter::Invisible(_)));
return (
open_spacing,
TokenStream::new(buf),
Expand Down
12 changes: 10 additions & 2 deletions compiler/rustc_parse/src/parser/expr.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3582,11 +3582,19 @@ impl<'a> Parser<'a> {
&& !self.token.is_reserved_ident()
&& self.look_ahead(1, |t| {
AssocOp::from_token(t).is_some()
|| matches!(t.kind, token::OpenDelim(_))
|| matches!(
t.kind,
token::OpenDelim(
Delimiter::Parenthesis
| Delimiter::Bracket
| Delimiter::Brace
)
)
|| *t == token::Dot
})
{
// Looks like they tried to write a shorthand, complex expression.
// Looks like they tried to write a shorthand, complex expression,
// E.g.: `n + m`, `f(a)`, `a[i]`, `S { x: 3 }`, or `x.y`.
e.span_suggestion_verbose(
self.token.span.shrink_to_lo(),
"try naming a field",
Expand Down
51 changes: 47 additions & 4 deletions compiler/rustc_parse/src/parser/nonterminal.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ use rustc_ast::ptr::P;
use rustc_ast::token::Nonterminal::*;
use rustc_ast::token::NtExprKind::*;
use rustc_ast::token::NtPatKind::*;
use rustc_ast::token::{self, Delimiter, NonterminalKind, Token};
use rustc_ast::token::{
self, Delimiter, InvisibleOrigin, MetaVarKind, Nonterminal, NonterminalKind, Token,
};
use rustc_ast_pretty::pprust;
use rustc_data_structures::sync::Lrc;
use rustc_errors::PResult;
Expand All @@ -22,7 +24,29 @@ impl<'a> Parser<'a> {
#[inline]
pub fn nonterminal_may_begin_with(kind: NonterminalKind, token: &Token) -> bool {
/// Checks whether the non-terminal may contain a single (non-keyword) identifier.
fn may_be_ident(nt: &token::Nonterminal) -> bool {
fn may_be_ident(kind: MetaVarKind) -> bool {
use MetaVarKind::*;
match kind {
Stmt
| Pat(_)
| Expr { .. }
| Ty
| Literal // `true`, `false`
| Meta
| Path => true,

Item
| Block
| Vis => false,

Ident
| Lifetime
| TT => unreachable!(),
}
}

/// Old variant of `may_be_ident`. Being phased out.
fn nt_may_be_ident(nt: &Nonterminal) -> bool {
match nt {
NtStmt(_)
| NtPat(_)
Expand Down Expand Up @@ -69,7 +93,8 @@ impl<'a> Parser<'a> {
| token::Ident(..)
| token::NtIdent(..)
| token::NtLifetime(..)
| token::Interpolated(_) => true,
| token::Interpolated(_)
| token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => true,
_ => token.can_begin_type(),
},
NonterminalKind::Block => match &token.kind {
Expand All @@ -79,11 +104,29 @@ impl<'a> Parser<'a> {
NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false,
},
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
MetaVarKind::Block
| MetaVarKind::Stmt
| MetaVarKind::Expr { .. }
| MetaVarKind::Literal => true,
MetaVarKind::Item
| MetaVarKind::Pat(_)
| MetaVarKind::Ty
| MetaVarKind::Meta
| MetaVarKind::Path
| MetaVarKind::Vis => false,
MetaVarKind::Lifetime | MetaVarKind::Ident | MetaVarKind::TT => {
unreachable!()
}
},
_ => false,
},
NonterminalKind::Path | NonterminalKind::Meta => match &token.kind {
token::PathSep | token::Ident(..) | token::NtIdent(..) => true,
token::Interpolated(nt) => may_be_ident(nt),
token::Interpolated(nt) => nt_may_be_ident(nt),
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
may_be_ident(*kind)
}
_ => false,
},
NonterminalKind::Pat(pat_kind) => token.can_begin_pattern(pat_kind),
Expand Down

0 comments on commit 7b1887d

Please sign in to comment.