Auto merge of #80100 - mark-i-m:pattORns-2, r=petrochenkov

or_patterns: implement :pat edition-specific behavior

cc #54883 `@joshtriplett`

This PR implements the edition-specific behavior of `:pat` wrt or-patterns, as determined by the crater runs and T-lang consensus in #54883 (comment).

I believe this can unblock stabilization of or_patterns.

r? `@petrochenkov`

Author: bors

compiler/rustc_expand/src/mbe/macro_parser.rs

@@ -77,9 +77,9 @@ use TokenTreeOrTokenTreeSlice::*;
 use crate::mbe::{self, TokenTree};
 
 use rustc_ast::token::{self, DocComment, Nonterminal, Token};
-use rustc_parse::parser::Parser;
+use rustc_parse::parser::{OrPatNonterminalMode, Parser};
 use rustc_session::parse::ParseSess;
-use rustc_span::symbol::MacroRulesNormalizedIdent;
+use rustc_span::{edition::Edition, symbol::MacroRulesNormalizedIdent};
 
 use smallvec::{smallvec, SmallVec};
 
@@ -414,6 +414,18 @@ fn token_name_eq(t1: &Token, t2: &Token) -> bool {
     }
 }
 
+/// In edition 2015/18, `:pat` can only match `pat<no_top_alt>` because otherwise, we have
+/// breakage. As of edition 2021, `:pat` matches `top_pat`.
+///
+/// See <https://github.com/rust-lang/rust/issues/54883> for more info.
+fn or_pat_mode(edition: Edition) -> OrPatNonterminalMode {
+    match edition {
+        Edition::Edition2015 | Edition::Edition2018 => OrPatNonterminalMode::NoTopAlt,
+        // FIXME(mark-i-m): uncomment this when edition 2021 machinery is added.
+        // Edition::Edition2021 =>  OrPatNonterminalMode::TopPat,
+    }
+}
+
 /// Process the matcher positions of `cur_items` until it is empty. In the process, this will
 /// produce more items in `next_items`, `eof_items`, and `bb_items`.
 ///
@@ -553,10 +565,14 @@ fn inner_parse_loop<'root, 'tt>(
 
                 // We need to match a metavar with a valid ident... call out to the black-box
                 // parser by adding an item to `bb_items`.
-                TokenTree::MetaVarDecl(_, _, kind) => {
-                    // Built-in nonterminals never start with these tokens,
-                    // so we can eliminate them from consideration.
-                    if Parser::nonterminal_may_begin_with(kind, token) {
+                TokenTree::MetaVarDecl(span, _, kind) => {
+                    // Built-in nonterminals never start with these tokens, so we can eliminate
+                    // them from consideration.
+                    //
+                    // We use the span of the metavariable declaration to determine any
+                    // edition-specific matching behavior for non-terminals.
+                    if Parser::nonterminal_may_begin_with(kind, token, or_pat_mode(span.edition()))
+                    {
                         bb_items.push(item);
                     }
                 }
@@ -717,7 +733,10 @@ pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> Na
             let mut item = bb_items.pop().unwrap();
             if let TokenTree::MetaVarDecl(span, _, kind) = item.top_elts.get_tt(item.idx) {
                 let match_cur = item.match_cur;
-                let nt = match parser.to_mut().parse_nonterminal(kind) {
+                // We use the span of the metavariable declaration to determine any
+                // edition-specific matching behavior for non-terminals.
+                let nt = match parser.to_mut().parse_nonterminal(kind, or_pat_mode(span.edition()))
+                {
                     Err(mut err) => {
                         err.span_label(
                             span,

compiler/rustc_parse/src/parser/expr.rs

@@ -1,4 +1,4 @@
-use super::pat::{GateOr, PARAM_EXPECTED};
+use super::pat::{GateOr, RecoverComma, PARAM_EXPECTED};
 use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
 use super::{BlockMode, Parser, PathStyle, Restrictions, TokenType};
 use super::{SemiColonMode, SeqSep, TokenExpectType};
@@ -1729,7 +1729,7 @@ impl<'a> Parser<'a> {
     /// The `let` token has already been eaten.
     fn parse_let_expr(&mut self, attrs: AttrVec) -> PResult<'a, P<Expr>> {
         let lo = self.prev_token.span;
-        let pat = self.parse_top_pat(GateOr::No)?;
+        let pat = self.parse_top_pat(GateOr::No, RecoverComma::Yes)?;
         self.expect(&token::Eq)?;
         let expr = self.with_res(self.restrictions | Restrictions::NO_STRUCT_LITERAL, |this| {
             this.parse_assoc_expr_with(1 + prec_let_scrutinee_needs_par(), None.into())
@@ -1792,7 +1792,7 @@ impl<'a> Parser<'a> {
             _ => None,
         };
 
-        let pat = self.parse_top_pat(GateOr::Yes)?;
+        let pat = self.parse_top_pat(GateOr::Yes, RecoverComma::Yes)?;
         if !self.eat_keyword(kw::In) {
             self.error_missing_in_for_loop();
         }
@@ -1902,7 +1902,7 @@ impl<'a> Parser<'a> {
     pub(super) fn parse_arm(&mut self) -> PResult<'a, Arm> {
         let attrs = self.parse_outer_attributes()?;
         let lo = self.token.span;
-        let pat = self.parse_top_pat(GateOr::No)?;
+        let pat = self.parse_top_pat(GateOr::No, RecoverComma::Yes)?;
         let guard = if self.eat_keyword(kw::If) {
             let if_span = self.prev_token.span;
             let cond = self.parse_expr()?;

compiler/rustc_parse/src/parser/mod.rs

@@ -12,6 +12,7 @@ mod ty;
 use crate::lexer::UnmatchedBrace;
 pub use diagnostics::AttemptLocalParseRecovery;
 use diagnostics::Error;
+pub use pat::OrPatNonterminalMode;
 pub use path::PathStyle;
 
 use rustc_ast::ptr::P;

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -4,14 +4,19 @@ use rustc_ast_pretty::pprust;
 use rustc_errors::PResult;
 use rustc_span::symbol::{kw, Ident};
 
+use crate::parser::pat::{GateOr, OrPatNonterminalMode, RecoverComma};
 use crate::parser::{FollowedByType, Parser, PathStyle};
 
 impl<'a> Parser<'a> {
     /// Checks whether a non-terminal may begin with a particular token.
     ///
     /// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that
     /// token. Be conservative (return true) if not sure.
-    pub fn nonterminal_may_begin_with(kind: NonterminalKind, token: &Token) -> bool {
+    pub fn nonterminal_may_begin_with(
+        kind: NonterminalKind,
+        token: &Token,
+        or_pat_mode: OrPatNonterminalMode,
+    ) -> bool {
         /// Checks whether the non-terminal may contain a single (non-keyword) identifier.
         fn may_be_ident(nt: &token::Nonterminal) -> bool {
             match *nt {
@@ -70,6 +75,8 @@ impl<'a> Parser<'a> {
                 token::ModSep |                     // path
                 token::Lt |                         // path (UFCS constant)
                 token::BinOp(token::Shl) => true,   // path (double UFCS)
+                // leading vert `|` or-pattern
+                token::BinOp(token::Or) =>  matches!(or_pat_mode, OrPatNonterminalMode::TopPat),
                 token::Interpolated(ref nt) => may_be_ident(nt),
                 _ => false,
             },
@@ -86,7 +93,12 @@ impl<'a> Parser<'a> {
         }
     }
 
-    pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, Nonterminal> {
+    /// Parse a non-terminal (e.g. MBE `:pat` or `:ident`).
+    pub fn parse_nonterminal(
+        &mut self,
+        kind: NonterminalKind,
+        or_pat_mode: OrPatNonterminalMode,
+    ) -> PResult<'a, Nonterminal> {
         // Any `Nonterminal` which stores its tokens (currently `NtItem` and `NtExpr`)
         // needs to have them force-captured here.
         // A `macro_rules!` invocation may pass a captured item/expr to a proc-macro,
@@ -130,7 +142,12 @@ impl<'a> Parser<'a> {
                 }
             }
             NonterminalKind::Pat => {
-                let (mut pat, tokens) = self.collect_tokens(|this| this.parse_pat(None))?;
+                let (mut pat, tokens) = self.collect_tokens(|this| match or_pat_mode {
+                    OrPatNonterminalMode::TopPat => {
+                        this.parse_top_pat(GateOr::Yes, RecoverComma::No)
+                    }
+                    OrPatNonterminalMode::NoTopAlt => this.parse_pat(None),
+                })?;
                 // We have have eaten an NtPat, which could already have tokens
                 if pat.tokens.is_none() {
                     pat.tokens = tokens;

compiler/rustc_parse/src/parser/pat.rs

@@ -26,11 +26,18 @@ pub(super) enum GateOr {
 
 /// Whether or not to recover a `,` when parsing or-patterns.
 #[derive(PartialEq, Copy, Clone)]
-enum RecoverComma {
+pub(super) enum RecoverComma {
     Yes,
     No,
 }
 
+/// Used when parsing a non-terminal (see `parse_nonterminal`) to determine if `:pat` should match
+/// `top_pat` or `pat<no_top_alt>`. See issue <https://github.com/rust-lang/rust/pull/78935>.
+pub enum OrPatNonterminalMode {
+    TopPat,
+    NoTopAlt,
+}
+
 impl<'a> Parser<'a> {
     /// Parses a pattern.
     ///
@@ -43,13 +50,17 @@ impl<'a> Parser<'a> {
 
     /// Entry point to the main pattern parser.
     /// Corresponds to `top_pat` in RFC 2535 and allows or-pattern at the top level.
-    pub(super) fn parse_top_pat(&mut self, gate_or: GateOr) -> PResult<'a, P<Pat>> {
+    pub(super) fn parse_top_pat(
+        &mut self,
+        gate_or: GateOr,
+        rc: RecoverComma,
+    ) -> PResult<'a, P<Pat>> {
         // Allow a '|' before the pats (RFCs 1925, 2530, and 2535).
         let gated_leading_vert = self.eat_or_separator(None) && gate_or == GateOr::Yes;
         let leading_vert_span = self.prev_token.span;
 
         // Parse the possibly-or-pattern.
-        let pat = self.parse_pat_with_or(None, gate_or, RecoverComma::Yes)?;
+        let pat = self.parse_pat_with_or(None, gate_or, rc)?;
 
         // If we parsed a leading `|` which should be gated,
         // and no other gated or-pattern has been parsed thus far,

compiler/rustc_parse/src/parser/stmt.rs

@@ -1,7 +1,7 @@
 use super::attr::DEFAULT_INNER_ATTR_FORBIDDEN;
 use super::diagnostics::{AttemptLocalParseRecovery, Error};
 use super::expr::LhsExpr;
-use super::pat::GateOr;
+use super::pat::{GateOr, RecoverComma};
 use super::path::PathStyle;
 use super::{BlockMode, Parser, Restrictions, SemiColonMode};
 use crate::maybe_whole;
@@ -185,7 +185,7 @@ impl<'a> Parser<'a> {
     /// Parses a local variable declaration.
     fn parse_local(&mut self, attrs: AttrVec) -> PResult<'a, P<Local>> {
         let lo = self.prev_token.span;
-        let pat = self.parse_top_pat(GateOr::Yes)?;
+        let pat = self.parse_top_pat(GateOr::Yes, RecoverComma::Yes)?;
 
         let (err, ty) = if self.eat(&token::Colon) {
             // Save the state of the parser before parsing type normally, in case there is a `:`

src/test/ui/macros/macro-pat-follow-2018.rs

@@ -0,0 +1,15 @@
+// run-pass
+// edition:2018
+
+macro_rules! pat_bar {
+    ($p:pat | $p2:pat) => {{
+        match Some(1u8) {
+            $p | $p2 => {}
+            _ => {}
+        }
+    }};
+}
+
+fn main() {
+    pat_bar!(Some(1u8) | None);
+}

src/test/ui/macros/macro-pat-follow.rs

@@ -3,29 +3,19 @@ macro_rules! pat_in {
     ($p:pat in $e:expr) => {{
         let mut iter = $e.into_iter();
         while let $p = iter.next() {}
-    }}
+    }};
 }
 
 macro_rules! pat_if {
     ($p:pat if $e:expr) => {{
         match Some(1u8) {
-            $p if $e => {},
+            $p if $e => {}
             _ => {}
         }
-    }}
-}
-
-macro_rules! pat_bar {
-    ($p:pat | $p2:pat) => {{
-        match Some(1u8) {
-            $p | $p2 => {},
-            _ => {}
-        }
-    }}
+    }};
 }
 
 fn main() {
     pat_in!(Some(_) in 0..10);
     pat_if!(Some(x) if x > 0);
-    pat_bar!(Some(1u8) | None);
 }

src/test/ui/or-patterns/or-patterns-syntactic-fail-2018.rs

@@ -0,0 +1,15 @@
+// Test that :pat doesn't accept top-level or-patterns in edition 2018.
+
+// edition:2018
+
+#![feature(or_patterns)]
+
+fn main() {}
+
+// Test the `pat` macro fragment parser:
+macro_rules! accept_pat {
+    ($p:pat) => {};
+}
+
+accept_pat!(p | q); //~ ERROR no rules expected the token `|`
+accept_pat!(|p| q); //~ ERROR no rules expected the token `|`

src/test/ui/or-patterns/or-patterns-syntactic-fail-2018.stderr

@@ -0,0 +1,20 @@
+error: no rules expected the token `|`
+  --> $DIR/or-patterns-syntactic-fail-2018.rs:14:15
+   |
+LL | macro_rules! accept_pat {
+   | ----------------------- when calling this macro
+...
+LL | accept_pat!(p | q);
+   |               ^ no rules expected this token in macro call
+
+error: no rules expected the token `|`
+  --> $DIR/or-patterns-syntactic-fail-2018.rs:15:13
+   |
+LL | macro_rules! accept_pat {
+   | ----------------------- when calling this macro
+...
+LL | accept_pat!(|p| q);
+   |             ^ no rules expected this token in macro call
+
+error: aborting due to 2 previous errors
+

src/test/ui/or-patterns/or-patterns-syntactic-fail.rs

@@ -5,16 +5,6 @@
 
 fn main() {}
 
-// Test the `pat` macro fragment parser:
-macro_rules! accept_pat {
-    ($p:pat) => {}
-}
-
-accept_pat!(p | q); //~ ERROR no rules expected the token `|`
-accept_pat!(| p | q); //~ ERROR no rules expected the token `|`
-
-// Non-macro tests:
-
 enum E { A, B }
 use E::*;