make declare macro a part of query system

does not compile after rebase (by @futile)

Co-authored-by: Felix Rath <[email protected]>

Cargo.lock

@@ -3689,6 +3689,7 @@ dependencies = [
  "rustc_lexer",
  "rustc_lint_defs",
  "rustc_macros",
+ "rustc_middle",
  "rustc_parse",
  "rustc_serialize",
  "rustc_session",

compiler/rustc_ast/src/ast.rs

@@ -2837,7 +2837,7 @@ impl UseTree {
 /// Distinguishes between `Attribute`s that decorate items and Attributes that
 /// are contained as statements within items. These two cases need to be
 /// distinguished for pretty-printing.
-#[derive(Clone, PartialEq, Encodable, Decodable, Debug, Copy, HashStable_Generic)]
+#[derive(Clone, PartialEq, Encodable, Decodable, Debug, Copy, HashStable_Generic, Hash)]
 pub enum AttrStyle {
     Outer,
     Inner,

compiler/rustc_ast/src/token.rs

@@ -1,5 +1,6 @@
 use std::borrow::Cow;
 use std::fmt;
+use std::hash::{Hash, Hasher};
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::Lrc;
@@ -21,7 +22,7 @@ use crate::ast;
 use crate::ptr::P;
 use crate::util::case::Case;
 
-#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic, Hash)]
 pub enum CommentKind {
     Line,
     Block,
@@ -66,7 +67,7 @@ pub enum Delimiter {
 // type. This means that float literals like `1f32` are classified by this type
 // as `Int`. Only upon conversion to `ast::LitKind` will such a literal be
 // given the `Float` kind.
-#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic, Hash)]
 pub enum LitKind {
     Bool, // AST only, must never appear in a `Token`
     Byte,
@@ -83,7 +84,7 @@ pub enum LitKind {
 }
 
 /// A literal token.
-#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic, Hash)]
 pub struct Lit {
     pub kind: LitKind,
     pub symbol: Symbol,
@@ -228,7 +229,7 @@ fn ident_can_begin_type(name: Symbol, span: Span, is_raw: IdentIsRaw) -> bool {
             .contains(&name)
 }
 
-#[derive(PartialEq, Encodable, Decodable, Debug, Copy, Clone, HashStable_Generic)]
+#[derive(PartialEq, Encodable, Decodable, Debug, Copy, Clone, HashStable_Generic, Hash)]
 pub enum IdentIsRaw {
     No,
     Yes,
@@ -248,7 +249,7 @@ impl From<IdentIsRaw> for bool {
 
 // SAFETY: due to the `Clone` impl below, all fields of all variants other than
 // `Interpolated` must impl `Copy`.
-#[derive(PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(PartialEq, Encodable, Decodable, Debug, HashStable_Generic, Hash)]
 pub enum TokenKind {
     /* Expression-operator symbols. */
     /// `=`
@@ -374,7 +375,7 @@ impl Clone for TokenKind {
     }
 }
 
-#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic, Hash)]
 pub struct Token {
     pub kind: TokenKind,
     pub span: Span,
@@ -910,7 +911,7 @@ pub enum Nonterminal {
     NtVis(P<ast::Visibility>),
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
+#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, Hash)]
 pub enum NonterminalKind {
     Item,
     Block,
@@ -1060,6 +1061,12 @@ where
     }
 }
 
+impl Hash for Nonterminal {
+    fn hash<H: Hasher>(&self, _state: &mut H) {
+        panic!("interpolated tokens should not be present in the HIR")
+    }
+}
+
 // Some types are used a lot. Make sure they don't unintentionally get bigger.
 #[cfg(target_pointer_width = "64")]
 mod size_asserts {

compiler/rustc_ast/src/tokenstream.rs

@@ -14,6 +14,7 @@
 //! ownership of the original.
 
 use std::borrow::Cow;
+use std::hash::{Hash, Hasher};
 use std::{cmp, fmt, iter};
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -28,7 +29,7 @@ use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind};
 use crate::{AttrVec, Attribute};
 
 /// Part of a `TokenStream`.
-#[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
+#[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic, Hash)]
 pub enum TokenTree {
     /// A single token. Should never be `OpenDelim` or `CloseDelim`, because
     /// delimiters are implicitly represented by `Delimited`.
@@ -106,6 +107,14 @@ where
     }
 }
 
+impl Hash for TokenStream {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        for sub_tt in self.trees() {
+            sub_tt.hash(state);
+        }
+    }
+}
+
 pub trait ToAttrTokenStream: sync::DynSend + sync::DynSync {
     fn to_attr_token_stream(&self) -> AttrTokenStream;
 }
@@ -300,7 +309,7 @@ pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>);
 /// compound token. Used for conversions to `proc_macro::Spacing`. Also used to
 /// guide pretty-printing, which is where the `JointHidden` value (which isn't
 /// part of `proc_macro::Spacing`) comes in useful.
-#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
+#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic, Hash)]
 pub enum Spacing {
     /// The token cannot join with the following token to form a compound
     /// token.
@@ -728,7 +737,7 @@ impl TokenTreeCursor {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
+#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic, Hash)]
 pub struct DelimSpan {
     pub open: Span,
     pub close: Span,
@@ -752,7 +761,7 @@ impl DelimSpan {
     }
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
+#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic, Hash)]
 pub struct DelimSpacing {
     pub open: Spacing,
     pub close: Spacing,

compiler/rustc_expand/Cargo.toml

@@ -20,6 +20,7 @@ rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_lexer = { path = "../rustc_lexer" }
 rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_parse = { path = "../rustc_parse" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }

compiler/rustc_expand/src/base.rs

@@ -15,6 +15,7 @@ use rustc_data_structures::sync::{self, Lrc};
 use rustc_errors::{DiagCtxtHandle, ErrorGuaranteed, PResult};
 use rustc_feature::Features;
 use rustc_lint_defs::{BufferedEarlyLint, RegisteredTools};
+use rustc_middle::expand::{CanRetry, TcxMacroExpander};
 use rustc_parse::parser::Parser;
 use rustc_parse::MACRO_ARGUMENTS;
 use rustc_session::config::CollapseMacroDebuginfo;
@@ -676,6 +677,11 @@ pub enum SyntaxExtensionKind {
         Box<dyn TTMacroExpander + sync::DynSync + sync::DynSend>,
     ),
 
+    TcxLegacyBang(
+        /// An expander with signature TokenStream -> AST.
+        Lrc<dyn TcxMacroExpander + sync::DynSync + sync::DynSend>,
+    ),
+
     /// A token-based attribute macro.
     Attr(
         /// An expander with signature (TokenStream, TokenStream) -> TokenStream.
@@ -754,7 +760,8 @@ impl SyntaxExtension {
         match self.kind {
             SyntaxExtensionKind::Bang(..)
             | SyntaxExtensionKind::LegacyBang(..)
-            | SyntaxExtensionKind::GlobDelegation(..) => MacroKind::Bang,
+            | SyntaxExtensionKind::GlobDelegation(..)
+            | SyntaxExtensionKind::TcxLegacyBang(..) => MacroKind::Bang,
             SyntaxExtensionKind::Attr(..)
             | SyntaxExtensionKind::LegacyAttr(..)
             | SyntaxExtensionKind::NonMacroAttr => MacroKind::Attr,
@@ -1072,6 +1079,13 @@ pub trait ResolverExpand {
         trait_def_id: DefId,
         impl_def_id: LocalDefId,
     ) -> Result<Vec<(Ident, Option<Ident>)>, Indeterminate>;
+
+    fn expand_legacy_bang(
+        &self,
+        invoc_id: LocalExpnId,
+        span: Span,
+        current_expansion: LocalExpnId,
+    ) -> Result<(TokenStream, usize), CanRetry>;
 }
 
 pub trait LintStoreExpand {

compiler/rustc_expand/src/expand.rs

@@ -12,13 +12,15 @@ use rustc_ast::visit::{self, try_visit, walk_list, AssocCtxt, Visitor, VisitorRe
 use rustc_ast::{
     AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, ExprKind, ForeignItemKind,
     HasAttrs, HasNodeId, Inline, ItemKind, MacStmtStyle, MetaItemKind, ModKind, NestedMetaItem,
-    NodeId, PatKind, StmtKind, TyKind,
+    NodeId, PatKind, StmtKind, TyKind, DUMMY_NODE_ID,
 };
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::PResult;
 use rustc_feature::Features;
+use rustc_middle::expand::CanRetry;
+use rustc_middle::ty::TyCtxt;
 use rustc_parse::parser::{
     AttemptLocalParseRecovery, CommaRecoveryMode, ForceCollect, Parser, RecoverColon, RecoverComma,
 };
@@ -31,6 +33,7 @@ use rustc_span::hygiene::SyntaxContext;
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::{ErrorGuaranteed, FileName, LocalExpnId, Span};
 use smallvec::SmallVec;
+use tracing::debug;
 
 use crate::base::*;
 use crate::config::StripUnconfigured;
@@ -40,6 +43,7 @@ use crate::errors::{
     WrongFragmentKind,
 };
 use crate::mbe::diagnostics::annotate_err_with_kind;
+use crate::mbe::macro_rules::{trace_macros_note, ParserAnyMacro};
 use crate::module::{mod_dir_path, parse_external_mod, DirOwnership, ParsedExternalMod};
 use crate::placeholders::{placeholder, PlaceholderExpander};
 
@@ -394,6 +398,18 @@ pub struct MacroExpander<'a, 'b> {
     monotonic: bool, // cf. `cx.monotonic_expander()`
 }
 
+pub fn expand_legacy_bang<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    key: (LocalExpnId, Span, LocalExpnId),
+) -> Result<(&'tcx TokenStream, usize), CanRetry> {
+    let (invoc_id, span, current_expansion) = key;
+    let map = tcx.macro_map.borrow();
+    let (arg, expander) = map.get(&invoc_id).as_ref().unwrap();
+    expander
+        .expand(&tcx.sess, span, arg.clone(), current_expansion)
+        .map(|(tts, i)| (tcx.arena.alloc(tts) as &TokenStream, i))
+}
+
 impl<'a, 'b> MacroExpander<'a, 'b> {
     pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self {
         MacroExpander { cx, monotonic }
@@ -679,6 +695,67 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                         Err(guar) => return ExpandResult::Ready(fragment_kind.dummy(span, guar)),
                     }
                 }
+                SyntaxExtensionKind::TcxLegacyBang(expander) => {
+                    // Macros defined in the current crate have a real node id,
+                    // whereas macros from an external crate have a dummy id.
+                    if self.cx.trace_macros() {
+                        let msg = format!(
+                            "expanding `{}! {{ {} }}`",
+                            expander.name(),
+                            pprust::tts_to_string(&mac.args.tokens)
+                        );
+                        trace_macros_note(&mut self.cx.expansions, span, msg);
+                    }
+
+                    // Macros defined in the current crate have a real node id,
+                    // whereas macros from an external crate have a dummy id.\
+                    let tok_result: Box<dyn MacResult> = match self.cx.resolver.expand_legacy_bang(
+                        invoc.expansion_data.id,
+                        span,
+                        self.cx.current_expansion.id,
+                    ) {
+                        Ok((tts, i)) => {
+                            if self.cx.trace_macros() {
+                                let msg = format!("to `{}`", pprust::tts_to_string(&tts));
+                                trace_macros_note(&mut self.cx.expansions, span, msg);
+                            }
+                            let is_local = expander.node_id() != DUMMY_NODE_ID;
+                            if is_local {
+                                self.cx.resolver.record_macro_rule_usage(expander.node_id(), i);
+                            }
+
+                            // Let the context choose how to interpret the result.
+                            // Weird, but useful for X-macros.
+                            Box::new(ParserAnyMacro::new(
+                                Parser::new(&self.cx.sess.psess, tts.clone(), None),
+                                // Pass along the original expansion site and the name of the macro,
+                                // so we can print a useful error message if the parse of the expanded
+                                // macro leaves unparsed tokens.
+                                span,
+                                expander.name(),
+                                self.cx.current_expansion.lint_node_id,
+                                self.cx.current_expansion.is_trailing_mac,
+                                expander.arm_span(i),
+                                is_local,
+                            ))
+                        }
+                        Err(CanRetry::No(guar)) => {
+                            debug!("Will not retry matching as an error was emitted already");
+                            DummyResult::any(span, guar)
+                        }
+                        Err(CanRetry::Yes) => {
+                            // Retry and emit a better error.
+                            DummyResult::any_valid(span)
+                        }
+                    };
+                    let result = if let Some(result) = fragment_kind.make_from(tok_result) {
+                        result
+                    } else {
+                        let guar = self.error_wrong_fragment_kind(fragment_kind, &mac, span);
+                        fragment_kind.dummy(span, guar)
+                    };
+                    result
+                }
                 SyntaxExtensionKind::LegacyBang(expander) => {
                     let tok_result = match expander.expand(self.cx, span, mac.args.tokens.clone()) {
                         ExpandResult::Ready(tok_result) => tok_result,

compiler/rustc_expand/src/lib.rs

@@ -26,6 +26,7 @@ mod placeholders;
 mod proc_macro_server;
 
 pub use mbe::macro_rules::compile_declarative_macro;
+use rustc_middle::query::Providers;
 pub mod base;
 pub mod config;
 pub mod expand;
@@ -34,4 +35,8 @@ pub mod module;
 #[allow(rustc::untranslatable_diagnostic)]
 pub mod proc_macro;
 
+pub fn provide(providers: &mut Providers) {
+    providers.expand_legacy_bang = expand::expand_legacy_bang;
+}
+
 rustc_fluent_macro::fluent_messages! { "../messages.ftl" }

compiler/rustc_expand/src/mbe/diagnostics.rs

@@ -18,7 +18,7 @@ use crate::mbe::macro_parser::ParseResult::*;
 use crate::mbe::macro_parser::{MatcherLoc, NamedParseResult, TtParser};
 use crate::mbe::macro_rules::{try_match_macro, Tracker};
 
-pub(super) fn failed_to_match_macro<'cx>(
+pub(crate) fn failed_to_match_macro<'cx>(
     cx: &'cx mut ExtCtxt<'_>,
     sp: Span,
     def_span: Span,