make declarative macro expansion a part of query system

Cargo.lock

@@ -3809,6 +3809,7 @@ dependencies = [
  "rustc_lexer",
  "rustc_lint_defs",
  "rustc_macros",
+ "rustc_middle",
  "rustc_parse",
  "rustc_serialize",
  "rustc_session",

compiler/rustc_ast/src/ast.rs

@@ -2769,7 +2769,7 @@ impl UseTree {
 /// Distinguishes between `Attribute`s that decorate items and Attributes that
 /// are contained as statements within items. These two cases need to be
 /// distinguished for pretty-printing.
-#[derive(Clone, PartialEq, Encodable, Decodable, Debug, Copy, HashStable_Generic)]
+#[derive(Clone, PartialEq, Encodable, Decodable, Debug, Copy, HashStable_Generic, Hash)]
 pub enum AttrStyle {
     Outer,
     Inner,

compiler/rustc_ast/src/token.rs

@@ -17,8 +17,9 @@ use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::{edition::Edition, ErrorGuaranteed, Span, DUMMY_SP};
 use std::borrow::Cow;
 use std::fmt;
+use std::hash::Hash;
 
-#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic, Hash)]
 pub enum CommentKind {
     Line,
     Block,
@@ -63,7 +64,7 @@ pub enum Delimiter {
 // type. This means that float literals like `1f32` are classified by this type
 // as `Int`. Only upon conversion to `ast::LitKind` will such a literal be
 // given the `Float` kind.
-#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic, Hash)]
 pub enum LitKind {
     Bool, // AST only, must never appear in a `Token`
     Byte,
@@ -80,7 +81,7 @@ pub enum LitKind {
 }
 
 /// A literal token.
-#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic, Hash)]
 pub struct Lit {
     pub kind: LitKind,
     pub symbol: Symbol,
@@ -224,7 +225,7 @@ fn ident_can_begin_type(name: Symbol, span: Span, is_raw: IdentIsRaw) -> bool {
             .contains(&name)
 }
 
-#[derive(PartialEq, Encodable, Decodable, Debug, Copy, Clone, HashStable_Generic)]
+#[derive(PartialEq, Encodable, Decodable, Debug, Copy, Clone, HashStable_Generic, Hash)]
 pub enum IdentIsRaw {
     No,
     Yes,
@@ -869,7 +870,7 @@ pub enum Nonterminal {
     NtVis(P<ast::Visibility>),
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
+#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, Hash)]
 pub enum NonterminalKind {
     Item,
     Block,

compiler/rustc_ast/src/tokenstream.rs

@@ -26,6 +26,7 @@ use rustc_span::{sym, Span, SpanDecoder, SpanEncoder, Symbol, DUMMY_SP};
 use smallvec::{smallvec, SmallVec};
 
 use std::borrow::Cow;
+use std::hash::Hash;
 use std::{cmp, fmt, iter};
 
 /// Part of a `TokenStream`.
@@ -107,6 +108,14 @@ where
     }
 }
 
+/*impl Hash for TokenStream {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        for sub_tt in self.trees() {
+            sub_tt.hash(state);
+        }
+    }
+}*/
+
 pub trait ToAttrTokenStream: sync::DynSend + sync::DynSync {
     fn to_attr_token_stream(&self) -> AttrTokenStream;
 }
@@ -297,7 +306,7 @@ pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>);
 /// compound token. Used for conversions to `proc_macro::Spacing`. Also used to
 /// guide pretty-printing, which is where the `JointHidden` value (which isn't
 /// part of `proc_macro::Spacing`) comes in useful.
-#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
+#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic, Hash)]
 pub enum Spacing {
     /// The token cannot join with the following token to form a compound
     /// token.
@@ -733,7 +742,7 @@ impl TokenTreeCursor {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
+#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic, Hash)]
 pub struct DelimSpan {
     pub open: Span,
     pub close: Span,
@@ -757,7 +766,7 @@ impl DelimSpan {
     }
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
+#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic, Hash)]
 pub struct DelimSpacing {
     pub open: Spacing,
     pub close: Spacing,

compiler/rustc_expand/Cargo.toml

@@ -20,6 +20,7 @@ rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_lexer = { path = "../rustc_lexer" }
 rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_parse = { path = "../rustc_parse" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }

compiler/rustc_expand/src/base.rs

@@ -16,6 +16,7 @@ use rustc_errors::{DiagCtxt, ErrorGuaranteed, PResult};
 use rustc_feature::Features;
 use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT;
 use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiag, RegisteredTools};
+use rustc_middle::expand::{CanRetry, TcxMacroExpander};
 use rustc_parse::{parser, MACRO_ARGUMENTS};
 use rustc_session::config::CollapseMacroDebuginfo;
 use rustc_session::{parse::ParseSess, Limit, Session};
@@ -676,6 +677,11 @@ pub enum SyntaxExtensionKind {
         Box<dyn TTMacroExpander + sync::DynSync + sync::DynSend>,
     ),
 
+    TcxLegacyBang(
+        /// An expander with signature TokenStream -> AST.
+        Lrc<dyn TcxMacroExpander + sync::DynSync + sync::DynSend>,
+    ),
+
     /// A token-based attribute macro.
     Attr(
         /// An expander with signature (TokenStream, TokenStream) -> TokenStream.
@@ -749,7 +755,9 @@ impl SyntaxExtension {
     /// Returns which kind of macro calls this syntax extension.
     pub fn macro_kind(&self) -> MacroKind {
         match self.kind {
-            SyntaxExtensionKind::Bang(..) | SyntaxExtensionKind::LegacyBang(..) => MacroKind::Bang,
+            SyntaxExtensionKind::Bang(..)
+            | SyntaxExtensionKind::LegacyBang(..)
+            | SyntaxExtensionKind::TcxLegacyBang(..) => MacroKind::Bang,
             SyntaxExtensionKind::Attr(..)
             | SyntaxExtensionKind::LegacyAttr(..)
             | SyntaxExtensionKind::NonMacroAttr => MacroKind::Attr,
@@ -1031,6 +1039,13 @@ pub trait ResolverExpand {
 
     /// Tools registered with `#![register_tool]` and used by tool attributes and lints.
     fn registered_tools(&self) -> &RegisteredTools;
+
+    fn expand_legacy_bang(
+        &self,
+        invoc_id: LocalExpnId,
+        span: Span,
+        current_expansion: LocalExpnId,
+    ) -> Result<(TokenStream, usize), CanRetry>;
 }
 
 pub trait LintStoreExpand {

compiler/rustc_expand/src/expand.rs

@@ -14,7 +14,9 @@ use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::visit::{self, try_visit, walk_list, AssocCtxt, Visitor, VisitorResult};
-use rustc_ast::{AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, ExprKind};
+use rustc_ast::{
+    AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, ExprKind, DUMMY_NODE_ID,
+};
 use rustc_ast::{ForeignItemKind, HasAttrs, HasNodeId};
 use rustc_ast::{Inline, ItemKind, MacStmtStyle, MetaItemKind, ModKind};
 use rustc_ast::{NestedMetaItem, NodeId, PatKind, StmtKind, TyKind};
@@ -35,11 +37,15 @@ use rustc_span::hygiene::SyntaxContext;
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::{ErrorGuaranteed, FileName, LocalExpnId, Span};
 
+use crate::mbe::macro_rules::{trace_macros_note, ParserAnyMacro};
+use rustc_middle::expand::CanRetry;
+use rustc_middle::ty::TyCtxt;
 use smallvec::SmallVec;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::rc::Rc;
 use std::{iter, mem};
+use tracing::debug;
 
 macro_rules! ast_fragments {
     (
@@ -387,6 +393,18 @@ pub struct MacroExpander<'a, 'b> {
     monotonic: bool, // cf. `cx.monotonic_expander()`
 }
 
+pub fn expand_legacy_bang<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    key: (LocalExpnId, Span, LocalExpnId),
+) -> Result<(&'tcx TokenStream, usize), CanRetry> {
+    let (invoc_id, span, current_expansion) = key;
+    let map = tcx.macro_map.borrow();
+    let (arg, expander) = map.get(&invoc_id).as_ref().unwrap();
+    expander
+        .expand(&tcx.sess, span, arg.clone(), current_expansion)
+        .map(|(tts, i)| (tcx.arena.alloc(tts) as &TokenStream, i))
+}
+
 impl<'a, 'b> MacroExpander<'a, 'b> {
     pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self {
         MacroExpander { cx, monotonic }
@@ -672,6 +690,67 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                         Err(guar) => return ExpandResult::Ready(fragment_kind.dummy(span, guar)),
                     }
                 }
+                SyntaxExtensionKind::TcxLegacyBang(expander) => {
+                    // Macros defined in the current crate have a real node id,
+                    // whereas macros from an external crate have a dummy id.
+                    if self.cx.trace_macros() {
+                        let msg = format!(
+                            "expanding `{}! {{ {} }}`",
+                            expander.name(),
+                            pprust::tts_to_string(&mac.args.tokens)
+                        );
+                        trace_macros_note(&mut self.cx.expansions, span, msg);
+                    }
+
+                    // Macros defined in the current crate have a real node id,
+                    // whereas macros from an external crate have a dummy id.\
+                    let tok_result: Box<dyn MacResult> = match self.cx.resolver.expand_legacy_bang(
+                        invoc.expansion_data.id,
+                        span,
+                        self.cx.current_expansion.id,
+                    ) {
+                        Ok((tts, i)) => {
+                            if self.cx.trace_macros() {
+                                let msg = format!("to `{}`", pprust::tts_to_string(&tts));
+                                trace_macros_note(&mut self.cx.expansions, span, msg);
+                            }
+                            let is_local = expander.node_id() != DUMMY_NODE_ID;
+                            if is_local {
+                                self.cx.resolver.record_macro_rule_usage(expander.node_id(), i);
+                            }
+
+                            // Let the context choose how to interpret the result.
+                            // Weird, but useful for X-macros.
+                            Box::new(ParserAnyMacro::new(
+                                Parser::new(&self.cx.sess.psess, tts.clone(), None),
+                                // Pass along the original expansion site and the name of the macro,
+                                // so we can print a useful error message if the parse of the expanded
+                                // macro leaves unparsed tokens.
+                                span,
+                                expander.name(),
+                                self.cx.current_expansion.lint_node_id,
+                                self.cx.current_expansion.is_trailing_mac,
+                                expander.arm_span(i),
+                                is_local,
+                            ))
+                        }
+                        Err(CanRetry::No(guar)) => {
+                            debug!("Will not retry matching as an error was emitted already");
+                            DummyResult::any(span, guar)
+                        }
+                        Err(CanRetry::Yes) => {
+                            // Retry and emit a better error.
+                            DummyResult::any_valid(span)
+                        }
+                    };
+                    let result = if let Some(result) = fragment_kind.make_from(tok_result) {
+                        result
+                    } else {
+                        let guar = self.error_wrong_fragment_kind(fragment_kind, &mac, span);
+                        fragment_kind.dummy(span, guar)
+                    };
+                    result
+                }
                 SyntaxExtensionKind::LegacyBang(expander) => {
                     let tok_result = match expander.expand(self.cx, span, mac.args.tokens.clone()) {
                         ExpandResult::Ready(tok_result) => tok_result,

compiler/rustc_expand/src/lib.rs

@@ -26,12 +26,16 @@ mod placeholders;
 mod proc_macro_server;
 
 pub use mbe::macro_rules::compile_declarative_macro;
-pub mod base;
+use rustc_middle::query::Providers;pub mod base;
 pub mod config;
 pub mod expand;
 pub mod module;
 // FIXME(Nilstrieb) Translate proc_macro diagnostics
 #[allow(rustc::untranslatable_diagnostic)]
 pub mod proc_macro;
 
+pub fn provide(providers: &mut Providers) {
+    providers.expand_legacy_bang = expand::expand_legacy_bang;
+}
+
 rustc_fluent_macro::fluent_messages! { "../messages.ftl" }

compiler/rustc_expand/src/mbe/diagnostics.rs

@@ -18,7 +18,7 @@ use tracing::debug;
 
 use super::macro_rules::{parser_from_cx, NoopTracker};
 
-pub(super) fn failed_to_match_macro<'cx>(
+pub(crate) fn failed_to_match_macro<'cx>(
     cx: &'cx mut ExtCtxt<'_>,
     sp: Span,
     def_span: Span,