Auto merge of #37542 - jseyfried:custom_derive_reexports, r=nrc

Support `#[macro_reexport]`ing custom derives

This is gated behind `#![feature(macro_reexport)]` and `#![feature(proc_macro)]`.
r? @nrc

src/librustc/middle/cstore.rs

@@ -34,6 +34,7 @@ use session::Session;
 use session::search_paths::PathKind;
 use util::nodemap::{NodeSet, DefIdMap};
 use std::path::PathBuf;
+use std::rc::Rc;
 use syntax::ast;
 use syntax::attr;
 use syntax::ext::base::SyntaxExtension;
@@ -106,6 +107,11 @@ pub enum InlinedItemRef<'a> {
     ImplItem(DefId, &'a hir::ImplItem)
 }
 
+pub enum LoadedMacro {
+    MacroRules(ast::MacroDef),
+    ProcMacro(Rc<SyntaxExtension>),
+}
+
 #[derive(Copy, Clone, Debug)]
 pub struct ExternCrate {
     /// def_id of an `extern crate` in the current crate that caused
@@ -211,7 +217,7 @@ pub trait CrateStore<'tcx> {
     fn relative_def_path(&self, def: DefId) -> Option<hir_map::DefPath>;
     fn struct_field_names(&self, def: DefId) -> Vec<ast::Name>;
     fn item_children(&self, did: DefId) -> Vec<def::Export>;
-    fn load_macro(&self, did: DefId, sess: &Session) -> ast::MacroDef;
+    fn load_macro(&self, did: DefId, sess: &Session) -> LoadedMacro;
 
     // misc. metadata
     fn maybe_get_item_ast<'a>(&'tcx self, tcx: TyCtxt<'a, 'tcx, 'tcx>, def: DefId)
@@ -383,7 +389,7 @@ impl<'tcx> CrateStore<'tcx> for DummyCrateStore {
     }
     fn struct_field_names(&self, def: DefId) -> Vec<ast::Name> { bug!("struct_field_names") }
     fn item_children(&self, did: DefId) -> Vec<def::Export> { bug!("item_children") }
-    fn load_macro(&self, did: DefId, sess: &Session) -> ast::MacroDef { bug!("load_macro") }
+    fn load_macro(&self, did: DefId, sess: &Session) -> LoadedMacro { bug!("load_macro") }
 
     // misc. metadata
     fn maybe_get_item_ast<'a>(&'tcx self, tcx: TyCtxt<'a, 'tcx, 'tcx>, def: DefId)
@@ -424,7 +430,6 @@ impl<'tcx> CrateStore<'tcx> for DummyCrateStore {
 }
 
 pub trait CrateLoader {
-    fn process_item(&mut self, item: &ast::Item, defs: &Definitions, load_macros: bool)
-                    -> Vec<(ast::Name, SyntaxExtension)>;
+    fn process_item(&mut self, item: &ast::Item, defs: &Definitions);
     fn postprocess(&mut self, krate: &ast::Crate);
 }

src/librustc_metadata/creader.rs

@@ -35,7 +35,6 @@ use syntax::ast;
 use syntax::abi::Abi;
 use syntax::attr;
 use syntax::ext::base::SyntaxExtension;
-use syntax::feature_gate::{self, emit_feature_err};
 use syntax::parse::token::{InternedString, intern};
 use syntax_pos::{Span, DUMMY_SP};
 use log;
@@ -285,15 +284,13 @@ impl<'a> CrateLoader<'a> {
 
         let cnum_map = self.resolve_crate_deps(root, &crate_root, &metadata, cnum, span, dep_kind);
 
-        if crate_root.macro_derive_registrar.is_some() {
-            self.sess.span_err(span, "crates of the `proc-macro` crate type \
-                                      cannot be linked at runtime");
-        }
-
         let cmeta = Rc::new(cstore::CrateMetadata {
             name: name.to_string(),
             extern_crate: Cell::new(None),
             key_map: metadata.load_key_map(crate_root.index),
+            proc_macros: crate_root.macro_derive_registrar.map(|_| {
+                self.load_derive_macros(&crate_root, dylib.clone().map(|p| p.0), span)
+            }),
             root: crate_root,
             blob: metadata,
             cnum_map: RefCell::new(cnum_map),
@@ -317,34 +314,48 @@ impl<'a> CrateLoader<'a> {
                      hash: Option<&Svh>,
                      span: Span,
                      kind: PathKind,
-                     dep_kind: DepKind)
+                     mut dep_kind: DepKind)
                      -> (CrateNum, Rc<cstore::CrateMetadata>) {
         info!("resolving crate `extern crate {} as {}`", name, ident);
-        let result = match self.existing_match(name, hash, kind) {
-            Some(cnum) => LoadResult::Previous(cnum),
-            None => {
-                info!("falling back to a load");
-                let mut locate_ctxt = locator::Context {
-                    sess: self.sess,
-                    span: span,
-                    ident: ident,
-                    crate_name: name,
-                    hash: hash.map(|a| &*a),
-                    filesearch: self.sess.target_filesearch(kind),
-                    target: &self.sess.target.target,
-                    triple: &self.sess.opts.target_triple,
-                    root: root,
+        let result = if let Some(cnum) = self.existing_match(name, hash, kind) {
+            LoadResult::Previous(cnum)
+        } else {
+            info!("falling back to a load");
+            let mut locate_ctxt = locator::Context {
+                sess: self.sess,
+                span: span,
+                ident: ident,
+                crate_name: name,
+                hash: hash.map(|a| &*a),
+                filesearch: self.sess.target_filesearch(kind),
+                target: &self.sess.target.target,
+                triple: &self.sess.opts.target_triple,
+                root: root,
+                rejected_via_hash: vec![],
+                rejected_via_triple: vec![],
+                rejected_via_kind: vec![],
+                rejected_via_version: vec![],
+                should_match_name: true,
+                is_proc_macro: Some(false),
+            };
+
+            self.load(&mut locate_ctxt).or_else(|| {
+                dep_kind = DepKind::MacrosOnly;
+
+                let mut proc_macro_locator = locator::Context {
+                    target: &self.sess.host,
+                    triple: config::host_triple(),
+                    filesearch: self.sess.host_filesearch(PathKind::Crate),
                     rejected_via_hash: vec![],
                     rejected_via_triple: vec![],
                     rejected_via_kind: vec![],
                     rejected_via_version: vec![],
-                    should_match_name: true,
+                    is_proc_macro: Some(true),
+                    ..locate_ctxt
                 };
-                match self.load(&mut locate_ctxt) {
-                    Some(result) => result,
-                    None => locate_ctxt.report_errs(),
-                }
-            }
+
+                self.load(&mut proc_macro_locator)
+            }).unwrap_or_else(|| locate_ctxt.report_errs())
         };
 
         match result {
@@ -431,6 +442,10 @@ impl<'a> CrateLoader<'a> {
                           dep_kind: DepKind)
                           -> cstore::CrateNumMap {
         debug!("resolving deps of external crate");
+        if crate_root.macro_derive_registrar.is_some() {
+            return cstore::CrateNumMap::new();
+        }
+
         // The map from crate numbers in the crate we're resolving to local crate
         // numbers
         let deps = crate_root.crate_deps.decode(metadata);
@@ -479,6 +494,7 @@ impl<'a> CrateLoader<'a> {
             rejected_via_kind: vec![],
             rejected_via_version: vec![],
             should_match_name: true,
+            is_proc_macro: None,
         };
         let library = self.load(&mut locate_ctxt).or_else(|| {
             if !is_cross {
@@ -525,51 +541,36 @@ impl<'a> CrateLoader<'a> {
     /// implemented as dynamic libraries, but we have a possible future where
     /// custom derive (and other macro-1.1 style features) are implemented via
     /// executables and custom IPC.
-    fn load_derive_macros(&mut self, item: &ast::Item, ekrate: &ExtensionCrate)
-                          -> Option<Vec<(ast::Name, SyntaxExtension)>> {
+    fn load_derive_macros(&mut self, root: &CrateRoot, dylib: Option<PathBuf>, span: Span)
+                          -> Vec<(ast::Name, Rc<SyntaxExtension>)> {
         use std::{env, mem};
         use proc_macro::TokenStream;
         use proc_macro::__internal::Registry;
         use rustc_back::dynamic_lib::DynamicLibrary;
         use syntax_ext::deriving::custom::CustomDerive;
 
-        let root = ekrate.metadata.get_root();
-        let index = match root.macro_derive_registrar {
-            Some(index) => index,
-            None => return None,
-        };
-        if !self.sess.features.borrow().proc_macro {
-            let issue = feature_gate::GateIssue::Language;
-            let msg = "loading custom derive macro crates is experimentally supported";
-            emit_feature_err(&self.sess.parse_sess, "proc_macro", item.span, issue, msg);
-        }
-
-        if ekrate.target_only {
-            let msg = format!("proc-macro crate is not available for triple `{}` (only found {})",
-                               config::host_triple(), self.sess.opts.target_triple);
-            self.sess.span_fatal(item.span, &msg);
-        }
-        let path = match ekrate.dylib.clone() {
+        let path = match dylib {
             Some(dylib) => dylib,
-            None => span_bug!(item.span, "proc-macro crate not dylib"),
+            None => span_bug!(span, "proc-macro crate not dylib"),
         };
         // Make sure the path contains a / or the linker will search for it.
         let path = env::current_dir().unwrap().join(path);
         let lib = match DynamicLibrary::open(Some(&path)) {
             Ok(lib) => lib,
-            Err(err) => self.sess.span_fatal(item.span, &err),
+            Err(err) => self.sess.span_fatal(span, &err),
         };
 
-        let sym = self.sess.generate_derive_registrar_symbol(&root.hash, index);
+        let sym = self.sess.generate_derive_registrar_symbol(&root.hash,
+                                                             root.macro_derive_registrar.unwrap());
         let registrar = unsafe {
             let sym = match lib.symbol(&sym) {
                 Ok(f) => f,
-                Err(err) => self.sess.span_fatal(item.span, &err),
+                Err(err) => self.sess.span_fatal(span, &err),
             };
             mem::transmute::<*mut u8, fn(&mut Registry)>(sym)
         };
 
-        struct MyRegistrar(Vec<(ast::Name, SyntaxExtension)>);
+        struct MyRegistrar(Vec<(ast::Name, Rc<SyntaxExtension>)>);
 
         impl Registry for MyRegistrar {
             fn register_custom_derive(&mut self,
@@ -580,7 +581,7 @@ impl<'a> CrateLoader<'a> {
                 let derive = SyntaxExtension::CustomDerive(
                     Box::new(CustomDerive::new(expand, attrs))
                 );
-                self.0.push((intern(trait_name), derive));
+                self.0.push((intern(trait_name), Rc::new(derive)));
             }
         }
 
@@ -590,7 +591,7 @@ impl<'a> CrateLoader<'a> {
         // Intentionally leak the dynamic library. We can't ever unload it
         // since the library can make things that will live arbitrarily long.
         mem::forget(lib);
-        Some(my_registrar.0)
+        my_registrar.0
     }
 
     /// Look for a plugin registrar. Returns library path, crate
@@ -928,35 +929,14 @@ impl<'a> middle::cstore::CrateLoader for CrateLoader<'a> {
         self.register_statically_included_foreign_items();
     }
 
-    fn process_item(&mut self, item: &ast::Item, definitions: &Definitions, load_macros: bool)
-                    -> Vec<(ast::Name, SyntaxExtension)> {
+    fn process_item(&mut self, item: &ast::Item, definitions: &Definitions) {
         match item.node {
             ast::ItemKind::ExternCrate(_) => {}
-            ast::ItemKind::ForeignMod(ref fm) => {
-                self.process_foreign_mod(item, fm);
-                return Vec::new();
-            }
-            _ => return Vec::new(),
+            ast::ItemKind::ForeignMod(ref fm) => return self.process_foreign_mod(item, fm),
+            _ => return,
         }
 
         let info = self.extract_crate_info(item).unwrap();
-        if load_macros {
-            let ekrate = self.read_extension_crate(item.span, &info);
-
-            // If this is a proc-macro crate, return here to avoid registering.
-            if let Some(custom_derives) = self.load_derive_macros(item, &ekrate) {
-                return custom_derives;
-            }
-
-            // Register crate now to avoid double-reading metadata
-            if let PMDSource::Owned(lib) = ekrate.metadata {
-                if ekrate.target_only || config::host_triple() == self.sess.opts.target_triple {
-                    let ExternCrateInfo { ref ident, ref name, dep_kind, .. } = info;
-                    self.register_crate(&None, ident, name, item.span, lib, dep_kind);
-                }
-            }
-        }
-
         let (cnum, ..) = self.resolve_crate(
             &None, &info.ident, &info.name, None, item.span, PathKind::Crate, info.dep_kind,
         );
@@ -968,7 +948,5 @@ impl<'a> middle::cstore::CrateLoader for CrateLoader<'a> {
             ExternCrate { def_id: def_id, span: item.span, direct: true, path_len: len };
         self.update_extern_crate(cnum, extern_crate, &mut FxHashSet());
         self.cstore.add_extern_mod_stmt_cnum(info.id, cnum);
-
-        Vec::new()
     }
 }

src/librustc_metadata/cstore.rs

@@ -28,6 +28,7 @@ use std::rc::Rc;
 use std::path::PathBuf;
 use flate::Bytes;
 use syntax::{ast, attr};
+use syntax::ext::base::SyntaxExtension;
 use syntax_pos;
 
 pub use rustc::middle::cstore::{NativeLibraryKind, LinkagePreference};
@@ -80,6 +81,8 @@ pub struct CrateMetadata {
 
     pub dep_kind: Cell<DepKind>,
     pub source: CrateSource,
+
+    pub proc_macros: Option<Vec<(ast::Name, Rc<SyntaxExtension>)>>,
 }
 
 pub struct CachedInlinedItem {

src/librustc_metadata/cstore_impl.rs

@@ -14,7 +14,7 @@ use locator;
 use schema;
 
 use rustc::middle::cstore::{InlinedItem, CrateStore, CrateSource, DepKind, ExternCrate};
-use rustc::middle::cstore::{NativeLibraryKind, LinkMeta, LinkagePreference};
+use rustc::middle::cstore::{NativeLibraryKind, LinkMeta, LinkagePreference, LoadedMacro};
 use rustc::hir::def::{self, Def};
 use rustc::middle::lang_items;
 use rustc::session::Session;
@@ -353,8 +353,13 @@ impl<'tcx> CrateStore<'tcx> for cstore::CStore {
         result
     }
 
-    fn load_macro(&self, id: DefId, sess: &Session) -> ast::MacroDef {
-        let (name, def) = self.get_crate_data(id.krate).get_macro(id.index);
+    fn load_macro(&self, id: DefId, sess: &Session) -> LoadedMacro {
+        let data = self.get_crate_data(id.krate);
+        if let Some(ref proc_macros) = data.proc_macros {
+            return LoadedMacro::ProcMacro(proc_macros[id.index.as_usize()].1.clone());
+        }
+
+        let (name, def) = data.get_macro(id.index);
         let source_name = format!("<{} macros>", name);
 
         // NB: Don't use parse_tts_from_source_str because it parses with quote_depth > 0.
@@ -379,15 +384,15 @@ impl<'tcx> CrateStore<'tcx> for cstore::CStore {
         sess.imported_macro_spans.borrow_mut()
             .insert(local_span, (def.name.as_str().to_string(), def.span));
 
-        ast::MacroDef {
+        LoadedMacro::MacroRules(ast::MacroDef {
             ident: ast::Ident::with_empty_ctxt(def.name),
             id: ast::DUMMY_NODE_ID,
             span: local_span,
             imported_from: None, // FIXME
             allow_internal_unstable: attr::contains_name(&def.attrs, "allow_internal_unstable"),
             attrs: def.attrs,
             body: body,
-        }
+        })
     }
 
     fn maybe_get_item_ast<'a>(&'tcx self,

src/librustc_metadata/decoder.rs

@@ -691,7 +691,15 @@ impl<'a, 'tcx> CrateMetadata {
     pub fn each_child_of_item<F>(&self, id: DefIndex, mut callback: F)
         where F: FnMut(def::Export)
     {
-        let macros_only = self.dep_kind.get() == DepKind::MacrosOnly;
+        if let Some(ref proc_macros) = self.proc_macros {
+            for (id, &(name, _)) in proc_macros.iter().enumerate() {
+                callback(def::Export {
+                    name: name,
+                    def: Def::Macro(DefId { krate: self.cnum, index: DefIndex::new(id), }),
+                })
+            }
+            return
+        }
 
         // Find the item.
         let item = match self.maybe_entry(id) {
@@ -700,6 +708,7 @@ impl<'a, 'tcx> CrateMetadata {
         };
 
         // Iterate over all children.
+        let macros_only = self.dep_kind.get() == DepKind::MacrosOnly;
         for child_index in item.children.decode(self) {
             if macros_only {
                 continue

src/librustc_metadata/locator.rs

@@ -262,6 +262,7 @@ pub struct Context<'a> {
     pub rejected_via_kind: Vec<CrateMismatch>,
     pub rejected_via_version: Vec<CrateMismatch>,
     pub should_match_name: bool,
+    pub is_proc_macro: Option<bool>,
 }
 
 pub struct ArchiveMetadata {
@@ -623,6 +624,12 @@ impl<'a> Context<'a> {
 
     fn crate_matches(&mut self, metadata: &MetadataBlob, libpath: &Path) -> Option<Svh> {
         let root = metadata.get_root();
+        if let Some(is_proc_macro) = self.is_proc_macro {
+            if root.macro_derive_registrar.is_some() != is_proc_macro {
+                return None;
+            }
+        }
+
         let rustc_version = rustc_version();
         if root.rustc_version != rustc_version {
             info!("Rejecting via version: expected {} got {}",