Remove tokens from foreign items in TokenStripper

compiler/rustc_interface/src/passes.rs

@@ -63,6 +63,13 @@ impl mut_visit::MutVisitor for TokenStripper {
         i.tokens = None;
         mut_visit::noop_flat_map_item(i, self)
     }
+    fn flat_map_foreign_item(
+        &mut self,
+        mut i: P<ast::ForeignItem>,
+    ) -> SmallVec<[P<ast::ForeignItem>; 1]> {
+        i.tokens = None;
+        mut_visit::noop_flat_map_foreign_item(i, self)
+    }
     fn visit_block(&mut self, b: &mut P<ast::Block>) {
         b.tokens = None;
         mut_visit::noop_visit_block(b, self);

src/test/ui/ast-json/issue-78398-foreign-ice.rs

@@ -0,0 +1,18 @@
+// Regression test for issue #78398
+// Tests that we don't ICE when trying to print the AST json
+// when we have capturd tokens for a foreign item
+
+// check-pass
+// compile-flags: -Zast-json
+
+fn main() {}
+
+macro_rules! mac_extern {
+    ($i:item) => {
+        extern "C" { $i }
+    }
+}
+
+mac_extern! {
+    fn foo();
+}

src/test/ui/ast-json/issue-78398-foreign-ice.stdout

@@ -0,0 +1 @@
+{"module":{"inner":{"lo":192,"hi":315},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":3,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":4,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":5,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":6,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":7,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":8,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":9,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":10,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":11,"span":{"lo":192,"hi":204},"vis":{"kind":"Inherited","span":{"lo":192,"hi":192},"tokens":null},"ident":{"name":"main","span":{"lo":195,"hi":199}},"kind":{"variant":"Fn","fields":["Final",{"header":{"unsafety":"No","asyncness":"No","constness":"No","ext":"None"},"decl":{"inputs":[],"output":{"variant":"Default","fields":[{"lo":202,"hi":202}]}},"span":{"lo":192,"hi":201}},{"params":[],"where_clause":{"has_where_token":false,"predicates":[],"span":{"lo":201,"hi":201}},"span":{"lo":199,"hi":199}},{"stmts":[],"id":12,"rules":"Default","span":{"lo":202,"hi":204},"tokens":null}]},"tokens":null},{"attrs":[],"id":13,"span":{"lo":206,"hi":284},"vis":{"kind":"Inherited","span":{"lo":206,"hi":206},"tokens":null},"ident":{"name":"mac_extern","span":{"lo":219,"hi":229}},"kind":{"variant":"MacroDef","fields":[{"body":{"variant":"Delimited","fields":[{"open":{"lo":230,"hi":231},"close":{"lo":283,"hi":284}},"Brace",{"0":[[{"variant":"Delimited","fields":[{"open":{"lo":236,"hi":237},"close":{"lo":244,"hi":245}},"Paren",{"0":[[{"variant":"Token","fields":[{"kind":"Dollar","span":{"lo":237,"hi":238}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["i",false]},"span":{"lo":238,"hi":239}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Colon","span":{"lo":239,"hi":240}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["item",false]},"span":{"lo":240,"hi":244}}]},"Alone"]]}]},"Alone"],[{"variant":"Token","fields":[{"kind":"FatArrow","span":{"lo":246,"hi":248}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":249,"hi":250},"close":{"lo":281,"hi":282}},"Brace",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["extern",false]},"span":{"lo":259,"hi":265}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"C","suffix":null}]},"span":{"lo":266,"hi":269}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":270,"hi":271},"close":{"lo":275,"hi":276}},"Brace",{"0":[[{"variant":"Token","fields":[{"kind":"Dollar","span":{"lo":272,"hi":273}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["i",false]},"span":{"lo":273,"hi":274}}]},"Alone"]]}]},"Alone"]]}]},"Alone"]]}]},"macro_rules":true}]},"tokens":null},{"attrs":[],"id":14,"span":{"lo":259,"hi":276},"vis":{"kind":"Inherited","span":{"lo":259,"hi":259},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"ForeignMod","fields":[{"unsafety":"No","abi":{"style":"Cooked","symbol":"C","suffix":null,"span":{"lo":266,"hi":269},"symbol_unescaped":"C"},"items":[{"attrs":[],"id":15,"span":{"lo":304,"hi":313},"vis":{"kind":"Inherited","span":{"lo":304,"hi":304},"tokens":null},"ident":{"name":"foo","span":{"lo":307,"hi":310}},"kind":{"variant":"Fn","fields":["Final",{"header":{"unsafety":"No","asyncness":"No","constness":"No","ext":"None"},"decl":{"inputs":[],"output":{"variant":"Default","fields":[{"lo":312,"hi":312}]}},"span":{"lo":304,"hi":313}},{"params":[],"where_clause":{"has_where_token":false,"predicates":[],"span":{"lo":312,"hi":312}},"span":{"lo":310,"hi":310}},null]},"tokens":null}]}]},"tokens":null}],"inline":true},"attrs":[],"span":{"lo":192,"hi":315},"proc_macros":[]}