Skip to content

Commit

Permalink
Merge branch 'master' into lexer-atn-prop
Browse files Browse the repository at this point in the history
  • Loading branch information
@parrt
parrt authored Dec 6, 2017
2 parents e22fbbc + b9adef5 commit dd4a1c8
Show file tree
Hide file tree
Showing 5 changed files with 263 additions and 4 deletions.
1 change: 1 addition & 0 deletions contributors.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,4 +174,5 @@ YYYY/MM/DD, github id, Full name, email
2017/11/02, jasonmoo, Jason Mooberry, jason.mooberry@gmail.com
2017/11/05, ajaypanyala, Ajay Panyala, ajay.panyala@gmail.com
2017/11/24, zqlu.cn, Zhiqiang Lu, zqlu.cn@gmail.com
2017/11/28, niccroad, Nicolas Croad, nic.croad@gmail.com
2017/12/03, oranoran, Oran Epelbaum, oran / epelbaum me
9 changes: 6 additions & 3 deletions doc/grammars.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,10 @@ $ grun MyELang stat
<= line 3:0 extraneous input ';' expecting {INT, ID}
```

If there were any `tokens` specifications, the main grammar would merge the token sets. Any named actions such as `@members` would be merged. In general, you should avoid named actions and actions within rules in imported grammars since that limits their reuse. ANTLR also ignores any options in imported grammars.
If there are modes in the main grammar or any of the imported grammars then the import process will import those modes and merge their rules where they are not overridden. In the event any mode becomes empty as all its
rules have been overridden by rules outside the mode this mode will be discarded.

If there were any `tokens` specifications, the main grammar would merge the token sets. If there were any `channel` specifications, the main grammar would merge the channel sets. Any named actions such as `@members` would be merged. In general, you should avoid named actions and actions within rules in imported grammars since that limits their reuse. ANTLR also ignores any options in imported grammars.

Imported grammars can also import other grammars. ANTLR pursues all imported grammars in a depth-first fashion. If two or more imported grammars define rule `r`, ANTLR chooses the first version of `r` it finds. In the following diagram, ANTLR examines grammars in the following order `Nested`, `G1`, `G3`, `G2`.

Expand All @@ -91,9 +94,9 @@ Imported grammars can also import other grammars. ANTLR pursues all imported gra

Not every kind of grammar can import every other kind of grammar:

* Lexer grammars can import lexers.
* Lexer grammars can import lexers, including lexers containing modes.
* Parsers can import parsers.
* Combined grammars can import lexers or parsers.
* Combined grammars can import parsers or lexers without modes.

ANTLR adds imported rules to the end of the rule list in a main lexer grammar. That means lexer rules in the main grammar get precedence over imported rules. For example, if a main grammar defines rule `IF : ’if’ ;` and an imported grammar defines rule `ID : [a-z]+ ;` (which also recognizes `if`), the imported `ID` won’t hide the main grammar’s `IF` token definition.

Expand Down
2 changes: 1 addition & 1 deletion runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Company>The ANTLR Organization</Company>
<Version>4.7.2</Version>
<Version>4.7.1.1</Version>
<NeutralLanguage>en-US</NeutralLanguage>
<TargetFrameworks>netstandard1.3;net35</TargetFrameworks>
<NoWarn>$(NoWarn);CS1591;CS1574;CS1580</NoWarn>
Expand Down
175 changes: 175 additions & 0 deletions tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,181 @@ public void testSetUp() throws Exception {
assertEquals(0, equeue.size());
}

@Test public void testImportIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);

String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a';\n" +
"B : 'b';\n";
writeFile(tmpdir, "M.g4", master);

String slave =
"lexer grammar S;\n" +
"C : 'c';\n";
writeFile(tmpdir, "S.g4", slave);

ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}

@Test public void testImportModesIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);

String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a' -> pushMode(X);\n" +
"B : 'b';\n";
writeFile(tmpdir, "M.g4", master);

String slave =
"lexer grammar S;\n" +
"D : 'd';\n" +
"mode X;\n" +
"C : 'c' -> popMode;\n";
writeFile(tmpdir, "S.g4", slave);

ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}

@Test public void testImportChannelsIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);

String master =
"lexer grammar M;\n" +
"import S;\n" +
"channels {CH_A, CH_B}\n" +
"A : 'a' -> channel(CH_A);\n" +
"B : 'b' -> channel(CH_B);\n";
writeFile(tmpdir, "M.g4", master);

String slave =
"lexer grammar S;\n" +
"C : 'c';\n";
writeFile(tmpdir, "S.g4", slave);

ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}

@Test public void testImportMixedChannelsIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);

String master =
"lexer grammar M;\n" +
"import S;\n" +
"channels {CH_A, CH_B}\n" +
"A : 'a' -> channel(CH_A);\n" +
"B : 'b' -> channel(CH_B);\n";
writeFile(tmpdir, "M.g4", master);

String slave =
"lexer grammar S;\n" +
"channels {CH_C}\n" +
"C : 'c' -> channel(CH_C);\n";
writeFile(tmpdir, "S.g4", slave);

ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}

@Test public void testImportClashingChannelsIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);

String master =
"lexer grammar M;\n" +
"import S;\n" +
"channels {CH_A, CH_B, CH_C}\n" +
"A : 'a' -> channel(CH_A);\n" +
"B : 'b' -> channel(CH_B);\n" +
"C : 'C' -> channel(CH_C);\n";
writeFile(tmpdir, "M.g4", master);

String slave =
"lexer grammar S;\n" +
"channels {CH_C}\n" +
"C : 'c' -> channel(CH_C);\n";
writeFile(tmpdir, "S.g4", slave);

ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}

@Test public void testMergeModesIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);

String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a' -> pushMode(X);\n" +
"mode X;\n" +
"B : 'b';\n";
writeFile(tmpdir, "M.g4", master);

String slave =
"lexer grammar S;\n" +
"D : 'd';\n" +
"mode X;\n" +
"C : 'c' -> popMode;\n";
writeFile(tmpdir, "S.g4", slave);

ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}

@Test public void testEmptyModesInLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);

String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a';\n" +
"C : 'e';\n" +
"B : 'b';\n";
writeFile(tmpdir, "M.g4", master);

String slave =
"lexer grammar S;\n" +
"D : 'd';\n" +
"mode X;\n" +
"C : 'c' -> popMode;\n";
writeFile(tmpdir, "S.g4", slave);

ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}

@Test public void testCombinedGrammarImportsModalLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);

String master =
"grammar M;\n" +
"import S;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"r : A B;\n";
writeFile(tmpdir, "M.g4", master);

String slave =
"lexer grammar S;\n" +
"D : 'd';\n" +
"mode X;\n" +
"C : 'c' -> popMode;\n";
writeFile(tmpdir, "S.g4", slave);

ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(1, equeue.errors.size());
ANTLRMessage msg = equeue.errors.get(0);
assertEquals(ErrorType.MODE_NOT_IN_LEXER, msg.getErrorType());
assertEquals("X", msg.getArgs()[0]);
assertEquals(3, msg.line);
assertEquals(5, msg.charPosition);
assertEquals("M.g4", new File(msg.fileName).getName());
}

@Test public void testDelegatesSeeSameTokenType() throws Exception {
String slaveS =
"parser grammar S;\n"+
Expand Down
80 changes: 80 additions & 0 deletions tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ public void integrateImportedGrammars(Grammar rootGrammar) {
GrammarAST id = (GrammarAST) root.getChild(0);
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(id.token.getInputStream());

GrammarAST channelsRoot = (GrammarAST)root.getFirstChildWithType(ANTLRParser.CHANNELS);
GrammarAST tokensRoot = (GrammarAST)root.getFirstChildWithType(ANTLRParser.TOKENS_SPEC);

List<GrammarAST> actionRoots = root.getNodesWithType(ANTLRParser.AT);
Expand All @@ -172,7 +173,39 @@ public void integrateImportedGrammars(Grammar rootGrammar) {
List<GrammarAST> rootRules = RULES.getNodesWithType(ANTLRParser.RULE);
for (GrammarAST r : rootRules) rootRuleNames.add(r.getChild(0).getText());

// make list of modes we have in root grammar
List<GrammarAST> rootModes = root.getNodesWithType(ANTLRParser.MODE);
Set<String> rootModeNames = new HashSet<String>();
for (GrammarAST m : rootModes) rootModeNames.add(m.getChild(0).getText());
List<GrammarAST> addedModes = new ArrayList<GrammarAST>();

for (Grammar imp : imports) {
// COPY CHANNELS
GrammarAST imp_channelRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.CHANNELS);
if ( imp_channelRoot != null) {
rootGrammar.tool.log("grammar", "imported channels: "+imp_channelRoot.getChildren());
if (channelsRoot==null) {
channelsRoot = imp_channelRoot.dupTree();
channelsRoot.g = rootGrammar;
root.insertChild(1, channelsRoot); // ^(GRAMMAR ID TOKENS...)
} else {
for (int c = 0; c < imp_channelRoot.getChildCount(); ++c) {
String channel = imp_channelRoot.getChild(c).getText();
boolean channelIsInRootGrammar = false;
for (int rc = 0; rc < channelsRoot.getChildCount(); ++rc) {
String rootChannel = channelsRoot.getChild(rc).getText();
if (rootChannel.equals(channel)) {
channelIsInRootGrammar = true;
break;
}
}
if (!channelIsInRootGrammar) {
channelsRoot.addChild(imp_channelRoot.getChild(c).dupNode());
}
}
}
}

// COPY TOKENS
GrammarAST imp_tokensRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.TOKENS_SPEC);
if ( imp_tokensRoot!=null ) {
Expand Down Expand Up @@ -242,7 +275,54 @@ public void integrateImportedGrammars(Grammar rootGrammar) {
}
}

// COPY MODES
// The strategy is to copy all the mode sections rules across to any
// mode section in the new grammar with the same name or a new
// mode section if no matching mode is resolved. Rules which are
// already in the new grammar are ignored for copy. If the mode
// section being added ends up empty it is not added to the merged
// grammar.
List<GrammarAST> modes = imp.ast.getNodesWithType(ANTLRParser.MODE);
if (modes != null) {
for (GrammarAST m : modes) {
rootGrammar.tool.log("grammar", "imported mode: " + m.toStringTree());
String name = m.getChild(0).getText();
boolean rootAlreadyHasMode = rootModeNames.contains(name);
GrammarAST destinationAST = null;
if (rootAlreadyHasMode) {
for (GrammarAST m2 : rootModes) {
if (m2.getChild(0).getText().equals(name)) {
destinationAST = m2;
break;
}
}
} else {
destinationAST = m.dupNode();
destinationAST.addChild(m.getChild(0).dupNode());
}

int addedRules = 0;
List<GrammarAST> modeRules = m.getAllChildrenWithType(ANTLRParser.RULE);
for (GrammarAST r : modeRules) {
rootGrammar.tool.log("grammar", "imported rule: "+r.toStringTree());
String ruleName = r.getChild(0).getText();
boolean rootAlreadyHasRule = rootRuleNames.contains(ruleName);
if (!rootAlreadyHasRule) {
destinationAST.addChild(r);
addedRules++;
rootRuleNames.add(ruleName);
}
}
if (!rootAlreadyHasMode && addedRules > 0) {
rootGrammar.ast.addChild(destinationAST);
rootModeNames.add(name);
rootModes.add(destinationAST);
}
}
}

// COPY RULES
// Rules copied in the mode copy phase are not copied again.
List<GrammarAST> rules = imp.ast.getNodesWithType(ANTLRParser.RULE);
if ( rules!=null ) {
for (GrammarAST r : rules) {
Expand Down

0 comments on commit dd4a1c8

Please sign in to comment.