Use int literals instead of refs for Python and JavaScript

Update getMultiTokenAlternativeDescriptor test fixes #3703 Signed-off-by: Ivan Kochurkin <kvanttt@gmail.com>
antlr · Jul 3, 2022 · 6caa65d · 6caa65d
1 parent e137adf
commit 6caa65d
Show file tree

Hide file tree

Showing 10 changed files with 61 additions and 22 deletions.
diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/CustomDescriptors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/CustomDescriptors.java
@@ -154,10 +154,11 @@ private static RuntimeTestDescriptor getAtnStatesSizeMoreThan65535Descriptor() {
 	private static RuntimeTestDescriptor getMultiTokenAlternativeDescriptor() {
 		final int tokensCount = 64;
 
-		StringBuilder rule = new StringBuilder("t: ");
+		StringBuilder rule = new StringBuilder("r1: ");
 		StringBuilder tokens = new StringBuilder();
 		StringBuilder input = new StringBuilder();
 		StringBuilder output = new StringBuilder();
+
 		for (int i = 0; i < tokensCount; i++) {
 			String currentToken = "T" + i;
 			rule.append(currentToken);
@@ -170,17 +171,21 @@ private static RuntimeTestDescriptor getMultiTokenAlternativeDescriptor() {
 			input.append(currentToken).append(" ");
 			output.append(currentToken);
 		}
+		String currentToken = "T" + tokensCount;
+		tokens.append(currentToken).append(": '").append(currentToken).append("';\n");
+		input.append(currentToken).append(" ");
+		output.append(currentToken);
 
 		String grammar = "grammar P;\n" +
-				"r: t+ EOF {<writeln(\"$text\")>};\n" +
+				"r: (r1 | T" + tokensCount + ")+ EOF {<writeln(\"$text\")>};\n" +
 				rule + "\n" +
 				tokens + "\n" +
 				"WS: [ ]+ -> skip;";
 
 		return new RuntimeTestDescriptor(
 				GrammarType.Parser,
-				"MultiTokenAlternative",
-				"https://github.com/antlr/antlr4/issues/3698",
+				"MultiTokenAlternative2",
+				"https://github.com/antlr/antlr4/issues/3698, https://github.com/antlr/antlr4/issues/3703",
 				input.toString(),
 				output + "\n",
 				"",

diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg
@@ -246,7 +246,7 @@ RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,fina
 
 <currentRule.escapedName>(<currentRule.args:{a | <a.escapedName>}; separator=", ">) {
     let localctx = new <currentRule.ctxType>(this, this._ctx, this.state<currentRule.args:{a | , <a.escapedName>}>);
-    this.enterRule(localctx, <currentRule.startState>, <parser.name>.RULE_<currentRule.name>);
+    this.enterRule(localctx, <currentRule.startState>, <currentRule.index>);
     <namedActions.init>
     <locals; separator="\n">
     try {
@@ -285,7 +285,7 @@ LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,
     let localctx = new <currentRule.ctxType>(this, this._ctx, _parentState<args:{a | , <a.escapedName>}>);
     let _prevctx = localctx;
     const _startState = <currentRule.startState>;
-    this.enterRecursionRule(localctx, <currentRule.startState>, <parser.name>.RULE_<currentRule.name>, _p);
+    this.enterRecursionRule(localctx, <currentRule.startState>, <currentRule.index>, _p);
     <namedActions.init>
     <locals; separator="\n">
     try {
@@ -477,11 +477,11 @@ offsetShiftType(shiftAmount, offset) ::= <%
 
 // produces more efficient bytecode when bits.ttypes contains at most two items
 bitsetInlineComparison(s, bits) ::= <%
-<bits.ttypes:{ttype | <s.varName>===<parser.name>.<ttype>}; separator=" || ">
+<bits.ttypes:{ttype | <s.varName>===<ttype>}; separator=" || ">
 %>
 
 cases(ttypes) ::= <<
-<ttypes:{t | case <parser.name>.<t>:}; separator="\n">
+<ttypes:{t | case <t>:}; separator="\n">
 >>
 
 InvokeRule(r, argExprsChunks) ::= <<
@@ -491,7 +491,7 @@ this.state = <r.stateNumber>;
 
 MatchToken(m) ::= <<
 this.state = <m.stateNumber>;
-<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>this.match(<parser.name>.<m.name>);
+<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>this.match(<m.ttype>);
 >>
 
 MatchSet(m, expr, capture) ::= "<CommonSetStuff(m, expr, capture, false)>"

diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg
@@ -253,7 +253,7 @@ RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,fina
 def <currentRule.escapedName>(self<currentRule.args:{a | , <a.escapedName>}>):
 
     localctx = <parser.name>.<currentRule.ctxType>(self, self._ctx, self.state<currentRule.args:{a | , <a.escapedName>}>)
-    self.enterRule(localctx, <currentRule.startState>, self.RULE_<currentRule.name>)
+    self.enterRule(localctx, <currentRule.startState>, <currentRule.index>)
     <namedActions.init>
     <locals; separator="\n">
     try:
@@ -288,7 +288,7 @@ def <currentRule.escapedName>(self, _p=0<if(currentRule.args)>, <args:{a | , <a>
     localctx = <parser.name>.<currentRule.ctxType>(self, self._ctx, _parentState<args:{a | , <a.escapedName>}>)
     _prevctx = localctx
     _startState = <currentRule.startState>
-    self.enterRecursionRule(localctx, <currentRule.startState>, self.RULE_<currentRule.name>, _p)
+    self.enterRecursionRule(localctx, <currentRule.startState>, <currentRule.index>, _p)
     <namedActions.init>
     <locals; separator="\n">
     try:
@@ -471,11 +471,11 @@ offsetShiftType(shiftAmount, offset) ::= <%
 
 // produces more efficient bytecode when bits.ttypes contains at most two items
 bitsetInlineComparison(s, bits) ::= <%
-<bits.ttypes:{ttype | <s.varName>==<parser.name>.<ttype>}; separator=" or ">
+<bits.ttypes:{ttype | <s.varName>==<ttype>}; separator=" or ">
 %>
 
 cases(ttypes) ::= <<
-if token in [<ttypes:{t | <parser.name>.<t>}; separator=", ">]:
+if token in [<ttypes:{t | <t>}; separator=", ">]:
 >>
 
 InvokeRule(r, argExprsChunks) ::= <<
@@ -485,7 +485,7 @@ self.state = <r.stateNumber>
 
 MatchToken(m) ::= <<
 self.state = <m.stateNumber>
-<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>self.match(<parser.name>.<m.name>)
+<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>self.match(<m.ttype>)
 >>
 
 MatchSet(m, expr, capture) ::= "<CommonSetStuff(m, expr, capture, false)>"

diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg
@@ -266,7 +266,7 @@ RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,fina
 def <currentRule.escapedName>(self<currentRule.args:{a | , <a.escapedName><if(a.type)>:<a.type><endif>}>):
 
     localctx = <parser.name>.<currentRule.ctxType>(self, self._ctx, self.state<currentRule.args:{a | , <a.escapedName>}>)
-    self.enterRule(localctx, <currentRule.startState>, self.RULE_<currentRule.name>)
+    self.enterRule(localctx, <currentRule.startState>, <currentRule.index>)
     <namedActions.init>
     <locals; separator="\n">
     try:
@@ -301,7 +301,7 @@ def <currentRule.escapedName>(self, _p:int=0<if(currentRule.args)>, <args:{a | ,
     localctx = <parser.name>.<currentRule.ctxType>(self, self._ctx, _parentState<args:{a | , <a.escapedName>}>)
     _prevctx = localctx
     _startState = <currentRule.startState>
-    self.enterRecursionRule(localctx, <currentRule.startState>, self.RULE_<currentRule.name>, _p)
+    self.enterRecursionRule(localctx, <currentRule.startState>, <currentRule.index>, _p)
     <namedActions.init>
     <locals; separator="\n">
     try:
@@ -484,11 +484,11 @@ offsetShiftType(shiftAmount, offset) ::= <%
 
 // produces more efficient bytecode when bits.ttypes contains at most two items
 bitsetInlineComparison(s, bits) ::= <%
-<bits.ttypes:{ttype | <s.varName>==<parser.name>.<ttype>}; separator=" or ">
+<bits.ttypes:{ttype | <s.varName>==<ttype>}; separator=" or ">
 %>
 
 cases(ttypes) ::= <<
-if token in [<ttypes:{t | <parser.name>.<t>}; separator=", ">]:
+if token in [<ttypes:{t | <t>}; separator=", ">]:
 >>
 
 InvokeRule(r, argExprsChunks) ::= <<
@@ -498,7 +498,7 @@ self.state = <r.stateNumber>
 
 MatchToken(m) ::= <<
 self.state = <m.stateNumber>
-<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>self.match(<parser.name>.<m.name>)
+<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>self.match(<m.ttype>)
 >>
 
 MatchSet(m, expr, capture) ::= "<CommonSetStuff(m, expr, capture, false)>"

diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java
@@ -655,4 +655,6 @@ public boolean isATNSerializedAsInts() {
 
 	/** @since 4.6 */
 	public boolean needsHeader() { return false; } // Override in targets that need header files.
+
+	public boolean supportsConstants() { return true; }
 }
diff --git a/tool/src/org/antlr/v4/codegen/model/Choice.java b/tool/src/org/antlr/v4/codegen/model/Choice.java
@@ -7,10 +7,13 @@
 package org.antlr.v4.codegen.model;
 
 import org.antlr.v4.codegen.OutputModelFactory;
+import org.antlr.v4.codegen.Target;
 import org.antlr.v4.codegen.model.decl.Decl;
 import org.antlr.v4.codegen.model.decl.TokenTypeDecl;
 import org.antlr.v4.misc.Utils;
+import org.antlr.v4.runtime.misc.IntegerList;
 import org.antlr.v4.runtime.misc.IntervalSet;
+import org.antlr.v4.tool.Grammar;
 import org.antlr.v4.tool.ast.GrammarAST;
 
 import java.util.ArrayList;
@@ -46,9 +49,21 @@ public void addPreambleOp(SrcOp op) {
 	}
 
 	public List<String[]> getAltLookaheadAsStringLists(IntervalSet[] altLookSets) {
-		List<String[]> altLook = new ArrayList<String[]>();
+		List<String[]> altLook = new ArrayList<>();
+		Target target = factory.getGenerator().getTarget();
+		Grammar grammar = factory.getGrammar();
 		for (IntervalSet s : altLookSets) {
-			altLook.add(factory.getGenerator().getTarget().getTokenTypesAsTargetLabels(factory.getGrammar(), s.toArray()));
+			if (target.supportsConstants()) {
+				altLook.add(target.getTokenTypesAsTargetLabels(grammar, s.toArray()));
+			}
+			else {
+				IntegerList list = s.toIntegerList();
+				String[] labels = new String[list.size()];
+				for (int i = 0; i < labels.length; i++) {
+					labels[i] = String.valueOf(list.get(i));
+				}
+				altLook.add(labels);
+			}
 		}
 		return altLook;
 	}

diff --git a/tool/src/org/antlr/v4/codegen/model/TestSetInline.java b/tool/src/org/antlr/v4/codegen/model/TestSetInline.java
@@ -49,7 +49,14 @@ private static Bitset[] createBitsets(OutputModelFactory factory,
 				bitsetList.add(current);
 			}
 
-			current.addToken(ttype, target.getTokenTypeAsTargetLabel(factory.getGrammar(), ttype));
+			String name;
+			if (target.supportsConstants()) {
+				name = target.getTokenTypeAsTargetLabel(factory.getGrammar(), ttype);
+			}
+			else  {
+				name = String.valueOf(ttype);
+			}
+			current.addToken(ttype, name);
 		}
 
 		return bitsetList.toArray(new Bitset[0]);

diff --git a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java
@@ -71,4 +71,8 @@ public boolean supportsOverloadedMethods() {
 	public boolean isATNSerializedAsInts() {
 		return true;
 	}
+
+
+	@Override
+	public boolean supportsConstants() { return false; }
 }
diff --git a/tool/src/org/antlr/v4/codegen/target/Python2Target.java b/tool/src/org/antlr/v4/codegen/target/Python2Target.java
@@ -88,5 +88,8 @@ public boolean wantsBaseVisitor() {
 	public boolean supportsOverloadedMethods() {
 		return false;
 	}
+
+	@Override
+	public boolean supportsConstants() { return false; }
 }
 
diff --git a/tool/src/org/antlr/v4/codegen/target/Python3Target.java b/tool/src/org/antlr/v4/codegen/target/Python3Target.java
@@ -87,4 +87,7 @@ public boolean wantsBaseVisitor() {
 	public boolean supportsOverloadedMethods() {
 		return false;
 	}
+
+	@Override
+	public boolean supportsConstants() { return false; }
 }