Fold constants for Python and JavaScript runtimes, fixes antlr#3698, f…

…ixes antlr#3699 Signed-off-by: Ivan Kochurkin <kvanttt@gmail.com>
KvanTTT · May 7, 2022 · f4b2fe0 · f4b2fe0
1 parent a9e0ccf
commit f4b2fe0
Show file tree

Hide file tree

Showing 5 changed files with 76 additions and 18 deletions.
diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/GeneratedLexerDescriptors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/GeneratedLexerDescriptors.java
@@ -127,4 +127,42 @@ static RuntimeTestDescriptor getAtnStatesSizeMoreThan65535Descriptor(String targ
 //		result.skipTargets.add("Go"); // syntax error
 		return result;
 	}
+
+	static RuntimeTestDescriptor getMultiTokenAlternativeDescriptor(String targetName) {
+		UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor();
+		result.name = "MultiTokenAlternative";
+		result.notes = "";
+		result.targetName = targetName;
+		result.testType = "Parser";
+
+		final int tokensCount = 64;
+
+		StringBuilder rule = new StringBuilder("t: ");
+		StringBuilder tokens = new StringBuilder();
+		StringBuilder input = new StringBuilder();
+		StringBuilder output = new StringBuilder();
+		for (int i = 0; i < tokensCount; i++) {
+			String currentToken = "T" + i;
+			rule.append(currentToken);
+			if (i < tokensCount - 1) {
+				rule.append(" | ");
+			} else {
+				rule.append(";");
+			}
+			tokens.append(currentToken).append(": '").append(currentToken).append("';\n");
+			input.append(currentToken).append(" ");
+			output.append(currentToken);
+		}
+
+		result.grammar = "grammar P;\n" +
+				"r: t+ EOF {<writeln(\"$text\")>};\n" +
+				rule + "\n" +
+				tokens + "\n" +
+				"WS: [ ]+ -> skip;";
+		result.grammarName = "P";
+		result.startRule = "r";
+		result.input = input.toString();
+		result.output = output + "\n";
+		return result;
+	}
 }
diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg
@@ -454,12 +454,12 @@ TestSetInline(s) ::= <<
 
 // Javascript language spec - shift operators are 32 bits long max
 testShiftInRange(shiftAmount) ::= <<
-((<shiftAmount>) & ~0x1f) == 0
+(<shiftAmount> & ~0x1f) == 0
 >>
 
 // produces smaller bytecode only when bits.ttypes contains more than two items
 bitsetBitfieldComparison(s, bits) ::= <%
-(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> && ((1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & (<bits.ttypes:{ttype | (1 \<\< <offsetShiftType(ttype, bits.shift)>)}; separator=" | ">)) !== 0)
+(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> && ((1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & <bits.calculated>) !== 0)
 %>
 
 isZero ::= [

diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg
@@ -448,12 +448,12 @@ TestSetInline(s) ::= <<
 
 // Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test
 testShiftInRange(shiftAmount) ::= <<
-((<shiftAmount>) & ~0x3f) == 0
+(<shiftAmount> & ~0x3f) == 0
 >>
 
 // produces smaller bytecode only when bits.ttypes contains more than two items
 bitsetBitfieldComparison(s, bits) ::= <%
-(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> and ((1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & (<bits.ttypes:{ttype | (1 \<\< <offsetShiftType(ttype, bits.shift)>)}; separator=" | ">)) != 0)
+(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> and ((1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & <bits.calculated>) != 0)
 %>
 
 isZero ::= [

diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg
@@ -461,12 +461,12 @@ TestSetInline(s) ::= <<
 
 // Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test
 testShiftInRange(shiftAmount) ::= <<
-((<shiftAmount>) & ~0x3f) == 0
+(<shiftAmount> & ~0x3f) == 0
 >>
 
 // produces smaller bytecode only when bits.ttypes contains more than two items
 bitsetBitfieldComparison(s, bits) ::= <%
-(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> and ((1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & (<bits.ttypes:{ttype | (1 \<\< <offsetShiftType(ttype, bits.shift)>)}; separator=" | ">)) != 0)
+(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> and (1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & <bits.calculated>) != 0
 %>
 
 isZero ::= [

diff --git a/tool/src/org/antlr/v4/codegen/model/TestSetInline.java b/tool/src/org/antlr/v4/codegen/model/TestSetInline.java
@@ -7,6 +7,7 @@
 package org.antlr.v4.codegen.model;
 
 import org.antlr.v4.codegen.OutputModelFactory;
+import org.antlr.v4.codegen.Target;
 import org.antlr.v4.runtime.misc.IntervalSet;
 import org.antlr.v4.tool.ast.GrammarAST;
 
@@ -15,9 +16,9 @@
 
 /** */
 public class TestSetInline extends SrcOp {
-	public int bitsetWordSize;
-	public String varName;
-	public Bitset[] bitsets;
+	public final int bitsetWordSize;
+	public final String varName;
+	public final Bitset[] bitsets;
 
 	public TestSetInline(OutputModelFactory factory, GrammarAST ast, IntervalSet set, int wordSize) {
 		super(factory, ast);
@@ -32,29 +33,48 @@ private static Bitset[] createBitsets(OutputModelFactory factory,
 										  IntervalSet set,
 										  int wordSize,
 										  boolean useZeroOffset) {
-		List<Bitset> bitsetList = new ArrayList<Bitset>();
+		List<Bitset> bitsetList = new ArrayList<>();
+		Target target = factory.getGenerator().getTarget();
+		Bitset current = null;
 		for (int ttype : set.toArray()) {
-			Bitset current = !bitsetList.isEmpty() ? bitsetList.get(bitsetList.size() - 1) : null;
 			if (current == null || ttype > (current.shift + wordSize-1)) {
-				current = new Bitset();
+				int shift;
 				if (useZeroOffset && ttype >= 0 && ttype < wordSize-1) {
-					current.shift = 0;
+					shift = 0;
 				}
 				else {
-					current.shift = ttype;
+					shift = ttype;
 				}
-
+				current = new Bitset(shift);
 				bitsetList.add(current);
 			}
 
-			current.ttypes.add(factory.getGenerator().getTarget().getTokenTypeAsTargetLabel(factory.getGrammar(), ttype));
+			current.addToken(ttype, target.getTokenTypeAsTargetLabel(factory.getGrammar(), ttype));
 		}
 
 		return bitsetList.toArray(new Bitset[0]);
 	}
 
 	public static final class Bitset {
-		public int shift;
-		public final List<String> ttypes = new ArrayList<String>();
+		public final int shift;
+		private final List<String> ttypes = new ArrayList<>();
+		private long calculated;
+
+		public Bitset(int shift) {
+			this.shift = shift;
+		}
+
+		public void addToken(int type, String name) {
+			ttypes.add(name);
+			calculated |= 1L << (type - shift);
+		}
+
+		public List<String> getTtypes() {
+			return ttypes;
+		}
+
+		public long getCalculated() {
+			return calculated;
+		}
 	}
 }