Fix lexer bugs with Unicode SMP

antlr · Jan 27, 2017 · fad55a2 · fad55a2
1 parent 4aa7738
commit fad55a2
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/tool/src/org/antlr/v4/automata/LexerATNFactory.java b/tool/src/org/antlr/v4/automata/LexerATNFactory.java
@@ -353,10 +353,12 @@ public Handle stringLiteral(TerminalAST stringLiteralAST) {
 		int n = chars.length();
 		ATNState prev = left;
 		right = null;
-		for (int i = 0; i < n; i++) {
+		for (int i = 0; i < n; ) {
 			right = newState(stringLiteralAST);
-			prev.addTransition(new AtomTransition(right, chars.charAt(i)));
+			int codePoint = chars.codePointAt(i);
+			prev.addTransition(new AtomTransition(right, codePoint));
 			prev = right;
+			i += Character.charCount(codePoint);
 		}
 		stringLiteralAST.atnState = left;
 		return new Handle(left, right);

diff --git a/tool/src/org/antlr/v4/misc/CharSupport.java b/tool/src/org/antlr/v4/misc/CharSupport.java
@@ -115,7 +115,7 @@ public static String getStringFromGrammarStringLiteral(String literal) {
 			if ( c==-1 ) {
 				return null; // invalid escape sequence.
 			}
-			else buf.append((char)c);
+			else buf.appendCodePoint(c);
 			i = end;
 		}
 		return buf.toString();