Merge pull request #3407 from parrt/runtime-test-descriptor-files

Reorg runtime tests (3rd generation); mostly to use descriptor files for tests not annotations in *Descriptor.java files (supports later versions of java)

.appveyor/workflow.yml

@@ -19,7 +19,7 @@ environment:
 matrix:
   fast_finish: false
 
-version: '4.9.3-SNAPSHOT+AppVeyor.{build}'
+version: '4.9.4-SNAPSHOT+AppVeyor.{build}'
 cache:
   - '%USERPROFILE%\.m2'
   - '%USERPROFILE%\.nuget\packages -> **\project.json'

.circleci/config.yml

@@ -61,6 +61,5 @@ workflows:
       - test_runtime:
           matrix:
             parameters:
-#              target: [ cpp, dotnet, swift ]
-              target: [ cpp, dotnet, dart ]
-              test-group: [ LEXER, PARSER, RECURSION ]
+              target: [ cpp, dotnet ]
+              test-group: [ LEXER, PARSER1, PARSER2, RECURSION ]

.circleci/scripts/run-tests-cpp.sh

@@ -10,8 +10,10 @@ pushd runtime-testsuite
   echo "running maven tests..."
   if [ $GROUP == "LEXER" ]; then
       mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=cpp.** test
-  elif [ $GROUP == "PARSER" ]; then
-      mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=cpp.** test
+  elif [ $GROUP == "PARSER1" ]; then
+      mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTestsGroup1" -Dtest=cpp.** test
+  elif [ $GROUP == "PARSER2" ]; then
+      mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTestsGroup2" -Dtest=cpp.** test
   elif [ $GROUP == "RECURSION" ]; then
       mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=cpp.** test
   else

.circleci/scripts/run-tests-dotnet.sh

@@ -6,8 +6,8 @@ pushd runtime-testsuite/
   echo "running maven tests..."
   if [ $GROUP == "LEXER" ]; then
       mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=csharp.** test
-  elif [ $GROUP == "PARSER" ]; then
-      mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=csharp.** test
+  elif [ $GROUP == "PARSER1" ]; then
+      mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTestsGroup1" -Dtest=csharp.** test
   elif [ $GROUP == "RECURSION" ]; then
       mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=csharp.** test
   else

.circleci/scripts/run-tests-swift.sh

@@ -16,8 +16,10 @@ if [ $rc == 0 ]; then
     echo "running maven tests..."
     if [ $GROUP == "LEXER" ]; then
         mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=swift.** test
-    elif [ $GROUP == "PARSER" ]; then
-        mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=swift.** test
+    elif [ $GROUP == "PARSER1" ]; then
+        mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTestsGroup1" -Dtest=swift.** test
+    elif [ $GROUP == "PARSER2" ]; then
+        mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTestsGroup2" -Dtest=swift.** test
     elif [ $GROUP == "RECURSION" ]; then
         mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=swift.** test
     else

.github/scripts/run-tests-cpp.sh

@@ -9,8 +9,10 @@ popd
 pushd runtime-testsuite
 if [ $GROUP == "LEXER" ]; then
     mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=cpp.** test
-elif [ $GROUP == "PARSER" ]; then
-    mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=cpp.** test
+elif [ $GROUP == "PARSER1" ]; then
+    mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTestsGroup1" -Dtest=cpp.** test
+elif [ $GROUP == "PARSER2" ]; then
+    mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTestsGroup2" -Dtest=cpp.** test
 elif [ $GROUP == "RECURSION" ]; then
     mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=cpp.** test
 else

.github/scripts/run-tests-dotnet.sh

@@ -14,8 +14,8 @@ cd runtime-testsuite/
 
 if [ $GROUP == "LEXER" ]; then
     mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=csharp.** test
-elif [ $GROUP == "PARSER" ]; then
-    mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=csharp.** test
+elif [ $GROUP == "PARSER1" ]; then
+    mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTestsGroup1" -Dtest=csharp.** test
 elif [ $GROUP == "RECURSION" ]; then
     mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=csharp.** test
 else

.github/scripts/run-tests-swift.sh

@@ -37,8 +37,10 @@ if [ $rc == 0 ]; then
   cd runtime-testsuite/
   if [ $GROUP == "LEXER" ]; then
       mvn -e -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest="swift.**" test
-  elif [ $GROUP == "PARSER" ]; then
-      mvn -e -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest="swift.**" test
+  elif [ $GROUP == "PARSER1" ]; then
+      mvn -e -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTestsGroup1" -Dtest="swift.**" test
+  elif [ $GROUP == "PARSER2" ]; then
+      mvn -e -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTestsGroup2" -Dtest="swift.**" test
   elif [ $GROUP == "RECURSION" ]; then
       mvn -e -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest="swift.**" test
   else

.github/workflows/macosx.yml

@@ -14,7 +14,7 @@ jobs:
       matrix:
 #        TARGET: [swift, cpp, dotnet] disabling dotnet which is unstable on M1
         TARGET: [swift, cpp]
-        GROUP: [LEXER, PARSER, RECURSION]
+        GROUP: [LEXER, PARSER1, PARSER2, RECURSION]
     steps:
     - uses: actions/checkout@v2
     - name: Set up JDK 1.8

doc/antlr-project-testing.md

@@ -6,52 +6,56 @@ Because ANTLR supports multiple target languages, the unit tests are broken into
 
 The runtime tests must be specified in a generic fashion to work across language targets. Furthermore, we must test the various targets from Java. This usually means Java launching processes to compile, say, C++ and run parsers.
 
-As of 4.6, we use [a Java descriptor object](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestDescriptor.java) to describe each runtime test.  Unit tests are grouped together into categories such as [ParserExecDescriptors](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserExecDescriptors.java), which has multiple nested descriptor objects, one per test. For example, here is the start of that file:
-
-```java
-public class ParserExecDescriptors {
-    public static class APlus extends BaseParserTestDescriptor {
-        public String input = "a b c";
-        public String output = "abc\n";
-        public String errors = "";
-        public String startRule = "a";
-        public String grammarName = "T";
-
-        /**
-         grammar T;
-         a : ID+ {
-         <writeln("$text")>
-         };
-         ID : 'a'..'z'+;
-         WS : (' '|'\n') -> skip;
-         */
-        @CommentHasStringValue
-        public String grammar;
-    }
+As of 4.9.4, we use a Java descriptor file held as an [UniversalRuntimeTestDescriptor.java object](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/UniversalRuntimeTestDescriptor.java) to represent each runtime test. Each test is described with a text file with various sections and resides in a group directory; see [directories under descriptors dir](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors). Here is a sample test descriptor:
+
 ```
+[notes]
+This is a regression test for blah blah blah...
+
+[type]
+Parser
+
+[grammar]
+grammar T;
+a : ID* {
+<writeln("$text")>
+};
+ID : 'a'..'z'+;
+WS : (' '|'\n') -> skip;
+
+[start]
+a
 
-The mysterious `@CommentHasStringValue` annotation is a bit of a hack that allows multi-line strings in Java. This kung fu is required so that we can use Java classes rather than StringTemplate group files to specify runtime tests (the legacy system used those and it was hard to get them right). Here are all the [Runtime test descriptors](https://github.com/antlr/antlr4/tree/master/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors) organized into groups.
+[input]
+a b c
+
+[output]
+"""abc
+"""
+```
 
 The grammars are strings representing StringTemplates (`ST` objects) so `<writeln("$text")>` will get replace when the unit test file is generated (`Test.java`, `Test.cs`, ...). The `writeln` template must be defined per target.  Here are all of the 
-[Target templates for runtime tests](https://github.com/antlr/antlr4/tree/master/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates).
+[Target templates for runtime tests](https://github.com/antlr/antlr4/tree/master/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates).  Use triple-quotes `"""` when whitespace matters (usually input/output sections).
 
 ## Requirements
 
+*out of date, at least for mono*
+
 In order to perform the tests on all target languages, you need to have the following languages installed:
 
 * `mono` (e.g., `brew install mono`) on non-Windows boxes (on Windows it uses the Microsoft .net stack). Also must [`xbuild` the runtime](https://github.com/antlr/antlr4/blob/master/doc/releasing-antlr.md) before tests will run; see below
 * `nodejs`
 * Python 2.7
 * Python 3.6
 * Go
-* Swift 4 (via XCode 10.x) tested currently only osx
+* Swift (via XCode) tested currently only osx
 * clang (for C++ target) 
-* 
+
 To **install into local repository** `~/.m2/repository/org/antlr`, do this:
 
 ```bash
-$ export MAVEN_OPTS="-Xmx1G"     # don't forget this on linux
-$ mvn install -DskipTests=true   # make sure all artifacts are visible on this machine
+$ export MAVEN_OPTS="-Xmx1G"  # don't forget this on linux
+$ mvn install -DskipTests     # make sure all artifacts are visible on this machine
 ```
 
 Now, make sure C# runtime is built and installed locally.
@@ -76,51 +80,6 @@ And the result of testing the entire subdirectory:
 
 <img src=images/python3-tests.png width=400>
 
-From `mvn`, on the commandline, you will see:
-
-```bash
-$ cd antlr4
-$ mvn test
-...
--------------------------------------------------------
- T E S T S
--------------------------------------------------------
-Running org.antlr.v4.test.runtime.csharp.TestCompositeLexers
-dir /var/folders/s1/h3qgww1x0ks3pb30l8t1wgd80000gn/T/TestCompositeLexers-1446068612451
-Starting build /usr/bin/xbuild /p:Configuration=Release /var/folders/s1/h3qgww1x0ks3pb30l8t1wgd80000gn/T/TestCompositeLexers-1446068612451/Antlr4.Test.mono.csproj
-dir /var/folders/s1/h3qgww1x0ks3pb30l8t1wgd80000gn/T/TestCompositeLexers-1446068615081
-Starting build /usr/bin/xbuild /p:Configuration=Release /var/folders/s1/h3qgww1x0ks3pb30l8t1wgd80000gn/T/TestCompositeLexers-1446068615081/Antlr4.Test.mono.csproj
-Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 3.451 sec
-Running org.antlr.v4.test.runtime.csharp.TestCompositeParsers
-dir /var/folders/s1/h3qgww1x0ks3pb30l8t1wgd80000gn/T/TestCompositeParsers-1446068615864
-antlr reports warnings from [-visitor, -Dlanguage=CSharp, -o, /var/folders/s1/h3qgww1x0ks3pb30l8t1wgd80000gn/T/TestCompositeParsers-1446068615864, -lib, /var/folders/s1/h3qgww1x0ks3pb30l8t1wgd80000gn/T/TestCompositeParsers-1446068615864, -encoding, UTF-8, /var/folders/s1/h3qgww1x0ks3pb30l8t1wgd80000gn/T/TestCompositeParsers-1446068615864/M.g4]
-...
-[INFO] ------------------------------------------------------------------------
-[INFO] Reactor Summary:
-[INFO] 
-[INFO] ANTLR 4 ............................................ SUCCESS [  0.445 s]
-[INFO] ANTLR 4 Runtime .................................... SUCCESS [  3.392 s]
-[INFO] ANTLR 4 Tool ....................................... SUCCESS [  1.373 s]
-[INFO] ANTLR 4 Maven plugin ............................... SUCCESS [  1.519 s]
-[INFO] ANTLR 4 Runtime Test Annotations ................... SUCCESS [  0.086 s]
-[INFO] ANTLR 4 Runtime Test Processors .................... SUCCESS [  0.014 s]
-[INFO] ANTLR 4 Runtime Tests (2nd generation) ............. SUCCESS [06:39 min]
-[INFO] ANTLR 4 Tool Tests ................................. SUCCESS [  6.922 s]
-[INFO] ------------------------------------------------------------------------
-[INFO] BUILD SUCCESS
-[INFO] ------------------------------------------------------------------------
-[INFO] Total time: 06:53 min
-[INFO] Finished at: 2016-11-16T15:36:56-08:00
-[INFO] Final Memory: 44M/458M
-[INFO] ------------------------------------------------------------------------
-```
-
-Note: That is actually result of running the much faster:
-
-```bash
-mvn -Dparallel=methods -DthreadCount=4 install
-```
-
 ## Running test subsets
 
 *From the `runtime-testsuite` dir*
@@ -190,68 +149,55 @@ Use this to run tests in parallel:
 
 ```bash
 $ export MAVEN_OPTS="-Xmx1G"
-$ mvn -Dparallel=methods -DthreadCount=4 test
+$ mvn -Dparallel=classes -DthreadCount=4 test
 ...
 -------------------------------------------------------
  T E S T S
 -------------------------------------------------------
-Concurrency config is parallel='methods', perCoreThreadCount=true, threadCount=4, useUnlimitedThreads=false
+Concurrency config is parallel='classes', perCoreThreadCount=true, threadCount=4, useUnlimitedThreads=false
 ...
 ```
 
 This can be combined with other `-D` above.
 
 ## Adding a runtime test
 
-To add a new runtime test, first determine which [group of tests](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors) it belongs to. Then, add a new [RuntimeTestDescriptor](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestDescriptor.java) implementation by subclassing one of:
-
-* [BaseParserTestDescriptor](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseParserTestDescriptor.java); see example [APlus](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserExecDescriptors.java#L7).
-* [BaseDiagnosticParserTestDescriptor](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseDiagnosticParserTestDescriptor) if you want to test parser diagnostic output; see [example output](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/FullContextParsingDescriptors.java#L16).
-* [BaseCompositeParserTestDescriptor](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseCompositeParserTestDescriptor.java); see example [BringInLiteralsFromDelegate](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/CompositeParsersDescriptors.java#L11)
-* [BaseLexerTestDescriptor](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseLexerTestDescriptor.java); see example [ActionPlacement](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LexerExecDescriptors.java#L12).
-* [BaseCompositeLexerTestDescriptor](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseCompositeLexerTestDescriptor.java); see example [LexerDelegatorInvokesDelegateRule](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/CompositeLexersDescriptors.java#L11)
+To add a new runtime test, first determine which [group (dir) of tests](https://github.com/antlr/antlr4/blob/master/runtime-testsuite/descriptors/org/antlr/v4/test/runtime/descriptors) it belongs to. Then, add a new descriptor file implementation by filling in one of these (omitting unused sections):
 
+```
+[notes]
+                                
+[type]
 
-Each descriptor object describes the following mandatory elements for the test:
+[grammar]
 
- * the test type
- * the grammar 
- * the start rule
- * the input text to parse or lex
- * the expected output
- * the expected errors
+[slaveGrammar]
 
-Your best bet is to find a similar test in the appropriate group and then copy and paste the descriptor object, creating a new nested class within the test group class. Modify the field definitions to suit your new problem.
+[start]
 
-If you need to create a whole new group of tests, it requires a new descriptor class; call it `XDescriptors`. Then, in each [target subdirectory](https://github.com/antlr/antlr4/tree/master/runtime-testsuite/test/org/antlr/v4/test/runtime), you need to create a new test rig `TestX.java` file:
+[input]
 
-```java
-package org.antlr.v4.test.runtime.java;
+[output]
 
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.ListenersDescriptors;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
+[errors]
 
-@RunWith(Parameterized.class)
-public class TestX extends BaseRuntimeTest {
-	public TestX(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new Base<TARGET>Test());
-	}
+[flags]
 
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(XDescriptors.class, "<TARGET>");
-	}
-}
+[skip]
 ```
 
-where `<TARGET>` is replaced with Java, Cpp, CSharp, Python2, ... in the various subdirectories.
+
+Your best bet is to find a similar test in the appropriate group and then copy and paste the descriptor file, creating a new file within the test group dir. Modify the sections to suit your new problem.
 
 ### Ignoring tests
 
-In order to turn off a test for a particular target, we need to use the `ignore` method. Given a target name, a descriptor object can decide whether to ignore the test. This is not always convenient but it is fully general and works well for the one case we have now where we have to ignore `Visitor` tests in all targets except JavaScript.
+In order to turn off a test for a particular target, we need to use the `skip` section in the descriptor file. For example, the following skips PHP and Dart targets:
+
+```
+[skip]
+PHP
+Dart
+```
 
 ### Target API/library testing
 

doc/building-antlr.md

@@ -46,7 +46,7 @@ $ if [[ "$?" != "0" ]]; then sudo apt install -y maven; fi
 The current maven build seems complicated to me because there is a dependency of the project on itself. The runtime tests naturally depend on the current version being available but it won't compile without the current version.  Once you have the generated/installed jar, mvn builds but otherwise there's a dependency on what you are going to build.  You will get this error when you try to clean but you can ignore it:
 
 ```
-[INFO] ANTLR 4 Runtime Tests (2nd generation) ............. FAILURE [  0.073 s]
+[INFO] ANTLR 4 Runtime Tests (3rd generation) ............. FAILURE [  0.073 s]
 ...
 [ERROR] Plugin org.antlr:antlr4-maven-plugin:4.9.4-SNAPSHOT or one of its dependencies could not be resolved: Could not find artifact org.antlr:antlr4-maven-plugin:jar:4.9.4-SNAPSHOT -> [Help 1]
 ```
@@ -79,8 +79,6 @@ You should see these jars (when building 4.6-SNAPSHOT):
 ```bash
 /Users/parrt/.m2/repository/org/antlr $ find antlr4* -name '*.jar'
 antlr4-maven-plugin/4.6-SNAPSHOT/antlr4-maven-plugin-4.6-SNAPSHOT.jar
-antlr4-runtime-test-annotation-processors/4.6-SNAPSHOT/antlr4-runtime-test-annotation-processors-4.6-SNAPSHOT.jar
-antlr4-runtime-test-annotations/4.6-SNAPSHOT/antlr4-runtime-test-annotations-4.6-SNAPSHOT.jar
 antlr4-runtime-testsuite/4.6-SNAPSHOT/antlr4-runtime-testsuite-4.6-SNAPSHOT-tests.jar
 antlr4-runtime-testsuite/4.6-SNAPSHOT/antlr4-runtime-testsuite-4.6-SNAPSHOT.jar
 antlr4-runtime/4.6-SNAPSHOT/antlr4-runtime-4.6-SNAPSHOT.jar

doc/releasing-antlr.md

@@ -178,7 +178,7 @@ Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antl
 [INFO] ANTLR 4 Maven plugin ............................... SUCCESS [  6.547 s]
 [INFO] ANTLR 4 Runtime Test Annotations ................... SUCCESS [  2.519 s]
 [INFO] ANTLR 4 Runtime Test Processors .................... SUCCESS [  2.385 s]
-[INFO] ANTLR 4 Runtime Tests (2nd generation) ............. SUCCESS [ 15.276 s]
+[INFO] ANTLR 4 Runtime Tests (3rd generation) ............. SUCCESS [ 15.276 s]
 [INFO] ANTLR 4 Tool Tests ................................. SUCCESS [  2.233 s]
 [INFO] ------------------------------------------------------------------------
 [INFO] BUILD SUCCESS

docker/Dockerfile

@@ -2,7 +2,7 @@ FROM adoptopenjdk/openjdk11:alpine AS builder
 
 WORKDIR /opt/antlr4
 
-ARG ANTLR_VERSION="4.9.3"
+ARG ANTLR_VERSION="4.9.4"
 ARG MAVEN_OPTS="-Xmx1G"