Add tracking of Java coverage

.travis.yml

@@ -5,6 +5,7 @@ sudo: false
 env:
   global:
     - secure: PrFbSr/qjhVXrpiOzrcEMUZrnjEWJmvx5DC9QdaAmnUKpG9/7mJBQVgNziVj+GWfW84WJpkOsQf+I2lx7/cQsWQKad/t8wtypl30wGesSTpAgl5fCiKLOBAbOghKXir+WIaxtcATPPffur9OLh3ogEcmIQbVi682YUKmA5zF11JZdpCR4QONc/u+DqB29FuHru/cFiesYP0Oz82A+M0UtMcYsurKIxdKMD4YK/uSG892PUrcZU6STXlukhgQuy3PitSWkYV2KGxXMVKzWGM7dJvRggN05r/S871pscuRwZ+Doxqr9b17B3umCHi3i4KXmNH+Esb0p1mvegs0iS/b7RyA5SENre+H24n3SOeXTa3wSpTnF90XxQrDEBbY5wV7lN7MJG+pHxkOvoZt6pS3f7x2VYR8Joa4J+Gf6IDvxZMiCd1v3N1kc9ygyvJmHf5wDmLMdupk0/frojApDXfJT6bqiVL3S0FqZpXSPGAsKYf8wfn30Xz/YUBsnfUQ/a21Zz52+OTqPbt32Hf1FGYIEJSkZJUN90Q8rHVJt9zPg37xKCDuf6bxlvT040KSzuuXtizLkOnHq2rhg4Oad/JTw3d4NzPoRVzUI9qDKPrA7RdUAjmrB04Z1f3g/I6w3h2B9JTSFAzBcMZ5NYZhIqE31GQukgStaqC98y32/zo9xFs=
+    - secure: "WalAXxkaD+B+mMBN+1QXrsQVl90984+giSLl7dgPwTPdQ/Ka/8AlX4n53HATztgmjwgJbM+Y53YhoSR6Zaj8CRUL6WhXGwNA2q7Xma8q0WNmX8LmoBsJlw44G50yGQOv20vTRw4SqPJp58f4PQanAAs4iPuNBOMGYji4eg60pC1bV5ZNyB5HalxQQQR5PyV647Ooa4WmP3YYLlx4FWqhh33kM3pt9RUOw8ZmVYwOqZ54qJur8Tl3Hp33R7rQ/YMNIenR1wnRunNlV/g8O8xLzUM2qjW1p+XKK08SVI/EyDn1ZtNxyzuhBp1OYxwPVLD28I4r6e/nIEr8+HnBeaUX3y43AjL6x/rzr4KTyPeCNhOYLgcdLM3A5tO/ae+KPKWXBXOwkmxjq7aF6v1Qj5uC/X1Vyb8u6EZETbrx7yNymBHyWpqM4OfljCFyU1c1JjOjsEkhWzE9YoNzhG2bMTBbpUz1gSA/6UCqjhK+3LzneTR/ZmN4RmMlsqTM98E6SMiaGGhqTENIu1UwuPpzl4pv0LJdGoxvR+2aIOZrUI/mD8iuqNoy/gQGU36OiuKencbAVjXBOJXjZueW/VcJ4ivX6Ch4rnFONLKvL2z04cu9tD8IJQfJEmjEieSI44ZCh0LXrMZRJ9bjLpIbhnRbtZ4LP6eBCOtxOwiyKIoueN1y8Fk="
 matrix:
   fast_finish: true
   include:

build.xml

@@ -1,4 +1,5 @@
-<project name="som" basedir="." default="compile-all">
+<project name="som" basedir="." default="compile-all"
+    xmlns:jacoco="antlib:org.jacoco.ant">
 
     <property name="src.dir"     value="src"/>
     <property name="src_gen.dir" value="src_gen"/>
@@ -11,15 +12,16 @@
     <property name="truffle.build" location="${truffle.dir}/mxbuild/dists" />
     <property name="somns-deps.version" value="0.3.3" />
     <property name="checkstyle.version" value="7.6.1" />
-
+    <property name="jacoco.version"     value="0.8.0" />
+
     <!-- <property name="lib.url" value="https://dl.bintray.com/smarr/SOM" /> -->
     <property name="lib.url" value="http://stefan-marr.de/dev" />
-    
+
     <property name="mx.cmd" value="../../mx/mx" />
 
     <property name="build.dir"   value="build"/>
     <property name="classes.dir" value="${build.dir}/classes"/>
-    
+
     <property environment="env"/>
 
     <path id="project.classpath">
@@ -40,18 +42,18 @@
         <delete dir="${build.dir}"/>
         <delete dir="${src_gen.dir}"/>
     </target>
-    
+
     <target name="clean-truffle" depends="check-truffle-available" if="truffle.present">
         <exec executable="${mx.cmd}" dir="${truffle.dir}">
             <arg value="clean"/>
         </exec>
     </target>
-    
+
     <target name="clobber" description="Do clean, and also clean truffle build" depends="clean,clean-truffle">
         <delete dir="${kompos.dir}/out"/>
         <delete dir="${kompos.dir}/node_modules"/>
     </target>
-    
+
     <target name="check-truffle-available">
         <available file="${lib.dir}/truffle/.git" property="truffle.present"/>
     </target>
@@ -67,7 +69,7 @@
             <arg value="--no-native"/>
         </exec>
     </target>
-    
+
     <target name="bd-libs"> <!-- implicit dependency on truffle-libs -->
         <ant dir="${bd.dir}" useNativeBasedir="true" target="libs-junit" inheritAll="false"/>
         <ant dir="${bd.dir}" useNativeBasedir="true" target="compile-nodeps" inheritAll="false">
@@ -121,7 +123,7 @@
             <arg line="${javafiles}"/>
         </exec>
     </target>
-    
+
     <target name="eclipseformat-check" depends="eclipseformat">
         <exec executable="git" dir="${basedir}">
             <arg value="status" />
@@ -141,7 +143,7 @@
             usetimestamp="true"
             dest="${lib.dir}/checkstyle-${checkstyle.version}-all.jar" />
     </target>
-    
+
     <target name="checkstyle" depends="checkstyle-jar" description="Check Code with Checkstyle">
         <taskdef resource="com/puppycrawl/tools/checkstyle/ant/checkstyle-ant-task.properties" classpath="${lib.dir}/checkstyle-${checkstyle.version}-all.jar" />
         <checkstyle config=".checkstyle_checks.xml">
@@ -150,6 +152,22 @@
         </checkstyle>
     </target>
 
+    <target name="jacoco-lib" description="Get JaCoCo dependency">
+        <get src="http://search.maven.org/remotecontent?filepath=org/jacoco/jacoco/${jacoco.version}/jacoco-${jacoco.version}.zip"
+            usetimestamp="true"
+            dest="${lib.dir}/jacoco-${jacoco.version}.zip" />
+        <unzip src="${lib.dir}/jacoco-${jacoco.version}.zip" dest="${lib.dir}/jacoco"/>
+        <taskdef uri="antlib:org.jacoco.ant"
+            resource="org/jacoco/ant/antlib.xml"
+            classpath="${lib.dir}/jacoco/lib/jacocoant.jar" />
+    </target>
+
+    <target name="codacy-coverage-lib" description="Report Coverage to Codacy">
+        <get src="https://oss.sonatype.org/service/local/repositories/releases/content/com/codacy/codacy-coverage-reporter/2.0.2/codacy-coverage-reporter-2.0.2-assembly.jar"
+            usetimestamp="true"
+            dest="${lib.dir}/codacy-coverage-reporter.jar" />
+    </target>
+
     <target name="compile-som" description="Compile SOMns">
         <mkdir dir="${build.dir}"/>
         <mkdir dir="${classes.dir}" />
@@ -169,7 +187,7 @@
           <classpath refid="project.classpath" />
         </javac>
     </target>
-    
+
     <target name="compile" depends="libs,compile-som" description="Compile SOMns and dependencies">
     </target>
 
@@ -179,17 +197,18 @@
             <arg value="install"/>
         </exec>
     </target>
-    
+
     <target name="compile-all" depends="compile,kompos" description="Build SOMns and Kompos">
     </target>
 
     <target name="jar" depends="compile" description="Package as JAR">
         <jar destfile="${build.dir}/som.jar" basedir="${classes.dir}"></jar>
     </target>
-
-    <target name="unit-tests" depends="compile" description="Execute tests">
+
+    <target name="unit-tests" depends="compile,jacoco-lib" description="Execute tests">
+      <jacoco:coverage>
         <junit haltonerror="false" haltonfailure="false" failureproperty="test.failed"
-            outputtoformatters="true">
+            outputtoformatters="true" fork="true" forkmode="once">
             <jvmarg value="-ea" />
             <jvmarg value="-esa" />
             <classpath refid="project.classpath" />
@@ -201,32 +220,44 @@
             </batchtest>
             <formatter type="brief" usefile="false"/>
         </junit>
+     </jacoco:coverage>
 
+     <jacoco:coverage>
         <java classname="som.VM" fork="true" failonerror="true">
             <classpath refid="project.classpath" />
             <jvmarg value="-ea" />
             <jvmarg value="-esa" />
             <arg line="core-lib/TestSuite/TestRunner.ns" />
         </java>
-        <fail message="Basic tests failed." if="test.failed" />
+     </jacoco:coverage>
+
+     <fail message="Basic tests failed." if="test.failed" />
     </target>
 
     <target name="som-tests" depends="compile">
-      <delete file="all.gcov" /> <!-- delete old coverage data, this should be the first task generating coverage data -->
+      <!-- delete old coverage data, this should be the first task generating coverage data -->
+      <delete file="all.gcov" />
+      <delete file="jacoco.exec" />
+
       <exec executable="./som" failonerror="true">
         <arg value="--coverage" />
         <arg value="all.gcov" />
+        <arg value="--java-coverage" />
+        <arg value="jacoco.exec" />
         <arg value="-G" />
         <arg value="core-lib/TestSuite/TestRunner.ns" />
       </exec>
+
       <exec executable="./som" failonerror="true">
         <arg value="-X" />
         <arg value="core-lib/TestSuite/TestRunner.ns" />
       </exec>
     </target>
 
     <target name="dynamic-metrics-tests" depends="compile-som">
-      <exec executable="tests/dym/test.sh" failonerror="true"></exec>
+      <exec executable="tests/dym/test.sh" failonerror="true">
+          <arg value="--coverage" />
+      </exec>
     </target>
 
     <target name="superinstructions-tests" depends="compile-som">
@@ -241,11 +272,43 @@
           <arg value="2" />
       </exec>
     </target>
-    
+
     <target name="core-tests" depends="unit-tests,som-tests,dynamic-metrics-tests,superinstructions-tests">
     </target>
 
-    <target name="coverage">
+    <target name="send-java-coverage" depends="jacoco-lib,codacy-coverage-lib">
+      <jacoco:report>
+        <executiondata>
+          <file file="jacoco.exec"/>
+        </executiondata>
+
+        <structure name="SOMns coverage">
+          <classfiles>
+            <fileset dir="${classes.dir}"/>
+            <fileset dir="${bd.dir}/build/classes"/>
+          </classfiles>
+          <sourcefiles encoding="UTF-8">
+            <fileset dir="src"/>
+            <fileset dir="${bd.dir}/src"/>
+          </sourcefiles>
+        </structure>
+
+        <!-- <html destdir="report"/> -->
+        <xml destfile="jacoco.xml"/>
+      </jacoco:report>
+
+      <java classname="com.codacy.CodacyCoverageReporter">
+        <classpath>
+          <pathelement location="${lib.dir}/codacy-coverage-reporter.jar" />
+        </classpath>
+        <arg value="-l" />
+        <arg value="Java" />
+        <arg value="-r" />
+        <arg value="jacoco.xml" />
+      </java>
+    </target>
+
+    <target name="send-somns-coverage">
         <!-- submit coverage data -->
         <java classname="coveralls.Report" fork="true" failonerror="true">
             <classpath refid="project.classpath" />
@@ -254,6 +317,8 @@
         </java>
     </target>
 
+    <target name="coverage" depends="send-java-coverage,send-somns-coverage" />
+
     <target name="tests" depends="core-tests,replay-tests,coverage">
     </target>
 </project>

docs/infrastructure.md

@@ -79,5 +79,9 @@ The benchmark execution is configured with `codespeed.conf` and are executed
 with the [ReBench](https://github.com/smarr/ReBench) tool.
 
 **SOMns Code Coverage:**
-To track SOMns code coverage, we use
-[Coveralls](https://coveralls.io/github/smarr/SOMns).
+To track code coverage of the SOMns code, we use
+[Coveralls](https://coveralls.io/github/smarr/SOMns).
+
+**Java Code Coverage:**
+To track code coverage of the Java code, we use
+[Codacy](https://app.codacy.com/app/smarr/SOMns/dashboard).

som

@@ -83,9 +83,10 @@ tools.add_argument('-TF', '--disable-trace-file', help='trace file wont be writt
                     dest='disable_trace_file', action='store_true', default=False)
 tools.add_argument('-r', '--replay', help='execution of the program is guided by an existing trace file',
                     dest='replay', action='store_true', default=False)
-tools.add_argument('--coverage', help='determine code coverage and store in given file',
+tools.add_argument('--coverage', help='determine SOMns code coverage and store in given file',
                    dest='coverage', default=None)
-
+tools.add_argument('--java-coverage', help='determine Java code coverage and store in given file',
+                   dest='java_coverage', default=None)
 
 parser.add_argument('-o', '--only', help='only compile give methods, comma separated list',
                     dest='only_compile', default=None)
@@ -248,6 +249,9 @@ if args.truffle_profile:
 if args.coverage:
     SOM_ARGS += ['--coverage', args.coverage]
 
+if args.java_coverage:
+    flags += ['-javaagent:libs/jacoco/lib/jacocoagent.jar=inclbootstrapclasses=true,destfile=' + args.java_coverage]
+
 if args.web_debugger:
     SOM_ARGS += ['--web-debug']
 if args.dynamic_metrics:

tests/dym/test.sh

@@ -13,6 +13,11 @@ popd > /dev/null
 
 SOM_DIR=$SCRIPT_PATH/../..
 
+if [ "$1" = "--coverage" ]
+then
+  COVERAGE="--java-coverage $SOM_DIR/jacoco.exec --coverage $SOM_DIR/all.gcov "
+fi
+
 ## create folder for new results
 mkdir -p $SCRIPT_PATH/results/
 
@@ -22,7 +27,7 @@ tar xf $SCRIPT_PATH/expected-results.tar.bz2 -C $SCRIPT_PATH/
 function runBenchmark {
   BENCH=$1
   HARNESS="$SOM_DIR/som -dm -Ddm.metrics=$SCRIPT_PATH/results/$BENCH \
-    --coverage $SOM_DIR/all.gcov \
+    $COVERAGE \
     -G $SOM_DIR/core-lib/Benchmarks/Harness.ns"
   echo $HARNESS $@
   $HARNESS $@

tests/java/som/tests/VMTests.java

@@ -58,10 +58,15 @@ public void testProcessArgumentsWithAppFileAndArgs() {
 
   @Test
   public void testProfileFlag() {
-    VmOptions opts = new VmOptions(
-        new String[] {"--profile"});
-    assertEquals(opts.platformFile, VmOptions.STANDARD_PLATFORM_FILE);
-    assertTrue(opts.profilingEnabled);
-    assertArrayEquals(opts.args, new String[0]);
+    try {
+      VmOptions opts = new VmOptions(
+          new String[] {"--profile"});
+      assertEquals(opts.platformFile, VmOptions.STANDARD_PLATFORM_FILE);
+      assertTrue(opts.profilingEnabled);
+      assertArrayEquals(opts.args, new String[0]);
+    } catch (IllegalStateException e) {
+      assertTrue("If it fails, expect that it complains that instrumentation is not enabled",
+          e.getMessage().contains(VmSettings.INSTRUMENTATION_PROP));
+    }
   }
 }