Skip to content

Commit

Permalink
[SUREFIRE-2095] Fork crash doesn't fail build with -Dmaven.test.failu…
Browse files Browse the repository at this point in the history
…re.ignore=true when run with failsafe (#545)

Verify goal should fail when test summary indicates a SurefireBooterForkException occurred
  • Loading branch information
br0nstein authored Feb 3, 2023
1 parent 2c8fe40 commit 59e096d
Show file tree
Hide file tree
Showing 8 changed files with 353 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.surefire.api.cli.CommandLineOption;
import org.apache.maven.surefire.api.suite.RunResult;
import org.apache.maven.surefire.booter.SurefireBooterForkException;
import org.codehaus.plexus.logging.Logger;

import java.io.File;
Expand Down Expand Up @@ -198,10 +199,27 @@ public void execute()
throw new MojoExecutionException( e.getMessage(), e );
}

reportExecution( this, summary, getConsoleLogger(), null );
reportExecution( this, summary, getConsoleLogger(), getBooterForkException( summary ) );
}
}

private Exception getBooterForkException( RunResult summary )
{
String firstForkExceptionFailureMessage =
String.format( "%s: " , SurefireBooterForkException.class.getName() );
if ( summary.getFailure() != null && summary.getFailure().contains( firstForkExceptionFailureMessage ) )
{
return new SurefireBooterForkException(
summary.getFailure().substring( firstForkExceptionFailureMessage.length() ) );
}
return null;
}

void setLogger( Logger logger )
{
this.logger = logger;
}

private PluginConsoleLogger getConsoleLogger()
{
if ( consoleLogger == null )
Expand Down Expand Up @@ -359,6 +377,16 @@ public void setReportsDirectory( File reportsDirectory )
this.reportsDirectory = reportsDirectory;
}

public File getSummaryFile()
{
return summaryFile;
}

public void setSummaryFile( File summaryFile )
{
this.summaryFile = summaryFile;
}

@Override
public boolean getFailIfNoTests()
{
Expand All @@ -383,6 +411,16 @@ public void setFailOnFlakeCount( int failOnFlakeCount )
this.failOnFlakeCount = failOnFlakeCount;
}

public MavenSession getSession()
{
return session;
}

public void setSession( MavenSession session )
{
this.session = session;
}

private boolean existsSummaryFile()
{
return summaryFile != null && summaryFile.isFile();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@
@SuiteClasses( {
IntegrationTestMojoTest.class,
MarshallerUnmarshallerTest.class,
RunResultTest.class
RunResultTest.class,
VerifyMojoTest.class
} )
@RunWith( Suite.class )
public class JUnit4SuiteTest
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package org.apache.maven.plugin.failsafe;

/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import java.io.File;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLDecoder;

import org.apache.maven.execution.MavenExecutionRequest;
import org.apache.maven.execution.MavenSession;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.codehaus.plexus.logging.Logger;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

/**
*/
public class VerifyMojoTest
{
private VerifyMojo mojo;

@Rule
public TemporaryFolder tempFolder = new TemporaryFolder();

@Before
public void init() throws UnsupportedEncodingException
{
mojo = new VerifyMojo();
mojo.setTestClassesDirectory( tempFolder.getRoot() );
mojo.setReportsDirectory( getTestBaseDir() );
}

private void setupExecuteMocks()
{
Logger logger = mock( Logger.class );
when( logger.isErrorEnabled() ).thenReturn( true );
when( logger.isWarnEnabled() ).thenReturn( true );
when( logger.isInfoEnabled() ).thenReturn( true );
when( logger.isDebugEnabled() ).thenReturn( false );
mojo.setLogger( logger );

MavenSession session = mock( MavenSession.class );
MavenExecutionRequest request = mock ( MavenExecutionRequest.class );
when( request.isShowErrors() ).thenReturn( true );
when( request.getReactorFailureBehavior() ).thenReturn( null );
when( session.getRequest() ).thenReturn( request );
mojo.setSession( session );
}

private File getTestBaseDir()
throws UnsupportedEncodingException
{
URL resource = getClass().getResource( "/verify-mojo" );
// URLDecoder.decode necessary for JDK 1.5+, where spaces are escaped to %20
return new File( URLDecoder.decode( resource.getPath(), "UTF-8" ) ).getAbsoluteFile();
}

@Test( expected = MojoExecutionException.class )
public void executeForForkError() throws MojoExecutionException, MojoFailureException, UnsupportedEncodingException
{
setupExecuteMocks();
mojo.setSummaryFile( new File( getTestBaseDir(), "failsafe-summary-booter-fork-error.xml" ) );
mojo.execute();
}

@Test( expected = MojoExecutionException.class )
public void executeForForkErrorTestFailureIgnore() throws MojoExecutionException, MojoFailureException,
UnsupportedEncodingException
{
setupExecuteMocks();
mojo.setSummaryFile( new File( getTestBaseDir(), "failsafe-summary-booter-fork-error.xml" ) );
mojo.setTestFailureIgnore( true );
mojo.execute();
}

@Test
public void executeForPassingTests() throws MojoExecutionException, MojoFailureException,
UnsupportedEncodingException
{
setupExecuteMocks();
mojo.setSummaryFile( new File( getTestBaseDir(), "failsafe-summary-success.xml" ) );
mojo.execute();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
<?xml version="1.0" encoding="UTF-8"?>
<failsafe-summary xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://maven.apache.org/surefire/maven-surefire-plugin/xsd/failsafe-summary.xsd" result="254" timeout="false">
<completed>0</completed>
<errors>0</errors>
<failures>0</failures>
<skipped>0</skipped>
<failureMessage>org.apache.maven.surefire.booter.SurefireBooterForkException: The forked VM terminated without properly saying goodbye. VM crash or System.exit called?
Command was /bin/sh -c cd &apos;/Users/aaron.braunstein/git/apache/maven-surefire/surefire-its/target/TestClass_testMethod&apos; &amp;&amp; &apos;/Library/Java/JavaVirtualMachines/temurin-11.jdk/Contents/Home/bin/java&apos; &apos;-Dfile.encoding=UTF-8&apos; &apos;-Duser.language=en&apos; &apos;-XFakeUnrecognizedOptionThatWillCrashJVM&apos; &apos;-Duser.region=US&apos; &apos;-showversion&apos; &apos;-Xmx6g&apos; &apos;-Xms2g&apos; &apos;-XX:+PrintGCDetails&apos; &apos;-jar&apos; &apos;/Users/aaron.braunstein/git/apache/maven-surefire/surefire-its/target/TestClass_testMethod/target/surefire/surefirebooter-20220606220315261_3.jar&apos; &apos;/Users/aaron.braunstein/git/apache/maven-surefire/surefire-its/target/TestClass_testMethod/target/surefire&apos; &apos;2022-06-06T22-03-11_910-jvmRun1&apos; &apos;surefire-20220606220315261_1tmp&apos; &apos;surefire_0-20220606220315261_2tmp&apos;
Error occurred in starting fork, check output in log
Process Exit Code: 1
at org.apache.maven.plugin.surefire.booterclient.ForkStarter.fork(ForkStarter.java:714)
at org.apache.maven.plugin.surefire.booterclient.ForkStarter.run(ForkStarter.java:311)
at org.apache.maven.plugin.surefire.booterclient.ForkStarter.run(ForkStarter.java:268)
at org.apache.maven.plugin.surefire.AbstractSurefireMojo.executeProvider(AbstractSurefireMojo.java:1334)
at org.apache.maven.plugin.surefire.AbstractSurefireMojo.executeAfterPreconditionsChecked(AbstractSurefireMojo.java:1167)
at org.apache.maven.plugin.surefire.AbstractSurefireMojo.execute(AbstractSurefireMojo.java:931)
at org.apache.maven.plugin.DefaultBuildPluginManager.executeMojo(DefaultBuildPluginManager.java:137)
at org.apache.maven.lifecycle.internal.MojoExecutor.doExecute(MojoExecutor.java:301)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:211)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:165)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:157)
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:121)
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:81)
at org.apache.maven.lifecycle.internal.builder.singlethreaded.SingleThreadedBuilder.build(SingleThreadedBuilder.java:56)
at org.apache.maven.lifecycle.internal.LifecycleStarter.execute(LifecycleStarter.java:127)
at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:294)
at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:192)
at org.apache.maven.DefaultMaven.execute(DefaultMaven.java:105)
at org.apache.maven.cli.MavenCli.execute(MavenCli.java:960)
at org.apache.maven.cli.MavenCli.doMain(MavenCli.java:293)
at org.apache.maven.cli.MavenCli.main(MavenCli.java:196)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced(Launcher.java:282)
at org.codehaus.plexus.classworlds.launcher.Launcher.launch(Launcher.java:225)
at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode(Launcher.java:406)
at org.codehaus.plexus.classworlds.launcher.Launcher.main(Launcher.java:347)
</failureMessage>
</failsafe-summary>
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<failsafe-summary xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://maven.apache.org/surefire/maven-surefire-plugin/xsd/failsafe-summary.xsd" result="254" timeout="false">
<completed>1</completed>
<errors>0</errors>
<failures>0</failures>
<skipped>0</skipped>
<failureMessage xsi:nil="true" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"/>
</failsafe-summary>
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package org.apache.maven.surefire.its.jiras;

/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import org.apache.maven.it.VerificationException;
import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
import org.junit.Test;

import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.is;

/**
* Test https://issues.apache.org/jira/browse/SUREFIRE-2095
*
*/
public class Surefire2095FailsafeJvmCrashShouldNotBeIgnoredIT
extends SurefireJUnit4IntegrationTestCase
{
@Test
public void mavenShouldFail() throws VerificationException
{
// Run failsafe with testFailureIgnore=true and an unknown JVM option that will cause a crash
unpack( "surefire-2095-failsafe-jvm-crash" )
.maven()
.withFailure()
.debugLogging()
.executeVerify()
.assertThatLogLine( containsString( "BUILD SUCCESS" ), is( 0 ) )
.verifyTextInLog( "BUILD FAILURE" );
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>org.apache.maven.plugins.surefire</groupId>
<artifactId>SUREFIRE-2095</artifactId>
<version>1.0-SNAPSHOT</version>
<name>SUREFIRE-2095</name>

<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>${surefire.version}</version>
<configuration>
<argLine>-Dfile.encoding=UTF-8 -Duser.language=en -XFakeUnrecognizedOptionThatWillCrashJVM -Duser.region=US -showversion -Xmx6g -Xms2g -XX:+PrintGCDetails</argLine>
<testFailureIgnore>true</testFailureIgnore>
</configuration>
<executions>
<execution>
<id>integration-test</id>
<goals>
<goal>integration-test</goal>
</goals>
</execution>
<execution>
<id>verify</id>
<goals>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

public class PojoIT
{
private static int calls;

public void setUp()
{
System.out.println( "setUp called " + ++calls );
}

public void tearDown()
{
System.out.println( "tearDown called " + calls );
}

public void testSuccess()
{
assert true;
}

public void testFailure()
{
assert false;
}

}

0 comments on commit 59e096d

Please sign in to comment.