From 3d2d1dc2ddec143cc510adcf4e912aaa2ef3e3b0 Mon Sep 17 00:00:00 2001 From: Matt Sicker Date: Thu, 15 Nov 2018 14:58:16 -0600 Subject: [PATCH] Add test retry logic Signed-off-by: Matt Sicker --- core/src/test/java/hudson/UtilTest.java | 6 +- core/src/test/java/jenkins/junit/Retry.java | 46 +++++++++++++ .../test/java/jenkins/junit/RetryRule.java | 65 +++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 core/src/test/java/jenkins/junit/Retry.java create mode 100644 core/src/test/java/jenkins/junit/RetryRule.java diff --git a/core/src/test/java/hudson/UtilTest.java b/core/src/test/java/hudson/UtilTest.java index f537bc7b9b..712151e36d 100644 --- a/core/src/test/java/hudson/UtilTest.java +++ b/core/src/test/java/hudson/UtilTest.java @@ -53,11 +53,12 @@ import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.startsWith; import static org.junit.Assert.*; +import jenkins.junit.Retry; +import jenkins.junit.RetryRule; import org.apache.commons.io.FileUtils; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.junit.Assume; -import org.junit.Ignore; import org.junit.Test; import org.jvnet.hudson.test.Issue; @@ -74,6 +75,7 @@ import com.google.common.collect.Lists; public class UtilTest { @Rule public TemporaryFolder tmp = new TemporaryFolder(); + @Rule public RetryRule retry = new RetryRule(); @Test public void testReplaceMacro() { @@ -353,7 +355,7 @@ public class UtilTest { } @Test - @Ignore("Frequently flaky test") + @Retry public void testDeleteContentsRecursive_onWindows() throws Exception { Assume.assumeTrue(Functions.isWindows()); final File dir = tmp.newFolder(); diff --git a/core/src/test/java/jenkins/junit/Retry.java b/core/src/test/java/jenkins/junit/Retry.java new file mode 100644 index 0000000000..affe1f17b5 --- /dev/null +++ b/core/src/test/java/jenkins/junit/Retry.java @@ -0,0 +1,46 @@ +/* + * The MIT License + * + * Copyright (c) 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package jenkins.junit; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Combined with {@link RetryRule}, retries failing test methods. Note that tests that fail with + * {@link org.junit.AssumptionViolatedException} are not retried. + */ +@Target({ElementType.METHOD, ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +public @interface Retry { + /** + * Number of times to retry test method or individual tests in a class before giving up. + * Set this to 0 to disable retries (e.g., to override a test class's @Retry annotation or similar). + */ + int value() default 2; +} diff --git a/core/src/test/java/jenkins/junit/RetryRule.java b/core/src/test/java/jenkins/junit/RetryRule.java new file mode 100644 index 0000000000..5697c91998 --- /dev/null +++ b/core/src/test/java/jenkins/junit/RetryRule.java @@ -0,0 +1,65 @@ +/* + * The MIT License + * + * Copyright (c) 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package jenkins.junit; + +import org.junit.AssumptionViolatedException; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +public class RetryRule implements TestRule { + + @Override + public Statement apply(Statement base, Description description) { + Retry retryForAllMethods = description.getTestClass().getAnnotation(Retry.class); + Retry retryMethod = description.getAnnotation(Retry.class); + int retries = 0; + if (retryMethod != null) { + retries += retryMethod.value(); + } else if (retryForAllMethods != null) { + retries += retryForAllMethods.value(); + } + int maxEvaluationAttempts = retries + 1; + return new Statement() { + @Override + public void evaluate() throws Throwable { + Throwable caught = null; + for (int i = 0; i < maxEvaluationAttempts; i++) { + try { + base.evaluate(); + return; + } catch (AssumptionViolatedException e) { + return; + } catch (Throwable e) { + e.printStackTrace(); + caught = e; + } + } + System.err.println(description.getDisplayName() + " retry count exhausted"); + throw caught; + } + }; + } +}