Add exponential backoff when downloading the metadata repository

common/utils/src/main/java/org/graalvm/buildtools/utils/ExponentialBackoff.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2003-2021 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.graalvm.buildtools.utils;
+
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+
+/**
+ * An utility class for exponential backoff of operations which
+ * can fail and can be retried.
+ */
+public class ExponentialBackoff {
+    private static final int DEFAULT_MAX_RETRIES = 3;
+    private static final Duration INITIAL_WAIT_PERIOD = Duration.of(250, ChronoUnit.MILLIS);
+
+    private final int maxRetries;
+    private final Duration initialWaitPeriod;
+
+    public ExponentialBackoff() {
+        this(DEFAULT_MAX_RETRIES, INITIAL_WAIT_PERIOD);
+    }
+
+    private ExponentialBackoff(int maxRetries, Duration initialWaitPeriod) {
+        if (maxRetries < 1) {
+            throw new IllegalArgumentException("Max retries must be at least 1");
+        }
+        if (initialWaitPeriod.isNegative() || initialWaitPeriod.isZero()) {
+            throw new IllegalArgumentException("Initial backoff wait delay must be strictly positive");
+        }
+        this.maxRetries = maxRetries;
+        this.initialWaitPeriod = initialWaitPeriod;
+    }
+
+    public static ExponentialBackoff get() {
+        return new ExponentialBackoff();
+    }
+
+    /**
+     * The maximum number of retries.
+     *
+     * @return an exponential backoff with the specified number of retries
+     */
+    public ExponentialBackoff withMaxRetries(int maxRetries) {
+        return new ExponentialBackoff(maxRetries, initialWaitPeriod);
+    }
+
+    /**
+     * The initial backoff duration, that is to say the time we will wait
+     * before the first retry (there's no wait for the initial attempt).
+     *
+     * @param duration the duration for the first retry
+     * @return an exponential backoff with the specified initial wait period
+     */
+    public ExponentialBackoff withInitialWaitPeriod(Duration duration) {
+        return new ExponentialBackoff(maxRetries, duration);
+    }
+
+    /**
+     * Executes an operation which returns a result. Retries a maximum number of
+     * times by multiplying the delay between each attempt by 2.
+     * @param supplier the operation to execute
+     * @return the result of the operation
+     * @param <T> the type of the result
+     */
+    public <T> T supply(FailableSupplier<T> supplier) {
+        int attempts = maxRetries + 1;
+        Duration waitPeriod = initialWaitPeriod;
+        Exception last = null;
+        while (attempts > 0) {
+            try {
+                return supplier.get();
+            } catch (Exception ex) {
+                last = ex;
+                attempts--;
+                try {
+                    Thread.sleep(waitPeriod.toMillis());
+                    waitPeriod = waitPeriod.multipliedBy(2);
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    throw new RetriableOperationFailedException("Thread was interrupted", e);
+                }
+            }
+        }
+        throw new RetriableOperationFailedException("Operation failed after " + maxRetries + " retries", last);
+    }
+
+    /**
+     * Executes an operation which doesn't return any result, until it passes,
+     * with this exponential backoff parameters.
+     * See {@link #supply(FailableSupplier)} for an operation which returns a result.
+     * @param operation the operation to execute.
+     */
+    public void execute(FailableOperation operation) {
+        supply(() -> {
+            operation.run();
+            return null;
+        });
+    }
+
+    @FunctionalInterface
+    public interface FailableOperation {
+        void run() throws Exception;
+    }
+
+    @FunctionalInterface
+    public interface FailableSupplier<T> {
+        T get() throws Exception;
+    }
+
+    public static final class RetriableOperationFailedException extends RuntimeException {
+                public RetriableOperationFailedException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+}

common/utils/src/test/java/org/graalvm/buildtools/utils/ExponentialBackoffTest.java

@@ -0,0 +1,69 @@
+package org.graalvm.buildtools.utils;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ExponentialBackoffTest {
+    @Test
+    @DisplayName("executed a passing operation")
+    void simpleExecution() {
+        AtomicBoolean success = new AtomicBoolean();
+        ExponentialBackoff.get().execute(() -> success.set(true));
+        assertTrue(success.get());
+    }
+
+    @ParameterizedTest
+    @ValueSource(ints = {1, 3})
+    @DisplayName("retries expected amount of times")
+    void countRetries(int retries) {
+        AtomicInteger count = new AtomicInteger();
+        assertThrows(ExponentialBackoff.RetriableOperationFailedException.class, () -> ExponentialBackoff.get().withMaxRetries(retries)
+            .execute(() -> {
+                count.incrementAndGet();
+                throw new RuntimeException();
+            }));
+        assertEquals(retries + 1, count.get());
+    }
+
+    @ParameterizedTest
+    @ValueSource(ints = {1, 3})
+    @DisplayName("passes after one retry")
+    void passAfterRetry(int retries) {
+        AtomicInteger count = new AtomicInteger();
+        int result = ExponentialBackoff.get().withMaxRetries(retries)
+            .supply(() -> {
+                if (count.getAndIncrement() == 0) {
+                    throw new RuntimeException();
+                }
+                return 200;
+            });
+        assertEquals(2, count.get());
+        assertEquals(200, result);
+    }
+
+    @Test
+    @DisplayName("can configure initial backoff time")
+    void canConfigureInitialBackoffTime() {
+        double sd = System.currentTimeMillis();
+        assertThrows(ExponentialBackoff.RetriableOperationFailedException.class, () -> ExponentialBackoff.get()
+            .withMaxRetries(4)
+            .withInitialWaitPeriod(Duration.of(1, ChronoUnit.MILLIS))
+            .execute(() -> {
+                throw new RuntimeException();
+            }));
+        double duration = System.currentTimeMillis() - sd;
+        assertTrue(duration < 100);
+    }
+
+}

docs/src/docs/asciidoc/index.adoc

@@ -19,6 +19,16 @@ If you are using alternative build systems, see <<alternative-build-systems.adoc
 [[changelog]]
 == Changelog
 
+=== Release 0.10.3
+
+==== Gradle plugin
+
+- Add retries when downloading the metadata repository when using a URL directly
+
+==== Maven plugin
+
+- Add retries when downloading the metadata repository when using a URL directly
+
 === Release 0.10.1
 
 - Mark additional JUnit 5 types for build-time initialization for compatibility with Native Image's `--strict-image-heap` option.

gradle/libs.versions.toml

@@ -1,6 +1,6 @@
 [versions]
 # Project versions
-nativeBuildTools = "0.10.2"
+nativeBuildTools = "0.10.3-SNAPSHOT"
 metadataRepository = "0.3.8"
 
 # External dependencies

native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/NativeImagePlugin.java

@@ -466,6 +466,12 @@ private Provider<GraalVMReachabilityMetadataService> graalVMReachabilityMetadata
                     spec.getParameters().getUri().set(repositoryExtension.getUri().map(serializableTransformerOf(configuredUri -> computeMetadataRepositoryUri(project, repositoryExtension, m -> logFallbackToDefaultUri(m, logger)))));
                     spec.getParameters().getCacheDir().set(
                             new File(project.getGradle().getGradleUserHomeDir(), "native-build-tools/repositories"));
+                    spec.getParameters().getBackoffMaxRetries().convention(
+                        GradleUtils.intProperty(project.getProviders(), "exponential.backoff.max.retries", 3)
+                    );
+                    spec.getParameters().getInitialBackoffMillis().convention(
+                        GradleUtils.intProperty(project.getProviders(), "exponential.backoff.initial.delay", 100)
+                    );
                 });
     }
 

native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/internal/GraalVMReachabilityMetadataService.java

@@ -40,6 +40,7 @@
  */
 package org.graalvm.buildtools.gradle.internal;
 
+import org.graalvm.buildtools.utils.ExponentialBackoff;
 import org.graalvm.buildtools.utils.FileUtils;
 import org.graalvm.reachability.DirectoryConfiguration;
 import org.graalvm.reachability.GraalVMReachabilityMetadataRepository;
@@ -66,6 +67,7 @@
 import java.nio.channels.ReadableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
@@ -85,6 +87,10 @@ public abstract class GraalVMReachabilityMetadataService implements BuildService
     protected abstract FileSystemOperations getFileOperations();
 
     public interface Params extends BuildServiceParameters {
+        Property<Integer> getBackoffMaxRetries();
+
+        Property<Integer> getInitialBackoffMillis();
+
         Property<LogLevel> getLogLevel();
 
         Property<URI> getUri();
@@ -124,14 +130,16 @@ private GraalVMReachabilityMetadataRepository newRepository(URI uri) throws URIS
                         throw new RuntimeException(e);
                     }
                 }
-
-                try (ReadableByteChannel readableByteChannel = Channels.newChannel(uri.toURL().openStream())) {
-                    try (FileOutputStream fileOutputStream = new FileOutputStream(zipped)) {
-                        fileOutputStream.getChannel().transferFrom(readableByteChannel, 0, Long.MAX_VALUE);
-                    }
-                } catch (IOException e) {
-                    throw new RuntimeException(e);
-                }
+                ExponentialBackoff.get()
+                    .withMaxRetries(getParameters().getBackoffMaxRetries().get())
+                    .withInitialWaitPeriod(Duration.ofMillis(getParameters().getInitialBackoffMillis().get()))
+                    .execute(() -> {
+                        try (ReadableByteChannel readableByteChannel = Channels.newChannel(uri.toURL().openStream())) {
+                            try (FileOutputStream fileOutputStream = new FileOutputStream(zipped)) {
+                                fileOutputStream.getChannel().transferFrom(readableByteChannel, 0, Long.MAX_VALUE);
+                            }
+                        }
+                    });
             }
             return newRepositoryFromZipFile(cacheKey, zipped, logLevel);
         }

native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/internal/GradleUtils.java

@@ -48,6 +48,8 @@
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.plugins.JavaPluginExtension;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.util.GradleVersion;
@@ -77,17 +79,28 @@ public static FileCollection transitiveProjectArtifacts(Project project, String
         ConfigurableFileCollection transitiveProjectArtifacts = project.getObjects().fileCollection();
         transitiveProjectArtifacts.from(findMainArtifacts(project));
         transitiveProjectArtifacts.from(findConfiguration(project, name)
-                .getIncoming()
-                .artifactView(view -> view.componentFilter(ProjectComponentIdentifier.class::isInstance))
-                .getFiles());
+            .getIncoming()
+            .artifactView(view -> view.componentFilter(ProjectComponentIdentifier.class::isInstance))
+            .getFiles());
         return transitiveProjectArtifacts;
     }
 
     public static FileCollection findMainArtifacts(Project project) {
         return findConfiguration(project, JavaPlugin.RUNTIME_ELEMENTS_CONFIGURATION_NAME)
-                .getOutgoing()
-                .getArtifacts()
-                .getFiles();
+            .getOutgoing()
+            .getArtifacts()
+            .getFiles();
     }
 
+    public static Provider<Integer> intProperty(ProviderFactory providers, String propertyName, int defaultValue) {
+        return stringProperty(providers, propertyName)
+            .map(Integer::parseInt)
+            .orElse(defaultValue);
+    }
+
+    private static Provider<String> stringProperty(ProviderFactory providers, String propertyName) {
+        return providers.systemProperty(propertyName)
+            .orElse(providers.gradleProperty(propertyName))
+            .orElse(providers.environmentVariable(propertyName.replace('.', '_').toUpperCase()));
+    }
 }

native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/AbstractNativeMojo.java

@@ -59,6 +59,7 @@
 import org.eclipse.aether.resolution.DependencyResult;
 import org.graalvm.buildtools.VersionInfo;
 import org.graalvm.buildtools.maven.config.MetadataRepositoryConfiguration;
+import org.graalvm.buildtools.utils.ExponentialBackoff;
 import org.graalvm.buildtools.utils.FileUtils;
 import org.graalvm.reachability.DirectoryConfiguration;
 import org.graalvm.reachability.GraalVMReachabilityMetadataRepository;
@@ -73,6 +74,7 @@
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.HashSet;
 import java.util.Optional;
 import java.util.Set;
@@ -104,6 +106,12 @@ public abstract class AbstractNativeMojo extends AbstractMojo {
     @Parameter(alias = "metadataRepository")
     protected MetadataRepositoryConfiguration metadataRepositoryConfiguration;
 
+    @Parameter(defaultValue = "3")
+    protected int metadataRepositoryMaxRetries;
+
+    @Parameter(defaultValue = "100")
+    protected int metadataRepositoryInitialBackoffMillis;
+
     protected final Set<DirectoryConfiguration> metadataRepositoryConfigurations;
 
     protected GraalVMReachabilityMetadataRepository metadataRepository;
@@ -207,7 +215,11 @@ private Path getRepo(Path destinationRoot) {
                 }
             }
 
-            return downloadMetadataRepo(destinationRoot, targetUrl);
+            URL finalTargetUrl = targetUrl;
+            return ExponentialBackoff.get()
+                .withMaxRetries(metadataRepositoryMaxRetries)
+                .withInitialWaitPeriod(Duration.ofMillis(metadataRepositoryInitialBackoffMillis))
+                .supply(() -> downloadMetadataRepo(destinationRoot, finalTargetUrl));
         }
     }