Update http-nio and wire its new settings

* it now has a retry mechanism so we're asking for a lot of retries

build.gradle

@@ -336,7 +336,7 @@ dependencies {
     implementation 'org.broadinstitute:gatk-bwamem-jni:1.0.4'
     implementation 'org.broadinstitute:gatk-fermilite-jni:1.2.0'
 
-    implementation 'org.broadinstitute:http-nio:0.1.0-rc1'
+    implementation 'org.broadinstitute:http-nio:1.0.0'
 
     // Required for COSMIC Funcotator data source:
     implementation 'org.xerial:sqlite-jdbc:3.44.1.0'

src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java

@@ -25,18 +25,17 @@
 import org.broadinstitute.hellbender.utils.Utils;
 import org.broadinstitute.hellbender.utils.io.IOUtils;
 import org.broadinstitute.http.nio.HttpFileSystemProvider;
+import org.broadinstitute.http.nio.HttpFileSystemProviderSettings;
 import org.broadinstitute.http.nio.HttpsFileSystemProvider;
 import com.google.api.gax.retrying.RetrySettings;
 import com.google.auth.oauth2.GoogleCredentials;
 import com.google.cloud.http.HttpTransportOptions;
+import org.broadinstitute.http.nio.RetryHandler;
 import org.threeten.bp.Duration;
 
 import java.io.*;
 import java.net.URL;
 import java.nio.channels.SeekableByteChannel;
-import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.net.URL;
 import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.UUID;
@@ -55,6 +54,8 @@ public final class BucketUtils {
 
     // slashes omitted since hdfs paths seem to only have 1 slash which would be weirder to include than no slashes
     public static final String FILE_PREFIX = "file:";
+    public static final int MAX_TIMEOUT = 120000;
+    public static final int MAX_ATTEMPTS = 15;
 
     private BucketUtils(){} //private so that no one will instantiate this class
 
@@ -387,14 +388,33 @@ public static String getPathWithoutBucket(String path) {
      * Sets max_reopens, requester_pays, and generous timeouts as the global default.
      * These will apply even to library code that creates its own paths to access with NIO.
      *
-     * @param maxReopens If the GCS bucket channel errors out, how many times it will attempt to
+     * @param maxReopens If the channel errors out, how many times it will attempt to
      *                   re-initiate the connection.
      * @param requesterProject Project to bill when accessing "requester pays" buckets. If unset,
      *                         these buckets cannot be accessed.
      */
     public static void setGlobalNIODefaultOptions(int maxReopens, String requesterProject) {
         CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(getCloudStorageConfiguration(maxReopens, requesterProject));
         CloudStorageFileSystemProvider.setStorageOptions(setGenerousTimeouts(StorageOptions.newBuilder()).build());
+        setGlobalHttpNioDefaultOptions(maxReopens);
+    }
+
+    /**
+     * Set the options for http-nio to have generous timeouts and defaults
+     */
+    public static void setGlobalHttpNioDefaultOptions(final int maxRetries) {
+        final HttpFileSystemProviderSettings.RetrySettings retrySettings = new HttpFileSystemProviderSettings.RetrySettings(
+                maxRetries,
+                RetryHandler.DEFAULT_RETRYABLE_HTTP_CODES,
+                RetryHandler.DEFAULT_RETRYABLE_EXCEPTIONS,
+                RetryHandler.DEFALT_RETRYABLE_MESSAGES,
+                e -> false);
+        final HttpFileSystemProviderSettings settings = new HttpFileSystemProviderSettings(
+                java.time.Duration.ofMillis(MAX_TIMEOUT),
+                HttpFileSystemProviderSettings.DEFAULT_SETTINGS.redirect(),
+                retrySettings);
+
+        HttpFileSystemProvider.setSettings(settings);
     }
 
     /**
@@ -437,11 +457,11 @@ public static CloudStorageConfiguration getCloudStorageConfiguration(int maxReop
     private static StorageOptions.Builder setGenerousTimeouts(StorageOptions.Builder builder) {
         return builder
             .setTransportOptions(HttpTransportOptions.newBuilder()
-                .setConnectTimeout(120000)
-                .setReadTimeout(120000)
+                .setConnectTimeout(MAX_TIMEOUT)
+                .setReadTimeout(MAX_TIMEOUT)
                 .build())
             .setRetrySettings(RetrySettings.newBuilder()
-                .setMaxAttempts(15)
+                .setMaxAttempts(MAX_ATTEMPTS)
                 .setMaxRetryDelay(Duration.ofMillis(256_000L))
                 .setTotalTimeout(Duration.ofMillis(4000_000L))
                 .setInitialRetryDelay(Duration.ofMillis(1000L))