From 3b8b5bf7928a97c31585b828cc25010555eeb4a9 Mon Sep 17 00:00:00 2001
From: Louis Bergelson <louisb@broadinstitute.org>
Date: Fri, 8 Dec 2023 19:36:23 -0500
Subject: [PATCH] Update http-nio and wire its new settings (#8611)

* Update http-nio and wire it so it's configured at startup along with GCS setttings.
---
 build.gradle                                  |  2 +-
 .../hellbender/utils/gcs/BucketUtils.java     | 31 +++++++++++++++----
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/build.gradle b/build.gradle
index fa68235fae8..55bb2dca57c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -336,7 +336,7 @@ dependencies {
     implementation 'org.broadinstitute:gatk-bwamem-jni:1.0.4'
     implementation 'org.broadinstitute:gatk-fermilite-jni:1.2.0'
 
-    implementation 'org.broadinstitute:http-nio:0.1.0-rc1'
+    implementation 'org.broadinstitute:http-nio:1.0.0'
 
     // Required for COSMIC Funcotator data source:
     implementation 'org.xerial:sqlite-jdbc:3.44.1.0'
diff --git a/src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java b/src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java
index 43f4ab688fd..9eeaffb6223 100644
--- a/src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java
+++ b/src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java
@@ -25,18 +25,17 @@
 import org.broadinstitute.hellbender.utils.Utils;
 import org.broadinstitute.hellbender.utils.io.IOUtils;
 import org.broadinstitute.http.nio.HttpFileSystemProvider;
+import org.broadinstitute.http.nio.HttpFileSystemProviderSettings;
 import org.broadinstitute.http.nio.HttpsFileSystemProvider;
 import com.google.api.gax.retrying.RetrySettings;
 import com.google.auth.oauth2.GoogleCredentials;
 import com.google.cloud.http.HttpTransportOptions;
+import org.broadinstitute.http.nio.RetryHandler;
 import org.threeten.bp.Duration;
 
 import java.io.*;
 import java.net.URL;
 import java.nio.channels.SeekableByteChannel;
-import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.net.URL;
 import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.UUID;
@@ -55,6 +54,7 @@ public final class BucketUtils {
 
     // slashes omitted since hdfs paths seem to only have 1 slash which would be weirder to include than no slashes
     public static final String FILE_PREFIX = "file:";
+    public static final int MAX_TIMEOUT = 120000;
 
     private BucketUtils(){} //private so that no one will instantiate this class
 
@@ -387,7 +387,7 @@ public static String getPathWithoutBucket(String path) {
      * Sets max_reopens, requester_pays, and generous timeouts as the global default.
      * These will apply even to library code that creates its own paths to access with NIO.
      *
-     * @param maxReopens If the GCS bucket channel errors out, how many times it will attempt to
+     * @param maxReopens If the channel errors out, how many times it will attempt to
      *                   re-initiate the connection.
      * @param requesterProject Project to bill when accessing "requester pays" buckets. If unset,
      *                         these buckets cannot be accessed.
@@ -395,6 +395,25 @@ public static String getPathWithoutBucket(String path) {
     public static void setGlobalNIODefaultOptions(int maxReopens, String requesterProject) {
         CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(getCloudStorageConfiguration(maxReopens, requesterProject));
         CloudStorageFileSystemProvider.setStorageOptions(setGenerousTimeouts(StorageOptions.newBuilder()).build());
+        setGlobalHttpNioDefaultOptions(maxReopens);
+    }
+
+    /**
+     * Set the options for http-nio to have generous timeouts and defaults
+     */
+    public static void setGlobalHttpNioDefaultOptions(final int maxRetries) {
+        final HttpFileSystemProviderSettings.RetrySettings retrySettings = new HttpFileSystemProviderSettings.RetrySettings(
+                maxRetries,
+                RetryHandler.DEFAULT_RETRYABLE_HTTP_CODES,
+                RetryHandler.DEFAULT_RETRYABLE_EXCEPTIONS,
+                RetryHandler.DEFALT_RETRYABLE_MESSAGES,
+                e -> false);
+        final HttpFileSystemProviderSettings settings = new HttpFileSystemProviderSettings(
+                java.time.Duration.ofMillis(MAX_TIMEOUT),
+                HttpFileSystemProviderSettings.DEFAULT_SETTINGS.redirect(),
+                retrySettings);
+
+        HttpFileSystemProvider.setSettings(settings);
     }
 
     /**
@@ -437,8 +456,8 @@ public static CloudStorageConfiguration getCloudStorageConfiguration(int maxReop
     private static StorageOptions.Builder setGenerousTimeouts(StorageOptions.Builder builder) {
         return builder
             .setTransportOptions(HttpTransportOptions.newBuilder()
-                .setConnectTimeout(120000)
-                .setReadTimeout(120000)
+                .setConnectTimeout(MAX_TIMEOUT)
+                .setReadTimeout(MAX_TIMEOUT)
                 .build())
             .setRetrySettings(RetrySettings.newBuilder()
                 .setMaxAttempts(15)