Allow remote retry max delay to be user configurable

src/main/java/com/google/devtools/build/lib/remote/RemoteRetrier.java

@@ -154,6 +154,7 @@ public static class ExponentialBackoff implements Backoff {
      */
     ExponentialBackoff(Duration initial, Duration max, double multiplier, double jitter,
         int maxAttempts) {
+      Preconditions.checkArgument(max.compareTo(initial) > 0, "max must be > initial");
       Preconditions.checkArgument(multiplier > 1, "multipler must be > 1");
       Preconditions.checkArgument(jitter >= 0 && jitter <= 1, "jitter must be in the range (0, 1)");
       Preconditions.checkArgument(maxAttempts >= 0, "maxAttempts must be >= 0");
@@ -167,7 +168,7 @@ public static class ExponentialBackoff implements Backoff {
     public ExponentialBackoff(RemoteOptions options) {
       this(
           /* initial = */ Duration.ofMillis(100),
-          /* max = */ Duration.ofSeconds(5),
+          /* max = */ options.remoteRetryMaxDelay,
           /* multiplier= */ 2,
           /* jitter= */ 0.1,
           options.remoteMaxRetryAttempts);

src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java

@@ -326,6 +326,19 @@ public String getTypeDescription() {
               + "If set to 0, retries are disabled.")
   public int remoteMaxRetryAttempts;
 
+  @Option(
+      name = "remote_retry_max_delay",
+      defaultValue = "5s",
+      documentationCategory = OptionDocumentationCategory.REMOTE,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      converter = RemoteTimeoutConverter.class,
+      help =
+          "The maximum backoff delay between remote retry attempts. Following units can be used:"
+              + " Days (d), hours (h), minutes (m), seconds (s), and milliseconds (ms). If"
+              + " the unit is omitted, the value is interpreted as seconds."
+          )
+  public Duration remoteRetryMaxDelay;
+
   @Option(
       name = "disk_cache",
       defaultValue = "null",