Clear up maximum_tries/maximum_retries for robust method

multiurl/__init__.py

@@ -10,7 +10,7 @@
 
 from .downloader import Downloader, download, robust
 
-__version__ = "0.3.5"
+__version__ = "0.3.7dev0"
 
 __all__ = [
     "download",

multiurl/http.py

@@ -460,18 +460,45 @@ def iterate_requests(chunk_size):
 )
 
 
-def robust(call, maximum_tries=500, retry_after=120, mirrors=None):
+def robust(
+    call,
+    maximum_retries: int = None,
+    retry_after: int = 120,
+    mirrors: dict = None,
+    maximum_tries: int = None
+):
+
+    # Clear up confusion between maximum_tries and maximum_retries
+    #  maximum_retries is the supprted kwarg, maximum_tries is for backward compatibility
+    maximum_retries = maximum_retries or maximum_tries or 500
+
     def retriable(code):
         return code in RETRIABLE
 
     def wrapped(url, *args, **kwargs):
         tries = 0
         main_url = url
 
+        if isinstance(retry_after, (list, tuple)):
+            sleep_min, sleep_max, sleep_incremental_ratio = retry_after
+        elif isinstance(retry_after, (int, float)):
+            sleep_min = sleep_max = retry_after
+            sleep_incremental_ratio = 1
+        else:
+            raise TypeError("retry_after must be int, float, tuple, or list")
+
+        assert sleep_min >= 0 and sleep_incremental_ratio > 0
+        assert (
+            sleep_min == sleep_max
+            if sleep_incremental_ratio == 1
+            else sleep_min < sleep_max
+        )
+        sleep = sleep_min if sleep_incremental_ratio >= 1 else sleep_max
+
         while True:
             tries += 1
 
-            if tries >= maximum_tries:
+            if tries >= maximum_retries:
                 # Last attempt, don't do anything
                 return call(main_url, *args, **kwargs)
 
@@ -488,7 +515,7 @@ def wrapped(url, *args, **kwargs):
                     "Recovering from connection error [%s], attempt %s of %s",
                     e,
                     tries,
-                    maximum_tries,
+                    maximum_retries,
                 )
 
             if r is not None:
@@ -499,7 +526,7 @@ def wrapped(url, *args, **kwargs):
                     r.status_code,
                     r.reason,
                     tries,
-                    maximum_tries,
+                    maximum_retries,
                 )
 
             alternate = None
@@ -517,8 +544,13 @@ def wrapped(url, *args, **kwargs):
                 LOG.warning("Retrying using mirror %s", mirror)
                 main_url = f"{mirror}{url[replace:]}"
             else:
-                LOG.warning("Retrying in %s seconds", retry_after)
-                time.sleep(retry_after)
+                LOG.warning("Retrying in %s seconds", sleep)
+                time.sleep(sleep)
+                sleep = (
+                    min(sleep * sleep_incremental_ratio, sleep_max)
+                    if sleep_incremental_ratio >= 1
+                    else max(sleep_min, sleep * sleep_incremental_ratio)
+                )
                 LOG.info("Retrying now...")
 
     return wrapped

tests/test_robust.py

@@ -14,9 +14,10 @@
 from contextlib import contextmanager
 
 import pytest
+import requests
 
 from multiurl import download
-from multiurl.http import RETRIABLE
+from multiurl.http import RETRIABLE, robust
 
 
 def handler(signum, frame):
@@ -47,6 +48,42 @@ def test_robust():
         )
 
 
+@pytest.mark.parametrize(
+    "retry_after,expected_logs",
+    [
+        [
+            0.1,
+            [
+                ("multiurl.http", 30, "Retrying in 0.1 seconds"),
+                ("multiurl.http", 30, "Retrying in 0.1 seconds"),
+                ("multiurl.http", 30, "Retrying in 0.1 seconds"),
+            ],
+        ],
+        [
+            (0.1, 0.2, 2),
+            [
+                ("multiurl.http", 30, "Retrying in 0.1 seconds"),
+                ("multiurl.http", 30, "Retrying in 0.2 seconds"),
+                ("multiurl.http", 30, "Retrying in 0.2 seconds"),
+            ],
+        ],
+        [
+            (0.1, 0.2, 0.5),
+            [
+                ("multiurl.http", 30, "Retrying in 0.2 seconds"),
+                ("multiurl.http", 30, "Retrying in 0.1 seconds"),
+                ("multiurl.http", 30, "Retrying in 0.1 seconds"),
+            ],
+        ],
+    ],
+)
+def test_robust_incremental_sleep(caplog, retry_after, expected_logs):
+    robust_get = robust(requests.get, retry_after=retry_after, maximum_tries=4)
+    codes = ",".join(map(str, RETRIABLE))
+    robust_get(f"http://httpbin.org/status/{codes}")
+    assert caplog.record_tuples[1::2] == expected_logs
+
+
 @pytest.mark.skipif(True, reason="Mirror disabled")
 def test_mirror():
     download(