Main into develop

multiurl/__init__.py

@@ -10,7 +10,7 @@
 
 from .downloader import Downloader, download, robust
 
-__version__ = "0.3.5"
+__version__ = "0.3.7dev0"
 
 __all__ = [
     "download",

multiurl/http.py

@@ -468,6 +468,22 @@ def wrapped(url, *args, **kwargs):
         tries = 0
         main_url = url
 
+        if isinstance(retry_after, (list, tuple)):
+            sleep_min, sleep_max, sleep_incremental_ratio = retry_after
+        elif isinstance(retry_after, (int, float)):
+            sleep_min = sleep_max = retry_after
+            sleep_incremental_ratio = 1
+        else:
+            raise TypeError("retry_after must be int, float, tuple, or list")
+
+        assert sleep_min >= 0 and sleep_incremental_ratio > 0
+        assert (
+            sleep_min == sleep_max
+            if sleep_incremental_ratio == 1
+            else sleep_min < sleep_max
+        )
+        sleep = sleep_min if sleep_incremental_ratio >= 1 else sleep_max
+
         while True:
             tries += 1
 
@@ -517,8 +533,13 @@ def wrapped(url, *args, **kwargs):
                 LOG.warning("Retrying using mirror %s", mirror)
                 main_url = f"{mirror}{url[replace:]}"
             else:
-                LOG.warning("Retrying in %s seconds", retry_after)
-                time.sleep(retry_after)
+                LOG.warning("Retrying in %s seconds", sleep)
+                time.sleep(sleep)
+                sleep = (
+                    min(sleep * sleep_incremental_ratio, sleep_max)
+                    if sleep_incremental_ratio >= 1
+                    else max(sleep_min, sleep * sleep_incremental_ratio)
+                )
                 LOG.info("Retrying now...")
 
     return wrapped

tests/test_robust.py

@@ -14,9 +14,10 @@
 from contextlib import contextmanager
 
 import pytest
+import requests
 
 from multiurl import download
-from multiurl.http import RETRIABLE
+from multiurl.http import RETRIABLE, robust
 
 
 def handler(signum, frame):
@@ -47,6 +48,42 @@ def test_robust():
         )
 
 
+@pytest.mark.parametrize(
+    "retry_after,expected_logs",
+    [
+        [
+            0.1,
+            [
+                ("multiurl.http", 30, "Retrying in 0.1 seconds"),
+                ("multiurl.http", 30, "Retrying in 0.1 seconds"),
+                ("multiurl.http", 30, "Retrying in 0.1 seconds"),
+            ],
+        ],
+        [
+            (0.1, 0.2, 2),
+            [
+                ("multiurl.http", 30, "Retrying in 0.1 seconds"),
+                ("multiurl.http", 30, "Retrying in 0.2 seconds"),
+                ("multiurl.http", 30, "Retrying in 0.2 seconds"),
+            ],
+        ],
+        [
+            (0.1, 0.2, 0.5),
+            [
+                ("multiurl.http", 30, "Retrying in 0.2 seconds"),
+                ("multiurl.http", 30, "Retrying in 0.1 seconds"),
+                ("multiurl.http", 30, "Retrying in 0.1 seconds"),
+            ],
+        ],
+    ],
+)
+def test_robust_incremental_sleep(caplog, retry_after, expected_logs):
+    robust_get = robust(requests.get, retry_after=retry_after, maximum_tries=4)
+    codes = ",".join(map(str, RETRIABLE))
+    robust_get(f"http://httpbin.org/status/{codes}")
+    assert caplog.record_tuples[1::2] == expected_logs
+
+
 @pytest.mark.skipif(True, reason="Mirror disabled")
 def test_mirror():
     download(