diff --git a/multiurl/http.py b/multiurl/http.py index 51f76ae..02e5447 100644 --- a/multiurl/http.py +++ b/multiurl/http.py @@ -468,6 +468,22 @@ def wrapped(url, *args, **kwargs): tries = 0 main_url = url + if isinstance(retry_after, (list, tuple)): + sleep_min, sleep_max, sleep_incremental_ratio = retry_after + elif isinstance(retry_after, (int, float)): + sleep_min = sleep_max = retry_after + sleep_incremental_ratio = 1 + else: + raise TypeError("retry_after must be int, float, tuple, or list") + + assert sleep_min >= 0 and sleep_incremental_ratio > 0 + assert ( + sleep_min == sleep_max + if sleep_incremental_ratio == 1 + else sleep_min < sleep_max + ) + sleep = sleep_min if sleep_incremental_ratio >= 1 else sleep_max + while True: tries += 1 @@ -517,8 +533,13 @@ def wrapped(url, *args, **kwargs): LOG.warning("Retrying using mirror %s", mirror) main_url = f"{mirror}{url[replace:]}" else: - LOG.warning("Retrying in %s seconds", retry_after) - time.sleep(retry_after) + LOG.warning("Retrying in %s seconds", sleep) + time.sleep(sleep) + sleep = ( + min(sleep * sleep_incremental_ratio, sleep_max) + if sleep_incremental_ratio >= 1 + else max(sleep_min, sleep * sleep_incremental_ratio) + ) LOG.info("Retrying now...") return wrapped diff --git a/tests/test_robust.py b/tests/test_robust.py index 55bd2a5..aea7afe 100644 --- a/tests/test_robust.py +++ b/tests/test_robust.py @@ -14,9 +14,10 @@ from contextlib import contextmanager import pytest +import requests from multiurl import download -from multiurl.http import RETRIABLE +from multiurl.http import RETRIABLE, robust def handler(signum, frame): @@ -47,6 +48,42 @@ def test_robust(): ) +@pytest.mark.parametrize( + "retry_after,expected_logs", + [ + [ + 0.1, + [ + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ], + ], + [ + (0.1, 0.2, 2), + [ + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ("multiurl.http", 30, "Retrying in 0.2 seconds"), + ("multiurl.http", 30, "Retrying in 0.2 seconds"), + ], + ], + [ + (0.1, 0.2, 0.5), + [ + ("multiurl.http", 30, "Retrying in 0.2 seconds"), + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ], + ], + ], +) +def test_robust_incremental_sleep(caplog, retry_after, expected_logs): + robust_get = robust(requests.get, retry_after=retry_after, maximum_tries=4) + codes = ",".join(map(str, RETRIABLE)) + robust_get(f"http://httpbin.org/status/{codes}") + assert caplog.record_tuples[1::2] == expected_logs + + @pytest.mark.skipif(True, reason="Mirror disabled") def test_mirror(): download(