From 6ef9bac89799b442e703adcf07ffcac823f61076 Mon Sep 17 00:00:00 2001 From: Eddy Comyn-Platt <53045993+EddyCMWF@users.noreply.github.com> Date: Wed, 16 Jul 2025 15:53:17 +0100 Subject: [PATCH 1/4] merge develop into main for release (#31) * implement incremental sleep --- multiurl/http.py | 25 +++++++++++++++++++++++-- tests/test_robust.py | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/multiurl/http.py b/multiurl/http.py index 51f76ae..02e5447 100644 --- a/multiurl/http.py +++ b/multiurl/http.py @@ -468,6 +468,22 @@ def wrapped(url, *args, **kwargs): tries = 0 main_url = url + if isinstance(retry_after, (list, tuple)): + sleep_min, sleep_max, sleep_incremental_ratio = retry_after + elif isinstance(retry_after, (int, float)): + sleep_min = sleep_max = retry_after + sleep_incremental_ratio = 1 + else: + raise TypeError("retry_after must be int, float, tuple, or list") + + assert sleep_min >= 0 and sleep_incremental_ratio > 0 + assert ( + sleep_min == sleep_max + if sleep_incremental_ratio == 1 + else sleep_min < sleep_max + ) + sleep = sleep_min if sleep_incremental_ratio >= 1 else sleep_max + while True: tries += 1 @@ -517,8 +533,13 @@ def wrapped(url, *args, **kwargs): LOG.warning("Retrying using mirror %s", mirror) main_url = f"{mirror}{url[replace:]}" else: - LOG.warning("Retrying in %s seconds", retry_after) - time.sleep(retry_after) + LOG.warning("Retrying in %s seconds", sleep) + time.sleep(sleep) + sleep = ( + min(sleep * sleep_incremental_ratio, sleep_max) + if sleep_incremental_ratio >= 1 + else max(sleep_min, sleep * sleep_incremental_ratio) + ) LOG.info("Retrying now...") return wrapped diff --git a/tests/test_robust.py b/tests/test_robust.py index 55bd2a5..aea7afe 100644 --- a/tests/test_robust.py +++ b/tests/test_robust.py @@ -14,9 +14,10 @@ from contextlib import contextmanager import pytest +import requests from multiurl import download -from multiurl.http import RETRIABLE +from multiurl.http import RETRIABLE, robust def handler(signum, frame): @@ -47,6 +48,42 @@ def test_robust(): ) +@pytest.mark.parametrize( + "retry_after,expected_logs", + [ + [ + 0.1, + [ + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ], + ], + [ + (0.1, 0.2, 2), + [ + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ("multiurl.http", 30, "Retrying in 0.2 seconds"), + ("multiurl.http", 30, "Retrying in 0.2 seconds"), + ], + ], + [ + (0.1, 0.2, 0.5), + [ + ("multiurl.http", 30, "Retrying in 0.2 seconds"), + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ], + ], + ], +) +def test_robust_incremental_sleep(caplog, retry_after, expected_logs): + robust_get = robust(requests.get, retry_after=retry_after, maximum_tries=4) + codes = ",".join(map(str, RETRIABLE)) + robust_get(f"http://httpbin.org/status/{codes}") + assert caplog.record_tuples[1::2] == expected_logs + + @pytest.mark.skipif(True, reason="Mirror disabled") def test_mirror(): download( From 2f9f5cf96095eb8f02ae4cf2f6a813d1a867ae2e Mon Sep 17 00:00:00 2001 From: Eddy Comyn-Platt <53045993+EddyCMWF@users.noreply.github.com> Date: Thu, 17 Jul 2025 13:10:20 +0100 Subject: [PATCH 2/4] Update __init__.py --- multiurl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multiurl/__init__.py b/multiurl/__init__.py index b05cf49..efe1164 100644 --- a/multiurl/__init__.py +++ b/multiurl/__init__.py @@ -10,7 +10,7 @@ from .downloader import Downloader, download, robust -__version__ = "0.3.5" +__version__ = "0.3.6" __all__ = [ "download", From 2354ca43e365fd131db006d92e231e04368af857 Mon Sep 17 00:00:00 2001 From: Eddy Comyn-Platt <53045993+EddyCMWF@users.noreply.github.com> Date: Thu, 17 Jul 2025 13:19:38 +0100 Subject: [PATCH 3/4] Back to develop version --- multiurl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multiurl/__init__.py b/multiurl/__init__.py index efe1164..c4bec96 100644 --- a/multiurl/__init__.py +++ b/multiurl/__init__.py @@ -10,7 +10,7 @@ from .downloader import Downloader, download, robust -__version__ = "0.3.6" +__version__ = "0.3.7dev0" __all__ = [ "download", From b94b1535255705679807e8d01cf3663c08861afc Mon Sep 17 00:00:00 2001 From: EddyCMWF Date: Thu, 17 Jul 2025 13:58:20 +0100 Subject: [PATCH 4/4] retries/tries --- multiurl/http.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/multiurl/http.py b/multiurl/http.py index 02e5447..4942eb6 100644 --- a/multiurl/http.py +++ b/multiurl/http.py @@ -460,7 +460,18 @@ def iterate_requests(chunk_size): ) -def robust(call, maximum_tries=500, retry_after=120, mirrors=None): +def robust( + call, + maximum_retries: int = None, + retry_after: int = 120, + mirrors: dict = None, + maximum_tries: int = None +): + + # Clear up confusion between maximum_tries and maximum_retries + # maximum_retries is the supprted kwarg, maximum_tries is for backward compatibility + maximum_retries = maximum_retries or maximum_tries or 500 + def retriable(code): return code in RETRIABLE @@ -487,7 +498,7 @@ def wrapped(url, *args, **kwargs): while True: tries += 1 - if tries >= maximum_tries: + if tries >= maximum_retries: # Last attempt, don't do anything return call(main_url, *args, **kwargs) @@ -504,7 +515,7 @@ def wrapped(url, *args, **kwargs): "Recovering from connection error [%s], attempt %s of %s", e, tries, - maximum_tries, + maximum_retries, ) if r is not None: @@ -515,7 +526,7 @@ def wrapped(url, *args, **kwargs): r.status_code, r.reason, tries, - maximum_tries, + maximum_retries, ) alternate = None