From 704a780faab4fa7958fa3443ab50be9080727a55 Mon Sep 17 00:00:00 2001 From: malmans2 Date: Mon, 14 Jul 2025 14:04:44 +0200 Subject: [PATCH 01/11] implement incremental sleep --- multiurl/http.py | 11 +++++++++-- tests/test_robust.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/multiurl/http.py b/multiurl/http.py index 51f76ae..590b9ed 100644 --- a/multiurl/http.py +++ b/multiurl/http.py @@ -468,6 +468,12 @@ def wrapped(url, *args, **kwargs): tries = 0 main_url = url + if isinstance(retry_after, tuple): + sleep_min, sleep_max, sleep_incremental_ratio = retry_after + else: + sleep_min = sleep_max = retry_after + sleep_incremental_ratio = 1 + while True: tries += 1 @@ -517,8 +523,9 @@ def wrapped(url, *args, **kwargs): LOG.warning("Retrying using mirror %s", mirror) main_url = f"{mirror}{url[replace:]}" else: - LOG.warning("Retrying in %s seconds", retry_after) - time.sleep(retry_after) + LOG.warning("Retrying in %s seconds", sleep_min) + time.sleep(sleep_min) + sleep_min = min(sleep_min * sleep_incremental_ratio, sleep_max) LOG.info("Retrying now...") return wrapped diff --git a/tests/test_robust.py b/tests/test_robust.py index 55bd2a5..49b0073 100644 --- a/tests/test_robust.py +++ b/tests/test_robust.py @@ -6,7 +6,7 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. # - +import requests import logging import os import random @@ -16,7 +16,7 @@ import pytest from multiurl import download -from multiurl.http import RETRIABLE +from multiurl.http import RETRIABLE, robust def handler(signum, frame): @@ -47,6 +47,34 @@ def test_robust(): ) +@pytest.mark.parametrize( + "retry_after,expected_logs", + [ + [ + 0.1, + [ + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ], + ], + [ + (0.1, 0.2, 2), + [ + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ("multiurl.http", 30, "Retrying in 0.2 seconds"), + ("multiurl.http", 30, "Retrying in 0.2 seconds"), + ], + ], + ], +) +def test_robust_incremental_sleep(caplog, retry_after, expected_logs): + robust_get = robust(requests.get, retry_after=retry_after, maximum_tries=4) + codes = ",".join(map(str, RETRIABLE)) + robust_get(f"http://httpbin.org/status/{codes}") + assert caplog.record_tuples[1::2] == expected_logs + + @pytest.mark.skipif(True, reason="Mirror disabled") def test_mirror(): download( From 091261cab374611947b1f202521beff983a10552 Mon Sep 17 00:00:00 2001 From: malmans2 Date: Mon, 14 Jul 2025 14:17:56 +0200 Subject: [PATCH 02/11] isort --- tests/test_robust.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_robust.py b/tests/test_robust.py index 49b0073..79b2d65 100644 --- a/tests/test_robust.py +++ b/tests/test_robust.py @@ -6,7 +6,6 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. # -import requests import logging import os import random @@ -14,6 +13,7 @@ from contextlib import contextmanager import pytest +import requests from multiurl import download from multiurl.http import RETRIABLE, robust From cdc60a76b0d01571bd53e2768fa0c440e4d5657e Mon Sep 17 00:00:00 2001 From: malmans2 Date: Tue, 15 Jul 2025 09:45:29 +0200 Subject: [PATCH 03/11] allow sleep to decrease --- multiurl/http.py | 13 ++++++++++--- tests/test_robust.py | 8 ++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/multiurl/http.py b/multiurl/http.py index 590b9ed..1854831 100644 --- a/multiurl/http.py +++ b/multiurl/http.py @@ -474,6 +474,9 @@ def wrapped(url, *args, **kwargs): sleep_min = sleep_max = retry_after sleep_incremental_ratio = 1 + assert (0 <= sleep_min <= sleep_max) and sleep_incremental_ratio > 0 + sleep = sleep_min if sleep_incremental_ratio >= 1 else sleep_max + while True: tries += 1 @@ -523,9 +526,13 @@ def wrapped(url, *args, **kwargs): LOG.warning("Retrying using mirror %s", mirror) main_url = f"{mirror}{url[replace:]}" else: - LOG.warning("Retrying in %s seconds", sleep_min) - time.sleep(sleep_min) - sleep_min = min(sleep_min * sleep_incremental_ratio, sleep_max) + LOG.warning("Retrying in %s seconds", sleep) + time.sleep(sleep) + sleep = ( + min(sleep * sleep_incremental_ratio, sleep_max) + if sleep_incremental_ratio >= 1 + else max(sleep_min, sleep * sleep_incremental_ratio) + ) LOG.info("Retrying now...") return wrapped diff --git a/tests/test_robust.py b/tests/test_robust.py index 79b2d65..d51a2a4 100644 --- a/tests/test_robust.py +++ b/tests/test_robust.py @@ -66,6 +66,14 @@ def test_robust(): ("multiurl.http", 30, "Retrying in 0.2 seconds"), ], ], + [ + (0.1, 0.2, 0.5), + [ + ("multiurl.http", 30, "Retrying in 0.2 seconds"), + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ("multiurl.http", 30, "Retrying in 0.1 seconds"), + ], + ], ], ) def test_robust_incremental_sleep(caplog, retry_after, expected_logs): From a9ccf80d4b9b51fe02bd4ccc9a508c9d04f823db Mon Sep 17 00:00:00 2001 From: malmans2 Date: Tue, 15 Jul 2025 09:50:04 +0200 Subject: [PATCH 04/11] better assertion --- multiurl/http.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/multiurl/http.py b/multiurl/http.py index 1854831..5b3a1f2 100644 --- a/multiurl/http.py +++ b/multiurl/http.py @@ -392,7 +392,7 @@ def mutate(self, *args, **kwargs): def split_large_requests(self, parts): ranges = [] for offset, length in parts: - ranges.append(f"{offset}-{offset+length-1}") + ranges.append(f"{offset}-{offset + length - 1}") # Nginx default is 4K # https://stackoverflow.com/questions/686217/maximum-on-http-header-values @@ -474,7 +474,12 @@ def wrapped(url, *args, **kwargs): sleep_min = sleep_max = retry_after sleep_incremental_ratio = 1 - assert (0 <= sleep_min <= sleep_max) and sleep_incremental_ratio > 0 + assert sleep_incremental_ratio > 0 + assert ( + sleep_min == sleep_max + if sleep_incremental_ratio == 1 + else (0 <= sleep_min <= sleep_max) + ) sleep = sleep_min if sleep_incremental_ratio >= 1 else sleep_max while True: From 2ad6615e34d1f0f42571def10e4a853f80922b4e Mon Sep 17 00:00:00 2001 From: malmans2 Date: Tue, 15 Jul 2025 09:51:05 +0200 Subject: [PATCH 05/11] restore formatting --- multiurl/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multiurl/http.py b/multiurl/http.py index 5b3a1f2..28d8375 100644 --- a/multiurl/http.py +++ b/multiurl/http.py @@ -392,7 +392,7 @@ def mutate(self, *args, **kwargs): def split_large_requests(self, parts): ranges = [] for offset, length in parts: - ranges.append(f"{offset}-{offset + length - 1}") + ranges.append(f"{offset}-{offset+length-1}") # Nginx default is 4K # https://stackoverflow.com/questions/686217/maximum-on-http-header-values From 33818a3a3944592167ef600707c9cad23372f457 Mon Sep 17 00:00:00 2001 From: malmans2 Date: Tue, 15 Jul 2025 09:57:52 +0200 Subject: [PATCH 06/11] fix assert --- multiurl/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multiurl/http.py b/multiurl/http.py index 28d8375..798ceeb 100644 --- a/multiurl/http.py +++ b/multiurl/http.py @@ -478,7 +478,7 @@ def wrapped(url, *args, **kwargs): assert ( sleep_min == sleep_max if sleep_incremental_ratio == 1 - else (0 <= sleep_min <= sleep_max) + else (0 <= sleep_min < sleep_max) ) sleep = sleep_min if sleep_incremental_ratio >= 1 else sleep_max From 00d95eab6369bd30f71841c60d9fda4a8b1d7c3d Mon Sep 17 00:00:00 2001 From: malmans2 Date: Tue, 15 Jul 2025 10:01:06 +0200 Subject: [PATCH 07/11] fix assertion --- multiurl/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multiurl/http.py b/multiurl/http.py index 798ceeb..b19bb59 100644 --- a/multiurl/http.py +++ b/multiurl/http.py @@ -474,11 +474,11 @@ def wrapped(url, *args, **kwargs): sleep_min = sleep_max = retry_after sleep_incremental_ratio = 1 - assert sleep_incremental_ratio > 0 + assert 0 <= sleep_min and sleep_incremental_ratio > 0 assert ( sleep_min == sleep_max if sleep_incremental_ratio == 1 - else (0 <= sleep_min < sleep_max) + else sleep_min < sleep_max ) sleep = sleep_min if sleep_incremental_ratio >= 1 else sleep_max From deda030027b165c747d7739c4678cefe5dce191e Mon Sep 17 00:00:00 2001 From: malmans2 Date: Tue, 15 Jul 2025 14:35:15 +0200 Subject: [PATCH 08/11] cleanup --- multiurl/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multiurl/http.py b/multiurl/http.py index b19bb59..46922e2 100644 --- a/multiurl/http.py +++ b/multiurl/http.py @@ -474,7 +474,7 @@ def wrapped(url, *args, **kwargs): sleep_min = sleep_max = retry_after sleep_incremental_ratio = 1 - assert 0 <= sleep_min and sleep_incremental_ratio > 0 + assert sleep_min >= 0 and sleep_incremental_ratio > 0 assert ( sleep_min == sleep_max if sleep_incremental_ratio == 1 From be082a5af75ef94c4f6917cb9797bfa0e56efa79 Mon Sep 17 00:00:00 2001 From: malmans2 Date: Tue, 15 Jul 2025 14:39:02 +0200 Subject: [PATCH 09/11] restore original empty line --- tests/test_robust.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_robust.py b/tests/test_robust.py index d51a2a4..aea7afe 100644 --- a/tests/test_robust.py +++ b/tests/test_robust.py @@ -6,6 +6,7 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. # + import logging import os import random From 6efec8c5a5c15f5f7221b80064dd22fe2dde1135 Mon Sep 17 00:00:00 2001 From: malmans2 Date: Wed, 16 Jul 2025 15:13:14 +0200 Subject: [PATCH 10/11] explicitly raise TypeError --- multiurl/http.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/multiurl/http.py b/multiurl/http.py index 46922e2..fafff46 100644 --- a/multiurl/http.py +++ b/multiurl/http.py @@ -392,7 +392,7 @@ def mutate(self, *args, **kwargs): def split_large_requests(self, parts): ranges = [] for offset, length in parts: - ranges.append(f"{offset}-{offset+length-1}") + ranges.append(f"{offset}-{offset + length - 1}") # Nginx default is 4K # https://stackoverflow.com/questions/686217/maximum-on-http-header-values @@ -468,11 +468,13 @@ def wrapped(url, *args, **kwargs): tries = 0 main_url = url - if isinstance(retry_after, tuple): + if isinstance(retry_after, (list, tuple)): sleep_min, sleep_max, sleep_incremental_ratio = retry_after - else: + elif isinstance(retry_after, (int, float)): sleep_min = sleep_max = retry_after sleep_incremental_ratio = 1 + else: + raise TypeError("retry_after must be int, float, tuple, or list") assert sleep_min >= 0 and sleep_incremental_ratio > 0 assert ( From 70adee78557843965a6647ecff74d1305a011438 Mon Sep 17 00:00:00 2001 From: malmans2 Date: Wed, 16 Jul 2025 15:14:47 +0200 Subject: [PATCH 11/11] original formatting --- multiurl/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multiurl/http.py b/multiurl/http.py index fafff46..02e5447 100644 --- a/multiurl/http.py +++ b/multiurl/http.py @@ -392,7 +392,7 @@ def mutate(self, *args, **kwargs): def split_large_requests(self, parts): ranges = [] for offset, length in parts: - ranges.append(f"{offset}-{offset + length - 1}") + ranges.append(f"{offset}-{offset+length-1}") # Nginx default is 4K # https://stackoverflow.com/questions/686217/maximum-on-http-header-values