From 7fcf952165091bae85bfe993773387a700554e6a Mon Sep 17 00:00:00 2001 From: buchhornm Date: Sat, 11 Jul 2026 11:40:31 +0200 Subject: [PATCH 1/2] Add retry logic with exponential backoff to handle exceptions in STAC search --- src/eo_processing/utils/catalogue_check.py | 38 ++++++++++++++++++---- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/src/eo_processing/utils/catalogue_check.py b/src/eo_processing/utils/catalogue_check.py index b609f10..4ada37a 100644 --- a/src/eo_processing/utils/catalogue_check.py +++ b/src/eo_processing/utils/catalogue_check.py @@ -6,6 +6,7 @@ import geojson from typing import TYPE_CHECKING import pystac_client +import time if TYPE_CHECKING: from eo_processing.config.data_formats import openEO_bbox_format @@ -217,8 +218,17 @@ def catalogue_check_CDSE_S1(orbit_direction: str, start: str, end: str, bbox: op # get the dates of all found matches results = [] - for item in search.items_as_dicts(): - results.append(item['properties']['datetime']) + + for attempt in range(3): + try: + for item in search.items_as_dicts(): + results.append(item['properties']['datetime']) + break + except Exception as e: + if attempt < 2: + time.sleep(2 ** attempt) # exponential backoff: 1s, 2s + else: + raise e # count the number of unique dates on which we have observations (resolved tile overlap) df = pd.DataFrame(results, columns=['date']) @@ -248,8 +258,16 @@ def catalogue_check_CDSE_S1(orbit_direction: str, start: str, end: str, bbox: op # get the dates of all found matches results = [] - for item in search.items_as_dicts(): - results.append(item['properties']['datetime']) + for attempt in range(3): + try: + for item in search.items_as_dicts(): + results.append(item['properties']['datetime']) + break + except Exception as e: + if attempt < 2: + time.sleep(2 ** attempt) # exponential backoff: 1s, 2s + else: + raise e # count the number of unique dates on which we have observations (resolved tile overlap) df = pd.DataFrame(results, columns=['date']) @@ -322,8 +340,16 @@ def catalogue_check_CDSE_S2(start: str, end: str, bbox: openEO_bbox_format, # get the dates of all found matches results = [] - for item in search.items_as_dicts(): - results.append(item['properties']['datetime']) + for attempt in range(3): + try: + for item in search.items_as_dicts(): + results.append(item['properties']['datetime']) + break + except Exception as e: + if attempt < 2: + time.sleep(2 ** attempt) # exponential backoff: 1s, 2s + else: + raise e # count the number of unique dates on which we have observations (resolved tile overlap) df = pd.DataFrame(results, columns=['date']) From 01b993a248339228cdcd30a4d04e2ed969911e1a Mon Sep 17 00:00:00 2001 From: buchhornm Date: Sat, 11 Jul 2026 11:40:38 +0200 Subject: [PATCH 2/2] Update `jobmanager.py` to use `file_prefix` instead of `title` for generating job metadata paths --- src/eo_processing/utils/jobmanager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/eo_processing/utils/jobmanager.py b/src/eo_processing/utils/jobmanager.py index 9e74897..5d62c0d 100644 --- a/src/eo_processing/utils/jobmanager.py +++ b/src/eo_processing/utils/jobmanager.py @@ -259,7 +259,10 @@ def on_job_done(self, job: openeo.BatchJob, row: pd.Series) -> None: job_metadata = job.describe() job_dir = self.get_job_dir(job.job_id) - title = os.path.splitext(job_metadata['title'])[0] + #title = os.path.splitext(job_metadata['title'])[0] + + title = row["file_prefix"] + file_ext = job_metadata['process']['process_graph']['saveresult1']['arguments']['format'].lower() metadata_path = self.get_job_metadata_path(job.job_id,title) job_graph_path = self.get_job_graph_path(job.job_id,title)