diff --git a/src/eo_processing/utils/catalogue_check.py b/src/eo_processing/utils/catalogue_check.py index b609f10..4ada37a 100644 --- a/src/eo_processing/utils/catalogue_check.py +++ b/src/eo_processing/utils/catalogue_check.py @@ -6,6 +6,7 @@ import geojson from typing import TYPE_CHECKING import pystac_client +import time if TYPE_CHECKING: from eo_processing.config.data_formats import openEO_bbox_format @@ -217,8 +218,17 @@ def catalogue_check_CDSE_S1(orbit_direction: str, start: str, end: str, bbox: op # get the dates of all found matches results = [] - for item in search.items_as_dicts(): - results.append(item['properties']['datetime']) + + for attempt in range(3): + try: + for item in search.items_as_dicts(): + results.append(item['properties']['datetime']) + break + except Exception as e: + if attempt < 2: + time.sleep(2 ** attempt) # exponential backoff: 1s, 2s + else: + raise e # count the number of unique dates on which we have observations (resolved tile overlap) df = pd.DataFrame(results, columns=['date']) @@ -248,8 +258,16 @@ def catalogue_check_CDSE_S1(orbit_direction: str, start: str, end: str, bbox: op # get the dates of all found matches results = [] - for item in search.items_as_dicts(): - results.append(item['properties']['datetime']) + for attempt in range(3): + try: + for item in search.items_as_dicts(): + results.append(item['properties']['datetime']) + break + except Exception as e: + if attempt < 2: + time.sleep(2 ** attempt) # exponential backoff: 1s, 2s + else: + raise e # count the number of unique dates on which we have observations (resolved tile overlap) df = pd.DataFrame(results, columns=['date']) @@ -322,8 +340,16 @@ def catalogue_check_CDSE_S2(start: str, end: str, bbox: openEO_bbox_format, # get the dates of all found matches results = [] - for item in search.items_as_dicts(): - results.append(item['properties']['datetime']) + for attempt in range(3): + try: + for item in search.items_as_dicts(): + results.append(item['properties']['datetime']) + break + except Exception as e: + if attempt < 2: + time.sleep(2 ** attempt) # exponential backoff: 1s, 2s + else: + raise e # count the number of unique dates on which we have observations (resolved tile overlap) df = pd.DataFrame(results, columns=['date']) diff --git a/src/eo_processing/utils/jobmanager.py b/src/eo_processing/utils/jobmanager.py index 9e74897..5d62c0d 100644 --- a/src/eo_processing/utils/jobmanager.py +++ b/src/eo_processing/utils/jobmanager.py @@ -259,7 +259,10 @@ def on_job_done(self, job: openeo.BatchJob, row: pd.Series) -> None: job_metadata = job.describe() job_dir = self.get_job_dir(job.job_id) - title = os.path.splitext(job_metadata['title'])[0] + #title = os.path.splitext(job_metadata['title'])[0] + + title = row["file_prefix"] + file_ext = job_metadata['process']['process_graph']['saveresult1']['arguments']['format'].lower() metadata_path = self.get_job_metadata_path(job.job_id,title) job_graph_path = self.get_job_graph_path(job.job_id,title)