From 47e2faecdf3598f9af9e11afa6667d89a5023ee6 Mon Sep 17 00:00:00 2001 From: smetsb Date: Tue, 28 Jul 2026 17:26:18 +0200 Subject: [PATCH 1/2] fixed issue to load nonEO_feature list from file --- src/eo_processing/utils/storage.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/eo_processing/utils/storage.py b/src/eo_processing/utils/storage.py index 7730214..c319101 100644 --- a/src/eo_processing/utils/storage.py +++ b/src/eo_processing/utils/storage.py @@ -815,6 +815,7 @@ def GenericQueryWithResult(self, sql_statement: str) -> List[tuple]: print('** get data from request') # create cursor cur = conn.cursor() + cur.itersize = 10000 cur.execute(sql_statement) # get data vResult = cur.fetchall() @@ -1355,7 +1356,7 @@ def upload_collection_to_catalog(self, collection: pystac.Collection, edit_flag: #get the auth auth_token = self.get_bearer_auth() #get catalog_url - catalog_url = self.get_catalog_url() + catalog_url = self.get_catalog_url().rstrip('/') # load the collection from the created collection. coll = collection.to_dict() @@ -1371,7 +1372,7 @@ def upload_collection_to_catalog(self, collection: pystac.Collection, edit_flag: # upload a new collection collection_url = f"{catalog_url}/collections/" resp = post(collection_url, auth=auth_token, json=coll) - if resp.status_code == 201: + if resp.status_code == 200 or resp.status_code == 201: coll_id = resp.json()["id"] if edit_flag: print(f"Collection edited: {coll_id}") @@ -1401,7 +1402,7 @@ def upload_items_to_collection(self, collection: pystac.Collection, #get the auth auth_token = self.get_bearer_auth() #get catalog_url - catalog_url = self.get_catalog_url() + catalog_url = self.get_catalog_url().rstrip('/') # check if there are items that need to be uploaded only if update is False if len(items_to_upload) == 0: From 2a4f21010289a0bd290cd59c29bfbe974b821ca9 Mon Sep 17 00:00:00 2001 From: smetsb Date: Wed, 29 Jul 2026 17:44:24 +0200 Subject: [PATCH 2/2] rename bands should become lower case (sql query returns lower case) --- src/eo_processing/openeo/preprocessing.py | 4 ++-- src/eo_processing/openeo/processing.py | 17 +++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/eo_processing/openeo/preprocessing.py b/src/eo_processing/openeo/preprocessing.py index 5970748..ff517c3 100644 --- a/src/eo_processing/openeo/preprocessing.py +++ b/src/eo_processing/openeo/preprocessing.py @@ -150,7 +150,7 @@ def extract_S1_datacube( # therefore, the integration has to be this ugly and not only in the final warper functions if get_NVBT: nvbt_band = bands.filter_bands(bands.metadata.band_names[0]).count_time().rename_labels("bands", - ["S1-NVBT"]) + ["S1-NVBT".lower()]) nvbt_band = nvbt_band.apply(lambda x: if_(x.is_nodata(), 0, x)) # Linearly interpolate missing values if wished @@ -319,7 +319,7 @@ def extract_S2_datacube( # Note: we want to create the NVBT here before any possible linear interpolation is done # therefore, the integration has to be this ugly and not only in the final warper functions if get_NVBT: - nvbt_band = bands.filter_bands(S2_bands[0]).count_time().rename_labels("bands", ["S2-NVBT"]) + nvbt_band = bands.filter_bands(S2_bands[0]).count_time().rename_labels("bands", ["S2-NVBT".lower()]) nvbt_band = nvbt_band.apply(lambda x: if_(x.is_nodata(), 0, x)) # Linearly interpolate missing values if wished diff --git a/src/eo_processing/openeo/processing.py b/src/eo_processing/openeo/processing.py index 30811da..34faa85 100644 --- a/src/eo_processing/openeo/processing.py +++ b/src/eo_processing/openeo/processing.py @@ -245,7 +245,7 @@ def calculate_features_cube(input_data: DataCube, chunk_size: int = CHUNK_SIZE) "TileSize": chunk_size}) # adapt the band names new_band_names = [ - band + "_" + stat + band.lower() + "_" + stat.lower() for band in input_data.metadata.band_names for stat in ["p2", "p5", "p25", "median", "p75", "p95", "p98", "mean", "sd", "sum", "iqr", "iqr0595"] ] @@ -254,11 +254,16 @@ def calculate_features_cube(input_data: DataCube, chunk_size: int = CHUNK_SIZE) # remove some bands which make no sense :) # mainly from S2REP --> sd, sum, iqr + #bands_keep = [band for band in features_cube.metadata.band_names if + # band not in ['S2REP_sd', 'S2REP_sum', 'S2REP_iqr', 'S2REP_iqr0595' , 'VV_sum', 'VH_sum', 'VHVVD_sum', + # 'S2-CLOUD-MASK_p2', 'S2-CLOUD-MASK_p5', 'S2-CLOUD-MASK_p25', 'S2-CLOUD-MASK_median', + # 'S2-CLOUD-MASK_p75','S2-CLOUD-MASK_p95', 'S2-CLOUD-MASK_p98', 'S2-CLOUD-MASK_mean', + # 'S2-CLOUD-MASK_sd','S2-CLOUD-MASK_iqr','S2-CLOUD-MASK_iqr0595', 'S2-CLOUD-MASK_sum']] bands_keep = [band for band in features_cube.metadata.band_names if - band not in ['S2REP_sd', 'S2REP_sum', 'S2REP_iqr', 'S2REP_iqr0595' , 'VV_sum', 'VH_sum', 'VHVVD_sum', - 'S2-CLOUD-MASK_p2', 'S2-CLOUD-MASK_p5', 'S2-CLOUD-MASK_p25', 'S2-CLOUD-MASK_median', - 'S2-CLOUD-MASK_p75','S2-CLOUD-MASK_p95', 'S2-CLOUD-MASK_p98', 'S2-CLOUD-MASK_mean', - 'S2-CLOUD-MASK_sd','S2-CLOUD-MASK_iqr','S2-CLOUD-MASK_iqr0595', 'S2-CLOUD-MASK_sum']] + band not in ['s2rep_sd', 's2rep_sum', 's2rep_iqr', 's2rep_iqr0595' , 'vv_sum', 'vh_sum', 'vhvvd_sum', + 's2-could-mask_p2', 's2-could-mask_p5', 's2-could-mask_p25', 's2-could-mask_median', + 's2-could-mask_p75','s2-could-mask_p95', 's2-could-mask_p98', 's2-could-mask_mean', + 's2-could-mask_sd','s2-could-mask_iqr','s2-could-mask_iqr0595', 's2-could-mask_sum']] features_cube = features_cube.filter_bands(bands=bands_keep) @@ -492,7 +497,7 @@ def generate_nonEO_feature_cube( # reduce the temporal domain since copernicus_30 collection is "special" and feature only are one time stamp nonEO_feature_cube = nonEO_feature_cube.reduce_dimension(dimension='t', reducer=lambda x: x.last(ignore_nodata=True)) - new_bands = [f"{collection}-{band}" for band in bands] + new_bands = [f"{collection.lower()}-{band.lower()}" for band in bands] # resample the cube to 10m and EPSG of corresponding 20x20km grid tile