From 554ef25f258fcf99d70309eb9d2cb35e5a703615 Mon Sep 17 00:00:00 2001 From: Matthew Deshotel Date: Sun, 29 Mar 2026 13:31:51 -0500 Subject: [PATCH 001/132] initial refactor of suppPrecipMod.py --- .../NextGen_Forcings_Engine/core/consts.py | 263 ++++++- .../core/suppPrecipMod.py | 657 ++++++++---------- 2 files changed, 538 insertions(+), 382 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py index 2fd37fe1..2b3567dd 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py @@ -1,6 +1,7 @@ from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core import ( regrid, time_handling, + timeInterpMod, ) from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.timeInterpMod import ( nearest_neighbor, @@ -74,6 +75,7 @@ "RAINRATE_NODE": ["Surface Precipitation Rate", "mm/s"], }, } + FORCING_EXTRACTION = { # Set mapping between InputForcings codes and forcing extraction scripts "forcing_src": { @@ -115,6 +117,7 @@ }, "extraction_script_path": "/ngen-app/ngen-forcing/Forcing_Extraction_Scripts", } + GEOMOD = { "GeoMeta": [ "nodeCoords", @@ -1005,20 +1008,254 @@ "precipBiasCorrectOpt", ], } + +SUPPPRECIPMOD = { + "SupplementalPrecip": { + "supp_precip_dirs", + "supp_precip_mandatory", + "product_name", + "supp_precip_file_types", + "nx_global", + "ny_global", + "nx_local", + "ny_local", + "x_lower_bound", + "x_upper_bound", + "y_lower_bound", + "y_upper_bound", + "regrid_opt_supp_pcp", + "suppTemporalInterp", + "esmf_lats", + "esmf_lons", + "esmf_grid_in", + "regridObj", + "esmf_field_in", + "esmf_field_out", + "esmf_field_out_elem", + "esmf_field_out_poly", + "regridded_precip1", + "regridded_precip2", + "regridded_rqi1", + "regridded_rqi2", + "regridded_mask", + "final_supp_precip", + "regridded_precip1_elem", + "regridded_precip2_elem", + "regridded_rqi1_elem", + "regridded_rqi2_elem", + "regridded_mask_elem", + "final_supp_precip_elem", + "file_in1", + "file_in2", + "rqiMethod", + "rqiThresh", + "rqi_file_in1", + "rqi_file_in2", + "pcp_hour1", + "pcp_hour2", + "pcp_date1", + "pcp_date2", + "fcst_hour1", + "fcst_hour2", + "input_frequency", + "netcdf_var_names", + "rqi_netcdf_var_names", + "grib_levels", + "grib_vars", + "tmpFile", + "supp_input_offsets", + "global_x_lower", + "global_y_lower", + "global_x_upper", + "global_y_upper", + }, + # Subclass init loops iterate these; empty because the base set above covers all needed attrs. + # To add subclass-specific attrs, add their names here and they will be set to None on init. + # NOTE: keyValue is intentionally excluded — it is set from a constructor parameter, not + # from config_options, so including it here would reset it to None after initialization. + "SupplementalPrecipGridded": set(), + "SupplementalPrecipHydrofabric": set(), + "SupplementalPrecipUnstructured": set(), + "PRODUCT_NAMES": { + 1: "MRMS_1HR_Radar_Only", + 2: "MRMS_1HR_Gage_Corrected", + 3: "WRF_ARW_Hawaii_2p5km_PCP", + 4: "WRF_ARW_PuertoRico_2p5km_PCP", + 5: "CONUS_MRMS_1HR_MultiSensor", + 6: "Hawaii_MRMS_1HR_MultiSensor", + 7: "MRMS_LiquidWaterFraction", + 8: "NBM_CORE_CONUS_APCP", + 9: "NBM_CORE_ALASKA_APCP", + 10: "AK_MRMS", + 11: "AK_Stage_IV_Precip-MRMS", + 12: "CONUS_Stage_IV_Precip-MRMS", + 13: "MRMS PrecipFlag", + 14: "Custom_Freq_Supp_Pcp", + 15: "NBM_CORE_PR_APCP", + 16: "NBM_CORE_HAWAII_APCP", + }, + "FILE_EXT": { + "GRIB1": ".grb", + "GRIB2": ".grib2", + "NETCDF": ".nc", + }, + "GRIB_VARS": { + 1: None, + 2: None, + 3: None, + 4: None, + 5: None, + 6: None, + 7: None, + 8: None, + 9: None, + 10: None, + 11: None, + 12: None, + 13: None, + 14: None, + 15: None, + 16: None, + }, + "GRIB_LEVELS": { + 1: ["BLAH"], + 2: ["BLAH"], + 3: ["BLAH"], + 4: ["BLAH"], + 5: ["BLAH"], + 6: ["BLAH"], + 7: ["BLAH"], + 8: ["BLAH"], + 9: ["BLAH"], + 10: ["BLAH"], + 11: ["BLAH"], + 12: ["BLAH"], + 13: ["BLAH"], + 14: ["BLAH"], + 15: ["BLAH"], + 16: ["BLAH"], + }, + "NET_CDF_VARS_NAMES": { + 1: ["RadarOnlyQPE01H_0mabovemeansealevel"], + 2: ["MultiSensorQPE01H_0mabovemeansealevel"], + 3: ["APCP_surface"], + 4: ["APCP_surface"], + 5: ["MultiSensorQPE01H_0mabovemeansealevel"], + 6: ["MultiSensorQPE01H_0mabovemeansealevel"], + 7: ["sbcv2_lwf"], + 8: ["APCP_surface"], + 9: ["APCP_surface"], + 10: ["MultiSensorQPE01H_0mabovemeansealevel"], + 11: [], # Set dynamically since we have have Stage IV and MRMS + 12: [], # Set dynamically since we have have Stage IV and MRMS + 13: ["PrecipFlag_0mabovemeansealevel"], + 14: ["PrecipFlag_0mabovemeansealevel"], + 15: ["APCP_surface"], + 16: ["APCP_surface"], + }, + "RQI_NETCDF_VAR_NAMES": { + 1: ["RadarQualityIndex_0mabovemeansealevel"], + 2: ["RadarQualityIndex_0mabovemeansealevel"], + 3: None, + 4: None, + 5: None, + 6: None, + 7: None, + 8: None, + 9: None, + 10: None, + 11: None, + 12: None, + 13: None, + 14: None, + 15: None, + 16: None, + }, + "OUTPUT_VAR_IDX": { + 1: 3, # RAINRATE + 2: 3, + 3: 3, + 4: 3, + 5: 3, + 6: 3, + 7: 8, # LQFRAC + 8: 3, + 9: 3, + 10: 3, + 11: 3, + 12: 3, + 13: 8, + 14: 3, + 15: 3, + 16: 3, + }, + "FIND_NEIGHBOR_FILES_MAP": { + 1: time_handling.find_hourly_mrms_radar_neighbors, + 2: time_handling.find_hourly_mrms_radar_neighbors, + 3: time_handling.find_hourly_wrf_arw_neighbors, + 4: time_handling.find_hourly_wrf_arw_neighbors, + 5: time_handling.find_hourly_mrms_radar_neighbors, + 6: time_handling.find_hourly_mrms_radar_neighbors, + 7: time_handling.find_sbcv2_lwf_neighbors, + 8: time_handling.find_hourly_nbm_neighbors, + 9: time_handling.find_hourly_nbm_neighbors, + 10: time_handling.find_hourly_mrms_radar_neighbors, + 11: time_handling.find_ak_ext_ana_precip_neighbors, + 12: time_handling.find_conus_ext_ana_precip_neighbors, + 13: time_handling.find_hourly_mrms_precip_flag, + 14: time_handling.find_custom_freq_neighbors, + 15: time_handling.find_hourly_nbm_neighbors, + 16: time_handling.find_hourly_nbm_neighbors, + }, + "REGRID_MAP": { + 1: regrid.regrid_mrms_hourly, + 2: regrid.regrid_mrms_hourly, + 3: regrid.regrid_hourly_wrf_arw_hi_res_pcp, + 4: regrid.regrid_hourly_wrf_arw_hi_res_pcp, + 5: regrid.regrid_mrms_hourly, + 6: regrid.regrid_mrms_hourly, + 7: regrid.regrid_sbcv2_liquid_water_fraction, + 8: regrid.regrid_hourly_nbm, + 9: regrid.regrid_hourly_nbm, + 10: regrid.regrid_mrms_hourly, + 11: regrid.regrid_ak_ext_ana_pcp, + 12: regrid.regrid_conus_ext_ana_pcp, + 13: regrid.regrid_mrms_precip_flag, + 14: regrid.regrid_mrms_hourly, + 15: regrid.regrid_hourly_nbm, + 16: regrid.regrid_hourly_nbm, + }, + "TEMPORAL_INTERPOLATE_INPUTS_MAP": { + 0: timeInterpMod.no_interpolation_supp_pcp, + 1: timeInterpMod.nearest_neighbor_supp_pcp, + 2: timeInterpMod.weighted_average_supp_pcp, + }, +} + TEST_UTILS = { "OLD_NEW_VAR_MAP": { - "q2dBiasCorrectOpt": "q2BiasCorrectOpt", - "paramDir": "dScaleParamDirs", - "border": "ignored_border_widths", - "regridOpt": "regrid_opt", - "userFcstHorizon": "fcst_input_horizons", - "inDir": "input_force_dirs", - "swDowscaleOpt": "swDownscaleOpt", - "t2dBiasCorrectOpt": "t2BiasCorrectOpt", - "userCycleOffset": "fcst_input_offsets", - "windBiasCorrectOpt": "windBiasCorrect", - "timeInterpOpt": "forceTemoralInterp", - "enforce": "input_force_mandatory", - "file_type": "input_force_types", + "forcingInputMod": { + "q2dBiasCorrectOpt": "q2BiasCorrectOpt", + "paramDir": "dScaleParamDirs", + "border": "ignored_border_widths", + "regridOpt": "regrid_opt", + "userFcstHorizon": "fcst_input_horizons", + "inDir": "input_force_dirs", + "swDowscaleOpt": "swDownscaleOpt", + "t2dBiasCorrectOpt": "t2BiasCorrectOpt", + "userCycleOffset": "fcst_input_offsets", + "windBiasCorrectOpt": "windBiasCorrect", + "timeInterpOpt": "forceTemoralInterp", + "enforce": "input_force_mandatory", + "file_type": "input_force_types", + }, + "SupplementalPrecip": { + "regridOpt": "regrid_opt_supp_pcp", + "enforce": "supp_precip_mandatory", + "timeInterpOpt": "suppTemporalInterp", + "inDir": "supp_precip_dirs", + "file_type": "supp_precip_file_types", + "userCycleOffset": "supp_input_offsets", + }, } } diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py index 6f1676bb..ca3462e3 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py @@ -1,110 +1,94 @@ """High-level module file that will handle supplemental analysis/observed precipitation grids that will replace precipitation in the final output files.""" +from __future__ import annotations + +import logging +from functools import cached_property +from typing import TYPE_CHECKING, Any + import numpy as np +from nextgen_forcings_ewts import MODULE_NAME + +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.consts import ( + SUPPPRECIPMOD, +) -from . import regrid, time_handling, timeInterpMod +if TYPE_CHECKING: + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( + ConfigOptions, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.geoMod import ( + GeoMeta, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import ( + MpiConfig, + ) +LOG = logging.getLogger(MODULE_NAME) -class supplemental_precip: + +class SupplementalPrecip: """Supplemental precipitation abstract class. This is an abstract class that will define all the parameters of a single supplemental precipitation product. """ - def __init__(self): + def __init__(self, idx: int, config_options: ConfigOptions, geo_meta: GeoMeta): """Initializie all attributes and objects to None.""" - self.keyValue = None - self.inDir = None - self.enforce = None - self.product_name = None - self.file_type = None - self.nx_global = None - self.ny_global = None - self.nx_local = None - self.ny_local = None - self.x_lower_bound = None - self.x_upper_bound = None - self.y_lower_bound = None - self.y_upper_bound = None - self.regridOpt = None - self.timeInterpOpt = None - self.esmf_lats = None - self.esmf_lons = None - self.esmf_grid_in = None self.regridComplete = False - self.regridObj = None - self.esmf_field_in = None - self.esmf_field_out = None - self.esmf_field_out_elem = None - self.esmf_field_out_poly = None - self.regridded_precip1 = None - self.regridded_precip2 = None - self.regridded_rqi1 = None - self.regridded_rqi2 = None - self.regridded_mask = None - self.final_supp_precip = None - self.regridded_precip1_elem = None - self.regridded_precip2_elem = None - self.regridded_rqi1_elem = None - self.regridded_rqi2_elem = None - self.regridded_mask_elem = None - self.final_supp_precip_elem = None - self.file_in1 = None - self.file_in2 = None - self.rqiMethod = None - self.rqiThresh = None - self.rqi_file_in1 = None - self.rqi_file_in2 = None - self.pcp_hour1 = None - self.pcp_hour2 = None - self.pcp_date1 = None - self.pcp_date2 = None - self.fcst_hour1 = None - self.fcst_hour2 = None - self.input_frequency = None - self.netcdf_var_names = None - self.rqi_netcdf_var_names = None - self.grib_levels = None - self.grib_vars = None - self.tmpFile = None - self.userCycleOffset = None - self.global_x_lower = None - self.global_y_lower = None - self.global_x_upper = None - self.global_y_upper = None self.has_cache = False + self._keyValue = config_options.supp_precip_forcings[idx] + self.idx = idx + self.config_options = config_options + self.geo_mdeta = geo_meta + for attr in SUPPPRECIPMOD[self.__class__.__base__.__name__]: + setattr(self, attr, None) - def define_product(self): - """Define the product name based on the mapping forcing key value. + self.initialize_config_options() - Function to define the product name based on the mapping - forcing key value. - :return: - """ - product_names = { - 1: "MRMS_1HR_Radar_Only", - 2: "MRMS_1HR_Gage_Corrected", - 3: "WRF_ARW_Hawaii_2p5km_PCP", - 4: "WRF_ARW_PuertoRico_2p5km_PCP", - 5: "CONUS_MRMS_1HR_MultiSensor", - 6: "Hawaii_MRMS_1HR_MultiSensor", - 7: "MRMS_LiquidWaterFraction", - 8: "NBM_CORE_CONUS_APCP", - 9: "NBM_CORE_ALASKA_APCP", - 10: "AK_MRMS", - 11: "AK_Stage_IV_Precip-MRMS", - 12: "CONUS_Stage_IV_Precip-MRMS", - 13: "MRMS PrecipFlag", - 14: "Custom_Freq_Supp_Pcp", - 15: "NBM_CORE_PR_APCP", - 16: "NBM_CORE_HAWAII_APCP", - # 17: "Alaska_MRMS_1HR_Radar_Only", - # 18: "Hawaii_MRMS_1HR_Radar_Only", - # 19: "Puerto_Rico_MRMS_1HR_Radar_Only", - # 20: "Puerto_Rico_MRMS_1HR_Gage_Corrected", - } - self.product_name = product_names[self.keyValue] + @property + def keyValue(self) -> int: + """Get the forcing key value.""" + if self._keyValue is None: + raise RuntimeError("keyValue has not yet been set") + return self._keyValue + + @keyValue.setter + def keyValue(self, val: int) -> int: + """Set the forcing key value.""" + if self._keyValue is not None: + raise RuntimeError(f"keyValue has already been set (to {self._keyValue}).") + self._keyValue = val + + def initialize_config_options(self) -> None: + """Initialize configuration options from the config_options attribute.""" + for key, val in list(vars(self.config_options).items()): + if ( + isinstance(val, list) + and len(val) > 0 + and key not in ["rqiMethod", "rqiThresh"] + ): + setattr(self, key, val[self.idx]) + LOG.info(key) + + @property + def rqiMethod(self) -> int | float: + if self.config_options.rqiMethod is not None: + return self.config_options.rqiMethod[self.idx] + else: + return 0 + + @property + def rqiThresh(self) -> int | float: + if self.config_options.rqiMethod is not None: + return self.config_options.rqiThresh[self.idx] + else: + return 1.0 + + @property + def product_name(self) -> str: + return SUPPPRECIPMOD["PRODUCT_NAMES"][self.keyValue] ## DEFINED IN CONFIG # product_types = { @@ -115,134 +99,38 @@ def define_product(self): # 5: "GRIB2" # } # self.file_type = product_types[self.keyValue] - if self.file_type == "GRIB1": - self.file_ext = ".grb" - elif self.file_type == "GRIB2": - self.file_ext = ".grib2" - elif self.file_type == "NETCDF": - self.file_ext = ".nc" - - grib_vars_in = { - 1: None, - 2: None, - 3: None, - 4: None, - 5: None, - 6: None, - 7: None, - 8: None, - 9: None, - 10: None, - 11: None, - 12: None, - 13: None, - 14: None, - 15: None, - 16: None, - # 17: None, - # 18: None, - # 19: None, - # 20: None, - } - self.grib_vars = grib_vars_in[self.keyValue] - - grib_levels_in = { - 1: ["BLAH"], - 2: ["BLAH"], - 3: ["BLAH"], - 4: ["BLAH"], - 5: ["BLAH"], - 6: ["BLAH"], - 7: ["BLAH"], - 8: ["BLAH"], - 9: ["BLAH"], - 10: ["BLAH"], - 11: ["BLAH"], - 12: ["BLAH"], - 13: ["BLAH"], - 14: ["BLAH"], - 15: ["BLAH"], - 16: ["BLAH"], - # 17: ["BLAH"], - # 18: ["BLAH"], - # 19: ["BLAH"], - # 20: ["BLAH"], - } - self.grib_levels = grib_levels_in[self.keyValue] - - netcdf_variables = { - 1: ["RadarOnlyQPE01H_0mabovemeansealevel"], - 2: ["MultiSensorQPE01H_0mabovemeansealevel"], - 3: ["APCP_surface"], - 4: ["APCP_surface"], - 5: ["MultiSensorQPE01H_0mabovemeansealevel"], - 6: ["MultiSensorQPE01H_0mabovemeansealevel"], - 7: ["sbcv2_lwf"], - 8: ["APCP_surface"], - 9: ["APCP_surface"], - 10: ["MultiSensorQPE01H_0mabovemeansealevel"], - 11: [], # Set dynamically since we have have Stage IV and MRMS - 12: [], # Set dynamically since we have have Stage IV and MRMS - 13: ["PrecipFlag_0mabovemeansealevel"], - 14: ["PrecipFlag_0mabovemeansealevel"], - 15: ["APCP_surface"], - 16: ["APCP_surface"], - # 17: ["RadarOnlyQPE01H_0mabovemeansealevel"], - # 18: ["RadarOnlyQPE01H_0mabovemeansealevel"], - # 19: ["RadarOnlyQPE01H_0mabovemeansealevel"], - # 20: ["MultiSensorQPE01H_0mabovemeansealevel"], - } - self.netcdf_var_names = netcdf_variables[self.keyValue] - - netcdf_rqi_variables = { - 1: ["RadarQualityIndex_0mabovemeansealevel"], - 2: ["RadarQualityIndex_0mabovemeansealevel"], - 3: None, - 4: None, - 5: None, - 6: None, - 7: None, - 8: None, - 9: None, - 10: None, - 11: None, - 12: None, - 13: None, - 14: None, - 15: None, - 16: None, - # 17: None, - # 18: None, - # 19: None, - # 20: None, - } - self.rqi_netcdf_var_names = netcdf_rqi_variables[self.keyValue] - - output_variables = { - 1: 3, # RAINRATE - 2: 3, - 3: 3, - 4: 3, - 5: 3, - 6: 3, - 7: 8, # LQFRAC - 8: 3, - 9: 3, - 10: 3, - 11: 3, - 12: 3, - 13: 8, - 14: 3, - 15: 3, - 16: 3, - # 17: 3, - # 18: 3, - # 19: 3, - # 20: 3, - } - self.output_var_idx = output_variables[self.keyValue] - - def calc_neighbor_files(self, ConfigOptions, dCurrent, MpiConfig): + + @property + def file_ext(self) -> str: + return SUPPPRECIPMOD["FILE_EXT"][self.file_type] + + @property + def grib_vars(self) -> None: + return SUPPPRECIPMOD["GRIB_VARS"][self.keyValue] + + @property + def grib_levels(self) -> list[str]: + return SUPPPRECIPMOD["GRIB_LEVELS"][self.keyValue] + + @property + def netcdf_var_names(self) -> list[str]: + return SUPPPRECIPMOD["NET_CDF_VARS_NAMES"][self.keyValue] + + @property + def rqi_netcdf_var_names(self) -> list[str] | None: + return SUPPPRECIPMOD["RQI_NETCDF_VAR_NAMES"][self.keyValue] + + @property + def output_var_idx(self) -> int: + return SUPPPRECIPMOD["OUTPUT_VAR_IDX"][self.keyValue] + + @property + def find_neighbor_files(self) -> dict: + return SUPPPRECIPMOD["FIND_NEIGHBOR_FILES_MAP"] + + def calc_neighbor_files( + self, config_options: ConfigOptions, dcurrent, mpi_config: MpiConfig + ) -> None: """Calculate neighbor supplemental precipitation files. Function that will calculate the last/next expected @@ -252,42 +140,17 @@ def calc_neighbor_files(self, ConfigOptions, dCurrent, MpiConfig): :param dCurrent: :return: """ - # First calculate the current input cycle date this - # WRF-Hydro output timestep corresponds to. - find_neighbor_files = { - 1: time_handling.find_hourly_mrms_radar_neighbors, - 2: time_handling.find_hourly_mrms_radar_neighbors, - 3: time_handling.find_hourly_wrf_arw_neighbors, - 4: time_handling.find_hourly_wrf_arw_neighbors, - 5: time_handling.find_hourly_mrms_radar_neighbors, - 6: time_handling.find_hourly_mrms_radar_neighbors, - 7: time_handling.find_sbcv2_lwf_neighbors, - 8: time_handling.find_hourly_nbm_neighbors, - 9: time_handling.find_hourly_nbm_neighbors, - 10: time_handling.find_hourly_mrms_radar_neighbors, - 11: time_handling.find_ak_ext_ana_precip_neighbors, - 12: time_handling.find_conus_ext_ana_precip_neighbors, - 13: time_handling.find_hourly_mrms_precip_flag, - 14: time_handling.find_custom_freq_neighbors, - 15: time_handling.find_hourly_nbm_neighbors, - 16: time_handling.find_hourly_nbm_neighbors, - # 17: time_handling.find_hourly_mrms_radar_neighbors, - # 18: time_handling.find_hourly_mrms_radar_neighbors, - # 19: time_handling.find_hourly_mrms_radar_neighbors, - # 20: time_handling.find_hourly_mrms_radar_neighbors, - } - - find_neighbor_files[self.keyValue](self, ConfigOptions, dCurrent, MpiConfig) - # try: - # find_neighbor_files[self.keyValue](self,ConfigOptions,dCurrent,MpiConfig) - # except TypeError: - # ConfigOptions.errMsg = "Unable to execute find_neighbor_files for " \ - # "supplemental precipitation: " + self.product_name - # raise - # except: - # raise - - def regrid_inputs(self, ConfigOptions, wrfHyroGeoMeta, MpiConfig): + self.find_neighbor_files_map[self.keyValue]( + self, config_options, dcurrent, mpi_config + ) + + @property + def regrid_map(self) -> dict: + return SUPPPRECIPMOD["REGRID_MAP"] + + def regrid_inputs( + self, config_options: ConfigOptions, geo_meta: GeoMeta, mpi_config: MpiConfig + ) -> None: """Polymorphic function that will regrid input forcings to the supplemental precipitation grids for this particular timestep. Polymorphic function that will regrid input forcings to the @@ -300,37 +163,15 @@ def regrid_inputs(self, ConfigOptions, wrfHyroGeoMeta, MpiConfig): """ # Establish a mapping dictionary that will point the # code to the functions to that will regrid the data. - regrid_inputs = { - 1: regrid.regrid_mrms_hourly, - 2: regrid.regrid_mrms_hourly, - 3: regrid.regrid_hourly_wrf_arw_hi_res_pcp, - 4: regrid.regrid_hourly_wrf_arw_hi_res_pcp, - 5: regrid.regrid_mrms_hourly, - 6: regrid.regrid_mrms_hourly, - 7: regrid.regrid_sbcv2_liquid_water_fraction, - 8: regrid.regrid_hourly_nbm, - 9: regrid.regrid_hourly_nbm, - 10: regrid.regrid_mrms_hourly, - 11: regrid.regrid_ak_ext_ana_pcp, - 12: regrid.regrid_conus_ext_ana_pcp, - 13: regrid.regrid_mrms_precip_flag, - 14: regrid.regrid_mrms_hourly, - 15: regrid.regrid_hourly_nbm, - 16: regrid.regrid_hourly_nbm, - # 17: regrid.regrid_mrms_hourly, - # 18: regrid.regrid_mrms_hourly, - # 19: regrid.regrid_mrms_hourly, - # 20: regrid.regrid_mrms_hourly, - } - regrid_inputs[self.keyValue](self, ConfigOptions, wrfHyroGeoMeta, MpiConfig) - # try: - # regrid_inputs[self.keyValue](self,ConfigOptions,MpiConfig) - # except: - # ConfigOptions.errMsg = "Unable to execute regrid_inputs for " + \ - # "input forcing: " + self.product_name - # raise - - def temporal_interpolate_inputs(self, ConfigOptions, MpiConfig): + self.regrid_map[self.keyValue](self, config_options, geo_meta, mpi_config) + + @property + def temporal_interpolate_inputs_map(self) -> dict: + return SUPPPRECIPMOD["TEMPORAL_INTERPOLATE_INPUTS_MAP"] + + def temporal_interpolate_inputs( + self, config_options: ConfigOptions, mpi_config: MpiConfig + ): """Polymorphic function that will run temporal interpolation of the supplemental precipitation grids that have been regridded. Polymorphic function that will run temporal interpolation of @@ -342,94 +183,172 @@ def temporal_interpolate_inputs(self, ConfigOptions, MpiConfig): :param MpiConfig: :return: """ - temporal_interpolate_inputs = { - 0: timeInterpMod.no_interpolation_supp_pcp, - 1: timeInterpMod.nearest_neighbor_supp_pcp, - 2: timeInterpMod.weighted_average_supp_pcp, - } - temporal_interpolate_inputs[self.timeInterpOpt](self, ConfigOptions, MpiConfig) - # temporal_interpolate_inputs[self.keyValue](self,ConfigOptions,MpiConfig) - # try: - # temporal_interpolate_inputs[self.timeInterpOpt](self,ConfigOptions,MpiConfig) - # except: - # ConfigOptions.errMsg = "Unable to execute temporal_interpolate_inputs " + \ - # " for input forcing: " + self.product_name - # raise - - -def initDict(ConfigOptions, GeoMetaWrfHydro): - """Initialize the supplemental precipitation input dictionary. + self.temporal_interpolate_inputs_map[self.keyValue][self.timeInterpOpt]( + self, config_options, mpi_config + ) - Initial function to create an supplemental dictionary, which - will contain an abstract class for each supplemental precip product. - This gets called one time by the parent calling program. - :param ConfigOptions: - :return: InputDict - A dictionary defining our inputs. - """ - # Initialize an empty dictionary - InputDict = {} - - for supp_pcp_tmp in range(0, ConfigOptions.number_supp_pcp): - supp_pcp_key = ConfigOptions.supp_precip_forcings[supp_pcp_tmp] - InputDict[supp_pcp_key] = supplemental_precip() - InputDict[supp_pcp_key].keyValue = supp_pcp_key - InputDict[supp_pcp_key].regridOpt = ConfigOptions.regrid_opt_supp_pcp[ - supp_pcp_tmp - ] - InputDict[supp_pcp_key].enforce = ConfigOptions.supp_precip_mandatory[ - supp_pcp_tmp - ] - InputDict[supp_pcp_key].timeInterpOpt = ConfigOptions.suppTemporalInterp[ - supp_pcp_tmp - ] - - InputDict[supp_pcp_key].inDir = ConfigOptions.supp_precip_dirs[supp_pcp_tmp] - InputDict[supp_pcp_key].file_type = ConfigOptions.supp_precip_file_types[ - supp_pcp_tmp - ] - InputDict[supp_pcp_key].define_product() - - if ConfigOptions.grid_type == "gridded": - # Initialize the local final grid of values - InputDict[supp_pcp_key].final_supp_precip = np.empty( - [GeoMetaWrfHydro.ny_local, GeoMetaWrfHydro.nx_local], np.float64 - ) - InputDict[supp_pcp_key].regridded_mask = np.empty( - [GeoMetaWrfHydro.ny_local, GeoMetaWrfHydro.nx_local], np.float32 - ) - elif ConfigOptions.grid_type == "unstructured": - # Initialize the local final grid of values - InputDict[supp_pcp_key].final_supp_precip = np.empty( - [GeoMetaWrfHydro.ny_local], np.float64 - ) - InputDict[supp_pcp_key].regridded_mask = np.empty( - [GeoMetaWrfHydro.ny_local], np.float32 - ) - InputDict[supp_pcp_key].final_supp_precip_elem = np.empty( - [GeoMetaWrfHydro.ny_local_elem], np.float64 - ) - InputDict[supp_pcp_key].regridded_mask_elem = np.empty( - [GeoMetaWrfHydro.ny_local_elem], np.float32 - ) - elif ConfigOptions.grid_type == "hydrofabric": - # Initialize the local final grid of values - # NOTE changed from np.empty to np.full for determinism of test data. - InputDict[supp_pcp_key].final_supp_precip = np.full( - [GeoMetaWrfHydro.ny_local], np.nan, dtype=np.float64 + +class SupplementalPrecipGridded(SupplementalPrecip): + def __init__( + self, + idx: int = None, + config_options: ConfigOptions = None, + geo_meta: GeoMeta = None, + ) -> None: + """Initialize InputForcingsUnstructured with configuration options, geospatial metadata, and MPI configuration.""" + super().__init__(idx, config_options, geo_meta) + for attr in SUPPPRECIPMOD[self.__class__.__name__]: + setattr(self, attr, None) + + @cached_property + def final_supp_precip(self) -> np.ndarray | Any: + if self._final_supp_precip is not None: + return self._final_supp_precip + else: + return np.full( + [self.geo_meta.ny_local, self.geo_meta.nx_local], + np.nan, + dtype=np.float64, ) - InputDict[supp_pcp_key].regridded_mask = np.full( - [GeoMetaWrfHydro.ny_local], np.nan, dtype=np.float32 + + @final_supp_precip.setter + def final_supp_precip(self, value: Any) -> Any: + """Setter for final_supp_precip.""" + self._final_supp_precip = value + + @cached_property + def regridded_mask(self) -> np.ndarray | Any: + if self._regridded_mask is not None: + return self._regridded_mask + else: + return np.full( + [self.geo_meta.ny_local, self.geo_meta.nx_local], np.nan, np.float32 ) - InputDict[supp_pcp_key].userCycleOffset = ConfigOptions.supp_input_offsets[ - supp_pcp_tmp - ] + @regridded_mask.setter + def regridded_mask(self, value: Any) -> Any: + """Setter for regridded_mask.""" + self._regridded_mask = value + + +class SupplementalPrecipHydrofabric(SupplementalPrecip): + def __init__( + self, + idx: int = None, + config_options: ConfigOptions = None, + geo_meta: GeoMeta = None, + ) -> None: + """Initialize InputForcingsUnstructured with configuration options, geospatial metadata, and MPI configuration.""" + super().__init__(idx, config_options, geo_meta) + for attr in SUPPPRECIPMOD[self.__class__.__name__]: + setattr(self, attr, None) + + @cached_property + def final_supp_precip(self) -> np.ndarray | Any: + if self._final_supp_precip is not None: + return self._final_supp_precip + else: + return np.full([self.geo_meta.ny_local], np.nan, dtype=np.float64) + + @final_supp_precip.setter + def final_supp_precip(self, value: Any) -> Any: + """Setter for final_supp_precip.""" + self._final_supp_precip = value + + @cached_property + def regridded_mask(self) -> np.ndarray | Any: + if self._regridded_mask is not None: + return self._regridded_mask + else: + return np.full([self.geo_meta.ny_local], np.nan, dtype=np.float32) + + @regridded_mask.setter + def regridded_mask(self, value: Any) -> Any: + """Setter for regridded_mask.""" + self._regridded_mask = value + + +class SupplementalPrecipUnstructured(SupplementalPrecip): + def __init__( + self, + idx: int = None, + config_options: ConfigOptions = None, + geo_meta: GeoMeta = None, + ) -> None: + """Initialize InputForcingsUnstructured with configuration options, geospatial metadata, and MPI configuration.""" + super().__init__(idx, config_options, geo_meta) + for attr in SUPPPRECIPMOD[self.__class__.__name__]: + setattr(self, attr, None) + + @cached_property + def final_supp_precip(self) -> np.ndarray | Any: + if self._final_supp_precip is not None: + return self._final_supp_precip + else: + return np.full([self.geo_meta.ny_local], np.nan, dtype=np.float64) + + @final_supp_precip.setter + def final_supp_precip(self, value: Any) -> Any: + """Setter for final_supp_precip.""" + self._final_supp_precip = value - if ConfigOptions.rqiMethod is not None: - InputDict[supp_pcp_key].rqiMethod = ConfigOptions.rqiMethod[supp_pcp_tmp] - InputDict[supp_pcp_key].rqiThresh = ConfigOptions.rqiThresh[supp_pcp_tmp] + @cached_property + def regridded_mask(self) -> np.ndarray | Any: + if self._regridded_mask is not None: + return self._regridded_mask else: - InputDict[supp_pcp_key].rqiMethod = 0 - InputDict[supp_pcp_key].rqiThresh = 1.0 + return np.full([self.geo_meta.ny_local], np.nan, dtype=np.float32) + + @regridded_mask.setter + def regridded_mask(self, value: Any) -> Any: + """Setter for regridded_mask.""" + self._regridded_mask = value + + @cached_property + def final_supp_precip_elem(self) -> np.ndarray | Any: + if self._final_supp_precip_elem is not None: + return self._final_supp_precip_elem + else: + return np.full([self.geo_meta.ny_local_elem], np.nan, dtype=np.float64) + + @final_supp_precip_elem.setter + def final_supp_precip_elem(self, value: Any) -> Any: + """Setter for final_supp_precip_elem.""" + self._final_supp_precip_elem = value + + @cached_property + def regridded_mask_elem(self) -> np.ndarray | Any: + if self._regridded_mask_elem is not None: + return self._regridded_mask_elem + else: + return np.full([self.geo_meta.ny_local_elem], np.nan, dtype=np.float32) + + @regridded_mask_elem.setter + def regridded_mask_elem(self, value: Any) -> Any: + """Setter for regridded_mask_elem.""" + self._regridded_mask_elem = value + - return InputDict +SUPPPRECIP = { + "gridded": SupplementalPrecipGridded, + "unstructured": SupplementalPrecipUnstructured, + "hydrofabric": SupplementalPrecipHydrofabric, +} + + +def init_dict(config_options: ConfigOptions, geo_meta: GeoMeta) -> dict: + """Initialize the supplemental precipitation input dictionary. + + Initial function to create an supplemental dictionary, which + will contain an abstract class for each supplemental precip product. + This gets called one time by the parent calling program. + :param ConfigOptions: + :return: input_dict - A dictionary defining our inputs. + """ + input_dict = {} + for idx in range(0, config_options.number_supp_pcp): + supp_pcp_key = config_options.supp_precip_forcings[idx] + input_dict[supp_pcp_key] = SUPPPRECIP[config_options.grid_type]( + idx, config_options, geo_meta + ) + return input_dict From 48cd232bb8940789d9a3b5a63bc7da4366430923 Mon Sep 17 00:00:00 2001 From: Matthew Deshotel Date: Sun, 29 Mar 2026 14:00:52 -0500 Subject: [PATCH 002/132] add doc strings --- .../core/suppPrecipMod.py | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py index ca3462e3..c8a2589c 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py @@ -28,10 +28,9 @@ class SupplementalPrecip: - """Supplemental precipitation abstract class. + """Supplemental precipitation class. - This is an abstract class that will define all the parameters - of a single supplemental precipitation product. + This class defines all the parameters of a single supplemental precipitation product. """ def __init__(self, idx: int, config_options: ConfigOptions, geo_meta: GeoMeta): @@ -74,6 +73,7 @@ def initialize_config_options(self) -> None: @property def rqiMethod(self) -> int | float: + """Get the RQI method for this supplemental precipitation product.""" if self.config_options.rqiMethod is not None: return self.config_options.rqiMethod[self.idx] else: @@ -81,6 +81,7 @@ def rqiMethod(self) -> int | float: @property def rqiThresh(self) -> int | float: + """Get the RQI threshold for this supplemental precipitation product.""" if self.config_options.rqiMethod is not None: return self.config_options.rqiThresh[self.idx] else: @@ -88,6 +89,7 @@ def rqiThresh(self) -> int | float: @property def product_name(self) -> str: + """Get the product name for this supplemental precipitation product.""" return SUPPPRECIPMOD["PRODUCT_NAMES"][self.keyValue] ## DEFINED IN CONFIG @@ -102,30 +104,37 @@ def product_name(self) -> str: @property def file_ext(self) -> str: + """Get the file extension for this supplemental precipitation product.""" return SUPPPRECIPMOD["FILE_EXT"][self.file_type] @property def grib_vars(self) -> None: + """Get the GRIB variable names for this supplemental precipitation product.""" return SUPPPRECIPMOD["GRIB_VARS"][self.keyValue] @property def grib_levels(self) -> list[str]: + """Get the GRIB levels for this supplemental precipitation product.""" return SUPPPRECIPMOD["GRIB_LEVELS"][self.keyValue] @property def netcdf_var_names(self) -> list[str]: + """Get the NetCDF variable names for this supplemental precipitation product.""" return SUPPPRECIPMOD["NET_CDF_VARS_NAMES"][self.keyValue] @property def rqi_netcdf_var_names(self) -> list[str] | None: + """Get the RQI NetCDF variable names for this supplemental precipitation product.""" return SUPPPRECIPMOD["RQI_NETCDF_VAR_NAMES"][self.keyValue] @property def output_var_idx(self) -> int: + """Get the output variable index for this supplemental precipitation product.""" return SUPPPRECIPMOD["OUTPUT_VAR_IDX"][self.keyValue] @property def find_neighbor_files(self) -> dict: + """Get the function to find neighbor supplemental precipitation files for this supplemental precipitation product.""" return SUPPPRECIPMOD["FIND_NEIGHBOR_FILES_MAP"] def calc_neighbor_files( @@ -146,6 +155,7 @@ def calc_neighbor_files( @property def regrid_map(self) -> dict: + """Get the function to regrid input forcings to the supplemental precipitation grids for this supplemental precipitation product.""" return SUPPPRECIPMOD["REGRID_MAP"] def regrid_inputs( @@ -167,6 +177,7 @@ def regrid_inputs( @property def temporal_interpolate_inputs_map(self) -> dict: + """Get the function to temporal interpolate input forcings to the supplemental precipitation grids for this supplemental precipitation product.""" return SUPPPRECIPMOD["TEMPORAL_INTERPOLATE_INPUTS_MAP"] def temporal_interpolate_inputs( @@ -189,19 +200,22 @@ def temporal_interpolate_inputs( class SupplementalPrecipGridded(SupplementalPrecip): + """Supplemental precipitation class for gridded products.""" + def __init__( self, idx: int = None, config_options: ConfigOptions = None, geo_meta: GeoMeta = None, ) -> None: - """Initialize InputForcingsUnstructured with configuration options, geospatial metadata, and MPI configuration.""" + """Initialize SupplementalPrecipGridded with configuration options, geospatial metadata, and MPI configuration.""" super().__init__(idx, config_options, geo_meta) for attr in SUPPPRECIPMOD[self.__class__.__name__]: setattr(self, attr, None) @cached_property def final_supp_precip(self) -> np.ndarray | Any: + """Get the final supplemental precipitation grid after regridding and temporal interpolation.""" if self._final_supp_precip is not None: return self._final_supp_precip else: @@ -218,6 +232,7 @@ def final_supp_precip(self, value: Any) -> Any: @cached_property def regridded_mask(self) -> np.ndarray | Any: + """Get the regridded mask after regridding input forcings to the supplemental precipitation grids.""" if self._regridded_mask is not None: return self._regridded_mask else: @@ -232,19 +247,22 @@ def regridded_mask(self, value: Any) -> Any: class SupplementalPrecipHydrofabric(SupplementalPrecip): + """Supplemental precipitation class for hydrofabric grids.""" + def __init__( self, idx: int = None, config_options: ConfigOptions = None, geo_meta: GeoMeta = None, ) -> None: - """Initialize InputForcingsUnstructured with configuration options, geospatial metadata, and MPI configuration.""" + """Initialize SupplementalPrecipHydrofabric with configuration options, geospatial metadata, and MPI configuration.""" super().__init__(idx, config_options, geo_meta) for attr in SUPPPRECIPMOD[self.__class__.__name__]: setattr(self, attr, None) @cached_property def final_supp_precip(self) -> np.ndarray | Any: + """Get the final supplemental precipitation grid after regridding and temporal interpolation.""" if self._final_supp_precip is not None: return self._final_supp_precip else: @@ -257,6 +275,7 @@ def final_supp_precip(self, value: Any) -> Any: @cached_property def regridded_mask(self) -> np.ndarray | Any: + """Get the regridded mask after regridding input forcings to the supplemental precipitation grids.""" if self._regridded_mask is not None: return self._regridded_mask else: @@ -269,6 +288,8 @@ def regridded_mask(self, value: Any) -> Any: class SupplementalPrecipUnstructured(SupplementalPrecip): + """Supplemental precipitation class for unstructured grids.""" + def __init__( self, idx: int = None, @@ -282,6 +303,7 @@ def __init__( @cached_property def final_supp_precip(self) -> np.ndarray | Any: + """Get the final supplemental precipitation grid after regridding and temporal interpolation.""" if self._final_supp_precip is not None: return self._final_supp_precip else: @@ -294,6 +316,7 @@ def final_supp_precip(self, value: Any) -> Any: @cached_property def regridded_mask(self) -> np.ndarray | Any: + """Get the regridded mask after regridding input forcings to the supplemental precipitation grids.""" if self._regridded_mask is not None: return self._regridded_mask else: @@ -306,6 +329,7 @@ def regridded_mask(self, value: Any) -> Any: @cached_property def final_supp_precip_elem(self) -> np.ndarray | Any: + """Get the final supplemental precipitation grid after regridding and temporal interpolation for unstructured grids.""" if self._final_supp_precip_elem is not None: return self._final_supp_precip_elem else: @@ -318,6 +342,7 @@ def final_supp_precip_elem(self, value: Any) -> Any: @cached_property def regridded_mask_elem(self) -> np.ndarray | Any: + """Get the regridded mask after regridding input forcings to the supplemental precipitation grids for unstructured grids.""" if self._regridded_mask_elem is not None: return self._regridded_mask_elem else: From d3a1af80a647b1e42fd6ddf04bcb0c4e53331bbb Mon Sep 17 00:00:00 2001 From: Matthew Deshotel Date: Mon, 30 Mar 2026 17:18:02 -0500 Subject: [PATCH 003/132] make _initialize_config_options a private method and increase verbosity of docstring --- .../NextGen_Forcings_Engine/core/suppPrecipMod.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py index c8a2589c..4c7fceef 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py @@ -44,7 +44,7 @@ def __init__(self, idx: int, config_options: ConfigOptions, geo_meta: GeoMeta): for attr in SUPPPRECIPMOD[self.__class__.__base__.__name__]: setattr(self, attr, None) - self.initialize_config_options() + self._initialize_config_options() @property def keyValue(self) -> int: @@ -60,8 +60,14 @@ def keyValue(self, val: int) -> int: raise RuntimeError(f"keyValue has already been set (to {self._keyValue}).") self._keyValue = val - def initialize_config_options(self) -> None: - """Initialize configuration options from the config_options attribute.""" + def _initialize_config_options(self) -> None: + """Initialize configuration options from the config_options attribute. + + Map attibutes from config_options to attibutes of this class if + they are a list with a length greater than 0. + + Check if the attibute allready exists before setting. + """ for key, val in list(vars(self.config_options).items()): if ( isinstance(val, list) From 01fd4627bb966585479ba12da08857b96bdb1d2c Mon Sep 17 00:00:00 2001 From: Matthew Deshotel Date: Mon, 30 Mar 2026 17:19:03 -0500 Subject: [PATCH 004/132] check if attibute allready exists --- .../NextGen_Forcings_Engine/core/suppPrecipMod.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py index 4c7fceef..a1145c72 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py @@ -74,8 +74,9 @@ def _initialize_config_options(self) -> None: and len(val) > 0 and key not in ["rqiMethod", "rqiThresh"] ): + if self.hasattr(self, key): + raise ValueError(f"Attribute {key} has already been set.") setattr(self, key, val[self.idx]) - LOG.info(key) @property def rqiMethod(self) -> int | float: From 6306463c1d4ce17d54a00ddf97757834e673239d Mon Sep 17 00:00:00 2001 From: Matthew Deshotel Date: Mon, 30 Mar 2026 17:19:46 -0500 Subject: [PATCH 005/132] add comment for initializing attributes to none --- .../NextGen_Forcings_Engine/core/suppPrecipMod.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py index a1145c72..3d14700b 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py @@ -41,6 +41,9 @@ def __init__(self, idx: int, config_options: ConfigOptions, geo_meta: GeoMeta): self.idx = idx self.config_options = config_options self.geo_mdeta = geo_meta + + # set list of attibutes from consts.py to None. + # These are indexed from the consts dictionary using the class name for attr in SUPPPRECIPMOD[self.__class__.__base__.__name__]: setattr(self, attr, None) From 23059c6a5a1bd375ea68a0ac7a09f529edaa4cdb Mon Sep 17 00:00:00 2001 From: Matthew Deshotel Date: Wed, 8 Apr 2026 11:57:43 -0500 Subject: [PATCH 006/132] fix manual caching of properties --- .../core/suppPrecipMod.py | 68 ++++++++++--------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py index 3d14700b..6e08d41f 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py @@ -226,14 +226,13 @@ def __init__( @cached_property def final_supp_precip(self) -> np.ndarray | Any: """Get the final supplemental precipitation grid after regridding and temporal interpolation.""" - if self._final_supp_precip is not None: - return self._final_supp_precip - else: - return np.full( + if self._final_supp_precip is None: + self._final_supp_precip = np.full( [self.geo_meta.ny_local, self.geo_meta.nx_local], np.nan, dtype=np.float64, ) + return self._final_supp_precip @final_supp_precip.setter def final_supp_precip(self, value: Any) -> Any: @@ -243,12 +242,11 @@ def final_supp_precip(self, value: Any) -> Any: @cached_property def regridded_mask(self) -> np.ndarray | Any: """Get the regridded mask after regridding input forcings to the supplemental precipitation grids.""" - if self._regridded_mask is not None: - return self._regridded_mask - else: - return np.full( + if self._regridded_mask is None: + self._regridded_mask = np.full( [self.geo_meta.ny_local, self.geo_meta.nx_local], np.nan, np.float32 ) + return self._regridded_mask @regridded_mask.setter def regridded_mask(self, value: Any) -> Any: @@ -273,10 +271,11 @@ def __init__( @cached_property def final_supp_precip(self) -> np.ndarray | Any: """Get the final supplemental precipitation grid after regridding and temporal interpolation.""" - if self._final_supp_precip is not None: - return self._final_supp_precip - else: - return np.full([self.geo_meta.ny_local], np.nan, dtype=np.float64) + if self._final_supp_precip is None: + self._final_supp_precip = np.full( + [self.geo_meta.ny_local], np.nan, dtype=np.float64 + ) + return self._final_supp_precip @final_supp_precip.setter def final_supp_precip(self, value: Any) -> Any: @@ -286,10 +285,11 @@ def final_supp_precip(self, value: Any) -> Any: @cached_property def regridded_mask(self) -> np.ndarray | Any: """Get the regridded mask after regridding input forcings to the supplemental precipitation grids.""" - if self._regridded_mask is not None: - return self._regridded_mask - else: - return np.full([self.geo_meta.ny_local], np.nan, dtype=np.float32) + if self._regridded_mask is None: + self._regridded_mask = np.full( + [self.geo_meta.ny_local], np.nan, dtype=np.float32 + ) + return self._regridded_mask @regridded_mask.setter def regridded_mask(self, value: Any) -> Any: @@ -314,10 +314,11 @@ def __init__( @cached_property def final_supp_precip(self) -> np.ndarray | Any: """Get the final supplemental precipitation grid after regridding and temporal interpolation.""" - if self._final_supp_precip is not None: - return self._final_supp_precip - else: - return np.full([self.geo_meta.ny_local], np.nan, dtype=np.float64) + if self._final_supp_precip is None: + self._final_supp_precip = np.full( + [self.geo_meta.ny_local], np.nan, dtype=np.float64 + ) + return self._final_supp_precip @final_supp_precip.setter def final_supp_precip(self, value: Any) -> Any: @@ -327,10 +328,11 @@ def final_supp_precip(self, value: Any) -> Any: @cached_property def regridded_mask(self) -> np.ndarray | Any: """Get the regridded mask after regridding input forcings to the supplemental precipitation grids.""" - if self._regridded_mask is not None: - return self._regridded_mask - else: - return np.full([self.geo_meta.ny_local], np.nan, dtype=np.float32) + if self._regridded_mask is None: + self._regridded_mask = np.full( + [self.geo_meta.ny_local], np.nan, dtype=np.float32 + ) + return self._regridded_mask @regridded_mask.setter def regridded_mask(self, value: Any) -> Any: @@ -340,10 +342,11 @@ def regridded_mask(self, value: Any) -> Any: @cached_property def final_supp_precip_elem(self) -> np.ndarray | Any: """Get the final supplemental precipitation grid after regridding and temporal interpolation for unstructured grids.""" - if self._final_supp_precip_elem is not None: - return self._final_supp_precip_elem - else: - return np.full([self.geo_meta.ny_local_elem], np.nan, dtype=np.float64) + if self._final_supp_precip_elem is None: + self._final_supp_precip_elem = np.full( + [self.geo_meta.ny_local_elem], np.nan, dtype=np.float64 + ) + return self._final_supp_precip_elem @final_supp_precip_elem.setter def final_supp_precip_elem(self, value: Any) -> Any: @@ -353,10 +356,11 @@ def final_supp_precip_elem(self, value: Any) -> Any: @cached_property def regridded_mask_elem(self) -> np.ndarray | Any: """Get the regridded mask after regridding input forcings to the supplemental precipitation grids for unstructured grids.""" - if self._regridded_mask_elem is not None: - return self._regridded_mask_elem - else: - return np.full([self.geo_meta.ny_local_elem], np.nan, dtype=np.float32) + if self._regridded_mask_elem is None: + self._regridded_mask_elem = np.full( + [self.geo_meta.ny_local_elem], np.nan, dtype=np.float32 + ) + return self._regridded_mask_elem @regridded_mask_elem.setter def regridded_mask_elem(self, value: Any) -> Any: From b59b83b8c29df57bba16cd514284a02e42abc108 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 10 Aug 2026 16:20:19 -0400 Subject: [PATCH 007/132] Fix broken symbol reference --- .../NextGen_Forcings_Engine/bmi_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 3c27eacb..90bae785 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -391,7 +391,7 @@ def initialize(self, config_file: str, output_path: str | None = None) -> None: # If we have specified supplemental precipitation products, initialize # the supp class. if self._job_meta.number_supp_pcp > 0: - self._supp_pcp_mod = suppPrecipMod.initDict(self._job_meta, self.geo_meta) + self._supp_pcp_mod = suppPrecipMod.init_dict(self._job_meta, self.geo_meta) else: self._supp_pcp_mod = None err_handler.check_program_status(self._job_meta, self._mpi_meta) From 20a552758126199563377bac8791c55f98e4189c Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 10 Aug 2026 16:28:41 -0400 Subject: [PATCH 008/132] Fix issues with handling of setters/getters, properties, usage of consts.py, symbol misspelling, and stale ewts reference. --- .../core/suppPrecipMod.py | 188 +++++++++++++----- 1 file changed, 140 insertions(+), 48 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py index 6e08d41f..92dfdf45 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py @@ -3,11 +3,9 @@ from __future__ import annotations import logging -from functools import cached_property from typing import TYPE_CHECKING, Any import numpy as np -from nextgen_forcings_ewts import MODULE_NAME from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.consts import ( SUPPPRECIPMOD, @@ -24,13 +22,23 @@ MpiConfig, ) -LOG = logging.getLogger(MODULE_NAME) +LOG = logging.getLogger("FORCING") class SupplementalPrecip: """Supplemental precipitation class. This class defines all the parameters of a single supplemental precipitation product. + + Three-tier attr initialization: + 1. Attrs set during init (keyValue, geo_meta, etc.) come from constructor params — must NOT be in SUPPPRECIPMOD. + 2. Attrs in SUPPPRECIPMOD[base class name] are then set to None as an "unset" sentinel. + 3. _initialize_config_options then sets list-valued attrs from config_options; + remaining attrs lazy-initialize via property getters on first access. + + NOTE: Lists are treated specially in config_options. When an attribute value in config_options + is a list, the idx of this instance is used to extract the corresponding element from that list. + This allows each SupplementalPrecip instance to reference its own value within a shared list structure. """ def __init__(self, idx: int, config_options: ConfigOptions, geo_meta: GeoMeta): @@ -40,10 +48,8 @@ def __init__(self, idx: int, config_options: ConfigOptions, geo_meta: GeoMeta): self._keyValue = config_options.supp_precip_forcings[idx] self.idx = idx self.config_options = config_options - self.geo_mdeta = geo_meta + self.geo_meta = geo_meta - # set list of attibutes from consts.py to None. - # These are indexed from the consts dictionary using the class name for attr in SUPPPRECIPMOD[self.__class__.__base__.__name__]: setattr(self, attr, None) @@ -59,48 +65,68 @@ def keyValue(self) -> int: @keyValue.setter def keyValue(self, val: int) -> int: """Set the forcing key value.""" - if self._keyValue is not None: - raise RuntimeError(f"keyValue has already been set (to {self._keyValue}).") self._keyValue = val def _initialize_config_options(self) -> None: """Initialize configuration options from the config_options attribute. - Map attibutes from config_options to attibutes of this class if - they are a list with a length greater than 0. - - Check if the attibute allready exists before setting. + For each attribute in SUPPPRECIPMOD["SupplementalPrecip"], check if the + same-named attribute exists in config_options as a list and set it on self. """ - for key, val in list(vars(self.config_options).items()): - if ( - isinstance(val, list) - and len(val) > 0 - and key not in ["rqiMethod", "rqiThresh"] - ): - if self.hasattr(self, key): - raise ValueError(f"Attribute {key} has already been set.") - setattr(self, key, val[self.idx]) + for attr in SUPPPRECIPMOD[self.__class__.__base__.__name__]: + if hasattr(self.config_options, attr): + val = getattr(self.config_options, attr) + if isinstance(val, list) and len(val) > 0: + setattr(self, attr, val[self.idx]) @property def rqiMethod(self) -> int | float: """Get the RQI method for this supplemental precipitation product.""" - if self.config_options.rqiMethod is not None: - return self.config_options.rqiMethod[self.idx] - else: - return 0 + if self._rqiMethod is None: + # config_options stores each product's values as a list (one entry per supp precip product). + # A non-list value means RQI is not configured (default rqiMethod to 0). + val = self.config_options.rqiMethod + if isinstance(val, list): + self._rqiMethod = val[self.idx] + elif val is None: + self._rqiMethod = 0 + else: + raise TypeError(f"Unexpected type for config_options.rqiMethod: {type(val)}") + return self._rqiMethod + + @rqiMethod.setter + def rqiMethod(self, val: int | float) -> None: + """Setter for grib_vars.""" + self._rqiMethod = val @property def rqiThresh(self) -> int | float: """Get the RQI threshold for this supplemental precipitation product.""" - if self.config_options.rqiMethod is not None: - return self.config_options.rqiThresh[self.idx] - else: - return 1.0 + if self._rqiThresh is None: + # config_options stores each product's values as a list (one entry per supp precip product). + # A non-list value means RQI is not configured (default rqiMethod to 1.0). + val = self.config_options.rqiThresh + if isinstance(val, list): + self._rqiThresh = val[self.idx] + elif val is None or isinstance(val, (int, float)): + # config.py initializes rqiThresh=1.0 as the no-RQI default. + # When RQI is configured, a scalar gets expanded to a list before reaching here. + self._rqiThresh = float(val) if val is not None else 1.0 + else: + raise TypeError(f"Unexpected type for config_options.rqiThresh: {type(val)}") + return self._rqiThresh + + @rqiThresh.setter + def rqiThresh(self, val: int | float) -> None: + """Setter for rqiThresh.""" + self._rqiThresh = val @property def product_name(self) -> str: """Get the product name for this supplemental precipitation product.""" - return SUPPPRECIPMOD["PRODUCT_NAMES"][self.keyValue] + if self._product_name is None: + self._product_name = SUPPPRECIPMOD["PRODUCT_NAMES"][self.keyValue] + return self._product_name ## DEFINED IN CONFIG # product_types = { @@ -112,30 +138,96 @@ def product_name(self) -> str: # } # self.file_type = product_types[self.keyValue] + @product_name.setter + def product_name(self, val: str) -> None: + """Setter for product_name.""" + self._product_name = val + + @property + def file_type(self) -> str: + """Get the file type; aliases supp_precip_file_types set by _initialize_config_options.""" + return self.supp_precip_file_types + + @file_type.setter + def file_type(self, val: str) -> None: + """Setter for file_type; writes through to supp_precip_file_types.""" + self.supp_precip_file_types = val + + # TODO: remove these aliases once time_handling.py and regrid.py are refactored to use new attribute names + @property + def inDir(self): + return self.supp_precip_dirs + + @property + def regridOpt(self): + return self.regrid_opt_supp_pcp + + @property + def enforce(self): + return self.supp_precip_mandatory + + @property + def timeInterpOpt(self): + return self.suppTemporalInterp + + @property + def userCycleOffset(self): + return self.supp_input_offsets + @property def file_ext(self) -> str: """Get the file extension for this supplemental precipitation product.""" return SUPPPRECIPMOD["FILE_EXT"][self.file_type] @property - def grib_vars(self) -> None: + def grib_vars(self) -> list[str]: """Get the GRIB variable names for this supplemental precipitation product.""" - return SUPPPRECIPMOD["GRIB_VARS"][self.keyValue] + if self._grib_vars is None: + self._grib_vars = SUPPPRECIPMOD["GRIB_VARS"][self.keyValue] + return self._grib_vars + + @grib_vars.setter + def grib_vars(self, val: list[str]) -> None: + """Setter for grib_vars.""" + self._grib_vars = val @property def grib_levels(self) -> list[str]: """Get the GRIB levels for this supplemental precipitation product.""" - return SUPPPRECIPMOD["GRIB_LEVELS"][self.keyValue] + if self._grib_levels is None: + self._grib_levels = SUPPPRECIPMOD["GRIB_LEVELS"][self.keyValue] + return self._grib_levels + + @grib_levels.setter + def grib_levels(self, val: list[str]) -> None: + """Setter for grib_levels.""" + self._grib_levels = val @property def netcdf_var_names(self) -> list[str]: """Get the NetCDF variable names for this supplemental precipitation product.""" - return SUPPPRECIPMOD["NET_CDF_VARS_NAMES"][self.keyValue] + if self._netcdf_var_names is None: + self._netcdf_var_names = SUPPPRECIPMOD["NET_CDF_VARS_NAMES"][self.keyValue] + return self._netcdf_var_names + + @netcdf_var_names.setter + def netcdf_var_names(self, val: list[str]) -> None: + """Setter for netcdf_var_names.""" + self._netcdf_var_names = val @property def rqi_netcdf_var_names(self) -> list[str] | None: """Get the RQI NetCDF variable names for this supplemental precipitation product.""" - return SUPPPRECIPMOD["RQI_NETCDF_VAR_NAMES"][self.keyValue] + if self._rqi_netcdf_var_names is None: + self._rqi_netcdf_var_names = SUPPPRECIPMOD["RQI_NETCDF_VAR_NAMES"][ + self.keyValue + ] + return self._rqi_netcdf_var_names + + @rqi_netcdf_var_names.setter + def rqi_netcdf_var_names(self, val: list[str] | None) -> None: + """Setter for rqi_netcdf_var_names.""" + self._rqi_netcdf_var_names = val @property def output_var_idx(self) -> int: @@ -159,7 +251,7 @@ def calc_neighbor_files( :param dCurrent: :return: """ - self.find_neighbor_files_map[self.keyValue]( + self.find_neighbor_files[self.keyValue]( self, config_options, dcurrent, mpi_config ) @@ -204,7 +296,7 @@ def temporal_interpolate_inputs( :param MpiConfig: :return: """ - self.temporal_interpolate_inputs_map[self.keyValue][self.timeInterpOpt]( + self.temporal_interpolate_inputs_map[self.timeInterpOpt]( self, config_options, mpi_config ) @@ -218,12 +310,12 @@ def __init__( config_options: ConfigOptions = None, geo_meta: GeoMeta = None, ) -> None: - """Initialize SupplementalPrecipGridded with configuration options, geospatial metadata, and MPI configuration.""" + """Initialize SupplementalPrecipGridded. Any subclass-specific attr names are sourced from SUPPPRECIPMOD[classname] in consts.py.""" super().__init__(idx, config_options, geo_meta) for attr in SUPPPRECIPMOD[self.__class__.__name__]: setattr(self, attr, None) - @cached_property + @property def final_supp_precip(self) -> np.ndarray | Any: """Get the final supplemental precipitation grid after regridding and temporal interpolation.""" if self._final_supp_precip is None: @@ -239,7 +331,7 @@ def final_supp_precip(self, value: Any) -> Any: """Setter for final_supp_precip.""" self._final_supp_precip = value - @cached_property + @property def regridded_mask(self) -> np.ndarray | Any: """Get the regridded mask after regridding input forcings to the supplemental precipitation grids.""" if self._regridded_mask is None: @@ -263,12 +355,12 @@ def __init__( config_options: ConfigOptions = None, geo_meta: GeoMeta = None, ) -> None: - """Initialize SupplementalPrecipHydrofabric with configuration options, geospatial metadata, and MPI configuration.""" + """Initialize SupplementalPrecipHydrofabric. Any subclass-specific attr names are sourced from SUPPPRECIPMOD[classname] in consts.py.""" super().__init__(idx, config_options, geo_meta) for attr in SUPPPRECIPMOD[self.__class__.__name__]: setattr(self, attr, None) - @cached_property + @property def final_supp_precip(self) -> np.ndarray | Any: """Get the final supplemental precipitation grid after regridding and temporal interpolation.""" if self._final_supp_precip is None: @@ -282,7 +374,7 @@ def final_supp_precip(self, value: Any) -> Any: """Setter for final_supp_precip.""" self._final_supp_precip = value - @cached_property + @property def regridded_mask(self) -> np.ndarray | Any: """Get the regridded mask after regridding input forcings to the supplemental precipitation grids.""" if self._regridded_mask is None: @@ -306,12 +398,12 @@ def __init__( config_options: ConfigOptions = None, geo_meta: GeoMeta = None, ) -> None: - """Initialize InputForcingsUnstructured with configuration options, geospatial metadata, and MPI configuration.""" + """Initialize SupplementalPrecipUnstructured. Any subclass-specific attr names are sourced from SUPPPRECIPMOD[classname] in consts.py.""" super().__init__(idx, config_options, geo_meta) for attr in SUPPPRECIPMOD[self.__class__.__name__]: setattr(self, attr, None) - @cached_property + @property def final_supp_precip(self) -> np.ndarray | Any: """Get the final supplemental precipitation grid after regridding and temporal interpolation.""" if self._final_supp_precip is None: @@ -325,7 +417,7 @@ def final_supp_precip(self, value: Any) -> Any: """Setter for final_supp_precip.""" self._final_supp_precip = value - @cached_property + @property def regridded_mask(self) -> np.ndarray | Any: """Get the regridded mask after regridding input forcings to the supplemental precipitation grids.""" if self._regridded_mask is None: @@ -339,7 +431,7 @@ def regridded_mask(self, value: Any) -> Any: """Setter for regridded_mask.""" self._regridded_mask = value - @cached_property + @property def final_supp_precip_elem(self) -> np.ndarray | Any: """Get the final supplemental precipitation grid after regridding and temporal interpolation for unstructured grids.""" if self._final_supp_precip_elem is None: @@ -353,7 +445,7 @@ def final_supp_precip_elem(self, value: Any) -> Any: """Setter for final_supp_precip_elem.""" self._final_supp_precip_elem = value - @cached_property + @property def regridded_mask_elem(self) -> np.ndarray | Any: """Get the regridded mask after regridding input forcings to the supplemental precipitation grids for unstructured grids.""" if self._regridded_mask_elem is None: From 6212790e1fdfa0814b7a47d1ecc46f39d507ca9a Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 10 Aug 2026 17:02:28 -0400 Subject: [PATCH 009/132] Add keys to supp precip expected test data --- ..._precip_after_update_n1_rank0__step_1.json | 48 +++++++++++++++++++ ..._precip_after_update_n1_rank0__step_2.json | 48 +++++++++++++++++++ ..._precip_after_update_n1_rank0__step_3.json | 48 +++++++++++++++++++ ..._precip_after_update_n2_rank0__step_1.json | 48 +++++++++++++++++++ ..._precip_after_update_n2_rank0__step_2.json | 48 +++++++++++++++++++ ..._precip_after_update_n2_rank0__step_3.json | 48 +++++++++++++++++++ ..._precip_after_update_n2_rank1__step_1.json | 48 +++++++++++++++++++ ..._precip_after_update_n2_rank1__step_2.json | 48 +++++++++++++++++++ ..._precip_after_update_n2_rank1__step_3.json | 48 +++++++++++++++++++ ...pected_supp_precip_finalize_n1_rank0_.json | 48 +++++++++++++++++++ ...pected_supp_precip_finalize_n2_rank0_.json | 48 +++++++++++++++++++ ...pected_supp_precip_finalize_n2_rank1_.json | 48 +++++++++++++++++++ ...t_expected_supp_precip_init_n1_rank0_.json | 48 +++++++++++++++++++ ...t_expected_supp_precip_init_n2_rank0_.json | 48 +++++++++++++++++++ ...t_expected_supp_precip_init_n2_rank1_.json | 48 +++++++++++++++++++ 15 files changed, 720 insertions(+) diff --git a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n1_rank0__step_1.json b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n1_rank0__step_1.json index d0a43866..938ee7b8 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n1_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n1_rank0__step_1.json @@ -317,6 +317,24 @@ 0.0001411111152265221 ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -326,6 +344,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": 60.0, "keyValue": 15, "netcdf_var_names": [ @@ -342,6 +361,25 @@ "pcp_hour2": null, "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask": "hash_4785517837256848181_len_31", "regridded_mask_elem": null, "regridded_precip1": "hash_7959657380107962273_len_31", @@ -357,6 +395,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": 0, "x_upper_bound": 339, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n1_rank0__step_2.json b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n1_rank0__step_2.json index 97415bc8..27e6eafa 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n1_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n1_rank0__step_2.json @@ -317,6 +317,24 @@ 0.0 ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -326,6 +344,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": 60.0, "keyValue": 15, "netcdf_var_names": [ @@ -342,6 +361,25 @@ "pcp_hour2": null, "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask": "hash_4785517837256848181_len_31", "regridded_mask_elem": null, "regridded_precip1": "hash_7959657380107962273_len_31", @@ -357,6 +395,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": 0, "x_upper_bound": 339, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n1_rank0__step_3.json b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n1_rank0__step_3.json index 23e8f1c0..7bf569c1 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n1_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n1_rank0__step_3.json @@ -317,6 +317,24 @@ 6.877352279843763e-05 ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -326,6 +344,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": 60.0, "keyValue": 15, "netcdf_var_names": [ @@ -342,6 +361,25 @@ "pcp_hour2": null, "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask": "hash_4785517837256848181_len_31", "regridded_mask_elem": null, "regridded_precip1": "hash_7959657380107962273_len_31", @@ -357,6 +395,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": 0, "x_upper_bound": 339, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank0__step_1.json b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank0__step_1.json index 71eadda7..55f8c191 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank0__step_1.json @@ -302,6 +302,24 @@ 2.606674752314575e-05 ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -311,6 +329,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": 60.0, "keyValue": 15, "netcdf_var_names": [ @@ -327,6 +346,25 @@ "pcp_hour2": null, "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask": "hash_-655538369282833494_len_16", "regridded_mask_elem": null, "regridded_precip1": "hash_4373573635414851039_len_16", @@ -342,6 +380,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": 0, "x_upper_bound": 339, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank0__step_2.json b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank0__step_2.json index 3ca4c3ed..6640f60c 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank0__step_2.json @@ -302,6 +302,24 @@ 0.0 ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -311,6 +329,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": 60.0, "keyValue": 15, "netcdf_var_names": [ @@ -327,6 +346,25 @@ "pcp_hour2": null, "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask": "hash_-655538369282833494_len_16", "regridded_mask_elem": null, "regridded_precip1": "hash_4373573635414851039_len_16", @@ -342,6 +380,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": 0, "x_upper_bound": 339, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank0__step_3.json b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank0__step_3.json index 96eed731..6496461c 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank0__step_3.json @@ -302,6 +302,24 @@ 7.055555761326104e-05 ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -311,6 +329,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": 60.0, "keyValue": 15, "netcdf_var_names": [ @@ -327,6 +346,25 @@ "pcp_hour2": null, "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask": "hash_-655538369282833494_len_16", "regridded_mask_elem": null, "regridded_precip1": "hash_4373573635414851039_len_16", @@ -342,6 +380,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": 0, "x_upper_bound": 339, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank1__step_1.json b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank1__step_1.json index 3ae99df1..1ff0c743 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank1__step_1.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank1__step_1.json @@ -301,6 +301,24 @@ 0.0001411111152265221 ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -310,6 +328,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": 60.0, "keyValue": 15, "netcdf_var_names": [ @@ -326,6 +345,25 @@ "pcp_hour2": null, "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask": "hash_-1465630676857108774_len_15", "regridded_mask_elem": null, "regridded_precip1": "hash_6600151672699777871_len_15", @@ -341,6 +379,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": 0, "x_upper_bound": 339, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank1__step_2.json b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank1__step_2.json index 448bff53..802a29b2 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank1__step_2.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank1__step_2.json @@ -301,6 +301,24 @@ 0.0 ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -310,6 +328,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": 60.0, "keyValue": 15, "netcdf_var_names": [ @@ -326,6 +345,25 @@ "pcp_hour2": null, "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask": "hash_-1465630676857108774_len_15", "regridded_mask_elem": null, "regridded_precip1": "hash_6600151672699777871_len_15", @@ -341,6 +379,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": 0, "x_upper_bound": 339, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank1__step_3.json b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank1__step_3.json index 605d5117..96d626ec 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank1__step_3.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_after_update_n2_rank1__step_3.json @@ -301,6 +301,24 @@ 6.877352279843763e-05 ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -310,6 +328,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": 60.0, "keyValue": 15, "netcdf_var_names": [ @@ -326,6 +345,25 @@ "pcp_hour2": null, "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask": "hash_-1465630676857108774_len_15", "regridded_mask_elem": null, "regridded_precip1": "hash_6600151672699777871_len_15", @@ -341,6 +379,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": 0, "x_upper_bound": 339, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_finalize_n1_rank0_.json b/tests/test_data/expected_results/test_expected_supp_precip_finalize_n1_rank0_.json index 23e8f1c0..7bf569c1 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_finalize_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_finalize_n1_rank0_.json @@ -317,6 +317,24 @@ 6.877352279843763e-05 ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -326,6 +344,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": 60.0, "keyValue": 15, "netcdf_var_names": [ @@ -342,6 +361,25 @@ "pcp_hour2": null, "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask": "hash_4785517837256848181_len_31", "regridded_mask_elem": null, "regridded_precip1": "hash_7959657380107962273_len_31", @@ -357,6 +395,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": 0, "x_upper_bound": 339, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_finalize_n2_rank0_.json b/tests/test_data/expected_results/test_expected_supp_precip_finalize_n2_rank0_.json index 96eed731..6496461c 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_finalize_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_finalize_n2_rank0_.json @@ -302,6 +302,24 @@ 7.055555761326104e-05 ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -311,6 +329,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": 60.0, "keyValue": 15, "netcdf_var_names": [ @@ -327,6 +346,25 @@ "pcp_hour2": null, "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask": "hash_-655538369282833494_len_16", "regridded_mask_elem": null, "regridded_precip1": "hash_4373573635414851039_len_16", @@ -342,6 +380,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": 0, "x_upper_bound": 339, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_finalize_n2_rank1_.json b/tests/test_data/expected_results/test_expected_supp_precip_finalize_n2_rank1_.json index 605d5117..96d626ec 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_finalize_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_finalize_n2_rank1_.json @@ -301,6 +301,24 @@ 6.877352279843763e-05 ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -310,6 +328,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": 60.0, "keyValue": 15, "netcdf_var_names": [ @@ -326,6 +345,25 @@ "pcp_hour2": null, "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask": "hash_-1465630676857108774_len_15", "regridded_mask_elem": null, "regridded_precip1": "hash_6600151672699777871_len_15", @@ -341,6 +379,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": 0, "x_upper_bound": 339, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_init_n1_rank0_.json b/tests/test_data/expected_results/test_expected_supp_precip_init_n1_rank0_.json index 085c27b9..9d6388e7 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_init_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_init_n1_rank0_.json @@ -78,6 +78,24 @@ NaN ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -87,6 +105,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": null, "keyValue": 15, "netcdf_var_names": [ @@ -104,6 +123,25 @@ "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, "regridObj": null, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask_elem": null, "regridded_precip1": null, "regridded_precip1_elem": null, @@ -118,6 +156,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": null, "x_upper_bound": null, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_init_n2_rank0_.json b/tests/test_data/expected_results/test_expected_supp_precip_init_n2_rank0_.json index eb614161..b1a72801 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_init_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_init_n2_rank0_.json @@ -63,6 +63,24 @@ NaN ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -72,6 +90,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": null, "keyValue": 15, "netcdf_var_names": [ @@ -89,6 +108,25 @@ "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, "regridObj": null, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask_elem": null, "regridded_precip1": null, "regridded_precip1_elem": null, @@ -103,6 +141,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": null, "x_upper_bound": null, diff --git a/tests/test_data/expected_results/test_expected_supp_precip_init_n2_rank1_.json b/tests/test_data/expected_results/test_expected_supp_precip_init_n2_rank1_.json index 1a40921e..e1055ece 100644 --- a/tests/test_data/expected_results/test_expected_supp_precip_init_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_supp_precip_init_n2_rank1_.json @@ -62,6 +62,24 @@ NaN ], "final_supp_precip_elem": null, + "find_neighbor_files": { + "1": "find_hourly_mrms_radar_neighbors", + "10": "find_hourly_mrms_radar_neighbors", + "11": "find_ak_ext_ana_precip_neighbors", + "12": "find_conus_ext_ana_precip_neighbors", + "13": "find_hourly_mrms_precip_flag", + "14": "find_custom_freq_neighbors", + "15": "find_hourly_nbm_neighbors", + "16": "find_hourly_nbm_neighbors", + "2": "find_hourly_mrms_radar_neighbors", + "3": "find_hourly_wrf_arw_neighbors", + "4": "find_hourly_wrf_arw_neighbors", + "5": "find_hourly_mrms_radar_neighbors", + "6": "find_hourly_mrms_radar_neighbors", + "7": "find_sbcv2_lwf_neighbors", + "8": "find_hourly_nbm_neighbors", + "9": "find_hourly_nbm_neighbors" + }, "global_x_lower": null, "global_x_upper": null, "global_y_lower": null, @@ -71,6 +89,7 @@ ], "grib_vars": null, "has_cache": false, + "idx": 0, "input_frequency": null, "keyValue": 15, "netcdf_var_names": [ @@ -88,6 +107,25 @@ "product_name": "NBM_CORE_PR_APCP", "regridComplete": false, "regridObj": null, + "regrid_map": { + "1": "regrid_mrms_hourly", + "10": "regrid_mrms_hourly", + "11": "regrid_ak_ext_ana_pcp", + "12": "regrid_conus_ext_ana_pcp", + "13": "regrid_mrms_precip_flag", + "14": "regrid_mrms_hourly", + "15": "regrid_hourly_nbm", + "16": "regrid_hourly_nbm", + "2": "regrid_mrms_hourly", + "3": "regrid_hourly_wrf_arw_hi_res_pcp", + "4": "regrid_hourly_wrf_arw_hi_res_pcp", + "5": "regrid_mrms_hourly", + "6": "regrid_mrms_hourly", + "7": "regrid_sbcv2_liquid_water_fraction", + "8": "regrid_hourly_nbm", + "9": "regrid_hourly_nbm" + }, + "regrid_opt_supp_pcp": 1, "regridded_mask_elem": null, "regridded_precip1": null, "regridded_precip1_elem": null, @@ -102,6 +140,16 @@ "rqi_file_in1": null, "rqi_file_in2": null, "rqi_netcdf_var_names": null, + "suppTemporalInterp": 0, + "supp_input_offsets": 0, + "supp_precip_dirs": "/ngen-app/data/raw_input/NBM_PR", + "supp_precip_file_types": "GRIB2", + "supp_precip_mandatory": 1, + "temporal_interpolate_inputs_map": { + "0": "no_interpolation_supp_pcp", + "1": "nearest_neighbor_supp_pcp", + "2": "weighted_average_supp_pcp" + }, "tmpFile": null, "x_lower_bound": null, "x_upper_bound": null, From 696e1af0df9de105fe7b45863efc2441461d4996 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 10 Aug 2026 22:01:17 -0400 Subject: [PATCH 010/132] Tests: add flexibility for equality assertions of keys "_coords", "lat_bounds", and "lon_bounds" (ignore hashes). --- tests/test_utils.py | 53 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 5e8fd204..05657c1c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,6 +3,7 @@ import json import logging import os +import re import typing from collections import OrderedDict from dataclasses import dataclass @@ -46,6 +47,54 @@ OS_VAR__CREATE_TEST_EXPECT_DATA = "FORCING_PYTEST_WRITE_TEST_EXPECTED_DATA" +_HASH_STRING_RE = re.compile(r"^hash_\d+_len_(\d+)$") + + +def _normalize_hash_str(s: str) -> str: + """Replace hash_{n}_len_{x} with hash_ANY_len_{x}, keeping only the length so that the hash value + can be optionally ignored, only asserting that the length matches. The intent of this is to handle + cases of large structures of coordinate values that end up getting hashed.""" + m = _HASH_STRING_RE.match(s) + return f"hash_ANY_len_{m.group(1)}" if m else s + + +_HASH_NORMALIZED_KEYS = frozenset({"_coords", "lat_bounds", "lon_bounds"}) + + +def _normalize_coords_hashes(data: typing.Any) -> typing.Any: + """Recursively normalize hash strings in known location keys so only the length is compared.""" + if isinstance(data, dict): + return { + k: ( + _apply_hash_normalization(v) + if k in _HASH_NORMALIZED_KEYS + else _normalize_coords_hashes(v) + ) + for k, v in data.items() + } + if isinstance(data, list): + return [_normalize_coords_hashes(item) for item in data] + return data + + +def _apply_hash_normalization(value: typing.Any) -> typing.Any: + """Normalize hash strings in a value (may be a string or nested lists of strings). + For _coords specifically, arrays of coordinates are dropped since + those values are already verified via other keys.""" + if isinstance(value, str): + return _normalize_hash_str(value) + if isinstance(value, list): + # Detect _coords structure: [list-of-hash-strings, coord-arrays]. Drop the coordinate arrays from the check. + if ( + len(value) == 2 + and isinstance(value[0], list) + and value[0] + and all(isinstance(s, str) and _HASH_STRING_RE.match(s) for s in value[0]) + ): + return [[_normalize_hash_str(s) for s in value[0]], None] + return [_apply_hash_normalization(item) for item in value] + return value + def remove_key(input_data: dict, keys_to_exclude: tuple = ()) -> dict: """Recursively remove keys from nested dicts. @@ -400,8 +449,8 @@ def compare(self, actual: dict, expected: dict) -> None: """Compare actual vs expected results.""" try: assert_equal_with_tol( - expect=expected, - actual=actual, + expect=_normalize_coords_hashes(expected), + actual=_normalize_coords_hashes(actual), new_keys_in_actual_ok=True, ) except ExpectVsActualError as e: From 01e9b414fe27f21ef62dd8532f3d0b4bbd3077be Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 10 Aug 2026 22:19:48 -0400 Subject: [PATCH 011/132] Format --- .../NextGen_Forcings_Engine/core/suppPrecipMod.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py index 92dfdf45..8a2c929b 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py @@ -91,7 +91,9 @@ def rqiMethod(self) -> int | float: elif val is None: self._rqiMethod = 0 else: - raise TypeError(f"Unexpected type for config_options.rqiMethod: {type(val)}") + raise TypeError( + f"Unexpected type for config_options.rqiMethod: {type(val)}" + ) return self._rqiMethod @rqiMethod.setter @@ -113,7 +115,9 @@ def rqiThresh(self) -> int | float: # When RQI is configured, a scalar gets expanded to a list before reaching here. self._rqiThresh = float(val) if val is not None else 1.0 else: - raise TypeError(f"Unexpected type for config_options.rqiThresh: {type(val)}") + raise TypeError( + f"Unexpected type for config_options.rqiThresh: {type(val)}" + ) return self._rqiThresh @rqiThresh.setter From e901ecdee57bd70dd2fcd794389cd498a1fc7573 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 10 Aug 2026 22:48:10 -0400 Subject: [PATCH 012/132] Tests: when performing equality tests (with tolerance) for a given key, travel recursively instead of falling back to pure equality tests after initial shallow inspection. This allows nested objects to have the numerical tolerances applied as intended. --- .../NextGen_Forcings_Engine/general_utils.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/general_utils.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/general_utils.py index 326ff7d7..5f928c03 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/general_utils.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/general_utils.py @@ -190,10 +190,21 @@ def assert_equal_with_tol( rtol=1e-10, ) except (TypeError, ValueError): - close = ( - v_expect[key_with_vals_not_matching] - == v_actual[key_with_vals_not_matching] - ) + sub_e = v_expect[key_with_vals_not_matching] + sub_a = v_actual[key_with_vals_not_matching] + if isinstance(sub_e, (dict, OrderedDict)) and isinstance(sub_a, (dict, OrderedDict)): + try: + assert_equal_with_tol( + expect=sub_e, + actual=sub_a, + absolute_tolerance=absolute_tolerance, + relative_tolerance=relative_tolerance, + ) + close = True + except ExpectVsActualError: + close = False + else: + close = sub_e == sub_a if not close: failing.append( ( From 244f9f1dd83caec7115930d0db5ef043aee1b0a2 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 10 Aug 2026 23:18:29 -0400 Subject: [PATCH 013/132] Tests data: handle negative sign after hash sentinel --- tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 05657c1c..e21a9d5c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -47,7 +47,7 @@ OS_VAR__CREATE_TEST_EXPECT_DATA = "FORCING_PYTEST_WRITE_TEST_EXPECTED_DATA" -_HASH_STRING_RE = re.compile(r"^hash_\d+_len_(\d+)$") +_HASH_STRING_RE = re.compile(r"^hash_-?\d+_len_(\d+)$") def _normalize_hash_str(s: str) -> str: From 8458c82469f43df813a1cb32a5096f37e93885bd Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Thu, 25 Jun 2026 12:57:50 -0500 Subject: [PATCH 014/132] use projected crs instead of silencing warning from geographic crs use --- .../NextGen_Forcings_Engine/historical_forcing.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py index 1e9b5178..e2c80bd5 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py @@ -32,7 +32,6 @@ from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import MpiConfig from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.general_utils import rand_str -warnings.filterwarnings("ignore", module="geopandas") LOG = logging.getLogger("FORCING") zarr.config.set({"async.concurrency": 100}) @@ -51,17 +50,17 @@ def __init__( self.config_options = config_options self.mpi_config = mpi_config self.wrf_hydro_geo_meta = wrf_hydro_geo_meta - self.dest_crs = CRS(4326) - self.buffer = 0.02 # degree buffer around bounding box + self._temp_crs = CRS(5070) + self.buffer = 3000 # m buffer around bounding box @cached_property def bounds(self) -> tuple[float, float, float, float]: """Get bounding box from geospatial dataframe. - Apply buffer in known crs/units (degrees) and then convert back to src_crs. + Apply buffer in known crs/units (km) and then convert back to src_crs. """ return ( - self.gdf.to_crs(self.dest_crs) + self.gdf.to_crs(self._temp_crs) .buffer(self.buffer) .to_crs(self.src_crs) .total_bounds From d3f13938876a682baf1390e80d8489e788eab247 Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Thu, 25 Jun 2026 13:01:38 -0500 Subject: [PATCH 015/132] fix doc string --- .../NextGen_Forcings_Engine/historical_forcing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py index e2c80bd5..58f82ffa 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py @@ -57,7 +57,7 @@ def __init__( def bounds(self) -> tuple[float, float, float, float]: """Get bounding box from geospatial dataframe. - Apply buffer in known crs/units (km) and then convert back to src_crs. + Apply buffer in known crs/units (m) and then convert back to src_crs. """ return ( self.gdf.to_crs(self._temp_crs) From 95712f87f1ee14e918198fbe6a24dc10e4fa8c4e Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Tue, 4 Aug 2026 08:17:32 -0500 Subject: [PATCH 016/132] adjust buffer to account for 4km NWM resolution --- .../NextGen_Forcings_Engine/historical_forcing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py index 58f82ffa..3eeb73b8 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py @@ -51,7 +51,6 @@ def __init__( self.mpi_config = mpi_config self.wrf_hydro_geo_meta = wrf_hydro_geo_meta self._temp_crs = CRS(5070) - self.buffer = 3000 # m buffer around bounding box @cached_property def bounds(self) -> tuple[float, float, float, float]: @@ -427,6 +426,7 @@ def __init__( self.x_label = "longitude" self.y_label = "latitude" self.time_label = "time" + self.buffer = 3000 # m buffer around bounding box @cached_property def src_crs(self) -> CRS: @@ -497,6 +497,7 @@ def __init__( self.x_label = "longitude" self.y_label = "latitude" self.time_label = "time" + self.buffer = 3000 # m buffer around bounding box @cached_property def src_crs(self): @@ -567,6 +568,7 @@ def __init__( self.x_label = "x" self.y_label = "y" self.time_label = "time" + self.buffer = 6000 # m buffer around bounding box @property def vars( From c1cb0768c66ac1851919886ce01bc81e71b39754 Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Tue, 4 Aug 2026 08:40:30 -0500 Subject: [PATCH 017/132] use domain specific projected CRS --- .../historical_forcing.py | 37 +++++++++++++++-- .../NextGen_Forcings_Engine/model.py | 40 +++++++++++-------- 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py index 3eeb73b8..dbe613a0 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py @@ -50,7 +50,6 @@ def __init__( self.config_options = config_options self.mpi_config = mpi_config self.wrf_hydro_geo_meta = wrf_hydro_geo_meta - self._temp_crs = CRS(5070) @cached_property def bounds(self) -> tuple[float, float, float, float]: @@ -427,6 +426,7 @@ def __init__( self.y_label = "latitude" self.time_label = "time" self.buffer = 3000 # m buffer around bounding box + self._temp_crs = CRS(5070) @cached_property def src_crs(self) -> CRS: @@ -498,6 +498,7 @@ def __init__( self.y_label = "latitude" self.time_label = "time" self.buffer = 3000 # m buffer around bounding box + self._temp_crs = CRS(3338) @cached_property def src_crs(self): @@ -598,6 +599,7 @@ def __init__( ): """Initialize NWM CONUS processor.""" super().__init__(config_options, mpi_config, wrf_hydro_geo_meta) + self._temp_crs = CRS(5070) def url(self, var: str) -> str: """Generate NWM S3 zarr URL for current variable. @@ -718,8 +720,36 @@ def s3_lazy_ds(self) -> xr.Dataset: return xr.open_zarr(ObjectStore(object_store)) +class NWMV3PuertoRicoProcessor(NWMV3OConusProcessor): + """Processor for NWM Puerto Rico data.""" + + def __init__( + self, + config_options: ConfigOptions, + mpi_config: MpiConfig, + wrf_hydro_geo_meta: dict, + ): + """Initialize NWM Puerto Rico processor.""" + super().__init__(config_options, mpi_config, wrf_hydro_geo_meta) + self._temp_crs = CRS(32161) + + +class NWMV3HawaiiProcessor(NWMV3OConusProcessor): + """Processor for NWM Hawaii data.""" + + def __init__( + self, + config_options: ConfigOptions, + mpi_config: MpiConfig, + wrf_hydro_geo_meta: dict, + ): + """Initialize NWM Hawaii processor.""" + super().__init__(config_options, mpi_config, wrf_hydro_geo_meta) + self._temp_crs = CRS(6628) + + class NWMV3AlaskaProcessor(NWMV3Processor): - """Processor for NWM OCONUS data.""" + """Processor for NWM Alaska data.""" def __init__( self, @@ -727,8 +757,9 @@ def __init__( mpi_config: MpiConfig, wrf_hydro_geo_meta: dict, ): - """Initialize NWM OCONUS processor.""" + """Initialize NWM Alaska processor.""" super().__init__(config_options, mpi_config, wrf_hydro_geo_meta) + self._temp_crs = CRS(3338) @cached_property def url(self) -> str: diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 9fbaa1f4..c9cc75f4 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -1,8 +1,9 @@ import datetime +import logging import os from contextlib import contextmanager from time import time -import logging + import numpy as np import pandas as pd from ewts import Payload as Pld @@ -29,7 +30,8 @@ AORCConusProcessor, NWMV3AlaskaProcessor, NWMV3ConusProcessor, - NWMV3OConusProcessor, + NWMV3HawaiiProcessor, + NWMV3PuertoRicoProcessor, ) LOG = logging.getLogger("FORCING") @@ -227,8 +229,8 @@ def determine_forecast( # If we're in an AnA configuration, then must offset the BMI future # timestamp to account for the "lookback" period being properly iterated # over between 3-28 hour look back time period and operation configuration - - #if config_options.input_forcings[0] in [20, 22]: + + # if config_options.input_forcings[0] in [20, 22]: # config_options.current_fcst_cycle = ( # config_options.b_date_proc # + pd.TimedeltaIndex( @@ -243,19 +245,19 @@ def determine_forecast( # ) # config_options.future_time = future_time # else: - + # Puerto Rico / Hawaii AnA: 1-hour lookback (based on 6-hourly forecast cycles) config_options.current_fcst_cycle = ( config_options.b_date_proc - + pd.TimedeltaIndex( - np.array([future_time - 3600.0], dtype=float), "s" - )[0] + + pd.TimedeltaIndex(np.array([future_time - 3600.0], dtype=float), "s")[ + 0 + ] ) config_options.current_time = ( config_options.b_date_proc - + pd.TimedeltaIndex( - np.array([future_time - 3600.0], dtype=float), "s" - )[0] + + pd.TimedeltaIndex(np.array([future_time - 3600.0], dtype=float), "s")[ + 0 + ] ) else: # Forecast-only mode — use BMI timestamp as-is @@ -467,13 +469,19 @@ def loop_through_forcing_products( self.source_data_processor = NWMV3ConusProcessor( config_options, mpi_config, wrf_hydro_geo_meta ) - elif config_options.nwm_domain in [ - "Hawaii", - "PR", - ]: - self.source_data_processor = NWMV3OConusProcessor( + elif config_options.nwm_domain == "Hawaii": + self.source_data_processor = NWMV3HawaiiProcessor( config_options, mpi_config, wrf_hydro_geo_meta ) + + elif config_options.nwm_domain == "PR": + self.source_data_processor = ( + NWMV3PuertoRicoProcessor( + config_options, + mpi_config, + wrf_hydro_geo_meta, + ) + ) elif config_options.nwm_domain == "Alaska": self.source_data_processor = NWMV3AlaskaProcessor( config_options, mpi_config, wrf_hydro_geo_meta From cb5691fda8d89c0e6a4058edaec14125a5e2176f Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 7 Aug 2026 13:26:18 -0400 Subject: [PATCH 018/132] Temporary CRS for making a buffered mask: use dynamic UTM CRS for Hawaii. Increase AORC buffer to 6km. Assert that the projection is meters --- .../NextGen_Forcings_Engine/core/geoMod.py | 8 +++++++ .../NextGen_Forcings_Engine/general_utils.py | 11 +++++++++ .../historical_forcing.py | 23 +++++++++++++++---- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py index 90a225cf..c219e2b0 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py @@ -250,6 +250,14 @@ def y_coords(self) -> np.ndarray: y_coords[:, :] = np.flipud(y_coords[:, :]) return y_coords + @property + def approx_centroid_global_xy(self) -> tuple[float, float]: + """The approximate centroid in global coordinates (a tuple of 2 floats: (x, y))""" + x_mean = np.mean([c[0] for c in self.elementcoords_global]) + y_mean = np.mean([c[1] for c in self.elementcoords_global]) + LOG.debug(f"Approximate centroid: ({x_mean}, {y_mean})") + return (x_mean, y_mean) + class GriddedGeoMeta(GeoMeta): """Class for handling information about the gridded domains for forcing.""" diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/general_utils.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/general_utils.py index 5f928c03..5c75641f 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/general_utils.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/general_utils.py @@ -7,6 +7,7 @@ from collections import OrderedDict import numpy as np +from pyproj import CRS JSON_NOT_SERIALIZABLE_SENTINEL = "ERR_NOT_JSON_SERIALIZABLE" JSON_NOT_SERIALIZABLE_FORMAT = JSON_NOT_SERIALIZABLE_SENTINEL + ":TYPE:{typ}" @@ -239,3 +240,13 @@ def rand_str(length: int) -> str: f"length requested was {length}, but this function only supports length 1 through 32" ) return str(uuid.uuid4()).replace("-", "")[:length] + + +def crs_assert_projected_horizontal_meters(crs: CRS) -> None: + """Assert that the CRS is projected and has horizontal units of meters.""" + if not crs.is_projected: + raise ValueError(f"CRS is not projected: {crs}") + if crs.axis_info[0].unit_conversion_factor != 1: + raise ValueError( + f"Expected crs.axis_info[0].unit_conversion_factor == 1, but got: {crs.axis_info[0].unit_conversion_factor}" + ) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py index dbe613a0..976e130f 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py @@ -30,7 +30,10 @@ ConfigOptions, ) from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import MpiConfig -from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.general_utils import rand_str +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.general_utils import ( + crs_assert_projected_horizontal_meters, + rand_str, +) LOG = logging.getLogger("FORCING") @@ -57,6 +60,10 @@ def bounds(self) -> tuple[float, float, float, float]: Apply buffer in known crs/units (m) and then convert back to src_crs. """ + LOG.debug( + f"Temporary CRS for creating a mask (will buffer AOI by {self.buffer} in this CRS): {self._temp_crs}" + ) + crs_assert_projected_horizontal_meters(self._temp_crs) return ( self.gdf.to_crs(self._temp_crs) .buffer(self.buffer) @@ -425,7 +432,7 @@ def __init__( self.x_label = "longitude" self.y_label = "latitude" self.time_label = "time" - self.buffer = 3000 # m buffer around bounding box + self.buffer = 6000 # m buffer around bounding box. Use 6km buffer in case someone applies this to legacy 4km AORC data instead of the newer 1km AORC data. self._temp_crs = CRS(5070) @cached_property @@ -497,7 +504,7 @@ def __init__( self.x_label = "longitude" self.y_label = "latitude" self.time_label = "time" - self.buffer = 3000 # m buffer around bounding box + self.buffer = 6000 # m buffer around bounding box. Use 6km buffer in case someone applies this to legacy 4km AORC data instead of the newer 1km AORC data. self._temp_crs = CRS(3338) @cached_property @@ -745,7 +752,15 @@ def __init__( ): """Initialize NWM Hawaii processor.""" super().__init__(config_options, mpi_config, wrf_hydro_geo_meta) - self._temp_crs = CRS(6628) + lon, lat = wrf_hydro_geo_meta.approx_centroid_global_xy + if not -180 < lon < 180: + raise ValueError(f"Unexpected (lon, lat) = ({lon}, {lat})") + utm_zone_number = int((lon + 180) / 6) + 1 + if utm_zone_number not in (1, 2, 3, 4, 5): + raise ValueError( + f"Unexpected UTM zone {utm_zone_number} for Hawaii. Expected zone 1 through 5. (lon, lat) = ({lon}, {lat})" + ) + self._temp_crs = CRS(f"EPSG:3260{utm_zone_number}") class NWMV3AlaskaProcessor(NWMV3Processor): From c1cfd434ea05734bf9c939b24a8c1b62c9631d5c Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Thu, 13 Aug 2026 22:20:11 -0400 Subject: [PATCH 019/132] Test data: include new centroid key and update for new mask buffer logic --- ...i_model_after_update_n1_rank0__step_1.json | 24 ++++++---- ...i_model_after_update_n1_rank0__step_2.json | 24 ++++++---- ...i_model_after_update_n1_rank0__step_3.json | 24 ++++++---- ...i_model_after_update_n2_rank0__step_1.json | 20 ++++---- ...i_model_after_update_n2_rank0__step_2.json | 20 ++++---- ...i_model_after_update_n2_rank0__step_3.json | 20 ++++---- ...i_model_after_update_n2_rank1__step_1.json | 20 ++++---- ...i_model_after_update_n2_rank1__step_2.json | 20 ++++---- ...i_model_after_update_n2_rank1__step_3.json | 20 ++++---- ...est_expected_bmi_model_init_n1_rank0_.json | 8 +++- ...est_expected_bmi_model_init_n2_rank0_.json | 4 ++ ...est_expected_bmi_model_init_n2_rank1_.json | 4 ++ ...options_after_update_n1_rank0__step_1.json | 8 ++-- ...options_after_update_n1_rank0__step_2.json | 8 ++-- ...options_after_update_n1_rank0__step_3.json | 8 ++-- ...options_after_update_n2_rank0__step_1.json | 8 ++-- ...options_after_update_n2_rank0__step_2.json | 8 ++-- ...options_after_update_n2_rank0__step_3.json | 8 ++-- ...options_after_update_n2_rank1__step_1.json | 8 ++-- ...options_after_update_n2_rank1__step_2.json | 8 ++-- ...options_after_update_n2_rank1__step_3.json | 8 ++-- ...ted_config_options_finalize_n1_rank0_.json | 8 ++-- ...ted_config_options_finalize_n2_rank0_.json | 8 ++-- ...ted_config_options_finalize_n2_rank1_.json | 8 ++-- ..._geomod_after_update_n1_rank0__step_1.json | 4 ++ ..._geomod_after_update_n1_rank0__step_2.json | 4 ++ ..._geomod_after_update_n1_rank0__step_3.json | 4 ++ ..._geomod_after_update_n2_rank0__step_1.json | 4 ++ ..._geomod_after_update_n2_rank0__step_2.json | 4 ++ ..._geomod_after_update_n2_rank0__step_3.json | 4 ++ ..._geomod_after_update_n2_rank1__step_1.json | 4 ++ ..._geomod_after_update_n2_rank1__step_2.json | 4 ++ ..._geomod_after_update_n2_rank1__step_3.json | 4 ++ ...st_expected_geomod_finalize_n1_rank0_.json | 4 ++ ...st_expected_geomod_finalize_n2_rank0_.json | 4 ++ ...st_expected_geomod_finalize_n2_rank1_.json | 4 ++ .../test_expected_geomod_init_n1_rank0_.json | 4 ++ .../test_expected_geomod_init_n2_rank0_.json | 4 ++ .../test_expected_geomod_init_n2_rank1_.json | 4 ++ ...forcing_after_update_n1_rank0__step_1.json | 40 ++++++++-------- ...forcing_after_update_n1_rank0__step_2.json | 40 ++++++++-------- ...forcing_after_update_n1_rank0__step_3.json | 40 ++++++++-------- ...forcing_after_update_n2_rank0__step_1.json | 40 ++++++++-------- ...forcing_after_update_n2_rank0__step_2.json | 40 ++++++++-------- ...forcing_after_update_n2_rank0__step_3.json | 40 ++++++++-------- ...forcing_after_update_n2_rank1__step_1.json | 48 +++++++++---------- ...forcing_after_update_n2_rank1__step_2.json | 48 +++++++++---------- ...forcing_after_update_n2_rank1__step_3.json | 48 +++++++++---------- ...cted_input_forcing_finalize_n1_rank0_.json | 40 ++++++++-------- ...cted_input_forcing_finalize_n2_rank0_.json | 40 ++++++++-------- ...cted_input_forcing_finalize_n2_rank1_.json | 48 +++++++++---------- 51 files changed, 492 insertions(+), 384 deletions(-) diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json index 3e36d322..40136e80 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json @@ -118,6 +118,10 @@ "time_step_seconds": 3600 }, "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "centerCoords": null, "config_options": { "ExactExtract": null, @@ -166,7 +170,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -176,7 +180,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -313,8 +317,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, @@ -530,7 +534,7 @@ "_coords": [ [ "hash_2904886930620536845_len_582", - "hash_2117205719057770460_len_582" + "hash_902249663378761532_len_582" ], [ [ @@ -685,7 +689,7 @@ "height_elem": null, "heights_global": null, "inds": null, - "lat_bounds": "hash_2117205719057770460_len_582", + "lat_bounds": "hash_902249663378761532_len_582", "latitude_grid": [ 41.67904491423198, 41.73961258049449, @@ -766,7 +770,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -776,7 +780,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -913,8 +917,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json index 8674813b..b495ac2e 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json @@ -118,6 +118,10 @@ "time_step_seconds": 3600 }, "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "centerCoords": null, "config_options": { "ExactExtract": null, @@ -166,7 +170,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -176,7 +180,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -313,8 +317,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, @@ -530,7 +534,7 @@ "_coords": [ [ "hash_2904886930620536845_len_582", - "hash_2117205719057770460_len_582" + "hash_902249663378761532_len_582" ], [ [ @@ -685,7 +689,7 @@ "height_elem": null, "heights_global": null, "inds": null, - "lat_bounds": "hash_2117205719057770460_len_582", + "lat_bounds": "hash_902249663378761532_len_582", "latitude_grid": [ 41.67904491423198, 41.73961258049449, @@ -766,7 +770,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -776,7 +780,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -913,8 +917,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json index b873eac6..f7a6b582 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json @@ -118,6 +118,10 @@ "time_step_seconds": 3600 }, "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "centerCoords": null, "config_options": { "ExactExtract": null, @@ -166,7 +170,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -176,7 +180,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -313,8 +317,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, @@ -530,7 +534,7 @@ "_coords": [ [ "hash_2904886930620536845_len_582", - "hash_2117205719057770460_len_582" + "hash_902249663378761532_len_582" ], [ [ @@ -685,7 +689,7 @@ "height_elem": null, "heights_global": null, "inds": null, - "lat_bounds": "hash_2117205719057770460_len_582", + "lat_bounds": "hash_902249663378761532_len_582", "latitude_grid": [ 41.67904491423198, 41.73961258049449, @@ -766,7 +770,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -776,7 +780,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -913,8 +917,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json index 8b9a53f4..b07e8be3 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json @@ -118,6 +118,10 @@ "time_step_seconds": 3600 }, "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "centerCoords": null, "config_options": { "ExactExtract": null, @@ -166,7 +170,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -176,7 +180,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -313,8 +317,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, @@ -748,7 +752,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -758,7 +762,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -895,8 +899,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json index cab5bbbd..26be099a 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json @@ -118,6 +118,10 @@ "time_step_seconds": 3600 }, "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "centerCoords": null, "config_options": { "ExactExtract": null, @@ -166,7 +170,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -176,7 +180,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -313,8 +317,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, @@ -748,7 +752,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -758,7 +762,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -895,8 +899,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json index 063c0dc0..211cc8ad 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json @@ -118,6 +118,10 @@ "time_step_seconds": 3600 }, "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "centerCoords": null, "config_options": { "ExactExtract": null, @@ -166,7 +170,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -176,7 +180,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -313,8 +317,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, @@ -748,7 +752,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -758,7 +762,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -895,8 +899,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json index 1a47d086..0669c8f9 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json @@ -118,6 +118,10 @@ "time_step_seconds": 3600 }, "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "centerCoords": null, "config_options": { "ExactExtract": null, @@ -166,7 +170,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -176,7 +180,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -313,8 +317,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, @@ -742,7 +746,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -752,7 +756,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -889,8 +893,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json index 17588365..49febe29 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json @@ -118,6 +118,10 @@ "time_step_seconds": 3600 }, "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "centerCoords": null, "config_options": { "ExactExtract": null, @@ -166,7 +170,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -176,7 +180,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -313,8 +317,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, @@ -742,7 +746,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -752,7 +756,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -889,8 +893,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json index b25c1a3f..9fbaea9a 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json @@ -118,6 +118,10 @@ "time_step_seconds": 3600 }, "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "centerCoords": null, "config_options": { "ExactExtract": null, @@ -166,7 +170,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -176,7 +180,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -313,8 +317,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, @@ -742,7 +746,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -752,7 +756,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -889,8 +893,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json index 4696fb3d..15342a9c 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json @@ -118,6 +118,10 @@ "time_step_seconds": 3600 }, "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "centerCoords": null, "config_options": { "ExactExtract": null, @@ -340,7 +344,7 @@ "_coords": [ [ "hash_2904886930620536845_len_582", - "hash_2117205719057770460_len_582" + "hash_902249663378761532_len_582" ], [ [ @@ -495,7 +499,7 @@ "height_elem": null, "heights_global": null, "inds": null, - "lat_bounds": "hash_2117205719057770460_len_582", + "lat_bounds": "hash_902249663378761532_len_582", "latitude_grid": [ 41.67904491423198, 41.73961258049449, diff --git a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json index 3669aea2..c3476459 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json @@ -118,6 +118,10 @@ "time_step_seconds": 3600 }, "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "centerCoords": null, "config_options": { "ExactExtract": null, diff --git a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json index 59148a38..18023934 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json @@ -118,6 +118,10 @@ "time_step_seconds": 3600 }, "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "centerCoords": null, "config_options": { "ExactExtract": null, diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json index 55de7770..bb89f16e 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json @@ -45,7 +45,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -55,7 +55,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -192,8 +192,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json index f1bfa774..ed3c2aaa 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json @@ -45,7 +45,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -55,7 +55,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -192,8 +192,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json index 764ed300..20e3ce1c 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json @@ -45,7 +45,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -55,7 +55,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -192,8 +192,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json index 55de7770..bb89f16e 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json @@ -45,7 +45,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -55,7 +55,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -192,8 +192,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json index f1bfa774..ed3c2aaa 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json @@ -45,7 +45,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -55,7 +55,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -192,8 +192,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json index 764ed300..20e3ce1c 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json @@ -45,7 +45,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -55,7 +55,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -192,8 +192,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json index 55de7770..bb89f16e 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json @@ -45,7 +45,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -55,7 +55,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -192,8 +192,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json index f1bfa774..ed3c2aaa 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json @@ -45,7 +45,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -55,7 +55,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -192,8 +192,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json index 764ed300..20e3ce1c 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json @@ -45,7 +45,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -55,7 +55,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -192,8 +192,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json index 0a968268..147309ce 100644 --- a/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json @@ -45,7 +45,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -55,7 +55,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -192,8 +192,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json index 0a968268..147309ce 100644 --- a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json @@ -45,7 +45,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -55,7 +55,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -192,8 +192,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json index 0a968268..147309ce 100644 --- a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json @@ -45,7 +45,7 @@ "long_name": "longitude", "units": "degrees_east" }, - "data": "hash_-7248319024934117860_len_14", + "data": "hash_-5399622063019475031_len_28", "dims": [ "x" ] @@ -55,7 +55,7 @@ "long_name": "latitude", "units": "degrees_north" }, - "data": "hash_5170267599511687273_len_29", + "data": "hash_-1457289664996315734_len_37", "dims": [ "y" ] @@ -192,8 +192,8 @@ } }, "dims": { - "x": 14, - "y": 29 + "x": 28, + "y": 37 } }, "aws_time": null, diff --git a/tests/test_data/expected_results/test_expected_geomod_after_update_n1_rank0__step_1.json b/tests/test_data/expected_results/test_expected_geomod_after_update_n1_rank0__step_1.json index 51e1d478..d2a22b9c 100644 --- a/tests/test_data/expected_results/test_expected_geomod_after_update_n1_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_geomod_after_update_n1_rank0__step_1.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_after_update_n1_rank0__step_2.json b/tests/test_data/expected_results/test_expected_geomod_after_update_n1_rank0__step_2.json index 51e1d478..d2a22b9c 100644 --- a/tests/test_data/expected_results/test_expected_geomod_after_update_n1_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_geomod_after_update_n1_rank0__step_2.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_after_update_n1_rank0__step_3.json b/tests/test_data/expected_results/test_expected_geomod_after_update_n1_rank0__step_3.json index 51e1d478..d2a22b9c 100644 --- a/tests/test_data/expected_results/test_expected_geomod_after_update_n1_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_geomod_after_update_n1_rank0__step_3.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank0__step_1.json b/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank0__step_1.json index b603aad6..1c67d2a2 100644 --- a/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank0__step_1.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank0__step_2.json b/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank0__step_2.json index b603aad6..1c67d2a2 100644 --- a/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank0__step_2.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank0__step_3.json b/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank0__step_3.json index b603aad6..1c67d2a2 100644 --- a/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank0__step_3.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank1__step_1.json b/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank1__step_1.json index 5ac2e4b1..e485ed6d 100644 --- a/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank1__step_1.json +++ b/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank1__step_1.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank1__step_2.json b/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank1__step_2.json index 5ac2e4b1..e485ed6d 100644 --- a/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank1__step_2.json +++ b/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank1__step_2.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank1__step_3.json b/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank1__step_3.json index 5ac2e4b1..e485ed6d 100644 --- a/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank1__step_3.json +++ b/tests/test_data/expected_results/test_expected_geomod_after_update_n2_rank1__step_3.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_finalize_n1_rank0_.json b/tests/test_data/expected_results/test_expected_geomod_finalize_n1_rank0_.json index 51e1d478..d2a22b9c 100644 --- a/tests/test_data/expected_results/test_expected_geomod_finalize_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_geomod_finalize_n1_rank0_.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_finalize_n2_rank0_.json b/tests/test_data/expected_results/test_expected_geomod_finalize_n2_rank0_.json index b603aad6..1c67d2a2 100644 --- a/tests/test_data/expected_results/test_expected_geomod_finalize_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_geomod_finalize_n2_rank0_.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_finalize_n2_rank1_.json b/tests/test_data/expected_results/test_expected_geomod_finalize_n2_rank1_.json index 5ac2e4b1..e485ed6d 100644 --- a/tests/test_data/expected_results/test_expected_geomod_finalize_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_geomod_finalize_n2_rank1_.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_init_n1_rank0_.json b/tests/test_data/expected_results/test_expected_geomod_init_n1_rank0_.json index 51e1d478..d2a22b9c 100644 --- a/tests/test_data/expected_results/test_expected_geomod_init_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_geomod_init_n1_rank0_.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_init_n2_rank0_.json b/tests/test_data/expected_results/test_expected_geomod_init_n2_rank0_.json index b603aad6..1c67d2a2 100644 --- a/tests/test_data/expected_results/test_expected_geomod_init_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_geomod_init_n2_rank0_.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_geomod_init_n2_rank1_.json b/tests/test_data/expected_results/test_expected_geomod_init_n2_rank1_.json index 5ac2e4b1..e485ed6d 100644 --- a/tests/test_data/expected_results/test_expected_geomod_init_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_geomod_init_n2_rank1_.json @@ -1,4 +1,8 @@ { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, diff --git a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n1_rank0__step_1.json b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n1_rank0__step_1.json index 19342cec..9f9fe657 100644 --- a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n1_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n1_rank0__step_1.json @@ -62,8 +62,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -75,8 +75,8 @@ "_rank": 2, "_size": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -92,8 +92,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -112,8 +112,8 @@ "_struct": {}, "_type": 6, "_upper_bounds": [ - 29, - 14 + 37, + 28 ], "_xd": 0 }, @@ -242,8 +242,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -255,8 +255,8 @@ "_rank": 2, "_size": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -272,8 +272,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -455,11 +455,11 @@ ], "nwmPRISM_denGrid": null, "nwmPRISM_numGrid": null, - "nx_global": 14, - "nx_local": 14, + "nx_global": 28, + "nx_local": 28, "nx_local_corner": null, - "ny_global": 29, - "ny_local": 29, + "ny_global": 37, + "ny_local": 37, "ny_local_corner": null, "outFreq": null, "precipBiasCorrectOpt": 0, @@ -694,10 +694,10 @@ "windBiasCorrect": 0, "x_lower_bound": 0, "x_lower_bound_corner": null, - "x_upper_bound": 14, + "x_upper_bound": 28, "x_upper_bound_corner": null, "y_lower_bound": 0, "y_lower_bound_corner": null, - "y_upper_bound": 29, + "y_upper_bound": 37, "y_upper_bound_corner": null } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n1_rank0__step_2.json b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n1_rank0__step_2.json index 17ebbc49..f424cac7 100644 --- a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n1_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n1_rank0__step_2.json @@ -62,8 +62,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -75,8 +75,8 @@ "_rank": 2, "_size": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -92,8 +92,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -112,8 +112,8 @@ "_struct": {}, "_type": 6, "_upper_bounds": [ - 29, - 14 + 37, + 28 ], "_xd": 0 }, @@ -242,8 +242,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -255,8 +255,8 @@ "_rank": 2, "_size": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -272,8 +272,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -455,11 +455,11 @@ ], "nwmPRISM_denGrid": null, "nwmPRISM_numGrid": null, - "nx_global": 14, - "nx_local": 14, + "nx_global": 28, + "nx_local": 28, "nx_local_corner": null, - "ny_global": 29, - "ny_local": 29, + "ny_global": 37, + "ny_local": 37, "ny_local_corner": null, "outFreq": null, "precipBiasCorrectOpt": 0, @@ -694,10 +694,10 @@ "windBiasCorrect": 0, "x_lower_bound": 0, "x_lower_bound_corner": null, - "x_upper_bound": 14, + "x_upper_bound": 28, "x_upper_bound_corner": null, "y_lower_bound": 0, "y_lower_bound_corner": null, - "y_upper_bound": 29, + "y_upper_bound": 37, "y_upper_bound_corner": null } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n1_rank0__step_3.json b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n1_rank0__step_3.json index 99e470aa..22c24695 100644 --- a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n1_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n1_rank0__step_3.json @@ -62,8 +62,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -75,8 +75,8 @@ "_rank": 2, "_size": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -92,8 +92,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -112,8 +112,8 @@ "_struct": {}, "_type": 6, "_upper_bounds": [ - 29, - 14 + 37, + 28 ], "_xd": 0 }, @@ -242,8 +242,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -255,8 +255,8 @@ "_rank": 2, "_size": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -272,8 +272,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -455,11 +455,11 @@ ], "nwmPRISM_denGrid": null, "nwmPRISM_numGrid": null, - "nx_global": 14, - "nx_local": 14, + "nx_global": 28, + "nx_local": 28, "nx_local_corner": null, - "ny_global": 29, - "ny_local": 29, + "ny_global": 37, + "ny_local": 37, "ny_local_corner": null, "outFreq": null, "precipBiasCorrectOpt": 0, @@ -694,10 +694,10 @@ "windBiasCorrect": 0, "x_lower_bound": 0, "x_lower_bound_corner": null, - "x_upper_bound": 14, + "x_upper_bound": 28, "x_upper_bound_corner": null, "y_lower_bound": 0, "y_lower_bound_corner": null, - "y_upper_bound": 29, + "y_upper_bound": 37, "y_upper_bound_corner": null } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank0__step_1.json b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank0__step_1.json index 4889b1cf..26675031 100644 --- a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank0__step_1.json @@ -62,8 +62,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -75,8 +75,8 @@ "_rank": 2, "_size": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -92,8 +92,8 @@ "_type": 6, "_upper_bounds": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -112,8 +112,8 @@ "_struct": {}, "_type": 6, "_upper_bounds": [ - 15, - 14 + 19, + 28 ], "_xd": 0 }, @@ -233,8 +233,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -246,8 +246,8 @@ "_rank": 2, "_size": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -263,8 +263,8 @@ "_type": 6, "_upper_bounds": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -419,11 +419,11 @@ ], "nwmPRISM_denGrid": null, "nwmPRISM_numGrid": null, - "nx_global": 14, - "nx_local": 14, + "nx_global": 28, + "nx_local": 28, "nx_local_corner": null, - "ny_global": 29, - "ny_local": 15, + "ny_global": 37, + "ny_local": 19, "ny_local_corner": null, "outFreq": null, "precipBiasCorrectOpt": 0, @@ -604,10 +604,10 @@ "windBiasCorrect": 0, "x_lower_bound": 0, "x_lower_bound_corner": null, - "x_upper_bound": 14, + "x_upper_bound": 28, "x_upper_bound_corner": null, "y_lower_bound": 0, "y_lower_bound_corner": null, - "y_upper_bound": 15, + "y_upper_bound": 19, "y_upper_bound_corner": null } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank0__step_2.json b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank0__step_2.json index cabce571..dcd4c02a 100644 --- a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank0__step_2.json @@ -62,8 +62,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -75,8 +75,8 @@ "_rank": 2, "_size": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -92,8 +92,8 @@ "_type": 6, "_upper_bounds": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -112,8 +112,8 @@ "_struct": {}, "_type": 6, "_upper_bounds": [ - 15, - 14 + 19, + 28 ], "_xd": 0 }, @@ -233,8 +233,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -246,8 +246,8 @@ "_rank": 2, "_size": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -263,8 +263,8 @@ "_type": 6, "_upper_bounds": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -419,11 +419,11 @@ ], "nwmPRISM_denGrid": null, "nwmPRISM_numGrid": null, - "nx_global": 14, - "nx_local": 14, + "nx_global": 28, + "nx_local": 28, "nx_local_corner": null, - "ny_global": 29, - "ny_local": 15, + "ny_global": 37, + "ny_local": 19, "ny_local_corner": null, "outFreq": null, "precipBiasCorrectOpt": 0, @@ -604,10 +604,10 @@ "windBiasCorrect": 0, "x_lower_bound": 0, "x_lower_bound_corner": null, - "x_upper_bound": 14, + "x_upper_bound": 28, "x_upper_bound_corner": null, "y_lower_bound": 0, "y_lower_bound_corner": null, - "y_upper_bound": 15, + "y_upper_bound": 19, "y_upper_bound_corner": null } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank0__step_3.json b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank0__step_3.json index 7f746758..d547bc93 100644 --- a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank0__step_3.json @@ -62,8 +62,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -75,8 +75,8 @@ "_rank": 2, "_size": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -92,8 +92,8 @@ "_type": 6, "_upper_bounds": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -112,8 +112,8 @@ "_struct": {}, "_type": 6, "_upper_bounds": [ - 15, - 14 + 19, + 28 ], "_xd": 0 }, @@ -233,8 +233,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -246,8 +246,8 @@ "_rank": 2, "_size": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -263,8 +263,8 @@ "_type": 6, "_upper_bounds": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -419,11 +419,11 @@ ], "nwmPRISM_denGrid": null, "nwmPRISM_numGrid": null, - "nx_global": 14, - "nx_local": 14, + "nx_global": 28, + "nx_local": 28, "nx_local_corner": null, - "ny_global": 29, - "ny_local": 15, + "ny_global": 37, + "ny_local": 19, "ny_local_corner": null, "outFreq": null, "precipBiasCorrectOpt": 0, @@ -604,10 +604,10 @@ "windBiasCorrect": 0, "x_lower_bound": 0, "x_lower_bound_corner": null, - "x_upper_bound": 14, + "x_upper_bound": 28, "x_upper_bound_corner": null, "y_lower_bound": 0, "y_lower_bound_corner": null, - "y_upper_bound": 15, + "y_upper_bound": 19, "y_upper_bound_corner": null } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank1__step_1.json b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank1__step_1.json index 9b377dfa..e968ddfd 100644 --- a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank1__step_1.json +++ b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank1__step_1.json @@ -48,7 +48,7 @@ "_has_corners": false, "_lower_bounds": [ [ - 15, + 19, 0 ], null, @@ -62,8 +62,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -75,8 +75,8 @@ "_rank": 2, "_size": [ [ - 14, - 14 + 18, + 28 ], null, null, @@ -92,8 +92,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -101,7 +101,7 @@ ] }, "_lower_bounds": [ - 15, + 19, 0 ], "_meta": {}, @@ -112,8 +112,8 @@ "_struct": {}, "_type": 6, "_upper_bounds": [ - 29, - 14 + 37, + 28 ], "_xd": 0 }, @@ -216,7 +216,7 @@ "_has_corners": false, "_lower_bounds": [ [ - 15, + 19, 0 ], null, @@ -230,8 +230,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -243,8 +243,8 @@ "_rank": 2, "_size": [ [ - 14, - 14 + 18, + 28 ], null, null, @@ -260,8 +260,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -407,11 +407,11 @@ ], "nwmPRISM_denGrid": null, "nwmPRISM_numGrid": null, - "nx_global": 14, - "nx_local": 14, + "nx_global": 28, + "nx_local": 28, "nx_local_corner": null, - "ny_global": 29, - "ny_local": 14, + "ny_global": 37, + "ny_local": 18, "ny_local_corner": null, "outFreq": null, "precipBiasCorrectOpt": 0, @@ -574,10 +574,10 @@ "windBiasCorrect": 0, "x_lower_bound": 0, "x_lower_bound_corner": null, - "x_upper_bound": 14, + "x_upper_bound": 28, "x_upper_bound_corner": null, - "y_lower_bound": 15, + "y_lower_bound": 19, "y_lower_bound_corner": null, - "y_upper_bound": 29, + "y_upper_bound": 37, "y_upper_bound_corner": null } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank1__step_2.json b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank1__step_2.json index 1bcce8db..b166c450 100644 --- a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank1__step_2.json +++ b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank1__step_2.json @@ -48,7 +48,7 @@ "_has_corners": false, "_lower_bounds": [ [ - 15, + 19, 0 ], null, @@ -62,8 +62,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -75,8 +75,8 @@ "_rank": 2, "_size": [ [ - 14, - 14 + 18, + 28 ], null, null, @@ -92,8 +92,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -101,7 +101,7 @@ ] }, "_lower_bounds": [ - 15, + 19, 0 ], "_meta": {}, @@ -112,8 +112,8 @@ "_struct": {}, "_type": 6, "_upper_bounds": [ - 29, - 14 + 37, + 28 ], "_xd": 0 }, @@ -216,7 +216,7 @@ "_has_corners": false, "_lower_bounds": [ [ - 15, + 19, 0 ], null, @@ -230,8 +230,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -243,8 +243,8 @@ "_rank": 2, "_size": [ [ - 14, - 14 + 18, + 28 ], null, null, @@ -260,8 +260,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -407,11 +407,11 @@ ], "nwmPRISM_denGrid": null, "nwmPRISM_numGrid": null, - "nx_global": 14, - "nx_local": 14, + "nx_global": 28, + "nx_local": 28, "nx_local_corner": null, - "ny_global": 29, - "ny_local": 14, + "ny_global": 37, + "ny_local": 18, "ny_local_corner": null, "outFreq": null, "precipBiasCorrectOpt": 0, @@ -574,10 +574,10 @@ "windBiasCorrect": 0, "x_lower_bound": 0, "x_lower_bound_corner": null, - "x_upper_bound": 14, + "x_upper_bound": 28, "x_upper_bound_corner": null, - "y_lower_bound": 15, + "y_lower_bound": 19, "y_lower_bound_corner": null, - "y_upper_bound": 29, + "y_upper_bound": 37, "y_upper_bound_corner": null } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank1__step_3.json b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank1__step_3.json index ab73724c..15369331 100644 --- a/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank1__step_3.json +++ b/tests/test_data/expected_results/test_expected_input_forcing_after_update_n2_rank1__step_3.json @@ -48,7 +48,7 @@ "_has_corners": false, "_lower_bounds": [ [ - 15, + 19, 0 ], null, @@ -62,8 +62,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -75,8 +75,8 @@ "_rank": 2, "_size": [ [ - 14, - 14 + 18, + 28 ], null, null, @@ -92,8 +92,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -101,7 +101,7 @@ ] }, "_lower_bounds": [ - 15, + 19, 0 ], "_meta": {}, @@ -112,8 +112,8 @@ "_struct": {}, "_type": 6, "_upper_bounds": [ - 29, - 14 + 37, + 28 ], "_xd": 0 }, @@ -216,7 +216,7 @@ "_has_corners": false, "_lower_bounds": [ [ - 15, + 19, 0 ], null, @@ -230,8 +230,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -243,8 +243,8 @@ "_rank": 2, "_size": [ [ - 14, - 14 + 18, + 28 ], null, null, @@ -260,8 +260,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -407,11 +407,11 @@ ], "nwmPRISM_denGrid": null, "nwmPRISM_numGrid": null, - "nx_global": 14, - "nx_local": 14, + "nx_global": 28, + "nx_local": 28, "nx_local_corner": null, - "ny_global": 29, - "ny_local": 14, + "ny_global": 37, + "ny_local": 18, "ny_local_corner": null, "outFreq": null, "precipBiasCorrectOpt": 0, @@ -574,10 +574,10 @@ "windBiasCorrect": 0, "x_lower_bound": 0, "x_lower_bound_corner": null, - "x_upper_bound": 14, + "x_upper_bound": 28, "x_upper_bound_corner": null, - "y_lower_bound": 15, + "y_lower_bound": 19, "y_lower_bound_corner": null, - "y_upper_bound": 29, + "y_upper_bound": 37, "y_upper_bound_corner": null } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_input_forcing_finalize_n1_rank0_.json b/tests/test_data/expected_results/test_expected_input_forcing_finalize_n1_rank0_.json index 99e470aa..22c24695 100644 --- a/tests/test_data/expected_results/test_expected_input_forcing_finalize_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_input_forcing_finalize_n1_rank0_.json @@ -62,8 +62,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -75,8 +75,8 @@ "_rank": 2, "_size": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -92,8 +92,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -112,8 +112,8 @@ "_struct": {}, "_type": 6, "_upper_bounds": [ - 29, - 14 + 37, + 28 ], "_xd": 0 }, @@ -242,8 +242,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -255,8 +255,8 @@ "_rank": 2, "_size": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -272,8 +272,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -455,11 +455,11 @@ ], "nwmPRISM_denGrid": null, "nwmPRISM_numGrid": null, - "nx_global": 14, - "nx_local": 14, + "nx_global": 28, + "nx_local": 28, "nx_local_corner": null, - "ny_global": 29, - "ny_local": 29, + "ny_global": 37, + "ny_local": 37, "ny_local_corner": null, "outFreq": null, "precipBiasCorrectOpt": 0, @@ -694,10 +694,10 @@ "windBiasCorrect": 0, "x_lower_bound": 0, "x_lower_bound_corner": null, - "x_upper_bound": 14, + "x_upper_bound": 28, "x_upper_bound_corner": null, "y_lower_bound": 0, "y_lower_bound_corner": null, - "y_upper_bound": 29, + "y_upper_bound": 37, "y_upper_bound_corner": null } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_input_forcing_finalize_n2_rank0_.json b/tests/test_data/expected_results/test_expected_input_forcing_finalize_n2_rank0_.json index 7f746758..d547bc93 100644 --- a/tests/test_data/expected_results/test_expected_input_forcing_finalize_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_input_forcing_finalize_n2_rank0_.json @@ -62,8 +62,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -75,8 +75,8 @@ "_rank": 2, "_size": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -92,8 +92,8 @@ "_type": 6, "_upper_bounds": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -112,8 +112,8 @@ "_struct": {}, "_type": 6, "_upper_bounds": [ - 15, - 14 + 19, + 28 ], "_xd": 0 }, @@ -233,8 +233,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -246,8 +246,8 @@ "_rank": 2, "_size": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -263,8 +263,8 @@ "_type": 6, "_upper_bounds": [ [ - 15, - 14 + 19, + 28 ], null, null, @@ -419,11 +419,11 @@ ], "nwmPRISM_denGrid": null, "nwmPRISM_numGrid": null, - "nx_global": 14, - "nx_local": 14, + "nx_global": 28, + "nx_local": 28, "nx_local_corner": null, - "ny_global": 29, - "ny_local": 15, + "ny_global": 37, + "ny_local": 19, "ny_local_corner": null, "outFreq": null, "precipBiasCorrectOpt": 0, @@ -604,10 +604,10 @@ "windBiasCorrect": 0, "x_lower_bound": 0, "x_lower_bound_corner": null, - "x_upper_bound": 14, + "x_upper_bound": 28, "x_upper_bound_corner": null, "y_lower_bound": 0, "y_lower_bound_corner": null, - "y_upper_bound": 15, + "y_upper_bound": 19, "y_upper_bound_corner": null } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_input_forcing_finalize_n2_rank1_.json b/tests/test_data/expected_results/test_expected_input_forcing_finalize_n2_rank1_.json index ab73724c..15369331 100644 --- a/tests/test_data/expected_results/test_expected_input_forcing_finalize_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_input_forcing_finalize_n2_rank1_.json @@ -48,7 +48,7 @@ "_has_corners": false, "_lower_bounds": [ [ - 15, + 19, 0 ], null, @@ -62,8 +62,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -75,8 +75,8 @@ "_rank": 2, "_size": [ [ - 14, - 14 + 18, + 28 ], null, null, @@ -92,8 +92,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -101,7 +101,7 @@ ] }, "_lower_bounds": [ - 15, + 19, 0 ], "_meta": {}, @@ -112,8 +112,8 @@ "_struct": {}, "_type": 6, "_upper_bounds": [ - 29, - 14 + 37, + 28 ], "_xd": 0 }, @@ -216,7 +216,7 @@ "_has_corners": false, "_lower_bounds": [ [ - 15, + 19, 0 ], null, @@ -230,8 +230,8 @@ null ], "_max_index": [ - 29, - 14 + 37, + 28 ], "_meta": {}, "_ndims": 2, @@ -243,8 +243,8 @@ "_rank": 2, "_size": [ [ - 14, - 14 + 18, + 28 ], null, null, @@ -260,8 +260,8 @@ "_type": 6, "_upper_bounds": [ [ - 29, - 14 + 37, + 28 ], null, null, @@ -407,11 +407,11 @@ ], "nwmPRISM_denGrid": null, "nwmPRISM_numGrid": null, - "nx_global": 14, - "nx_local": 14, + "nx_global": 28, + "nx_local": 28, "nx_local_corner": null, - "ny_global": 29, - "ny_local": 14, + "ny_global": 37, + "ny_local": 18, "ny_local_corner": null, "outFreq": null, "precipBiasCorrectOpt": 0, @@ -574,10 +574,10 @@ "windBiasCorrect": 0, "x_lower_bound": 0, "x_lower_bound_corner": null, - "x_upper_bound": 14, + "x_upper_bound": 28, "x_upper_bound_corner": null, - "y_lower_bound": 15, + "y_lower_bound": 19, "y_lower_bound_corner": null, - "y_upper_bound": 29, + "y_upper_bound": 37, "y_upper_bound_corner": null } \ No newline at end of file From 67b2358adcfc60c1d1ec60581a4dc3c4a95cd914 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 10 Jun 2026 11:59:25 -0400 Subject: [PATCH 020/132] Explicit cleanup call on BMI finalize() --- .../NextGen_Forcings_Engine/bmi_model.py | 3 + .../NextGen_Forcings_Engine/core/mpi_utils.py | 17 +- .../NextGen_Forcings_Engine/core/parallel.py | 253 ++++-------------- .../NextGen_Forcings_Engine/nc_utils.py | 37 --- 4 files changed, 59 insertions(+), 251 deletions(-) delete mode 100644 NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/nc_utils.py diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 90bae785..1afafb9a 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -584,6 +584,9 @@ def finalize(self): # process exits gc.collect() # make sure objects are deleted from memory LOG.info(Pld(St.COMPLETE, msg="Finishing BMI finalize()", modnm=MODNM)) + mpi_meta: MpiConfig = getattr(self, "_mpi_meta", None) + if mpi_meta is not None: + mpi_meta.cleanup() # ------------------------------------------------------------------- # ------------------------------------------------------------------- diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/mpi_utils.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/mpi_utils.py index f8157f27..dfee83b1 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/mpi_utils.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/mpi_utils.py @@ -1,24 +1,21 @@ import uuid - +import numpy as np import mpi4py mpi4py.rc.threads = False -from mpi4py import MPI - -import numpy as np +from mpi4py import MPI # noqa: E402 -# def get_new_broadcasted_uid(comm: MPI.Comm) -> str: -def get_new_broadcasted_uid() -> str: +def get_new_broadcasted_uid(comm: MPI.Comm | None = None) -> str: """Broadcast a random uint64 then return the hash of that. Used for generating a random string shared among all ranks.""" - # if not isinstance(comm, MPI.Comm): - # raise TypeError(f"Expected comm to be type MPI.Comm, got: {type(comm)}") + if comm is None: + comm = MPI.COMM_WORLD rand_uint64 = None - if MPI.COMM_WORLD.rank == 0: + if comm.rank == 0: rng = np.random.default_rng() rand_uint64 = rng.integers(0, 2**64, dtype=np.uint64) - rand_uint64 = MPI.COMM_WORLD.bcast(rand_uint64, root=0) + rand_uint64 = comm.bcast(rand_uint64, root=0) # uuid.UUID expects a built-in Python int. Convert the NumPy uint64 uid_64bit_hex = uuid.UUID(int=int(rand_uint64)).hex diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index e0e48891..72a13a0f 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -1,16 +1,20 @@ -import atexit +from __future__ import annotations from functools import partial import os -import uuid import signal import sys +import typing +if typing.TYPE_CHECKING: + from typing import Any, Type, TypeVar + from .geoMod import GriddedGeoMeta + T = TypeVar("T") import mpi4py import numpy as np mpi4py.rc.threads = False -from mpi4py import MPI +from mpi4py import MPI # noqa: E402 from .config import ConfigOptions from . import err_handler @@ -46,7 +50,6 @@ def __init__(self, config_options: ConfigOptions): self.log_debug = partial(err_handler.log_msg, self.config_options, self, True) self.log_info = partial(err_handler.log_msg, self.config_options, self, False) self.log_warning = partial(err_handler.log_warning, self.config_options, self) - self.__register_exit_handlers() def initialize_comm(self, comm=None): """Initialize MPI communication. @@ -57,25 +60,29 @@ def initialize_comm(self, comm=None): try: self.comm = comm if comm is not None else MPI.COMM_WORLD self.comm.Set_errhandler(MPI.ERRORS_ARE_FATAL) - except AttributeError as ae: + except AttributeError: self.config_options.errMsg = ( "Unable to initialize the MPI Communicator object" ) - raise ae + raise try: self.size = self.comm.Get_size() - except MPI.Exception as mpi_exception: + except MPI.Exception: self.config_options.errMsg = "Unable to retrieve the MPI size." - raise mpi_exception + raise try: self.rank = self.comm.Get_rank() - except MPI.Exception as mpi_exception: + except MPI.Exception: self.config_options.errMsg = "Unable to retrieve the MPI processor rank." - raise mpi_exception + raise - self.__broadcast_new_64bit_uid() + try: + self.uid64 = mpi_utils.get_new_broadcasted_uid() + except Exception: + self.config_options.errMsg = "Unable to generate a global unique ID." + raise wait_for_debug = os.getenv("WAIT_FOR_DEBUGPY", "") if wait_for_debug.lower() in ("true", "1"): @@ -107,93 +114,18 @@ def abort_with_cleanup(self, errorcode: int) -> None: """Call cleanup methods, before calling MPI Abort. Do not make direct calls to MPI Abort without this method. Use this method for all MPI abort needs.""" - comm = getattr(self, "comm", None) + comm: MPI.Comm = getattr(self, "comm", None) if comm is None: raise RuntimeError("comm is not initialized") - # if self.rank == 0: - if True: - self._cleanup() - err_handler.log_msg( - self.config_options, self, debug=True, msg="About to MPI Abort" - ) - comm.Abort(errorcode) - comm.Barrier() # For testing case of only rank 0 aborting. - raise RuntimeError("At bottom of abort_with_cleanup, should not get here.") - - def __signals_handled(self) -> tuple[int]: - """Return a tuple of signals to be handled by cleanup routine.""" - ### signal.valid_signals() contains many that are unrelated to stoppage / interruption / error. - # sigs = [s for s in signal.valid_signals() if s not in (signal.SIGKILL, signal.SIGSTOP)] - sigs = ( - signal.SIGINT, - signal.SIGTERM, - signal.SIGHUP, - signal.SIGQUIT, - signal.SIGSEGV, - signal.SIGABRT, - signal.SIGFPE, - signal.SIGBUS, - signal.SIGILL, - ) - return sigs - - def __register_exit_handlers(self) -> None: - """Register exit handlers for unhandled exceptions, signals, and regular exits. - TODO: consider WCOSS gating. - TODO: note that when non-0 ranks call Abort directly, the rank 0 exit handler is still invoked, at least in some cases, - so there may be opportunities to streamline this further to have only rank 0 perform the cleanup. Would need to test - against potential deadlock conditions to be sure (would need to confirm that a non-0 rank initiating an abort would cause - rank 0 break out of a collective call if it happens to be waiting at one).""" - # Exceptions - sys.exepthook = self.__excepthook - # Regular exits - atexit.register(self._cleanup) - # Signals - for sig in self.__signals_handled(): - signal.signal(sig, self.__signal_handler) - - def __excepthook(self, ex_type, value, tb) -> None: - """Custom excepthook which follows these steps: - 1. Call Python's built-in excepthook. - 2. Log .errMsg as CRITICAL (unless it is None). - 3. Cleanup. - 4. MPI Abort. - - To apply, set `sys.excepthook` to this method.""" - sys.__excepthook__(ex_type, value, tb) - if self.config_options.errMsg is not None: - err_handler.log_critical( - self.config_options, - self, - msg=f"In excepthook, found errMsg = {repr(self.config_options.errMsg)}", - ) - self.abort_with_cleanup(1) - - def __signal_handler(self, signum, frame) -> None: - """Handle termination signals by cleaning up before exit.""" - ### Unregister the signal handler - for s in self.__signals_handled(): - signal.signal(s, signal.SIG_DFL) - ### Cleanup and re-send the original signal to itself - # self._cleanup() - # os.kill(os.getpid(), signum) - ### Cleanup and abort directly - self.abort_with_cleanup(signum) - - def _cleanup(self) -> None: - """High-level cleanup routine called by exit handlers.""" - # if self.rank != 0: - # return - err_handler.log_msg( - self.config_options, self, debug=True, msg="About to clean up" - ) + self.cleanup() + self.log_debug("About to MPI Abort") + comm.Abort(errorcode) + + def cleanup(self) -> None: + """High-level cleanup routine called during BMI finalization.""" + self.log_debug("About to clean up") self._cleanup_scratch_dir() self._cleanup_geogrid() - # TODO: Consider if this can be gated for non-wcoss only - try: - atexit.unregister(self._cleanup) - except Exception: - pass def _cleanup_scratch_dir(self) -> None: """Remove contents of scratch dir. @@ -222,7 +154,7 @@ def _cleanup_scratch_dir(self) -> None: # # Only delete files that don't start with either of these skip_starts = (".nfs", "NextGen_Forcings_Engine") - to_delete = [_ for _ in contents if not _.startswith(skip_starts)] + to_delete = [n for n in contents if not n.startswith(skip_starts)] for fn in to_delete: fp = os.path.join(self.config_options.scratch_dir, fn) if os.path.isfile(fp): @@ -342,10 +274,6 @@ def __test_exit(self, mode: str, rank: int) -> None: self.log_debug(msg) raise RuntimeError(msg) - def __broadcast_new_64bit_uid(self): - """Broadcast a random uint64 then save the hash of that to self.uid64, which effectively broadcasts the same unique string to all ranks.""" - self.uid64 = mpi_utils.get_new_broadcasted_uid() - def wait_for_debugpy_client(self): """Block until the debugpy clients have attached to cppdbg/gdb. @@ -357,7 +285,7 @@ def wait_for_debugpy_client(self): debugpy.listen(("localhost", 5678 + self.rank)) debugpy.wait_for_client() - def broadcast_parameter(self, value_broadcast, config_options, param_type): + def broadcast_parameter(self, value_broadcast: Any, config_options: ConfigOptions, param_type: Type[T]) -> T: """Broadcast a single parameter value to all processors. Generic function for sending a parameter value out to the processors. @@ -380,74 +308,14 @@ def broadcast_parameter(self, value_broadcast, config_options, param_type): return None return param.item(0) - def scatter_array_logan(self, geoMeta, array_broadcast, ConfigOptions): + def scatter_array(self, geo_meta: GriddedGeoMeta, src_array: np.ndarray, config_options: ConfigOptions): """Scatter an array based on the input dataset type. Generic function for calling scatter functons based on the input dataset type. - :param geoMeta: + :param geo_meta: :param array_broadcast: - :param ConfigOptions: - :return: - """ - # Determine which type of input array we have based on the - # type of numpy array. - data_type_flag = -1 - if self.rank == 0: - if array_broadcast.dtype == np.float32: - data_type_flag = 1 - if array_broadcast.dtype == np.float64: - data_type_flag = 2 - - # Broadcast the numpy datatype to the other processors. - if self.rank == 0: - tmpDict = {"varTmp": data_type_flag} - else: - tmpDict = None - try: - tmpDict = self.comm.bcast(tmpDict, root=0) - except Exception: - ConfigOptions.errMsg = ( - "Unable to broadcast numpy datatype value from rank 0" - ) - err_handler.log_critical(ConfigOptions, self) - return None - data_type_flag = tmpDict["varTmp"] - - # Broadcast the global array to the child processors, then - if self.rank == 0: - arrayGlobalTmp = array_broadcast - else: - if data_type_flag == 1: - arrayGlobalTmp = np.empty( - [geoMeta.ny_global, geoMeta.nx_global], np.float32 - ) - else: # data_type_flag == 2: - arrayGlobalTmp = np.empty( - [geoMeta.ny_global, geoMeta.nx_global], np.float64 - ) - try: - self.comm.Bcast(arrayGlobalTmp, root=0) - except Exception: - ConfigOptions.errMsg = ( - "Unable to broadcast a global numpy array from rank 0" - ) - err_handler.log_critical(ConfigOptions, self) - return None - arraySub = arrayGlobalTmp[ - geoMeta.y_lower_bound : geoMeta.y_upper_bound, - geoMeta.x_lower_bound : geoMeta.x_upper_bound, - ] - return arraySub - - def scatter_array_scatterv_no_cache(self, geoMeta, src_array, ConfigOptions): - """Scatter an array based on the input dataset type. - - Generic function for calling scatter functons based on - the input dataset type. - :param geoMeta: - :param array_broadcast: - :param ConfigOptions: + :param config_options: :return: """ # Determine which type of input array we have based on the @@ -469,34 +337,33 @@ def scatter_array_scatterv_no_cache(self, geoMeta, src_array, ConfigOptions): try: self.comm.Bcast(data_type_buffer, root=0) - except: - ConfigOptions.errMsg = ( + except Exception: + config_options.errMsg = ( "Unable to broadcast numpy datatype value from rank 0" ) - err_handler.err_out(ConfigOptions) + err_handler.err_out(config_options) return None data_type_flag = data_type_buffer[0] - data_type_buffer = None # gather buffer offsets and bounds to rank 0 bounds = np.array( [ - np.int32(geoMeta.x_lower_bound), - np.int32(geoMeta.y_lower_bound), - np.int32(geoMeta.x_upper_bound), - np.int32(geoMeta.y_upper_bound), + np.int32(geo_meta.x_lower_bound), + np.int32(geo_meta.y_lower_bound), + np.int32(geo_meta.x_upper_bound), + np.int32(geo_meta.y_upper_bound), ] ) global_bounds = np.zeros((self.size * 4), np.int32) try: self.comm.Allgather([bounds, MPI.INTEGER], [global_bounds, MPI.INTEGER]) - except: - ConfigOptions.errMsg = "Failed all gathering global bounds at rank" + str( + except Exception: + config_options.errMsg = "Failed all gathering global bounds at rank" + str( self.rank ) - err_handler.err_out(ConfigOptions) + err_handler.err_out(config_options) return None # create slices for x and y bounds arrays @@ -544,9 +411,9 @@ def scatter_array_scatterv_no_cache(self, geoMeta, src_array, ConfigOptions): # scatter the data try: self.comm.Scatterv([sendbuf, counts, offsets, data_type], recvbuf, root=0) - except: - ConfigOptions.errMsg = "Failed Scatterv from rank 0" - err_handler.error_out(ConfigOptions) + except Exception: + config_options.errMsg = "Failed Scatterv from rank 0" + err_handler.err_out(config_options) return None subarray = np.reshape( @@ -558,10 +425,7 @@ def scatter_array_scatterv_no_cache(self, geoMeta, src_array, ConfigOptions): ).copy() return subarray - # use scatterv based scatter_array - scatter_array = scatter_array_scatterv_no_cache - - def merge_slabs_gatherv(self, local_slab, options, allgather: bool = False): + def merge_slabs_gatherv(self, local_slab: np.ndarray, options: ConfigOptions, allgather: bool = False): """If allgather is True, then Allgatherv will be used instead of Gatherv, which causes all ranks to be distributed to all other ranks. This is necessary for the hydrofabric case, to handle how ngen's hydrologic @@ -581,13 +445,10 @@ def merge_slabs_gatherv(self, local_slab, options, allgather: bool = False): try: self.comm.Allgather([shapes, MPI.INTEGER], [global_shapes, MPI.INTEGER]) - except: + except Exception: options.errMsg = "Failed all gathering slab shapes at rank" + str(self.rank) err_handler.log_critical(options, self) - global_bounds = None - - # options.errMsg = "All gather for global shapes complete" - # err_handler.log_msg(options,self) + return None if len(local_slab.shape) == 2: # check that all slabes are the same width and sum the number of rows @@ -601,14 +462,7 @@ def merge_slabs_gatherv(self, local_slab, options, allgather: bool = False): + str(i) ) err_handler.log_critical(options, self) - # TODO why was there an abort here? - # Switched it to a new wrapped/cleanup abort, - # but would like to remove that call too if - # there is no reason to keep it. - self.abort_with_cleanup(1) - - # options.errMsg = "Checking of Rows and Columns complete" - # err_handler.log_msg(options,self) + return None # generate counts counts = [ @@ -621,9 +475,6 @@ def merge_slabs_gatherv(self, local_slab, options, allgather: bool = False): for i in range(0, len(counts) - 1): offsets.append(offsets[i] + counts[i]) - # options.errMsg = "Counts and Offsets generated" - # err_handler.log_msg(options,self) - # create the receive buffer if allgather or self.rank == 0: recvbuf = np.empty([total_rows, width], local_slab.dtype) @@ -638,9 +489,6 @@ def merge_slabs_gatherv(self, local_slab, options, allgather: bool = False): for i in range(0, len(counts) - 1): offsets.append(offsets[i] + counts[i]) - # options.errMsg = "Counts and Offsets generated" - # err_handler.log_msg(options,self) - # create the receive buffer if allgather or self.rank == 0: recvbuf = np.empty([sum(global_shapes)], local_slab.dtype) @@ -668,12 +516,9 @@ def merge_slabs_gatherv(self, local_slab, options, allgather: bool = False): recvbuf=[recvbuf, counts, offsets, data_type], root=0, ) - except: + except Exception: options.errMsg = "Failed to Gatherv to rank 0 from rank " + str(self.rank) err_handler.log_critical(options, self) return None - # options.errMsg = "Gatherv complete" - # err_handler.log_msg(options,self) - return recvbuf diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/nc_utils.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/nc_utils.py deleted file mode 100644 index 1148f24f..00000000 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/nc_utils.py +++ /dev/null @@ -1,37 +0,0 @@ -import types - -import netCDF4 - -from . import retry_utils -from .core.config import ConfigOptions -from .core.parallel import MpiConfig - - -@retry_utils.retry_w_mpi_context( - abort=True, num_retries=3, sleep_start=1, sleep_factor=3 -) -def nc_Dataset_retry( - mpi_config: MpiConfig, - config_options: ConfigOptions, - err_handler: types.ModuleType, - *args, - **kwargs, -): - """netCDF4 Dataset open with MPI retry logic.""" - return netCDF4.Dataset(*args, **kwargs) - - -@retry_utils.retry_w_mpi_context( - abort=True, num_retries=3, sleep_start=1, sleep_factor=3 -) -def nc_read_var_retry( - mpi_config: MpiConfig, - config_options: ConfigOptions, - err_handler: types.ModuleType, - nc_var: netCDF4.Variable, - slices=None, -): - """Read NetCDF variable data with MPI retry logic.""" - if slices is None: - return nc_var[:].data - return nc_var[slices].data From 64dca62f915ad8393a60696936c8f1009ffbbc36 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 14 Jul 2026 11:17:41 -0400 Subject: [PATCH 021/132] Simplify broadcast_parameter to use bcast --- .../core/disaggregateMod.py | 8 ++--- .../NextGen_Forcings_Engine/core/parallel.py | 33 ++++++++----------- .../NextGen_Forcings_Engine/core/regrid.py | 28 +++++++--------- 3 files changed, 26 insertions(+), 43 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/disaggregateMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/disaggregateMod.py index d7a3d473..c4d8b9af 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/disaggregateMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/disaggregateMod.py @@ -150,9 +150,7 @@ def ak_ext_ana_disaggregate( date_iter += timedelta(hours=1) - found_target_hh = mpi_config.broadcast_parameter( - found_target_hh, config_options, param_type=bool - ) + found_target_hh = mpi_config.broadcast_parameter(found_target_hh) err_handler.check_program_status(config_options, mpi_config) if not found_target_hh: if mpi_config.rank == 0: @@ -167,9 +165,7 @@ def ak_ext_ana_disaggregate( supplemental_precip.regridded_precip2[:] = config_options.globalNdv return - read_hours = mpi_config.broadcast_parameter( - read_hours, config_options, param_type=int - ) + read_hours = mpi_config.broadcast_parameter(read_hours) err_handler.check_program_status(config_options, mpi_config) if read_hours != 6: if mpi_config.rank == 0: diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index 72a13a0f..749586c1 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -4,11 +4,6 @@ import signal import sys import typing -if typing.TYPE_CHECKING: - from typing import Any, Type, TypeVar - from .geoMod import GriddedGeoMeta - T = TypeVar("T") - import mpi4py import numpy as np @@ -26,6 +21,10 @@ mpi4py.rc.initialize = False mpi4py.rc.finalize = False +if typing.TYPE_CHECKING: + from typing import Any, Type, TypeVar + from .geoMod import GriddedGeoMeta + T = TypeVar("T") class MpiConfig: """MPI config class. @@ -40,7 +39,7 @@ def __init__(self, config_options: ConfigOptions): NOTE: this class overrides the system excepthook so that cleanup steps and MPI abort can be triggered on unhandled exceptions. """ - self.comm = None + self.comm: MPI.Intracomm = None self.rank = None self.size = None self.uid64: str | None = ( @@ -285,7 +284,7 @@ def wait_for_debugpy_client(self): debugpy.listen(("localhost", 5678 + self.rank)) debugpy.wait_for_client() - def broadcast_parameter(self, value_broadcast: Any, config_options: ConfigOptions, param_type: Type[T]) -> T: + def broadcast_parameter(self, value_broadcast: T) -> T: """Broadcast a single parameter value to all processors. Generic function for sending a parameter value out to the processors. @@ -293,20 +292,14 @@ def broadcast_parameter(self, value_broadcast: Any, config_options: ConfigOption :param config_options: :return: """ - dtype = np.dtype(param_type) - - if self.rank == 0: - param = np.asarray(value_broadcast, dtype=dtype) - else: - param = np.empty(dtype=dtype, shape=()) - + if self.size == 1: + return value_broadcast try: - self.comm.Bcast(param, root=0) - except MPI.Exception: - config_options.errMsg = "Unable to broadcast single value from rank 0." - err_handler.log_critical(config_options, self) - return None - return param.item(0) + return self.comm.bcast(value_broadcast, root=0) + except Exception as e: + self.config_options.errMsg = f"Unable to broadcst single value {value_broadcast} from rank 0: {e.__class__.__name__} -- {e}" + err_handler.log_critical(self.config_options, self) + raise def scatter_array(self, geo_meta: GriddedGeoMeta, src_array: np.ndarray, config_options: ConfigOptions): """Scatter an array based on the input dataset type. diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py index 5dd3a109..fda67297 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py @@ -213,11 +213,11 @@ def regrid_ak_ext_ana(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co input_forcings.nx_global = ds.dimensions["x"].size input_forcings.ny_global = mpi_config.broadcast_parameter( - input_forcings.ny_global, config_options, param_type=int + input_forcings.ny_global ) err_handler.check_program_status(config_options, mpi_config) input_forcings.nx_global = mpi_config.broadcast_parameter( - input_forcings.nx_global, config_options, param_type=int + input_forcings.nx_global ) err_handler.check_program_status(config_options, mpi_config) @@ -10499,7 +10499,7 @@ def regrid_ndfd(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config): ) # look to see if current time is in file: - skip_file = np.ubyte(0) + skip_file = False if mpi_config.rank == 0: times = [datetime.utcfromtimestamp(t) for t in id_tmp["time"][:]] if ndfd_var != "qpf": @@ -10523,16 +10523,14 @@ def regrid_ndfd(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config): # TODO: qpf special handling if forecast_time > times[-1] - timedelta(hours=6): pt.log_debug("Forecast time beyond NDFD precip range, skipping") - skip_file = 1 + skip_file = True else: time_index = int(hour // 6) pt.log_debug( f"Forecast hour {forecast_time} will use precip from {times[time_index] - timedelta(hours=6)} to {times[time_index]}" ) - skip_file = mpi_config.broadcast_parameter( - skip_file, config_options, param_type=np.ubyte - ) + skip_file = mpi_config.broadcast_parameter(skip_file) err_handler.check_program_status(config_options, mpi_config) if skip_file: @@ -11399,9 +11397,7 @@ def check_regrid_status( # mpi_config.comm.barrier() # Broadcast the flag to the other processors. - calc_regrid_flag = mpi_config.broadcast_parameter( - calc_regrid_flag, config_options, param_type=bool - ) + calc_regrid_flag = mpi_config.broadcast_parameter(calc_regrid_flag) err_handler.check_program_status(config_options, mpi_config) return calc_regrid_flag @@ -11611,9 +11607,7 @@ def check_supp_pcp_regrid_status( # mpi_config.comm.barrier() # Broadcast the flag to the other processors. - calc_regrid_flag = mpi_config.broadcast_parameter( - calc_regrid_flag, config_options, param_type=bool - ) + calc_regrid_flag = mpi_config.broadcast_parameter(calc_regrid_flag) mpi_config.comm.barrier() return calc_regrid_flag @@ -11862,11 +11856,11 @@ def calculate_weights( # Broadcast the forcing nx/ny values input_forcings.ny_global = mpi_config.broadcast_parameter( - input_forcings.ny_global, config_options, param_type=int + input_forcings.ny_global ) err_handler.check_program_status(config_options, mpi_config) input_forcings.nx_global = mpi_config.broadcast_parameter( - input_forcings.nx_global, config_options, param_type=int + input_forcings.nx_global ) err_handler.check_program_status(config_options, mpi_config) @@ -12245,10 +12239,10 @@ def calculate_supp_pcp_weights( # Broadcast the forcing nx/ny values supplemental_precip.ny_global = mpi_config.broadcast_parameter( - supplemental_precip.ny_global, config_options, param_type=int + supplemental_precip.ny_global ) supplemental_precip.nx_global = mpi_config.broadcast_parameter( - supplemental_precip.nx_global, config_options, param_type=int + supplemental_precip.nx_global ) # mpi_config.comm.barrier() From 72be130e30f05d52a5dda46f0e143be09ba04c5e Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 14 Jul 2026 12:30:40 -0400 Subject: [PATCH 022/132] Remove use of err_handler.err_out --- .../NextGen_Forcings_Engine/core/parallel.py | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index 749586c1..f30492be 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -304,7 +304,7 @@ def broadcast_parameter(self, value_broadcast: T) -> T: def scatter_array(self, geo_meta: GriddedGeoMeta, src_array: np.ndarray, config_options: ConfigOptions): """Scatter an array based on the input dataset type. - Generic function for calling scatter functons based on + Generic function for calling scatter functions based on the input dataset type. :param geo_meta: :param array_broadcast: @@ -330,12 +330,12 @@ def scatter_array(self, geo_meta: GriddedGeoMeta, src_array: np.ndarray, config_ try: self.comm.Bcast(data_type_buffer, root=0) - except Exception: + except Exception as e: config_options.errMsg = ( - "Unable to broadcast numpy datatype value from rank 0" + f"Unable to broadcast numpy datatype value from rank 0: {e.__class__.__name__} -- {e}" ) - err_handler.err_out(config_options) - return None + err_handler.log_critical(config_options, self) + raise data_type_flag = data_type_buffer[0] @@ -352,12 +352,10 @@ def scatter_array(self, geo_meta: GriddedGeoMeta, src_array: np.ndarray, config_ try: self.comm.Allgather([bounds, MPI.INTEGER], [global_bounds, MPI.INTEGER]) - except Exception: - config_options.errMsg = "Failed all gathering global bounds at rank" + str( - self.rank - ) - err_handler.err_out(config_options) - return None + except Exception as e: + config_options.errMsg = f"Failed all gathering global bounds at rank {self.rank}: {e.__class__.__name__} -- {e}" + err_handler.log_critical(config_options, self) + raise # create slices for x and y bounds arrays x_lower = global_bounds[0 : (self.size * 4) + 0 : 4] @@ -404,10 +402,10 @@ def scatter_array(self, geo_meta: GriddedGeoMeta, src_array: np.ndarray, config_ # scatter the data try: self.comm.Scatterv([sendbuf, counts, offsets, data_type], recvbuf, root=0) - except Exception: - config_options.errMsg = "Failed Scatterv from rank 0" - err_handler.err_out(config_options) - return None + except Exception as e: + config_options.errMsg = f"Failed Scatterv from rank 0: {e.__class__.__name__} -- {e}" + err_handler.log_critical(config_options, self) + raise subarray = np.reshape( recvbuf, From baaf86429d97557efcd7b3816724242de23c5714 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 14 Jul 2026 14:51:05 -0400 Subject: [PATCH 023/132] Documentation updates --- .../NextGen_Forcings_Engine/core/parallel.py | 167 ++++++------------ 1 file changed, 53 insertions(+), 114 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index f30492be..ee0032d4 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -1,8 +1,6 @@ from __future__ import annotations from functools import partial import os -import signal -import sys import typing import mpi4py import numpy as np @@ -11,7 +9,6 @@ from mpi4py import MPI # noqa: E402 -from .config import ConfigOptions from . import err_handler from . import mpi_utils @@ -22,39 +19,52 @@ mpi4py.rc.finalize = False if typing.TYPE_CHECKING: - from typing import Any, Type, TypeVar + from typing import TypeVar from .geoMod import GriddedGeoMeta + from .config import ConfigOptions + T = TypeVar("T") + class MpiConfig: """MPI config class. - Abstract class for defining the MPI parameters, + Class for defining the MPI parameters, along with initialization of the MPI communication handle from mpi4py. """ + comm: MPI.Intercomm + rank: int + """MPI rank of the process of this instance.""" + size: int + """The number of MPI processes on this run.""" + uid64: str + """Random 16 chars based on random uint64 shared between all processes in a run.""" + config_options: ConfigOptions + """Forcing Engine configurations options. Messaging will use the instance of ConfigOptions passed to it in the constructor.""" + def __init__(self, config_options: ConfigOptions): """Initialize the MPI abstract class that will contain basic information and communication handles. - NOTE: this class overrides the system excepthook so that - cleanup steps and MPI abort can be triggered on unhandled exceptions. + + NOTE: Temporary files that are created during a normal forcing run should be cleaned up using the `cleanup` method. """ - self.comm: MPI.Intracomm = None + self.comm = None self.rank = None self.size = None - self.uid64: str | None = ( - None # broadcasted random 16 chars based on random uint64 - ) + self.uid64 = None self.config_options = config_options self.log_debug = partial(err_handler.log_msg, self.config_options, self, True) self.log_info = partial(err_handler.log_msg, self.config_options, self, False) self.log_warning = partial(err_handler.log_warning, self.config_options, self) - def initialize_comm(self, comm=None): - """Initialize MPI communication. + def initialize_comm(self, comm: MPI.Intercomm | None = None) -> None: + """Initialize MPI communication, including getting MPI rank and size. + Also generates the UID for the run. - Initial function to initialize MPI. - :return: + Usage note: if an exception is thrown, an error messsage is added to the + `config_options` object with the expectation the caller will handle + logging the error message generated. The error will be reraised. """ try: self.comm = comm if comm is not None else MPI.COMM_WORLD @@ -87,38 +97,15 @@ def initialize_comm(self, comm=None): if wait_for_debug.lower() in ("true", "1"): self.wait_for_debugpy_client() - # self._test_exit() - - # ------------------------------------------------------ - # Exit handling, exception handling, cleanup, and abort. - # ------------------------------------------------------ - - def _test_exit(self) -> None: - """Various methods for testing potential exit conditions""" - self.__test_exit("exception", 0) - # self.__test_exit("exception", 1) - # self.__test_exit("signal", 0) - ### Signal on rank 1 causes a deadlock iff abort_with_cleanup only allows rank 0 to abort, so all ranks need to be able to abort. - # self.__test_exit("signal", 1) - # self.__test_exit("sysexit1", 0) - # self.__test_exit("sysexit1", 1) - # self.__test_exit("check_program_status", 0) - # self.__test_exit("check_program_status", 1) - # self.__test_exit("err_out_screen", 0) - # self.__test_exit("err_out_screen", 1) - # self.__test_exit("err_out_screen_para", 0) - # self.__test_exit("err_out_screen_para", 1) - def abort_with_cleanup(self, errorcode: int) -> None: """Call cleanup methods, before calling MPI Abort. Do not make direct calls to MPI Abort without this method. Use this method for all MPI abort needs.""" - comm: MPI.Comm = getattr(self, "comm", None) - if comm is None: + if self.comm is None: raise RuntimeError("comm is not initialized") self.cleanup() self.log_debug("About to MPI Abort") - comm.Abort(errorcode) + self.comm.Abort(errorcode) def cleanup(self) -> None: """High-level cleanup routine called during BMI finalization.""" @@ -175,21 +162,18 @@ def _cleanup_geogrid(self) -> None: self.try_delete_file_no_reraise(geogrid) else: self.log_debug("Cleanup: config_options.geogrid is not set") - return def try_list_dir_no_reraise(self, dir_path: str) -> list[str]: """Try to list the directory and return a list of its contents. Do not reraise an exception if it fails due to FileNotFoundError or NotADirectoryError""" self.log_debug(f"Trying to list directory: {dir_path}") try: - contents = os.listdir(dir_path) + return os.listdir(dir_path) except (FileNotFoundError, NotADirectoryError) as e: self.log_debug( f"Could not list (it may have already been deleted): {dir_path}: {e}" ) return [] - else: - return contents def try_delete_file_no_reraise(self, file_path: str) -> None: """Try to delete the file, do not reraise an exception if it fails due to OSError""" @@ -215,64 +199,6 @@ def try_remove_empty_dir_no_reraise(self, dir_path: str) -> None: else: self.log_info(f"Removed directory: {dir_path}") - def __test_exit(self, mode: str, rank: int) -> None: - """Intentionally exit in a particular way, for testing exit/cleanup behavior. - `mode` : str. Mode of exit. See match/case block below for accepted values. - `rank` : int. Rank to perform the mode of exit. Can be 0 or 1. They have different""" - self.log_debug(f"__test_exit(): provided: mode={repr(mode)}, rank={repr(rank)}") - if rank not in (0, 1): - raise ValueError(f"__test_exit(): unsupported value for rank: {repr(rank)}") - - if self.rank == rank: - match mode: - case "exception": - msg = "__test_exit(): raising intentional RuntimeError" - self.log_debug(msg) - self.config_options.errMsg = "TEST" - raise RuntimeError(msg) - - case "signal": - # msg = f"__test_exit(): sending signal.SIGHUP ({signal.SIGHUP})" - msg = f"__test_exit(): sending signal.SIGTERM ({signal.SIGTERM})" - self.log_debug(msg) - # os.kill(os.getpid(), signal.SIGHUP) - os.kill(os.getpid(), signal.SIGTERM) - - case "sysexit1": - msg = "__test_exit(): calling sys.exit(1)" - self.log_debug(msg) - sys.exit(1) - - case "check_program_status": - msg = "__test_exit(): setting critical msg before calling check_program_status()" - self.log_debug(msg) - err_handler.log_critical( - self.config_options, self, msg="TESTING EXIT HANDLING" - ) - - case "err_out_screen": - msg = "__test_exit(): calling err_out_screen()" - self.log_debug(msg) - err_handler.err_out_screen(msg) - - case "err_out_screen_para": - msg = "__test_exit(): calling err_out_screen_para()" - self.log_debug(msg) - err_handler.err_out_screen_para(msg, self) - - case _: - raise ValueError(f"Unsupported mode={repr(mode)} for __test_exit()") - - self.log_debug("__test_exit(): reaching check_program_status()") - err_handler.check_program_status(self.config_options, self) - - self.log_debug("__test_exit(): reaching MPI Barrier") - self.comm.Barrier() - - msg = "__test_exit(): got past MPI Barrier (should not get here)" - self.log_debug(msg) - raise RuntimeError(msg) - def wait_for_debugpy_client(self): """Block until the debugpy clients have attached to cppdbg/gdb. @@ -301,15 +227,21 @@ def broadcast_parameter(self, value_broadcast: T) -> T: err_handler.log_critical(self.config_options, self) raise - def scatter_array(self, geo_meta: GriddedGeoMeta, src_array: np.ndarray, config_options: ConfigOptions): - """Scatter an array based on the input dataset type. + def scatter_array( + self, + geo_meta: GriddedGeoMeta, + src_array: np.ndarray, + config_options: ConfigOptions, + ): + """Scatter an array based on the input dataset type from rank 0 to all other ranks. Generic function for calling scatter functions based on the input dataset type. - :param geo_meta: - :param array_broadcast: + + :param geo_meta: GriddedGeoMeta instance used to determine the extent of the data received. + :param src_array: Data to be shared with other MPI ranks. :param config_options: - :return: + :return: The results of the scattered data filtered to the extent of `geo_meta` """ # Determine which type of input array we have based on the # type of numpy array. @@ -331,9 +263,7 @@ def scatter_array(self, geo_meta: GriddedGeoMeta, src_array: np.ndarray, config_ try: self.comm.Bcast(data_type_buffer, root=0) except Exception as e: - config_options.errMsg = ( - f"Unable to broadcast numpy datatype value from rank 0: {e.__class__.__name__} -- {e}" - ) + config_options.errMsg = f"Unable to broadcast numpy datatype value from rank 0: {e.__class__.__name__} -- {e}" err_handler.log_critical(config_options, self) raise @@ -403,7 +333,9 @@ def scatter_array(self, geo_meta: GriddedGeoMeta, src_array: np.ndarray, config_ try: self.comm.Scatterv([sendbuf, counts, offsets, data_type], recvbuf, root=0) except Exception as e: - config_options.errMsg = f"Failed Scatterv from rank 0: {e.__class__.__name__} -- {e}" + config_options.errMsg = ( + f"Failed Scatterv from rank 0: {e.__class__.__name__} -- {e}" + ) err_handler.log_critical(config_options, self) raise @@ -416,11 +348,18 @@ def scatter_array(self, geo_meta: GriddedGeoMeta, src_array: np.ndarray, config_ ).copy() return subarray - def merge_slabs_gatherv(self, local_slab: np.ndarray, options: ConfigOptions, allgather: bool = False): - """If allgather is True, then Allgatherv will be used instead of Gatherv, which causes all ranks to be distributed to all other ranks. + def merge_slabs_gatherv( + self, local_slab: np.ndarray, options: ConfigOptions, allgather: bool = False + ) -> np.ndarray: + """Gather arrays from all processes. The returned array will have the gathered data if `self.rank == 0` or `allgather` is `True`. - This is necessary for the hydrofabric case, to handle how ngen's hydrologic + The use of `allgather` is necessary for the hydrofabric case, to handle how ngen's hydrologic catchment partitionining differs from ESMF's arbitrary partitioning. + + :param local_slab: Data that will be gathered from all processes. + :param options: + :param allgather: Boolean on whether the gathered array should be broadcasted to all processes instead of just rank 0. + :return: Numpy array of the data gathered from all processes. """ # Filter based on dimensionality of array if len(local_slab.shape) == 2: From f817a33a36efcbfe9f444d47ee54d6d8b3ff662b Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 14 Jul 2026 15:21:16 -0400 Subject: [PATCH 024/132] Always use default ConfigOptions for messaging --- .../NextGen_Forcings_Engine/core/ioMod.py | 4 ---- .../NextGen_Forcings_Engine/core/parallel.py | 15 +++++++-------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/ioMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/ioMod.py index 75bce435..96bf60b6 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/ioMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/ioMod.py @@ -990,12 +990,10 @@ def gather_global_outputs(self, ConfigOptions, geoMetaWrfHydro, MpiConfig): self.output_local[ output_variable_attribute_dict[varTmp][0], :, : ], - ConfigOptions, ) elif ConfigOptions.grid_type == "hydrofabric": dataOutTmp = MpiConfig.merge_slabs_gatherv( self.output_local[output_variable_attribute_dict[varTmp][0], :], - ConfigOptions, allgather=True, ) # NOTE this assumes that the var order here matches var order elsewhere. @@ -1006,14 +1004,12 @@ def gather_global_outputs(self, ConfigOptions, geoMetaWrfHydro, MpiConfig): self.output_local_elem[ output_variable_attribute_dict[varTmp][0], : ], - ConfigOptions, ) else: dataOutTmp = MpiConfig.merge_slabs_gatherv( self.output_local[ output_variable_attribute_dict[varTmp][0], : ], - ConfigOptions, ) else: raise ValueError(f"Invalid grid_type: {ConfigOptions.grid_type}") diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index ee0032d4..fa7d7c44 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -349,7 +349,7 @@ def scatter_array( return subarray def merge_slabs_gatherv( - self, local_slab: np.ndarray, options: ConfigOptions, allgather: bool = False + self, local_slab: np.ndarray, allgather: bool = False ) -> np.ndarray: """Gather arrays from all processes. The returned array will have the gathered data if `self.rank == 0` or `allgather` is `True`. @@ -357,7 +357,6 @@ def merge_slabs_gatherv( catchment partitionining differs from ESMF's arbitrary partitioning. :param local_slab: Data that will be gathered from all processes. - :param options: :param allgather: Boolean on whether the gathered array should be broadcasted to all processes instead of just rank 0. :return: Numpy array of the data gathered from all processes. """ @@ -376,8 +375,8 @@ def merge_slabs_gatherv( try: self.comm.Allgather([shapes, MPI.INTEGER], [global_shapes, MPI.INTEGER]) except Exception: - options.errMsg = "Failed all gathering slab shapes at rank" + str(self.rank) - err_handler.log_critical(options, self) + self.config_options.errMsg = "Failed all gathering slab shapes at rank" + str(self.rank) + err_handler.log_critical(self.config_options, self) return None if len(local_slab.shape) == 2: @@ -387,11 +386,11 @@ def merge_slabs_gatherv( for i in range(0, self.size): total_rows += global_shapes[2 * i] if global_shapes[(2 * i) + 1] != width: - options.errMsg = ( + self.config_options.errMsg = ( "Error: slabs with differing widths detected on slab for rank" + str(i) ) - err_handler.log_critical(options, self) + err_handler.log_critical(self.config_options, self) return None # generate counts @@ -447,8 +446,8 @@ def merge_slabs_gatherv( root=0, ) except Exception: - options.errMsg = "Failed to Gatherv to rank 0 from rank " + str(self.rank) - err_handler.log_critical(options, self) + self.config_options.errMsg = "Failed to Gatherv to rank 0 from rank " + str(self.rank) + err_handler.log_critical(self.config_options, self) return None return recvbuf From 283c24f22a4ff00450226ba7ea707ac6cd379db0 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 3 Aug 2026 11:46:16 -0400 Subject: [PATCH 025/132] Intercomm/Intracomm mixup --- .../NextGen_Forcings_Engine/core/parallel.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index fa7d7c44..b0c6b4ec 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -2,6 +2,7 @@ from functools import partial import os import typing +from typing import TypeVar import mpi4py import numpy as np @@ -19,11 +20,10 @@ mpi4py.rc.finalize = False if typing.TYPE_CHECKING: - from typing import TypeVar from .geoMod import GriddedGeoMeta from .config import ConfigOptions - T = TypeVar("T") +_T = TypeVar("_T") class MpiConfig: @@ -34,7 +34,7 @@ class MpiConfig: handle from mpi4py. """ - comm: MPI.Intercomm + comm: MPI.Intracomm rank: int """MPI rank of the process of this instance.""" size: int @@ -58,7 +58,7 @@ def __init__(self, config_options: ConfigOptions): self.log_info = partial(err_handler.log_msg, self.config_options, self, False) self.log_warning = partial(err_handler.log_warning, self.config_options, self) - def initialize_comm(self, comm: MPI.Intercomm | None = None) -> None: + def initialize_comm(self, comm: MPI.Intracomm | None = None) -> None: """Initialize MPI communication, including getting MPI rank and size. Also generates the UID for the run. @@ -210,7 +210,7 @@ def wait_for_debugpy_client(self): debugpy.listen(("localhost", 5678 + self.rank)) debugpy.wait_for_client() - def broadcast_parameter(self, value_broadcast: T) -> T: + def broadcast_parameter(self, value_broadcast: _T) -> _T: """Broadcast a single parameter value to all processors. Generic function for sending a parameter value out to the processors. From 303377997097f5a4c09de744d0d6d1caf1b3324c Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 6 Aug 2026 11:14:30 -0400 Subject: [PATCH 026/132] Reintroduce atexit cleanup for signal handling --- .../NextGen_Forcings_Engine/bmi_model.py | 3 - .../NextGen_Forcings_Engine/core/parallel.py | 64 +++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 1afafb9a..90bae785 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -584,9 +584,6 @@ def finalize(self): # process exits gc.collect() # make sure objects are deleted from memory LOG.info(Pld(St.COMPLETE, msg="Finishing BMI finalize()", modnm=MODNM)) - mpi_meta: MpiConfig = getattr(self, "_mpi_meta", None) - if mpi_meta is not None: - mpi_meta.cleanup() # ------------------------------------------------------------------- # ------------------------------------------------------------------- diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index b0c6b4ec..213585c8 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -1,6 +1,9 @@ from __future__ import annotations +import atexit from functools import partial import os +import signal +import sys import typing from typing import TypeVar import mpi4py @@ -57,6 +60,7 @@ def __init__(self, config_options: ConfigOptions): self.log_debug = partial(err_handler.log_msg, self.config_options, self, True) self.log_info = partial(err_handler.log_msg, self.config_options, self, False) self.log_warning = partial(err_handler.log_warning, self.config_options, self) + self.__register_exit_handlers() def initialize_comm(self, comm: MPI.Intracomm | None = None) -> None: """Initialize MPI communication, including getting MPI rank and size. @@ -451,3 +455,63 @@ def merge_slabs_gatherv( return None return recvbuf + + def __register_exit_handlers(self) -> None: + """Register exit handlers for unhandled exceptions, signals, and regular exits. + TODO: consider WCOSS gating. + TODO: note that when non-0 ranks call Abort directly, the rank 0 exit handler is still invoked, at least in some cases, + so there may be opportunities to streamline this further to have only rank 0 perform the cleanup. Would need to test + against potential deadlock conditions to be sure (would need to confirm that a non-0 rank initiating an abort would cause + rank 0 break out of a collective call if it happens to be waiting at one).""" + # Exceptions + sys.excepthook = self.__excepthook + # Regular exits + atexit.register(self.cleanup) + # Signals + for sig in self.__signals_handled(): + signal.signal(sig, self.__signal_handler) + + def __excepthook(self, ex_type, value, tb) -> None: + """Custom excepthook which follows these steps: + 1. Call Python's built-in excepthook. + 2. Log .errMsg as CRITICAL (unless it is None). + 3. Cleanup. + 4. MPI Abort. + + To apply, set `sys.excepthook` to this method.""" + sys.__excepthook__(ex_type, value, tb) + if self.config_options.errMsg is not None: + err_handler.log_critical( + self.config_options, + self, + msg=f"In excepthook, found errMsg = {repr(self.config_options.errMsg)}", + ) + self.abort_with_cleanup(1) + + def __signal_handler(self, signum, frame) -> None: + """Handle termination signals by cleaning up before exit.""" + ### Unregister the signal handler + for s in self.__signals_handled(): + signal.signal(s, signal.SIG_DFL) + ### Cleanup and re-send the original signal to itself + # self._cleanup() + # os.kill(os.getpid(), signum) + ### Cleanup and abort directly + self.abort_with_cleanup(signum) + + def __signals_handled(self) -> tuple[int]: + """Return a tuple of signals to be handled by cleanup routine.""" + ### signal.valid_signals() contains many that are unrelated to stoppage / interruption / error. + # sigs = [s for s in signal.valid_signals() if s not in (signal.SIGKILL, signal.SIGSTOP)] + sigs = ( + signal.SIGINT, + signal.SIGTERM, + signal.SIGHUP, + signal.SIGQUIT, + signal.SIGSEGV, + signal.SIGABRT, + signal.SIGFPE, + signal.SIGBUS, + signal.SIGILL, + ) + return sigs From ef17cb491c024e5c83a9f7b88625ad64786f05d0 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 7 Aug 2026 14:04:16 -0400 Subject: [PATCH 027/132] Update a commented-out block --- .../NextGen_Forcings_Engine/core/parallel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index 213585c8..742e4511 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -494,7 +494,7 @@ def __signal_handler(self, signum, frame) -> None: for s in self.__signals_handled(): signal.signal(s, signal.SIG_DFL) ### Cleanup and re-send the original signal to itself - # self._cleanup() + # self.cleanup() # os.kill(os.getpid(), signum) ### Cleanup and abort directly self.abort_with_cleanup(signum) From 4da06a7028c6ae0e1733c234068c8ea054803ea8 Mon Sep 17 00:00:00 2001 From: Matthew Deshotel Date: Wed, 15 Jul 2026 17:32:14 -0500 Subject: [PATCH 028/132] add type hints; rename vars --- .../core/forecastMod.py | 368 +++++++++--------- 1 file changed, 194 insertions(+), 174 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py index ffa03745..1f7d8af9 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py @@ -1,11 +1,56 @@ +from __future__ import annotations + import datetime import os - -from . import bias_correction, disaggregateMod, downscale, err_handler, layeringMod +from typing import TYPE_CHECKING + +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.bias_correction import ( + run_bias_correction, +) +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.disaggregateMod import ( + disaggregate_factory, +) +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.downscale import ( + run_downscaling, +) +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.err_handler import ( + check_forcing_bounds, + check_program_status, + check_supp_pcp_bounds, + err_out_screen_para, + log_critical, + log_msg, +) +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.layeringMod import ( + layer_final_forcings, + layer_supplemental_forcing, +) + +if TYPE_CHECKING: + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( + ConfigOptions, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.forcingInputMod import ( + InputForcings, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.geoMod import ( + GeoMeta, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import ( + MPIConfig, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.suppPrecipMod import ( + supplemental_precip, + ) def process_forecasts( - ConfigOptions, wrfHydroGeoMeta, inputForcingMod, suppPcpMod, MpiConfig, OutputObj + config_options: ConfigOptions, + geo_meta: GeoMeta, + input_forcing: InputForcings, + supp_precip: supplemental_precip, + mpi_config: MPIConfig, + output_obj, ): """Process forecasts. @@ -34,72 +79,62 @@ def process_forecasts( # checked upon the beginning of this program to see if we # need to process any files. - disaggregate_fun = disaggregateMod.disaggregate_factory(ConfigOptions) + disaggregate_fun = disaggregate_factory(config_options) - for fcstCycleNum in range(ConfigOptions.nFcsts): - ConfigOptions.current_fcst_cycle = ( - ConfigOptions.b_date_proc - + datetime.timedelta(seconds=ConfigOptions.fcst_freq * 60 * fcstCycleNum) + for fcst_cycle_num in range(config_options.nFcsts): + config_options.current_fcst_cycle = ( + config_options.b_date_proc + + datetime.timedelta(seconds=config_options.fcst_freq * 60 * fcst_cycle_num) ) - if ConfigOptions.first_fcst_cycle is None: - ConfigOptions.first_fcst_cycle = ConfigOptions.current_fcst_cycle + if config_options.first_fcst_cycle is None: + config_options.first_fcst_cycle = config_options.current_fcst_cycle - if ConfigOptions.ana_flag: - fcstCycleOutDir = ( - f"{ConfigOptions.output_dir}/{ConfigOptions.e_date_proc.strftime('%Y%m%d%H')}" - ) + if config_options.ana_flag: + fcst_cycle_out_dir = f"{config_options.output_dir}/{config_options.e_date_proc.strftime('%Y%m%d%H')}" else: - fcstCycleOutDir = ( - f"{ConfigOptions.output_dir}/{ConfigOptions.current_fcst_cycle.strftime('%Y%m%d%H')}" - ) + fcst_cycle_out_dir = f"{config_options.output_dir}/{config_options.current_fcst_cycle.strftime('%Y%m%d%H')}" # reset skips if present - for forceKey in ConfigOptions.input_forcings: - inputForcingMod[forceKey].skip = False + for force_key in config_options.input_forcings: + input_forcing[force_key].skip = False # put all AnA output in the same directory - if ConfigOptions.ana_flag: - if ConfigOptions.ana_out_dir is None: - ConfigOptions.ana_out_dir = fcstCycleOutDir - fcstCycleOutDir = ConfigOptions.ana_out_dir - - # completeFlag = ConfigOptions.scratch_dir + "/WrfHydroForcing.COMPLETE" - completeFlag = f"{fcstCycleOutDir}/WrfHydroForcing.COMPLETE" - if os.path.isfile(completeFlag): - ConfigOptions.statusMsg = ( - f"Forecast Cycle: {ConfigOptions.current_fcst_cycle.strftime('%Y-%m-%d %H:%M')} has already completed." - ) - err_handler.log_msg(ConfigOptions, MpiConfig) + if config_options.ana_flag: + if config_options.ana_out_dir is None: + config_options.ana_out_dir = fcst_cycle_out_dir + fcst_cycle_out_dir = config_options.ana_out_dir + + # completeFlag = config_options.scratch_dir + "/WrfHydroForcing.COMPLETE" + complete_flag = f"{fcst_cycle_out_dir}/WrfHydroForcing.COMPLETE" + if os.path.isfile(complete_flag): + config_options.statusMsg = f"Forecast Cycle: {config_options.current_fcst_cycle.strftime('%Y-%m-%d %H:%M')} has already completed." + log_msg(config_options, mpi_config) # We have already completed processing this cycle, # move on. continue - if not ConfigOptions.ana_flag: - if MpiConfig.rank == 0: + if not config_options.ana_flag: + if mpi_config.rank == 0: # If the cycle directory doesn't exist, create it. - if not os.path.isdir(fcstCycleOutDir): + if not os.path.isdir(fcst_cycle_out_dir): try: - os.mkdir(fcstCycleOutDir) + os.mkdir(fcst_cycle_out_dir) except Exception: - ConfigOptions.errMsg = ( - f"Unable to create output directory: {fcstCycleOutDir}" + config_options.errMsg = ( + f"Unable to create output directory: {fcst_cycle_out_dir}" ) - err_handler.err_out_screen_para(ConfigOptions.errMsg, MpiConfig) - err_handler.check_program_status(ConfigOptions, MpiConfig) + err_out_screen_para(config_options.errMsg, mpi_config) + check_program_status(config_options, mpi_config) # Log information about this forecast cycle - if MpiConfig.rank == 0: - ConfigOptions.statusMsg = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" - err_handler.log_msg(ConfigOptions, MpiConfig) - ConfigOptions.statusMsg = ( - f"Processing Forecast Cycle: {ConfigOptions.current_fcst_cycle.strftime('%Y-%m-%d %H:%M')}" - ) - err_handler.log_msg(ConfigOptions, MpiConfig) - ConfigOptions.statusMsg = ( - f"Forecast Cycle Length is: {ConfigOptions.cycle_length_minutes!s} minutes" - ) - err_handler.log_msg(ConfigOptions, MpiConfig) - # MpiConfig.comm.barrier() + if mpi_config.rank == 0: + config_options.statusMsg = "X" * 38 + log_msg(config_options, mpi_config) + config_options.statusMsg = f"Processing Forecast Cycle: {config_options.current_fcst_cycle.strftime('%Y-%m-%d %H:%M')}" + log_msg(config_options, mpi_config) + config_options.statusMsg = f"Forecast Cycle Length is: {config_options.cycle_length_minutes!s} minutes" + log_msg(config_options, mpi_config) + # mpi_config.comm.barrier() # Loop through each output timestep. Perform the following functions: # 1.) Calculate all necessary input files per user options. @@ -107,69 +142,63 @@ def process_forecasts( # 3.) Regrid the forcings, and temporally interpolate. # 4.) Downscale. # 5.) Layer, and output as necessary. - ana_factor = 1 if ConfigOptions.ana_flag is False else 0 + ana_factor = 1 if config_options.ana_flag is False else 0 show_message = True - for outStep in range(1, ConfigOptions.num_output_steps + 1): + for out_step in range(1, config_options.num_output_steps + 1): # Reset out final grids to missing values. - OutputObj.output_local[:, :, :] = -9999.0 + output_obj.output_local[:, :, :] = -9999.0 - ConfigOptions.current_output_step = outStep - OutputObj.outDate = ConfigOptions.current_fcst_cycle + datetime.timedelta( - seconds=ConfigOptions.output_freq * 60 * outStep + config_options.current_output_step = out_step + output_obj.outDate = config_options.current_fcst_cycle + datetime.timedelta( + seconds=config_options.output_freq * 60 * out_step ) - ConfigOptions.current_output_date = OutputObj.outDate + config_options.current_output_date = output_obj.outDate # if AnA, adjust file date for analysis vs forecast - if ConfigOptions.ana_flag: - file_date = OutputObj.outDate - datetime.timedelta( - seconds=ConfigOptions.output_freq * 60 + if config_options.ana_flag: + file_date = output_obj.outDate - datetime.timedelta( + seconds=config_options.output_freq * 60 ) else: - file_date = OutputObj.outDate + file_date = output_obj.outDate # Calculate the previous output timestep. This is used in potential downscaling routines. - if outStep == ana_factor: - ConfigOptions.prev_output_date = ConfigOptions.current_output_date + if out_step == ana_factor: + config_options.prev_output_date = config_options.current_output_date else: - ConfigOptions.prev_output_date = ( - ConfigOptions.current_output_date - - datetime.timedelta(seconds=ConfigOptions.output_freq * 60) - ) - if MpiConfig.rank == 0 and show_message: - ConfigOptions.statusMsg = "=========================================" - err_handler.log_msg(ConfigOptions, MpiConfig, True) - ConfigOptions.statusMsg = ( - f"Processing for output timestep: {file_date.strftime('%Y-%m-%d %H:%M')}" + config_options.prev_output_date = ( + config_options.current_output_date + - datetime.timedelta(seconds=config_options.output_freq * 60) ) - err_handler.log_msg(ConfigOptions, MpiConfig, True) - # MpiConfig.comm.barrier() + if mpi_config.rank == 0 and show_message: + config_options.statusMsg = "=========================================" + log_msg(config_options, mpi_config, True) + config_options.statusMsg = f"Processing for output timestep: {file_date.strftime('%Y-%m-%d %H:%M')}" + log_msg(config_options, mpi_config, True) + # mpi_config.comm.barrier() # Compose the expected path to the output file. Check to see if the file exists, # if so, continue to the next time step. Also initialize our output arrays if necessary. - OutputObj.outPath = ( - f"{fcstCycleOutDir}/{file_date.strftime('%Y%m%d%H%M')}.LDASIN_DOMAIN1" - ) - # MpiConfig.comm.barrier() - - if os.path.isfile(OutputObj.outPath): - if MpiConfig.rank == 0: - ConfigOptions.statusMsg = ( - f"Output file: {OutputObj.outPath} exists. Moving to the next output timestep." - ) - err_handler.log_msg(ConfigOptions, MpiConfig) - err_handler.check_program_status(ConfigOptions, MpiConfig) + output_obj.outPath = f"{fcst_cycle_out_dir}/{file_date.strftime('%Y%m%d%H%M')}.LDASIN_DOMAIN1" + # mpi_config.comm.barrier() + + if os.path.isfile(output_obj.outPath): + if mpi_config.rank == 0: + config_options.statusMsg = f"Output file: {output_obj.outPath} exists. Moving to the next output timestep." + log_msg(config_options, mpi_config) + check_program_status(config_options, mpi_config) continue else: - ConfigOptions.currentForceNum = 0 - ConfigOptions.currentCustomForceNum = 0 + config_options.currentForceNum = 0 + config_options.currentCustomForceNum = 0 # Loop over each of the input forcings specifed. - for forceKey in ConfigOptions.input_forcings: - input_forcings = inputForcingMod[forceKey] + for force_key in config_options.input_forcings: + input_forcings = input_forcing[force_key] # Calculate the previous and next input cycle files from the inputs. input_forcings.calc_neighbor_files( - ConfigOptions, OutputObj.outDate, MpiConfig + config_options, output_obj.outDate, mpi_config ) - err_handler.check_program_status(ConfigOptions, MpiConfig) + check_program_status(config_options, mpi_config) # break loop if done early if input_forcings.skip is True: @@ -177,16 +206,12 @@ def process_forecasts( break # Regrid forcings. - input_forcings.regrid_inputs( - ConfigOptions, wrfHydroGeoMeta, MpiConfig - ) - err_handler.check_program_status(ConfigOptions, MpiConfig) + input_forcings.regrid_inputs(config_options, geo_meta, mpi_config) + check_program_status(config_options, mpi_config) # Run check on regridded fields for reasonable values that are not missing values. - err_handler.check_forcing_bounds( - ConfigOptions, input_forcings, MpiConfig - ) - err_handler.check_program_status(ConfigOptions, MpiConfig) + check_forcing_bounds(config_options, input_forcings, mpi_config) + check_program_status(config_options, mpi_config) # If we are restarting a forecast cycle, re-calculate the neighboring files, and regrid the # next set of forcings as the previous step just regridded the previous forcing. @@ -202,131 +227,126 @@ def process_forecasts( # Re-calculate the neighbor files. input_forcings.calc_neighbor_files( - ConfigOptions, OutputObj.outDate, MpiConfig + config_options, output_obj.outDate, mpi_config ) - err_handler.check_program_status(ConfigOptions, MpiConfig) + check_program_status(config_options, mpi_config) # Regrid the forcings for the end of the window. input_forcings.regrid_inputs( - ConfigOptions, wrfHydroGeoMeta, MpiConfig + config_options, geo_meta, mpi_config ) - err_handler.check_program_status(ConfigOptions, MpiConfig) + check_program_status(config_options, mpi_config) input_forcings.rstFlag = 0 # Run temporal interpolation on the grids. - input_forcings.temporal_interpolate_inputs(ConfigOptions, MpiConfig) - err_handler.check_program_status(ConfigOptions, MpiConfig) + input_forcings.temporal_interpolate_inputs( + config_options, mpi_config + ) + check_program_status(config_options, mpi_config) # Run bias correction. - bias_correction.run_bias_correction( - input_forcings, ConfigOptions, wrfHydroGeoMeta, MpiConfig + run_bias_correction( + input_forcings, config_options, geo_meta, mpi_config ) - err_handler.check_program_status(ConfigOptions, MpiConfig) + check_program_status(config_options, mpi_config) # Run downscaling on grids for this output timestep. - downscale.run_downscaling( - input_forcings, ConfigOptions, wrfHydroGeoMeta, MpiConfig + run_downscaling( + input_forcings, config_options, geo_meta, mpi_config ) - err_handler.check_program_status(ConfigOptions, MpiConfig) + check_program_status(config_options, mpi_config) # Layer in forcings from this product. - layeringMod.layer_final_forcings( - OutputObj, input_forcings, ConfigOptions, MpiConfig + layer_final_forcings( + output_obj, input_forcings, config_options, mpi_config ) - err_handler.check_program_status(ConfigOptions, MpiConfig) + check_program_status(config_options, mpi_config) - ConfigOptions.currentForceNum = ConfigOptions.currentForceNum + 1 + config_options.currentForceNum = config_options.currentForceNum + 1 - if forceKey == 10: - ConfigOptions.currentCustomForceNum = ( - ConfigOptions.currentCustomForceNum + 1 + if force_key == 10: + config_options.currentCustomForceNum = ( + config_options.currentCustomForceNum + 1 ) else: # Process supplemental precipitation if we specified in the configuration file. - if ConfigOptions.number_supp_pcp > 0: - for suppPcpKey in ConfigOptions.supp_precip_forcings: + if config_options.number_supp_pcp > 0: + for supp_pcp_key in config_options.supp_precip_forcings: # Like with input forcings, calculate the neighboring files to use. - suppPcpMod[suppPcpKey].calc_neighbor_files( - ConfigOptions, OutputObj.outDate, MpiConfig + supp_precip[supp_pcp_key].calc_neighbor_files( + config_options, output_obj.outDate, mpi_config ) - err_handler.check_program_status(ConfigOptions, MpiConfig) + check_program_status(config_options, mpi_config) # Regrid the supplemental precipitation. - suppPcpMod[suppPcpKey].regrid_inputs( - ConfigOptions, wrfHydroGeoMeta, MpiConfig + supp_precip[supp_pcp_key].regrid_inputs( + config_options, geo_meta, mpi_config ) - err_handler.check_program_status(ConfigOptions, MpiConfig) + check_program_status(config_options, mpi_config) if ( - suppPcpMod[suppPcpKey].regridded_precip1 is not None - and suppPcpMod[suppPcpKey].regridded_precip2 is not None + supp_precip[supp_pcp_key].regridded_precip1 is not None + and supp_precip[supp_pcp_key].regridded_precip2 + is not None ): - # if np.any(suppPcpMod[suppPcpKey].regridded_precip1) and \ - # np.any(suppPcpMod[suppPcpKey].regridded_precip2): + # if np.any(supp_precip[supp_pcp_key].regridded_precip1) and \ + # np.any(supp_precip[supp_pcp_key].regridded_precip2): # Run check on regridded fields for reasonable values that are not missing values. - err_handler.check_supp_pcp_bounds( - ConfigOptions, suppPcpMod[suppPcpKey], MpiConfig - ) - err_handler.check_program_status( - ConfigOptions, MpiConfig + check_supp_pcp_bounds( + config_options, + supp_precip[supp_pcp_key], + mpi_config, ) + check_program_status(config_options, mpi_config) disaggregate_fun( input_forcings, - suppPcpMod[suppPcpKey], - ConfigOptions, - MpiConfig, - ) - err_handler.check_program_status( - ConfigOptions, MpiConfig + supp_precip[supp_pcp_key], + config_options, + mpi_config, ) + check_program_status(config_options, mpi_config) # Run temporal interpolation on the grids. - suppPcpMod[suppPcpKey].temporal_interpolate_inputs( - ConfigOptions, MpiConfig - ) - err_handler.check_program_status( - ConfigOptions, MpiConfig + supp_precip[supp_pcp_key].temporal_interpolate_inputs( + config_options, mpi_config ) + check_program_status(config_options, mpi_config) # Layer in the supplemental precipitation into the current output object. - layeringMod.layer_supplemental_forcing( - OutputObj, - suppPcpMod[suppPcpKey], - ConfigOptions, - MpiConfig, - ) - err_handler.check_program_status( - ConfigOptions, MpiConfig + layer_supplemental_forcing( + output_obj, + supp_precip[supp_pcp_key], + config_options, + mpi_config, ) + check_program_status(config_options, mpi_config) # Call the output routines # adjust date for AnA if necessary - if ConfigOptions.ana_flag: - OutputObj.outDate = file_date + if config_options.ana_flag: + output_obj.outDate = file_date - OutputObj.output_final_ldasin( - ConfigOptions, wrfHydroGeoMeta, MpiConfig - ) - err_handler.check_program_status(ConfigOptions, MpiConfig) + output_obj.output_final_ldasin(config_options, geo_meta, mpi_config) + check_program_status(config_options, mpi_config) - if (not ConfigOptions.ana_flag) or (fcstCycleNum == (ConfigOptions.nFcsts - 1)): - if MpiConfig.rank == 0: - ConfigOptions.statusMsg = ( - f"Forcings complete for forecast cycle: {ConfigOptions.current_fcst_cycle.strftime('%Y-%m-%d %H:%M')}" - ) - err_handler.log_msg(ConfigOptions, MpiConfig) - err_handler.check_program_status(ConfigOptions, MpiConfig) + if (not config_options.ana_flag) or ( + fcst_cycle_num == (config_options.nFcsts - 1) + ): + if mpi_config.rank == 0: + config_options.statusMsg = f"Forcings complete for forecast cycle: {config_options.current_fcst_cycle.strftime('%Y-%m-%d %H:%M')}" + log_msg(config_options, mpi_config) + check_program_status(config_options, mpi_config) # Success.... Now touch an empty complete file for this forecast cycle to indicate # completion in case the code is re-ran. try: - open(completeFlag, "a").close() + open(complete_flag, "a").close() except Exception: - ConfigOptions.errMsg = ( - f"Unable to create completion file: {completeFlag}" + config_options.errMsg = ( + f"Unable to create completion file: {complete_flag}" ) - err_handler.log_critical(ConfigOptions, MpiConfig) - err_handler.check_program_status(ConfigOptions, MpiConfig) + log_critical(config_options, mpi_config) + check_program_status(config_options, mpi_config) From 90616b3b5e9ccc4d755311a5c2acecdc7c9b56e0 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 11 Aug 2026 15:29:56 -0400 Subject: [PATCH 029/132] Deprecate forecastMod.py --- .../NextGen_Forcings_Engine/core/forecastMod.py | 4 ++++ NextGen_Forcings_Engine_BMI/README.md | 2 ++ 2 files changed, 6 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py index 1f7d8af9..8e166c48 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py @@ -1,3 +1,6 @@ + +raise NotImplementedError(f"This file, {__file__}, is deprecated.") + from __future__ import annotations import datetime @@ -78,6 +81,7 @@ def process_forecasts( # 'WrfHydroForcing.COMPLETE' flag in the directory. This will be # checked upon the beginning of this program to see if we # need to process any files. + raise NotImplementedError(f"This file, {__file__}, is deprecated.") disaggregate_fun = disaggregate_factory(config_options) diff --git a/NextGen_Forcings_Engine_BMI/README.md b/NextGen_Forcings_Engine_BMI/README.md index 01226edc..96a0357f 100644 --- a/NextGen_Forcings_Engine_BMI/README.md +++ b/NextGen_Forcings_Engine_BMI/README.md @@ -60,6 +60,8 @@ # Overview bullet points for modifying the original NWM Forcings Engine into a BMI complaint NextGen Forcings Engine capable of handling any domain types • To streamline the NWMv3.0 Forcings engine into a Basic Model Interface application, we’ve had to initialize the BMI model using the same approach highlighted in the “genForcing.py” module and then directly reconfigure the forecast module (forecastMod.py) workflow to streamline the ability to update and produce gridded forcings for the WRFHydro domain based on a specified time stamp within the standard BMI functionality (“model_update_until”). This is all completed within the “model.py” module, which essentially mimics the “forecastMod.py” module within the “core” directory as a BMI-compliant module. + • **2026 Update**: `forecastMod.py` is no longer referenced by the `ngen-forcing` codebase nor the `ngen` stack, and has been deprecated with a `NotImplementedError`. + • Once source code modifications were implemented, we were able to demonstrate the ability for the NextGen Forcings Engine to advertise gridded and unstructured mesh forcings back to the NextGen model engine. • We've optimized the NextGen Forcings Engine source code within the BMI to include I/O functionality for producing netcdf forcing files across any domain configuration (gridded, hydrofabric, unstructured) and also clear out data production within its scratch directory once the BMI execution is complete. From e6f1f1364412358ac108bee4adfe8e3e2d0a20d1 Mon Sep 17 00:00:00 2001 From: Matthew Deshotel Date: Tue, 5 May 2026 16:33:54 -0400 Subject: [PATCH 030/132] move consts to consts.py --- .../NextGen_Forcings_Engine/core/consts.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py index 2b3567dd..b442aecd 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py @@ -1259,3 +1259,117 @@ }, } } + +CONFIGOPTIONS = { + "ConfigOptions": [ + "bmi_time", + "current_time", + "input_forcings", + "supp_precip_forcings", + "input_force_dirs", + "input_force_types", + "supp_precip_dirs", + "supp_precip_file_types", + "supp_precip_param_dir", + "input_force_mandatory", + "supp_precip_mandatory", + "supp_pcp_max_hours", + "number_inputs", + "number_supp_pcp", + "output_freq", + "sub_output_hour", + "sub_output_freq", + "scratch_dir", + "num_output_steps", + "num_supp_output_steps", + "actual_output_steps", + "realtime_flag", + "refcst_flag", + "ana_flag", + "e_date_proc", + "first_fcst_cycle", + "current_fcst_cycle", + "current_output_step", + "cycle_length_minutes", + "prev_output_date", + "current_output_date", + "look_back", + "future_time", + "fcst_freq", + "nFcsts", + "fcst_shift", + "fcst_input_horizons", + "fcst_input_offsets", + "process_window", + "spatial_meta", + "grid_type", + "grid_meta", + "ExactExtract", + "lat_var", + "lon_var", + "hgt_var", + "cosalpha_var", + "sinalpha_var", + "slope_var", + "slope_azimuth_var", + "slope_var_elem", + "slope_azimuth_var_elem", + "nodecoords_var", + "elemcoords_var", + "elemconn_var", + "numelemconn_var", + "element_id_var", + "hgt_elem_var", + "ignored_border_widths", + "regrid_opt", + "weightsDir", + "regrid_opt_supp_pcp", + "errMsg", + "statusMsg", + "logFile", + "logHandle", + "dScaleParamDirs", + "paramFlagArray", + "forceTemoralInterp", + "suppTemporalInterp", + "t2dDownscaleOpt", + "swDownscaleOpt", + "psfcDownscaleOpt", + "precipDownscaleOpt", + "q2dDownscaleOpt", + "t2BiasCorrectOpt", + "psfcBiasCorrectOpt", + "q2BiasCorrectOpt", + "windBiasCorrect", + "swBiasCorrectOpt", + "lwBiasCorrectOpt", + "precipBiasCorrectOpt", + "cfsv2EnsMember", + "customSuppPcpFreq", + "customFcstFreq", + "rqiMethod", + "nwmVersion", + "nwmConfig", + "forcing_output", + "aws", + "aws_obj", + "aws_time", + "nwm_geogrid", + "geopackage", + "uid64", + ], + "var_rename_map": {"config_path": "cfg_bmi"}, + "cfg_bmi_to_attrs_map": { + "SuppPcp": "supp_precip_forcings", + "OutputFrequency": "output_freq", + "SubOutputHour": "sub_output_hour", + "SubOutFreq": "sub_output_freq", + "ScratchDir": "scratch_dir", + "compressOutput": "useCompression", + "AnAFlag": "ana_flag", + "LookBack": "look_back", + "ForecastFrequency": "fcst_freq", + "SpatialMetaIn": "spatial_meta", + }, + "file_types": ["GRIB1", "GRIB2", "NETCDF", "NETCDF4", "NWM", "ZARR"], +} From 1f4288e93cd44228f857df5c6a24a09c8970ce00 Mon Sep 17 00:00:00 2001 From: Matthew Deshotel Date: Tue, 5 May 2026 16:34:18 -0400 Subject: [PATCH 031/132] break intialize method into properties with setters to validate. --- .../NextGen_Forcings_Engine/core/config.py | 3096 +++++++---------- 1 file changed, 1209 insertions(+), 1887 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 17bb9f8b..4c7b3add 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -5,10 +5,16 @@ import re import uuid from datetime import datetime, timedelta, timezone +from functools import cached_property # Use the Error, Warning, and Trapping System Package for logging import numpy as np +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core import mpi_utils +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.consts import ( + CONFIGOPTIONS, + FORCINGINPUTMOD, +) from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.err_handler import ( err_out_screen, ) @@ -16,125 +22,35 @@ calculate_lookback_window, ) -from . import mpi_utils - LOG = logging.getLogger("FORCING") -FORCE_COUNT = 27 class ConfigOptions: """Configuration abstract class for configuration options read in from the file specified by the user.""" - def __init__(self, config: dict, b_date=None, geogrid_arg=None): + def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> None: """Initialize the configuration class to empty None attributes. - param config: The user-specified path to the configuration file. + The attributes of this class are populated by the validate_config function, which reads in the configuration file and checks that all necessary options are provided and properly formatted. The attributes of this class are used to control the flow of the program and the processing of input forcings. + + Args: + cfg_bmi (dict): The configuration dictionary read in from the configuration file specified by the user. This should be read in using the config_utils.read_config function, which also handles any necessary preprocessing of the configuration file. + b_date (str, optional): The beginning date of processing in the format YYYYMMDDHHMM. This is used to calculate the processing window for realtime simulations. If not provided, it will be read from the configuration file. + geogrid (str, optional): The filepath to the geogrid file to be used for processing. This is used to specify the grid information for regridding input forcings. If not provided, it will be read from the configuration file. + """ - self.bmi_time = None - self.current_time = None self.bmi_time_index = 0 - self.input_forcings = None self.precip_only_flag = False - self.supp_precip_forcings = None - self.input_force_dirs = None - self.input_force_types = None - self.supp_precip_dirs = None - self.supp_precip_file_types = None - self.supp_precip_param_dir = None - self.input_force_mandatory = None - self.supp_precip_mandatory = None - self.supp_pcp_max_hours = None - self.number_inputs = None - self.number_supp_pcp = None self.number_custom_inputs = 0 - self.output_freq = None - self.sub_output_hour = None - self.sub_output_freq = None - self.scratch_dir = None self.useCompression = 0 self.useFloats = 0 - self.num_output_steps = None - self.num_supp_output_steps = None - self.actual_output_steps = None - self.realtime_flag = None - self.refcst_flag = None - self.ana_flag = None - self.b_date_proc = b_date - self.e_date_proc = None - self.first_fcst_cycle = None - self.current_fcst_cycle = None - self.current_output_step = None - self.cycle_length_minutes = None - self.prev_output_date = None - self.current_output_date = None - self.look_back = None - self.future_time = None - self.fcst_freq = None - self.nFcsts = None - self.fcst_shift = None - self.fcst_input_horizons = None - self.fcst_input_offsets = None - self.process_window = None - self.spatial_meta = None - self.grid_type = None - self.grid_meta = None - self.ExactExtract = None - self.lat_var = None - self.lon_var = None - self.hgt_var = None - self.cosalpha_var = None - self.sinalpha_var = None - self.slope_var = None - self.slope_azimuth_var = None - self.slope_var_elem = None - self.slope_azimuth_var_elem = None - self.nodecoords_var = None - self.elemcoords_var = None - self.elemconn_var = None - self.numelemconn_var = None - self.element_id_var = None - self.hgt_elem_var = None - self.ignored_border_widths = None - self.regrid_opt = None - self.weightsDir = None - self.regrid_opt_supp_pcp = None - self.config_path = config - self.errMsg = None - self.statusMsg = None - self.logFile = None - self.logHandle = None - self.dScaleParamDirs = None - self.paramFlagArray = None - self.forceTemoralInterp = None - self.suppTemporalInterp = None - self.t2dDownscaleOpt = None - self.swDownscaleOpt = None - self.psfcDownscaleOpt = None - self.precipDownscaleOpt = None - self.q2dDownscaleOpt = None - self.t2BiasCorrectOpt = None - self.psfcBiasCorrectOpt = None - self.q2BiasCorrectOpt = None - self.windBiasCorrect = None - self.swBiasCorrectOpt = None - self.lwBiasCorrectOpt = None - self.precipBiasCorrectOpt = None + self._b_date_proc = b_date + self._cfg_bmi = cfg_bmi self.runCfsNldasBiasCorrect = False - self.cfsv2EnsMember = None - self.customSuppPcpFreq = None - self.customFcstFreq = None - self.rqiMethod = None self.rqiThresh = 1.0 self.globalNdv = -9999.0 self.d_program_init = datetime.now(timezone.utc) self.errFlag = 0 - self.nwmVersion = None - self.nwmConfig = None - self.include_lqfrac = False - self.forcing_output = None - self.aws = None - self.aws_obj = None - self.aws_time = None self.aorc_conus_source = "s3://noaa-nws-aorc-v1-1-1km" self.aorc_conus_year_url = "{source}/{year}.zarr" self.aorc_alaska_source = "s3://ngwpc-data/AORC/Alaska" @@ -142,20 +58,159 @@ def __init__(self, config: dict, b_date=None, geogrid_arg=None): "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4" ) self.nwm_source = "s3://noaa-nwm-retrospective-3-0-pds" - - self.nwm_geogrid = None - self.geogrid = geogrid_arg - self.geopackage = None - - self.uid64 = None + self._geogrid = geogrid self.broadcast_new_64bit_uid() self._scratch_dir_has_been_uniquefied = False + # set list of attibutes from consts.py to None. + # These are indexed from the consts dictionary using the class name + for attr in CONFIGOPTIONS[self.__class__.__name__]: + setattr(self, attr, None) + self._validate_config() + + @property + def cfg_bmi(self) -> dict: + """Return the configuration dictionary read in from the configuration file specified by the user.""" + return self._cfg_bmi + + @cfg_bmi.setter + def cfg_bmi(self, value: dict) -> None: + """Set the configuration dictionary read in from the configuration file specified by the user.""" + if not isinstance(value, dict): + raise TypeError( + f"Expected dict, got {type(value)} for type of cfg_bmi: {value}" + ) + self._validate_config() + self._cfg_bmi = value + + @property + def force_count(self) -> int: + """Calculate the number of total possible input forcing options based on the length of the InputForcings list in the consts.py file. This is used for error checking to ensure users specify valid input forcing options in the configuration file.""" + return len(FORCINGINPUTMOD["InputForcings"]["PRODUCT_NAME"]) + + @property + def supp_precip_count(self) -> int: + """Calculate the number of total possible supplemental precip forcing options based on the length of the SuppPrecipForcings list in the consts.py file. This is used for error checking to ensure users specify valid supplemental precip forcing options in the configuration file.""" + # TODO make this dynamic based on the length of the SUPPPRECIPMOD list in consts.py, but for now hardcoding to 15 since that is the number of options currently available in consts.py and this will avoid any issues with the formatting of the consts.py file causing errors in the program. This is used for error checking to ensure users specify valid supplemental precip forcing options in the configuration file. + # return len(SUPPPRECIPMOD["suppPrecipMod"]["PRODUCT_NAMES"]) + return 15 + + @property + def number_supp_pcp(self) -> int: + """Calculate the number of supplemental precip forcings specified by the user in the configuration file.""" + return len(self.supp_precip_forcings) + + @property + def precip_only_flag(self) -> bool: + """Flag to indicate whether the user has chosen to run the supplemental precip forcings module only, which will trigger some different processing pathways and error checking for certain configuration options.""" + if self.number_supp_pcp == 1: + if int(self.supp_precip_forcings[0]) == 14: + return True + + def set_attrs(self): + """Set the attributes of the class based on the configuration file. This is used to populate the attributes of the class after they have been read in and validated from the configuration file.""" + for cfg_bmi_attr, config_options_attr in CONFIGOPTIONS[ + "cfg_bmi_to_attrs_map" + ].items(): + setattr( + self, config_options_attr, self.extract_input_variable(cfg_bmi_attr) + ) + + if self.output_freq <= 0: + err_out_screen( + "Please specify an OutputFrequency that is greater than zero minutes." + ) + + def extract_input_variable(self, variable_name: str) -> str: + """Extract the variable name from the configuration file for a given variable.""" + try: + return self.cfg_bmi[variable_name] + except ValueError as e: + err_out_screen( + f"Improper {variable_name} value specified in the configuration file. Error: {e}" + ) + except (KeyError, configparser.NoOptionError) as e: + err_out_screen( + f"Unable to locate {variable_name} in the configuration file. Error: {e}" + ) + except json.decoder.JSONDecodeError as e: + err_out_screen( + f"Improper {variable_name} file option specified in configuration file. Error: {e}", + e, + ) + + def extract_input_variable_set_default(self, variable_name: str, default=0) -> str: + """Extract the variable name from the configuration file for a given variable, and set it to a default value if it is not found.""" + try: + variable = self.cfg_bmi[variable_name] + except (KeyError, configparser.NoOptionError) as e: + variable = default + except ValueError as e: + err_out_screen( + f"Improper {variable_name} value: {self.cfg_bmi[variable_name]}", e + ) + if variable not in [0, 1]: + err_out_screen(f"Please choose a {variable_name} value of 0 or 1.") + return variable + + def try_config_get(self, variable_name: str, default=None) -> str: + """Try to get a variable from the configuration file, and return a default value if it is not found.""" + try: + var = self.cfg_bmi.get(variable_name, default) + if var is None: + err_out_screen( + f"Unable to locate {variable_name} in the configuration file." + ) + return var + except (KeyError, configparser.NoOptionError) as e: + err_out_screen( + f"Unable to locate {variable_name} in the configuration file.", e + ) + + def check_number_of_inputs( + self, value: list, variable_name: str, input_type: str + ) -> None: + """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable.""" + if len(value) != self.number_inputs: + err_out_screen( + f"Number of {variable_name} values must match the number of {input_type} in the configuration file." + ) + + def check_number_of_inputs_forcings(self, value: list, variable_name: str) -> None: + """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable, specifically for input forcings variables which should match the number of input forcing options specified by the user in the configuration file.""" + return self.check_number_of_inputs(value, variable_name, " InputForcings") + + def check_number_of_inputs_supp_pcp(self, value: list, variable_name: str) -> None: + """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable, specifically for supplemental precip forcing variables which should match the number of supplemental precip forcing options specified by the user in the configuration file.""" + return self.check_number_of_inputs( + value, variable_name, " supplemental precip forcings" + ) + + def check_input_values_in_range( + self, value: list, variable_name: str, valid_input_options: list + ) -> None: + """Check that the input values specified by the user in the configuration file are within a valid range for a given variable.""" + for val in value: + if val in valid_input_options: + err_out_screen( + f"Invalid {variable_name} value '{val}' specified in configuration file. Please specify valid values: {valid_input_options}." + ) + + def check_input_values_positive(self, value: list, variable_name: str) -> None: + """Check that the input values specified by the user in the configuration file are positive for a given variable.""" + for val in value: + if val <= 0: + err_out_screen( + f"Invalid {variable_name} value '{val}' specified in configuration file. Please specify values greater than zero." + ) + def uniquefy_scratch_dir_as_child(self, uid: str) -> None: """Modify the existing scratch dir by adding the UID string available to all ranks from the MpiConfig class. + This may only be called once. Subsequent calls will result in an error. - This must be called by all ranks, once.""" + This must be called by all ranks, once. + """ LOG.debug(f"Uniquefying scratch dir: adding suffix {uid} to {self.scratch_dir}") if not isinstance(uid, str): raise TypeError(f"Expected str, got {type(uid)} for type of uid: {uid}") @@ -174,56 +229,88 @@ def make_scratch_dir(self) -> None: os.makedirs(self.scratch_dir, exist_ok=True) LOG.debug(f"Scratch dir: {self.scratch_dir}") - def broadcast_new_64bit_uid(self): + def broadcast_new_64bit_uid(self) -> None: """Broadcast a random uint64 then save the hash of that to self.uid64, which effectively broadcasts the same unique string to all ranks. - Should be called once to avoid confusion.""" + + Should be called once to avoid confusion. + """ if self.uid64 is not None: raise RuntimeError("self.uid64 has already been initialized.") self.uid64 = mpi_utils.get_new_broadcasted_uid() - def validate_config(self, cfg_bmi: dict) -> None: - """Validate in options from the configuration file and check that proper options were provided.""" - # Ensure b_date_proc is set; if not, read from the configuration file - if self.b_date_proc is None: + @property + def b_date_proc(self) -> str: + """Get the beginning date of processing for reforecast simulations. This is used to calculate the processing window for reforecast simulations, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" + return self._bdate_proc + + @b_date_proc.setter + def b_date_proc(self, value: str | datetime) -> None: + """Set the beginning date of processing for reforecast simulations. This is used to calculate the processing window for reforecast simulations, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" + if value is None: + value = self.try_config_get("RefcstBDateProc") + if isinstance(value, datetime): + self._b_date_proc = value + if value != -9999: + if isinstance(value, str) and len(value) != 12: + err_out_screen( + "Improper RefcstBDateProc length entered into the configuration file. Please check your entry." + ) try: - self.b_date_proc = cfg_bmi.get( - "RefcstBDateProc", None - ) # Default to None if not found - if self.b_date_proc is None: - err_out_screen( - "Unable to locate RefcstBDateProc under Logistics section in configuration file." - ) - except KeyError as e: + self._b_date_proc = datetime.strptime(value, "%Y%m%d%H%M") + except ValueError as e: err_out_screen( - "Unable to locate RefcstBDateProc under Logistics section in configuration file.", + "Improper RefcstBDateProc value entered into the configuration file. Please check your entry.", e, ) + else: + self._b_date_proc = -9999 + LOG.info(f"Begin date: {value}") - # Ensure geopackage is set; if not, read from the configuration file - if self.geopackage is None: - try: - self.geopackage = cfg_bmi.get( - "Geopackage", None - ) # Default to None if not found - if self.geopackage is None: - err_out_screen( - "Unable to locate Geopackage in the configuration file." - ) - except KeyError as e: - err_out_screen( - "Unable to locate Geopackage in the configuration file.", e - ) + @property + def realtime_flag(self) -> bool: + """Flag to indicate whether the user has chosen to run a realtime simulation, which will trigger some different processing pathways and error checking for certain configuration options, and will also control how the processing window is calculated.""" + if self.look_back == -9999: + return False + elif self.b_date_proc == -9999: + return True + else: + return False - # Ensure geogrid is set; if not, read from the configuration file - if self.geogrid is None: - try: - geogrid_base = cfg_bmi.get( - "GeogridIn", None - ) # Default to None if not found - except KeyError as e: - err_out_screen( - "Unable to locate GeogridIn in the configuration file.", e - ) + @property + def refcst_flag(self) -> bool: + """Flag to indicate whether the user has chosen to run a reforecast simulation, which will trigger some different processing pathways and error checking for certain configuration options, and will also control how the processing window is calculated.""" + if self.look_back == -9999: + return True + elif self.b_date_proc == -9999: + return True + else: + return False + + @property + def geopackage(self) -> str: + """Get the pathway to the geopackage file to be used for processing. This is used to specify the grid information for regridding input forcings, and is only necessary if the user is running a simulation that requires regridding of input forcings.""" + return self._geopackage + + @geopackage.setter + def geopackage(self, value: str) -> None: + """Set the pathway to the geopackage file to be used for processing. This is used to specify the grid information for regridding input forcings, and is only necessary if the user is running a simulation that requires regridding of input forcings.""" + if value is not None: + self._geopackage = value + else: + self._geopackage = self.try_config_get("Geopackage") + + @property + def geogrid(self) -> str: + """Get the pathway to the geogrid file to be used for processing. This is used to specify the grid information for regridding input forcings, and is only necessary if the user is running a simulation that requires regridding of input forcings.""" + return self._geogrid + + @geogrid.setter + def geogrid(self, value: str) -> None: + """Set the pathway to the geogrid file to be used for processing. This is used to specify the grid information for regridding input forcings, and is only necessary if the user is running a simulation that requires regridding of input forcings.""" + if value is not None: + self._geogrid = value + else: + geogrid_base = self.try_config_get("GeogridIn") if geogrid_base is None: err_out_screen("Unable to locate GeogridIn in the configuration file.") self.geogrid = None @@ -232,1467 +319,723 @@ def validate_config(self, cfg_bmi: dict) -> None: geogrid_filename = os.path.basename(geogrid_base) if self.uid64 is None: raise ValueError("self.uid64 cannot be None, please initialize it.") - self.geogrid = os.path.join( + self._geogrid = os.path.join( geogrid_parent, f"{self.uid64}_{geogrid_filename}" ) - # Create directory for esmf_mesh file - if not os.path.isdir(geogrid_parent): - try: - os.makedirs(geogrid_parent, exist_ok=True) - LOG.debug(f"Created esmf mesh directory: {geogrid_parent}") - except OSError as e: - err_out_screen( - f"Unable to create esmf_mesh directory: {geogrid_parent}. Error: {e}" - ) - - # Read in the base input forcing options as an array of values to map. - try: - self.supp_precip_forcings = cfg_bmi["SuppPcp"] - except KeyError as e: - err_out_screen( - "Unable to locate SuppPcp under SuppForcing section in configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate SuppPcp under SuppForcing section in configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen("Improper SuppPcp option specified in configuration file", e) - - self.number_supp_pcp = len(self.supp_precip_forcings) - - if self.number_supp_pcp == 1: - if int(self.supp_precip_forcings[0]) == 14: - self.precip_only_flag = True + self.try_make_dir(geogrid_parent, " esmf_mesh") - if not self.precip_only_flag: - # Read in the base input forcing options as an array of values to map. + def try_make_dir(self, directory: str, optional_str: str = "") -> None: + """Try to make a directory, and catch any errors.""" + if not os.path.isdir(directory): try: - self.input_forcings = cfg_bmi["InputForcings"] - except KeyError as e: - err_out_screen( - "Unable to locate InputForcings under Input section in configuration file.", - e, - ) - except configparser.NoOptionError as e: + os.makedirs(directory, exist_ok=True) + LOG.debug(f"Created{optional_str} directory: {directory}") + except OSError as e: err_out_screen( - "Unable to locate InputForcings under Input section in configuration file.", - e, + f"Unable to create{optional_str} directory: {directory}. Error: {e}" ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper InputForcings option specified in configuration file", e + + @property + def input_forcing_options(self) -> list: + """Get the list of input forcing options specified by the user in the configuration file. This is used to control which input forcings are processed and how they are processed based on the other configuration options specified for each input forcing.""" + return self._input_forcing_options + + @input_forcing_options.setter + def input_forcing_options(self, value: list) -> None: + """Set the list of input forcing options specified by the user in the configuration file. This is used to control which input forcings are processed and how they are processed based on the other configuration options specified for each input forcing.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("InputForcings") + if not self.precip_only_flag: + for force_opt in value: + self.check_input_values_in_range( + value, "InputForcings", list(range(1, self.force_count + 1)) ) + self._input_forcing_options = value + + @property + def number_inputs(self) -> int: + """Calculate the number of input forcing options specified by the user in the configuration file. This is used for error checking to ensure users specify valid input forcing options in the configuration file, and to control the flow of the program based on how many input forcings are being processed.""" + if not self.precip_only_flag: if len(self.input_forcings) == 0: err_out_screen( "Please choose at least one InputForcings dataset to process" ) - self.number_inputs = len(self.input_forcings) + return len(self.input_forcing_options) - # Check to make sure forcing options make sense - for force_opt in self.input_forcings: - if force_opt < 0 or force_opt > FORCE_COUNT: - err_out_screen( - f"Please specify InputForcings values between 1 and {FORCE_COUNT}." - ) - - # Keep tabs on how many custom input forcings we have. + @property + def number_custom_inputs(self) -> int: + """Calculate the number of custom input forcing options specified by the user in the configuration file. This is used to control the flow of the program based on how many custom input forcings are being processed, since custom input forcings require some different processing pathways.""" + if not self.precip_only_flag: + count = 0 + for force_opt in self.input_forcing_options: if force_opt == 10: - self.number_custom_inputs = self.number_custom_inputs + 1 + count += 1 + return count - # Flag to force mandatory configuration option to specify the NWM geogrid file if user requests - # NWM forcing files to be regridded to a given domain configuration - if force_opt == 27: - try: - self.nwm_geogrid = cfg_bmi["NWM_Geogrid"] - except KeyError as e: - err_out_screen( - "Unable to locate NWM Geogrid file required for the NWM forcings module. Need to specify the pathway to the NWM geo_em_DOMAIN.nc file to the NWM_Geogrid configuration input option within the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate NWM Geogrid file required for the NWM forcings module. Need to specify the pathway to the NWM geo_em_DOMAIN.nc file to the NWM_Geogrid configuration input option within the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper NWM Geogrid file option specified in configuration file", - e, - ) + @property + def nwm_geogrid(self) -> str: + """Get the pathway to the NWM geogrid file specified by the user in the configuration file. This is used to specify the grid information for regridding NWM input forcings, and is only necessary if the user has chosen to regrid NWM input forcings in the configuration file.""" + return self._nwm_geogrid + + @nwm_geogrid.setter + def nwm_geogrid(self, value: str) -> None: + """Set the pathway to the NWM geogrid file specified by the user in the configuration file. This is used to specify the grid information for regridding NWM input forcings, and is only necessary if the user has chosen to regrid NWM input forcings in the configuration file.""" + if value is None and not self.precip_only_flag: + if 27 in self.input_forcing_options: + value = self.extract_input_variable("NWM_Geogrid") + self._nwm_geogrid = value - # Read in the input forcings types (GRIB[1|2], NETCDF) - try: - # self.input_force_types = config.get('Input', 'InputForcingTypes').strip("[]").split(',') - # self.input_force_types = [ftype.strip() for ftype in self.input_force_types] - self.input_force_types = cfg_bmi["InputForcingTypes"] - if self.input_force_types == [""]: - self.input_force_types = [] - except KeyError as e: - err_out_screen( - "Unable to locate InputForcingTypes in Input section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate InputForcingTypes in Input section in the configuration file.", - e, - ) - if len(self.input_force_types) != self.number_inputs: - err_out_screen( - "Number of InputForcingTypes must match the number " - "of InputForcings in the configuration file." - ) - for file_type in self.input_force_types: - if file_type not in [ - "GRIB1", - "GRIB2", - "NETCDF", - "NETCDF4", - "NWM", - "ZARR", - "GRIB2_CFS", - ]: - err_out_screen( - f'Invalid forcing file type "{file_type}" specified. ' - "Only GRIB1, GRIB2, NETCDF, NWM, ZARR, and GRIB2_CFS are supported" - ) + @property + def input_force_types(self) -> list: + """Get the list of input forcing file types specified by the user in the configuration file. This is used to control how input forcings are read in and processed based on the file type specified for each input forcing in the configuration file.""" + return self._input_force_types + + @input_force_types.setter + def input_force_types(self, value: list) -> None: + """Set the list of input forcing file types specified by the user in the configuration file. This is used to control how input forcings are read in and processed based on the file type specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("InputForcingTypes") + if not self.precip_only_flag: + if value == [""]: + value = [] + self.check_number_of_inputs_forcings(value, "InputForcingTypes") + self.check_input_values_in_range( + value, "InputForcingTypes", self.file_types + ) + self._input_force_types = value - # Read in the input directories for each forcing option. - try: - self.input_force_dirs = cfg_bmi["InputForcingDirectories"] - except KeyError as e: - err_out_screen( - "Unable to locate InputForcingDirectories in Input section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate InputForcingDirectories in Input section in the configuration file.", - e, - ) - if len(self.input_force_dirs) != self.number_inputs: - err_out_screen( - "Number of InputForcingDirectories must match the number " - "of InputForcings in the configuration file." - ) + @property + def file_types(self): + """Get the list of input forcing file types specified by the user in the configuration file. This is used to control how input forcings are read in and processed based on the file type specified for each input forcing in the configuration file.""" + return self.CONFIGOPTIONS["file_types"] + + @property + def input_force_dirs(self) -> list: + """Get the list of input forcing directories specified by the user in the configuration file. This is used to control where input forcings are read in from for each input forcing specified by the user in the configuration file.""" + return self._input_force_dirs + + @input_force_dirs.setter + def input_force_dirs(self, value: list) -> None: + """Set the list of input forcing directories specified by the user in the configuration file. This is used to control where input forcings are read in from for each input forcing specified by the user in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("InputForcingDirectories") + if not self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "InputForcingDirectories") # Loop through and ensure all input directories exist. Also strip out any whitespace # or new line characters. - for dir_tmp in range(0, len(self.input_force_dirs)): - self.input_force_dirs[dir_tmp] = self.input_force_dirs[dir_tmp].strip() - - dir_path = self.input_force_dirs[dir_tmp] - forcing_type = self.input_forcings[dir_tmp] + for dir_tmp in range(0, len(value)): + value[dir_tmp] = value[dir_tmp].strip() + dir_path = value[dir_tmp] + forcing_type = self.input_forcing_options[dir_tmp] is_aws_forcing = forcing_type in [12, 21, 27] if not os.path.isdir(dir_path): if is_aws_forcing: self.aws = True else: - try: - os.makedirs(dir_path, exist_ok=True) - LOG.debug(f"Created missing forcing directory: {dir_path}") - except OSError as e: - err_out_screen( - f"Unable to create forcing directory: {dir_path}. Error: {e}" - ) - - # Read in the mandatory enforcement options for input forcings. - try: - self.input_force_mandatory = cfg_bmi["InputMandatory"] - except KeyError as e: - err_out_screen( - "Unable to locate InputMandatory under Input section in configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate InputMandatory under Input section in configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper InputMandatory option specified in configuration file", e - ) + self.try_make_dir(dir_path, " forcing") + self._input_force_dirs = value + + def input_force_mandatory(self) -> list: + """Get the list of input forcing mandatory flags specified by the user in the configuration file. This is used to control whether the program should raise an error if input forcings for a given forecast cycle are not found for each input forcing specified by the user in the configuration file.""" + return self._input_force_mandatory + + @input_force_mandatory.setter + def input_force_mandatory(self, value: list) -> None: + """Set the list of input forcing mandatory flags specified by the user in the configuration file. This is used to control whether the program should raise an error if input forcings for a given forecast cycle are not found for each input forcing specified by the user in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("InputMandatory") + self.check_number_of_inputs_forcings(value, "InputMandatory") + self.check_input_values_in_range(value, "InputMandatory", [0, 1]) + self._input_force_mandatory = value + + def customSuppPcpFreq(self) -> int: + """Get the custom supplemental precip output frequency specified by the user in the configuration file. This is used to control the output frequency of supplemental precip forcings if the user has chosen to run the supplemental precip forcings module only.""" + return self._customSuppPcpFreq + + @customSuppPcpFreq.setter + def customSuppPcpFreq(self, value: int) -> None: + """Set the custom supplemental precip output frequency specified by the user in the configuration file. This is used to control the output frequency of supplemental precip forcings if the user has chosen to run the supplemental precip forcings module only.""" + if value is None and self.precip_only_flag: + value = self.extract_input_variable("customSuppPcpFreq") + self.check_input_values_positive([value], "customSuppPcpFreq") + self._customSuppPcpFreq = value - if len(self.input_force_mandatory) != self.number_inputs: - err_out_screen( - "Please specify InputMandatory values for each corresponding input " - "forcings in the configuration file." - ) - # Check to make sure enforcement options makes sense. - for enforce_opt in self.input_force_mandatory: - if enforce_opt < 0 or enforce_opt > 1: - err_out_screen( - "Invalid InputMandatory chosen in the configuration file. Please choose a value of 0 or 1 for each corresponding input forcing." - ) + @property + def include_lqfrac(self): + """Get the flag for whether to include the liquid/solid precipitation fraction variable in the output files specified by the user in the configuration file. This is used to control whether the liquid/solid precipitation fraction variable is included in the output files.""" + return self._include_lqfrac - # Read in the output frequency - try: - self.output_freq = cfg_bmi["OutputFrequency"] - except ValueError as e: - err_out_screen( - "Improper OutputFrequency value specified in the configuration file." - ) - except KeyError as e: - err_out_screen( - "Unable to locate OutputFrequency in the configuration file." - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate OutputFrequency in the configuration file." - ) - if self.output_freq <= 0: - err_out_screen( - "Please specify an OutputFrequency that is greater than zero minutes." - ) + @include_lqfrac.setter + def include_lqfrac(self, value): + """Set the flag for whether to include the liquid/solid precipitation fraction variable in the output files specified by the user in the configuration file. This is used to control whether the liquid/solid precipitation fraction variable is included in the output files.""" + if value is None: + value = self.extract_input_variable_set_default("includeLQFrac", default=0) - if self.precip_only_flag: - # Read in the custom supp output frequency - try: - self.customSuppPcpFreq = int(cfg_bmi["customSuppPcpFreq"]) - except ValueError as e: - err_out_screen( - "Improper customSuppPcpFreq value specified in the configuration file.", - e, - ) - except KeyError as e: - err_out_screen( - "Unable to locate customSuppPcpFreq in the configuration file.", e - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate customSuppPcpFreq in the configuration file.", e - ) - if self.output_freq <= 0: - err_out_screen( - "Please specify an customSuppPcpFreq that is greater than zero minutes." - ) + @property + def include_lqfrac(self): + """Get the flag for whether to include the liquid/solid precipitation fraction variable in the output files specified by the user in the configuration file. This is used to control whether the liquid/solid precipitation fraction variable is included in the output files.""" + return self._include_lqfrac - # Read in the sub output hour - try: - self.sub_output_hour = int(cfg_bmi["SubOutputHour"]) - except ValueError as e: - err_out_screen( - "Improper SubOutputHour value specified in the configuration file.", e - ) - except KeyError as e: - err_out_screen( - "Unable to locate SubOutputHour in the configuration file.", e - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate SubOutputHour in the configuration file.", e - ) - if self.sub_output_hour < 0: - err_out_screen( - "Please specify an SubOutputHour that is greater than zero minutes." - ) - if self.sub_output_hour == 0: - self.sub_output_hour = None - # Read in the output frequency - try: - self.sub_output_freq = int(cfg_bmi["SubOutFreq"]) - except ValueError as e: - err_out_screen( - "Improper SubOutFreq value specified in the configuration file.", e - ) - except KeyError as e: - err_out_screen("Unable to locate SubOutFreq in the configuration file.", e) - except configparser.NoOptionError as e: - err_out_screen("Unable to locate SubOutFreq in the configuration file.", e) - if self.sub_output_freq < 0: - err_out_screen( - "Please specify an SubOutFreq that is greater than zero minutes." - ) - if self.sub_output_freq == 0: - self.sub_output_freq = None + @include_lqfrac.setter + def include_lqfrac(self, value): + if value is None: + value = self.extract_input_variable_set_default("includeLQFrac", default=0) + self._include_lqfrac = value - # TODO Can this be a /tmp directory? - # Read in the scratch temporary directory, which also may contain output forcing file if requested. - try: - self.scratch_dir = cfg_bmi["ScratchDir"] - except ValueError as e: - err_out_screen( - "Improper ScratchDir specified in the configuration file.", e - ) - except KeyError as e: - err_out_screen("Unable to locate ScratchDir in the configuration file.", e) - except configparser.NoOptionError as e: - err_out_screen("Unable to locate ScratchDir in the configuration file.", e) + @property + def forcing_output(self) -> int: + """Get the flag for whether to output the input forcings specified by the user in the configuration file. This is used to control whether the input forcings are output in addition to the processed forcings.""" + return self._forcing_output + + @forcing_output.setter + def forcing_output(self, value: int) -> None: + if value is None: + value = self.extract_input_variable_set_default("Output", default=0) + self._forcing_output = value + + def fcst_shift(self) -> int: + """Get the forecast shift specified by the user in the configuration file. This is used to control the calculation of the processing window for realtime simulations.""" + return self._fcst_shift + + @fcst_shift.setter + def fcst_shift(self, value: int) -> None: + if value is None: + value = self.extract_input_variable("ForecastShift") + self.check_input_values_positive([value], "ForecastShift") + self._fcst_shift = value - self.make_scratch_dir() + @property + def fcst_input_horizons(self) -> list: + """Get the list of forecast input horizons specified by the user in the configuration file. This is used to control the calculation of the forecast cycle length and the processing of input forcings based on the forecast time horizons specified for each input forcing.""" + return self._fcst_input_horizons + + @fcst_input_horizons.setter + def fcst_input_horizons(self, value: list) -> None: + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("ForecastInputHorizons") + if not self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "ForecastInputHorizons") + self.check_input_values_positive(value, "ForecastInputHorizons") + else: + if value is None: + value = self.extract_input_variable("ForecastInputHorizons") + self._fcst_input_horizons = value - # Read in compression option - try: - self.useCompression = cfg_bmi["compressOutput"] - except KeyError as e: - err_out_screen("Unable to locate compressOut in the configuration file.", e) - except configparser.NoOptionError as e: - err_out_screen("Unable to locate compressOut in the configuration file.", e) - except ValueError as e: - err_out_screen("Improper compressOut value.", e) - if self.useCompression < 0 or self.useCompression > 1: - err_out_screen("Please choose a compressOut value of 0 or 1.") + @property + def fcst_input_offsets(self): + """Get the list of forecast input offsets specified by the user in the configuration file. This is used to control the calculation of the processing window for both realtime and reforecast simulations based on the forecast time horizons and input offsets specified for each input forcing.""" + return self._fcst_input_offsets + + @fcst_input_offsets.setter + def fcst_input_offsets(self, value: list) -> None: + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("ForecastInputOffsets") + if not self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "ForecastInputOffsets") + self.check_input_values_positive(value, "ForecastInputOffsets") + self._fcst_input_offsets = value - # Read in floating-point option - try: - self.useFloats = cfg_bmi["floatOutput"] - except KeyError as e: - # err_out_screen('Unable to locate floatOutput in the configuration file.', e) - self.useFloats = 0 - except configparser.NoOptionError as e: - # err_out_screen('Unable to locate floatOutput in the configuration file.', e) - self.useFloats = 0 - except ValueError as e: - err_out_screen( - "Improper floatOutput value: {}".format(cfg_bmi["includeLQFraq"]) - ) - if self.useFloats < 0 or self.useFloats > 1: - err_out_screen("Please choose a floatOutput value of 0 or 1.") + @property + def cycle_length_minutes(self) -> int: + """Get the forecast cycle length in minutes, which is calculated based on the maximum of the forecast input horizons specified by the user in the configuration file. - # Read in lqfrac option - try: - self.include_lqfrac = cfg_bmi["includeLQFrac"] - except KeyError as e: - # err_out_screen('Unable to locate includeLQFraq in the configuration file.', e) - self.include_lqfrac = 0 - except configparser.NoOptionError as e: - # err_out_screen('Unable to locate includeLQFraq in the configuration file.', e) - self.useFinclude_lqfracloats = 0 - except ValueError as e: + Ensure the number maximum cycle length is an equal divider of the output time step specified by the user. + """ + cycle_len = max(self.fcst_input_horizons) + if cycle_len % self.output_freq != 0: err_out_screen( - "Improper includeLQFrac value: {}".format(cfg_bmi["includeLQFraq"]), e + "Please specify an output time step that is an equal divider of the maximum of the forecast time horizons specified." ) - if self.include_lqfrac < 0 or self.include_lqfrac > 1: - err_out_screen("Please choose an includeLQFrac value of 0 or 1.") + return cycle_len - # Read in Forcing output option - try: - self.forcing_output = cfg_bmi["Output"] - except KeyError as e: - self.forcing_output = 0 - except configparser.NoOptionError as e: - self.forcing_output = 0 - except ValueError as e: - err_out_screen( - "Improper Forcing Output value: {}".format(cfg_bmi["Output"]), e - ) - if self.forcing_output < 0 or self.forcing_output > 1: - err_out_screen( - "Please choose a Forcing Output value of 0 (No output) or 1 (output)." + def num_output_steps(self) -> int: + """Calculate the number of output time steps per forecast cycle based on the forecast cycle length and the output frequency specified by the user in the configuration file.""" + if self.sub_output_hour is None: + num_steps = int(self.cycle_length_minutes / self.output_freq) + else: + num_steps = ( + int( + (self.cycle_length_minutes - (self.sub_output_hour * 60)) + / self.sub_output_freq + ) + + int((self.sub_output_hour * 60) / self.output_freq) + - 1 ) + return num_steps - # Read AnA flag option - try: - # check both the Forecast section and if it's not there, the old BiasCorrection location - self.ana_flag = int(cfg_bmi["AnAFlag"]) - except KeyError as e: - err_out_screen("Unable to locate AnAFlag in the configuration file.", e) - except configparser.NoOptionError as e: - err_out_screen("Unable to locate AnAFlag in the configuration file.", e) - except ValueError as e: - err_out_screen("Improper AnAFlag value ", e) - if self.ana_flag < 0 or self.ana_flag > 1: - err_out_screen("Please choose a AnAFlag value of 0 or 1.") + def num_supp_output_steps(self) -> int: + """Calculate the number of supplemental precip output time steps per forecast cycle based on the forecast cycle length and the custom supplemental precip output frequency specified by the user in the configuration file.""" + if self.precip_only_flag: + return int(self.cycle_length_minutes / self.customSuppPcpFreq) - # For the NextGen Forcings Engine BMI, we are assuming a realtime or reforecast simulation. - try: - self.look_back = cfg_bmi["LookBack"] - if self.look_back <= 0 and self.look_back != -9999: - err_out_screen( - "Please specify a positive LookBack or -9999 for realtime." - ) - except ValueError as e: - err_out_screen( - "Improper LookBack value entered into the configuration file. Please check your entry.", - e, - ) - except KeyError as e: - err_out_screen( - "Unable to locate LookBack in the configuration file. Please verify entries exist.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate LookBack in the configuration file. Please verify entries exist.", - e, - ) + def actual_output_steps(self) -> int: + """Calculate the actual number of output time steps per forecast cycle based on whether the user has chosen to run a reforecast simulation with a specified processing window, which will only output time steps for which input forcings are available based on the processing window and forecast time horizons specified by the user in the configuration file.""" + if self.ana_flag: + return np.int32(self.nFcsts) + else: + return np.int32(self.num_output_steps) + + @property + def grid_type(self) -> str: + """Get the grid type specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings based on the grid type specified by the user in the configuration file.""" + return self._grid_type + + @grid_type.setter + def grid_type(self, value: str) -> None: + """Set the grid type specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings based on the grid type specified by the user in the configuration file.""" + if value is None: + value = self.extract_input_variable("GRID_TYPE") + self.check_input_values_in_range( + [value], "GRID_TYPE", ["gridded", "unstructured", "hydrofabric"] + ) + self._grid_type = value.lower() - # Process the beginning date of reforecast forcings to process + def raise_grid_type_error(self, grid_type: str, variable_name: str) -> None: + """Raise an error if a variable is requested that is not valid for the given grid type.""" + err_out_screen( + f"{variable_name} is not a valid variable for grid type {grid_type}. Please check your configuration file." + ) - if self.b_date_proc: - beg_date_tmp = self.b_date_proc - e = "" + @property + def lon_var(self) -> str: + """Get the longitude variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen a gridded grid type in the configuration file.""" + if self.grid_type == "gridded": + return self.extract_input_variable("LONVAR") else: - try: - beg_date_tmp = cfg_bmi["RefcstBDateProc"] - except KeyError as e: - err_out_screen( - "Unable to locate RefcstBDateProc under Logistics section in configuration file.", - e, - ) - beg_date_tmp = None - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate RefcstBDateProc under Logistics section in configuration file.", - e, - ) - beg_date_tmp = None + self.raise_grid_type_error(self.grid_type, "LONVAR") - if beg_date_tmp != -9999: - if isinstance(beg_date_tmp, str) and len(beg_date_tmp) != 12: - err_out_screen( - "Improper RefcstBDateProc length entered into the configuration file. Please check your entry.", - e, - ) - try: - self.b_date_proc = datetime.strptime(beg_date_tmp, "%Y%m%d%H%M") - except ValueError as e: - err_out_screen( - "Improper RefcstBDateProc value entered into the configuration file. Please check your entry.", - e, - ) + @property + def lat_var(self) -> str: + """Get the latitude variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen a gridded grid type in the configuration file.""" + if self.grid_type == "gridded": + return self.extract_input_variable("LATVAR") else: - self.b_date_proc = -9999 + self.raise_grid_type_error(self.grid_type, "LATVAR") - LOG.info(f"Begin date: {beg_date_tmp}") + @property + def nodecoords_var(self) -> str: + """Get the node coordinates variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen an unstructured or hydrofabric grid type in the configuration file.""" + if self.grid_type in ["unstructured", "hydrofabric"]: + return self.extract_input_variable("NodeCoords") + else: + self.raise_grid_type_error(self.grid_type, "NodeCoords") - # If the Retro flag is off, and lookback is off, then we assume we are - # running a reforecast. - if self.look_back == -9999: - self.realtime_flag = False - self.refcst_flag = True - elif self.b_date_proc == -9999: - self.realtime_flag = True - self.refcst_flag = True + @property + def elemcoords_var(self) -> str: + """Get the element coordinates variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen an unstructured or hydrofabric grid type in the configuration file.""" + if self.grid_type in ["unstructured", "hydrofabric"]: + return self.extract_input_variable("ElemCoords") else: - # The processing window will be calculated based on current time and the - # lookback option since this is a realtime instance. - self.realtime_flag = False - self.refcst_flag = False - # self.b_date_proc = -9999 - # self.e_date_proc = -9999 + self.raise_grid_type_error(self.grid_type, "ElemCoords") - # Calculate the delta time between the beginning and ending time of processing. - # self.process_window = self.e_date_proc - self.b_date_proc + @property + def elemconn_var(self) -> str: + """Get the element connectivity variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen an unstructured or hydrofabric grid type in the configuration file.""" + if self.grid_type in ["unstructured", "hydrofabric"]: + return self.extract_input_variable("ElemConn") + else: + self.raise_grid_type_error(self.grid_type, "ElemConn") - # Read in the ForecastFrequency option. - try: - self.fcst_freq = cfg_bmi["ForecastFrequency"] - except ValueError as e: - err_out_screen( - "Improper ForecastFrequency value entered into the configuration file. Please check your entry.", - e, - ) - except KeyError as e: - err_out_screen( - "Unable to locate ForecastFrequency in the configuration file. Please verify entries exist.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate ForecastFrequency in the configuration file. Please verify entries exist.", - e, - ) - if self.fcst_freq <= 0: - err_out_screen( - "Please specify a ForecastFrequency in the configuration file greater than zero." - ) - # Currently, we only support daily or sub-daily forecasts. Any other iterations should - # be done using custom config files for each forecast cycle. - if self.fcst_freq > 1440: - err_out_screen( - "Only forecast cycles of daily or sub-daily are supported at this time" - ) + @property + def numelemconn_var(self) -> str: + """Get the number of element connectivity variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen an unstructured or hydrofabric grid type in the configuration file.""" + if self.grid_type in ["unstructured", "hydrofabric"]: + return self.extract_input_variable("NumElemConn") + else: + self.raise_grid_type_error(self.grid_type, "NumElemConn") - # Read in the ForecastShift option. This is ONLY done for the realtime instance as - # it's used to calculate the beginning of the processing window. - if True: # was: self.realtime_flag: - try: - self.fcst_shift = cfg_bmi["ForecastShift"] - except ValueError as e: - err_out_screen( - "Improper ForecastShift value entered into the configuration file. Please check your entry.", - e, - ) - except KeyError as e: - err_out_screen( - "Unable to locate ForecastShift in the configuration file. Please verify entries exist.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate ForecastShift in the configuration file. Please verify entries exist.", - e, - ) - if self.fcst_shift < 0: + @property + def element_id_var(self) -> str: + """Get the element ID variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen a hydrofabric grid type in the configuration file.""" + if self.grid_type == "hydrofabric": + return self.extract_input_variable("ElemID") + else: + self.raise_grid_type_error(self.grid_type, "ElemID") + + @property + def ignored_border_widths(self) -> list: + """Get the list of ignored border widths specified by the user in the configuration file. This is used to control how the program processes input forcings based on the ignored border widths specified for each input forcing in the configuration file.""" + return self._ignored_border_widths + + @ignored_border_widths.setter + def ignored_border_widths(self, value: list) -> None: + """Set the list of ignored border widths specified by the user in the configuration file. This is used to control how the program processes input forcings based on the ignored border widths specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("IgnoredBorderWidths") + if self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "IgnoredBorderWidths") + self.check_input_values_positive(value, "IgnoredBorderWidths") + self._ignored_border_widths = value + + @property + def regrid_opt(self): + """Get the list of regridding options specified by the user in the configuration file. This is used to control how input forcings are regridded based on the regridding option specified for each input forcing in the configuration file.""" + return self._regrid_opt + + @regrid_opt.setter + def regrid_opt(self, value: list) -> None: + """Set the list of regridding options specified by the user in the configuration file. This is used to control how input forcings are regridded based on the regridding option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("RegridOpt") + if self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "RegridOpt") + self.check_input_values_in_range(value, "RegridOpt", [1, 2, 3]) + self._regrid_opt = value + + @property + def weightsDir(self) -> str: + """Get the pathway to the ESMF weights directory specified by the user in the configuration file. This is used to control where the program looks for ESMF weights files if the user has chosen to use pre-generated ESMF weights files for regridding input forcings in the configuration file.""" + return self._weightsDir + + @weightsDir.setter + def weightsDir(self, value: str) -> None: + """Set the pathway to the ESMF weights directory specified by the user in the configuration file. This is used to control where the program looks for ESMF weights files if the user has chosen to use pre-generated ESMF weights files for regridding input forcings in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.try_config_get("RegridWeightsDir") + if self.precip_only_flag: + if value is not None and not os.path.exists(value): err_out_screen( - "Please specify a ForecastShift in the configuration file greater than or equal to zero." + f"ESMF Weights file directory specified ({value}) but does not exist" ) + self._weightsDir = value - # Calculate the beginning/ending processing dates if we are running realtime - if self.realtime_flag: - calculate_lookback_window(self) - - # if self.refcst_flag: - # Calculate the number of forecasts to issue, and verify the user has chosen a - # correct divider based on the dates - # dt_tmp = self.e_date_proc - self.b_date_proc - # if (dt_tmp.days * 1440 + dt_tmp.seconds / 60.0) % self.fcst_freq != 0: - # err_out_screen('Please choose an equal divider forecast frequency for your ' - # 'specified reforecast range.') - # self.nFcsts = int((dt_tmp.days * 1440 + dt_tmp.seconds / 60.0) / self.fcst_freq) + @property + def forceTemoralInterp(self) -> list: + """Get the list of forcing temporal interpolation options specified by the user in the configuration file. This is used to control how input forcings are temporally interpolated based on the temporal interpolation option specified for each input forcing in the configuration file.""" + return self._forceTemoralInterp + + @forceTemoralInterp.setter + def forceTemoralInterp(self, value: list) -> None: + """Set the list of forcing temporal interpolation options specified by the user in the configuration file. This is used to control how input forcings are temporally interpolated based on the temporal interpolation option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("ForcingTemporalInterpolation") + if not self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "ForcingTemporalInterpolation") + self.check_input_values_in_range( + value, "ForcingTemporalInterpolation", [0, 1, 2] + ) + self._forceTemoralInterp = value - # Flag to constrain AORC forcing data cycle output - # for optTmp in self.input_forcings: - # if optTmp == 12: - # self.nFcsts = 1 - self.nFcsts = 1 + @property + def forceTemoralInterp(self): + """Get the list of forcing temporal interpolation options specified by the user in the configuration file. This is used to control how input forcings are temporally interpolated based on the temporal interpolation option specified for each input forcing in the configuration file.""" + return self._forceTemoralInterp + + @forceTemoralInterp.setter + def forceTemoralInterp(self, value): + """Set the list of forcing temporal interpolation options specified by the user in the configuration file. This is used to control how input forcings are temporally interpolated based on the temporal interpolation option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("ForcingTemporalInterpolation") + if not self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "ForcingTemporalInterpolation") + self.check_input_values_in_range( + value, "ForcingTemporalInterpolation", [0, 1, 2] + ) + self._forceTemoralInterp = value - if self.look_back != -9999: - calculate_lookback_window(self) + @property + def t2dDownscaleOpt(self) -> list: + """Get the list of temperature downscaling options specified by the user in the configuration file. This is used to control how temperature input forcings are downscaled based on the temperature downscaling option specified for each input forcing in the configuration file.""" + return self._t2dDownscaleOpt + + @t2dDownscaleOpt.setter + def t2dDownscaleOpt(self, value: list) -> None: + """Set the list of temperature downscaling options specified by the user in the configuration file. This is used to control how temperature input forcings are downscaled based on the temperature downscaling option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("TemperatureDownscaling") + if not self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "TemperatureDownscaling") + self.check_input_values_in_range(value, "TemperatureDownscaling", [0, 1, 2]) + self._t2dDownscaleOpt = value + @property + def psfcDownscaleOpt(self) -> list: + """Get the list of pressure downscaling options specified by the user in the configuration file. This is used to control how pressure input forcings are downscaled based on the pressure downscaling option specified for each input forcing in the configuration file.""" + return self._psfcDownscaleOpt + + @psfcDownscaleOpt.setter + def psfcDownscaleOpt(self, value: list) -> None: + """Set the list of pressure downscaling options specified by the user in the configuration file. This is used to control how pressure input forcings are downscaled based on the pressure downscaling option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("PressureDownscaling") if not self.precip_only_flag: - # Read in the ForecastInputHorizons options. - try: - self.fcst_input_horizons = cfg_bmi["ForecastInputHorizons"] - except KeyError as e: - err_out_screen( - "Unable to locate ForecastInputHorizons under Forecast section in configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate ForecastInputHorizons under Forecast section in configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper ForecastInputHorizons option specified in configuration file", - e, - ) - if len(self.fcst_input_horizons) != self.number_inputs: - err_out_screen( - "Please specify ForecastInputHorizon values for each corresponding input forcings for forecasts." - ) + self.check_number_of_inputs_forcings(value, "PressureDownscaling") + self.check_input_values_in_range(value, "PressureDownscaling", [0, 1]) + self._psfcDownscaleOpt = value - # Check to make sure the horizons options make sense. There will be additional - # checking later when input choices are mapped to input products. - for horizonOpt in self.fcst_input_horizons: - if horizonOpt <= 0: - err_out_screen( - "Please specify ForecastInputHorizon values greater than zero." - ) - else: - # Read in the ForecastInputHorizons options. - try: - self.fcst_input_horizons = cfg_bmi["ForecastInputHorizons"] - except KeyError as e: - err_out_screen( - "Unable to locate ForecastInputHorizons under Forecast section in configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate ForecastInputHorizons under Forecast section in configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper ForecastInputHorizons option specified in configuration file", - e, - ) - if len(self.fcst_input_horizons) != 1: - err_out_screen( - "Please specify ForecastInputHorizon values for each corresponding input forcings for forecasts." - ) + @property + def swDownscaleOpt(self) -> list: + """Get the list of shortwave downscaling options specified by the user in the configuration file. This is used to control how shortwave radiation input forcings are downscaled based on the shortwave downscaling option specified for each input forcing in the configuration file.""" + return self._swDownscaleOpt + + @swDownscaleOpt.setter + def swDownscaleOpt(self, value: list) -> None: + """Set the list of shortwave downscaling options specified by the user in the configuration file. This is used to control how shortwave radiation input forcings are downscaled based on the shortwave downscaling option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("ShortwaveDownscaling") + if not self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "ShortwaveDownscaling") + self.check_input_values_in_range(value, "ShortwaveDownscaling", [0, 1]) + self._swDownscaleOpt = value + @property + def q2dDownscaleOpt(self) -> list: + """Get the list of humidity downscaling options specified by the user in the configuration file. This is used to control how humidity input forcings are downscaled based on the humidity downscaling option specified for each input forcing in the configuration file.""" + return self._q2dDownscaleOpt + + @q2dDownscaleOpt.setter + def q2dDownscaleOpt(self, value: list) -> None: + """Set the list of humidity downscaling options specified by the user in the configuration file. This is used to control how humidity input forcings are downscaled based on the humidity downscaling option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("HumidityDownscaling") if not self.precip_only_flag: - # Read in the ForecastInputOffsets options. - try: - self.fcst_input_offsets = cfg_bmi["ForecastInputOffsets"] - except KeyError as e: - err_out_screen( - "Unable to locate ForecastInputOffsets under Forecast section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate ForecastInputOffsets under Forecast section in the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper ForecastInputOffsets option specified in the configuration file.", - e, - ) - if len(self.fcst_input_offsets) != self.number_inputs: - err_out_screen( - "Please specify ForecastInputOffset values for each corresponding input forcings for forecasts." - ) - # Check to make sure the input offset options make sense. There will be additional - # checking later when input choices are mapped to input products. - for inputOffset in self.fcst_input_offsets: - if inputOffset < 0: - err_out_screen( - "Please specify ForecastInputOffset values greater than or equal to zero." - ) + self.check_number_of_inputs_forcings(value, "HumidityDownscaling") + self.check_input_values_in_range(value, "HumidityDownscaling", [0, 1]) + self._q2dDownscaleOpt = value - # Calculate the length of the forecast cycle, based on the maximum - # length of the input forcing length chosen by the user. - self.cycle_length_minutes = max(self.fcst_input_horizons) + @property + def precipDownscaleOpt(self) -> list: + """Get the list of precipitation downscaling options specified by the user in the configuration file. This is used to control how precipitation input forcings are downscaled based on the precipitation downscaling option specified for each input forcing in the configuration file.""" + return self._precipDownscaleOpt - # Ensure the number maximum cycle length is an equal divider of the output - # time step specified by the user. - if self.cycle_length_minutes % self.output_freq != 0: - err_out_screen( - "Please specify an output time step that is an equal divider of the maximum of the forecast time horizons specified." - ) + @precipDownscaleOpt.setter + def precipDownscaleOpt(self, value: list) -> None: + """Set the list of precipitation downscaling options specified by the user in the configuration file. This is used to control how precipitation input forcings are downscaled based on the precipitation downscaling option specified for each input forcing in the configuration file.""" + if value is None: + value = self.extract_input_variable("PrecipDownscaling") + self.check_number_of_inputs_forcings(value, "PrecipDownscaling") + self.check_input_values_in_range(value, "PrecipDownscaling", [0, 1]) - if self.sub_output_hour is None: - # Calculate the number of output time steps per forecast cycle. - self.num_output_steps = int(self.cycle_length_minutes / self.output_freq) - if self.precip_only_flag: - self.num_supp_output_steps = ( - int(self.cycle_length_minutes) / self.customSuppPcpFreq - ) - if self.ana_flag: - self.actual_output_steps = np.int32(self.nFcsts) - else: - self.actual_output_steps = np.int32(self.num_output_steps) - else: - # Calculate the number of output time steps per forecast cycle. - self.num_output_steps = ( - int( - (self.cycle_length_minutes - (self.sub_output_hour * 60)) - / self.sub_output_freq - ) - + int((self.sub_output_hour * 60) / self.output_freq) - - 1 - ) - if self.precip_only_flag: - self.num_supp_output_steps = ( - int(self.cycle_length_minutes) / self.customSuppPcpFreq - ) - if self.ana_flag: - self.actual_output_steps = np.int32(self.nFcsts) - else: - self.actual_output_steps = np.int32(self.num_output_steps) + self._precipDownscaleOpt = value - # Process the grid type - try: - self.grid_type = cfg_bmi["GRID_TYPE"] - except KeyError as e: - err_out_screen("Unable to locate GRID_TYPE in the configuration file.", e) - except configparser.NoOptionError as e: - err_out_screen("Unable to locate GRID_TYPE in the configuration file.", e) + @property + def dScaleParamDirs(self) -> list: + """Get the list of downscaling parameter directories specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for each input forcing based on the downscaling parameter directory specified for each input forcing in the configuration file.""" + return self._dScaleParamDirs + + @dScaleParamDirs.setter + def dScaleParamDirs(self, value: list) -> None: + """Set the list of downscaling parameter directories specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for each input forcing based on the downscaling parameter directory specified for each input forcing in the configuration file.""" + if value is None: + value = self.extract_input_variable("DownscalingParamDirs") + self.check_number_of_inputs_forcings(value, "DownscalingParamDirs") + for dirTmp in range(0, len(value)): + dir_path = value[dirTmp] + if not os.path.isdir(dir_path): + err_out_screen( + f"Unable to locate parameter directory: {os.path.abspath(dir_path)}" + ) + self._dScaleParamDirs = value + + def perform_downscaling(self) -> bool: + """Determine whether downscaling of input forcings is necessary based on the downscaling options specified by the user for each input forcing in the configuration file.""" if ( - self.grid_type.lower() != "gridded" - and self.grid_type.lower() != "unstructured" - and self.grid_type.lower() != "hydrofabric" + 1 in self.q2dDownscaleOpt + or 1 in self.swDownscaleOpt + or 1 in self.psfcDownscaleOpt + or 1 in self.t2dDownscaleOpt + or 2 in self.t2dDownscaleOpt ): - err_out_screen( - 'GRID_TYPE in the configuration file only accepts "unstructured", "gridded", or "hydrofabric" as options.' + return True + + @property + def sinalpha_var(self) -> str: + """Get the sine of the grid orientation variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the grid orientation variable specified for each input forcing in the configuration file.""" + if self.perform_downscaling: + return self.extract_input_variable("SINALPHA") + + @property + def cosalpha_var(self) -> str: + """Get the cosine of the grid orientation variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the grid orientation variable specified for each input forcing in the configuration file.""" + if self.perform_downscaling: + return self.extract_input_variable("COSALPHA") + + @property + def slope_var(self) -> str: + """Get the slope variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the slope variable specified for each input forcing in the configuration file.""" + if self.perform_downscaling: + return self.extract_input_variable("SLOPE") + + @property + def slope_azimuth_var(self) -> str: + """Get the slope azimuth variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the slope azimuth variable specified for each input forcing in the configuration file.""" + if self.perform_downscaling: + return self.extract_input_variable("SLOPE_AZIMUTH") + + @property + def slope_var_elem(self) -> str: + """Get the slope variable name specified by the user in the configuration file for element-based grids. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the slope variable specified for each input forcing in the configuration file for element-based grids.""" + if self.perform_downscaling: + if self.grid_type == "unstructured": + return self.extract_input_variable("SLOPE_ELEM") + else: + self.raise_grid_type_error(self.grid_type, "SLOPE_ELEM") + + @property + def slope_azimuth_var_elem(self) -> str: + """Get the slope azimuth variable name specified by the user in the configuration file for element-based grids. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the slope azimuth variable specified for each input forcing in the configuration file for element-based grids.""" + if self.perform_downscaling: + if self.grid_type == "unstructured": + return self.extract_input_variable("SLOPE_AZIMUTH_ELEM") + else: + self.raise_grid_type_error(self.grid_type, "SLOPE_AZIMUTH_ELEM") + + @property + def hgt_elem_var(self) -> str: + """Get the height variable name specified by the user in the configuration file for element-based grids. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the height variable specified for each input forcing in the configuration file for element-based grids.""" + if self.perform_downscaling: + if self.grid_type == "unstructured": + return self.extract_input_variable("HGT_ELEM") + else: + self.raise_grid_type_error(self.grid_type, "HGT_ELEM") + + @property + def hgt_var(self) -> str: + """Get the height variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the height variable specified for each input forcing in the configuration file.""" + if self.perform_downscaling: + return self.extract_input_variable("HGT") + + @property + def t2BiasCorrectOpt(self) -> list: + """Get the list of temperature bias correction options specified by the user in the configuration file. This is used to control how temperature input forcings are bias corrected based on the temperature bias correction option specified for each input forcing in the configuration file.""" + return self._t2BiasCorrectOpt + + @t2BiasCorrectOpt.setter + def t2BiasCorrectOpt(self, value: list) -> None: + """Set the list of temperature bias correction options specified by the user in the configuration file. This is used to control how temperature input forcings are bias corrected based on the temperature bias correction option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("TemperatureBiasCorrection") + if not self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "TemperatureBiasCorrection") + self.check_input_values_in_range( + value, "TemperatureBiasCorrection", [0, 1, 2, 3, 4] ) + self._t2BiasCorrectOpt = value - if self.grid_type.lower() == "gridded": - # Process the geogrid variable information - try: - self.lon_var = cfg_bmi["LONVAR"] - except KeyError as e: - err_out_screen("Unable to locate LONVAR in the configuration file.", e) - except configparser.NoOptionError as e: - err_out_screen("Unable to locate LONVAR in the configuration file.", e) - try: - self.lat_var = cfg_bmi["LATVAR"] - except KeyError as e: - err_out_screen("Unable to locate LATVAR in the configuration file.", e) - except configparser.NoOptionError as e: - err_out_screen("Unable to locate LATVAR in the configuration file.", e) - - elif self.grid_type.lower() == "unstructured": - # Process the geogrid variable information - try: - self.nodecoords_var = cfg_bmi["NodeCoords"] - except KeyError as e: - err_out_screen( - "Unable to locate NodeCoords for unstructured mesh in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate NodeCoords for unstructured mesh in the configuration file.", - e, - ) - try: - self.elemcoords_var = cfg_bmi["ElemCoords"] - except KeyError as e: - err_out_screen( - "Unable to locate ElemCoords for unstructured mesh in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate ElemCoords for unstructured mesh in the configuration file.", - e, - ) - try: - self.elemconn_var = cfg_bmi["ElemConn"] - except KeyError as e: - err_out_screen( - "Unable to locate ElemConn for unstructured mesh in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate ElemConn for unstructured mesh in the configuration file.", - e, - ) - try: - self.numelemconn_var = cfg_bmi["NumElemConn"] - except KeyError as e: - err_out_screen( - "Unable to locate NumElemConn for unstructured mesh in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate NumElemConn for unstructured mesh in the configuration file.", - e, - ) - - elif self.grid_type.lower() == "hydrofabric": - # Process the geogrid variable information - try: - self.nodecoords_var = cfg_bmi["NodeCoords"] - except KeyError as e: - err_out_screen( - "Unable to locate NodeCoords for unstructured mesh in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate NodeCoords for unstructured mesh in the configuration file.", - e, - ) - try: - self.elemcoords_var = cfg_bmi["ElemCoords"] - except KeyError as e: - err_out_screen( - "Unable to locate ElemCoords for unstructured mesh in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate ElemCoords for unstructured mesh in the configuration file.", - e, - ) - try: - self.element_id_var = cfg_bmi["ElemID"] - except KeyError as e: - err_out_screen( - "Unable to locate ElemID for unstructured mesh in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate ElemID for unstructured mesh in the configuration file.", - e, - ) - try: - self.elemconn_var = cfg_bmi["ElemConn"] - except KeyError as e: - err_out_screen( - "Unable to locate ElemConn for unstructured mesh in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate ElemConn for unstructured mesh in the configuration file.", - e, - ) - try: - self.numelemconn_var = cfg_bmi["NumElemConn"] - except KeyError as e: - err_out_screen( - "Unable to locate NumElemConn for unstructured mesh in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate NumElemConn for unstructured mesh in the configuration file.", - e, - ) - - # Process geospatial information - - if self.geogrid: - LOG.debug(f"Geogrid: {self.geogrid}") - else: - try: - self.geogrid = cfg_bmi["GeogridIn"] - except KeyError as e: - err_out_screen( - "Unable to locate GeogridIn in the configuration file.", e - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate GeogridIn in the configuration file.", e - ) - - # Check for the optional geospatial land metadata file. - try: - self.spatial_meta = cfg_bmi["SpatialMetaIn"] - except KeyError as e: - err_out_screen( - "Unable to locate SpatialMetaIn in the configuration file.", e - ) - if len(self.spatial_meta) == 0: - # No spatial metadata file found. - self.spatial_meta = None - else: - if not os.path.isfile(self.spatial_meta): - err_out_screen( - "Unable to locate optional spatial metadata file: " - + self.spatial_meta - ) + @property + def psfcBiasCorrectOpt(self) -> list: + """Get the list of pressure bias correction options specified by the user in the configuration file. This is used to control how pressure input forcings are bias corrected based on the pressure bias correction option specified for each input forcing in the configuration file.""" + return self._psfcBiasCorrectOpt + + @psfcBiasCorrectOpt.setter + def psfcBiasCorrectOpt(self, value: list) -> None: + """Set the list of pressure bias correction options specified by the user in the configuration file. This is used to control how pressure input forcings are bias corrected based on the pressure bias correction option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("PressureBiasCorrection") + if not self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "PressureBiasCorrection") + self.check_input_values_in_range(value, "PressureBiasCorrection", [0, 1]) + self._psfcBiasCorrectOpt = value + @property + def q2BiasCorrectOpt(self): + """Get the list of humidity bias correction options specified by the user in the configuration file. This is used to control how humidity input forcings are bias corrected based on the humidity bias correction option specified for each input forcing in the configuration file.""" + return self._q2BiasCorrectOpt + + @q2BiasCorrectOpt.setter + def q2BiasCorrectOpt(self, value): + """Set the list of humidity bias correction options specified by the user in the configuration file. This is used to control how humidity input forcings are bias corrected based on the humidity bias correction option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("HumidityBiasCorrection") if not self.precip_only_flag: - # Check for the IgnoredBorderWidths - try: - self.ignored_border_widths = cfg_bmi["IgnoredBorderWidths"] - except (KeyError, configparser.NoOptionError): - # if didn't specify, no worries, just set to 0 - self.ignored_border_widths = [0.0] * self.number_inputs - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper IgnoredBorderWidths option specified in the configuration file." - "({} was supplied".format( - cfg_bmi["Geospatial"]["IgnoredBorderWidths"] - ), - e, - ) - if len(self.ignored_border_widths) != self.number_inputs: - err_out_screen( - "Please specify IgnoredBorderWidths values for each " - "corresponding input forcings for SuppForcing." - "({} was supplied".format(self.ignored_border_widths) - ) - if any(map(lambda x: x < 0, self.ignored_border_widths)): - err_out_screen( - "Please specify IgnoredBorderWidths values greater than or equal to zero:" - "({} was supplied".format(self.ignored_border_widths) - ) + self.check_number_of_inputs_forcings(value, "HumidityBiasCorrection") + self.check_input_values_in_range(value, "HumidityBiasCorrection", [0, 1, 2]) + self._q2BiasCorrectOpt = value + @property + def windBiasCorrect(self): + """Get the list of wind bias correction options specified by the user in the configuration file. This is used to control how wind input forcings are bias corrected based on the wind bias correction option specified for each input forcing in the configuration file.""" + return self._windBiasCorrect + + @windBiasCorrect.setter + def windBiasCorrect(self, value): + """Set the list of wind bias correction options specified by the user in the configuration file. This is used to control how wind input forcings are bias corrected based on the wind bias correction option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("WindBiasCorrection") if not self.precip_only_flag: - # Process regridding options. - try: - self.regrid_opt = cfg_bmi["RegridOpt"] - except KeyError as e: - err_out_screen( - "Unable to locate RegridOpt under the Regridding section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate RegridOpt under the Regridding section in the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper RegridOpt options specified in the configuration file.", e - ) - if len(self.regrid_opt) != self.number_inputs: - err_out_screen( - "Please specify RegridOpt values for each corresponding input forcings in the configuration file.", - e, - ) - # Check to make sure regridding options makes sense. - for regridOpt in self.regrid_opt: - if regridOpt < 1 or regridOpt > 3: - err_out_screen( - "Invalid RegridOpt chosen in the configuration file. Please choose a " - "value of 1-2 for each corresponding input forcing." - ) - try: - # Read weight file directory (optional) - self.weightsDir = cfg_bmi["RegridWeightsDir"] - except Exception: - # Set wieghtsDir to None; this will create regrid object in memory - self.weightsDir = None - if self.weightsDir: - # if we do have one specified, make sure it exists - if not os.path.exists(self.weightsDir): - err_out_screen( - "ESMF Weights file directory specified ({}) but does not exist" - ).format(self.weightsDir) + self.check_number_of_inputs_forcings(value, "WindBiasCorrection") + self.check_input_values_in_range(value, "WindBiasCorrection", [0, 4]) + self._windBiasCorrect = value - # Calculate the beginning/ending processing dates if we are running realtime - if self.realtime_flag: - calculate_lookback_window(self) - - # Create temporary array to hold flags if we need input parameter files. - param_flag = np.empty([len(self.input_forcings)], int) - param_flag[:] = 0 + @property + def swBiasCorrectOpt(self) -> list: + """Get the list of shortwave radiation bias correction options specified by the user in the configuration file. This is used to control how shortwave radiation input forcings are bias corrected based on the shortwave radiation bias correction option specified for each input forcing in the configuration file.""" + return self._swBiasCorrectOpt + + @swBiasCorrectOpt.setter + def swBiasCorrectOpt(self, value: list) -> None: + """Set the list of shortwave radiation bias correction options specified by the user in the configuration file. This is used to control how shortwave radiation input forcings are bias corrected based on the shortwave radiation bias correction option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("ShortwaveBiasCorrection") if not self.precip_only_flag: - # Read in temporal interpolation options. - try: - self.forceTemoralInterp = cfg_bmi["ForcingTemporalInterpolation"] - except KeyError as e: - err_out_screen( - "Unable to locate ForcingTemporalInterpolation under the Interpolation section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate ForcingTemporalInterpolation under the Interpolation section in the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper ForcingTemporalInterpolation options specified in the configuration file.", - e, - ) - if len(self.forceTemoralInterp) != self.number_inputs: - err_out_screen( - "Please specify ForcingTemporalInterpolation values for each corresponding input forcings in the configuration file." - ) - # Ensure the forcingTemporalInterpolation values make sense. - for temporalInterpOpt in self.forceTemoralInterp: - if temporalInterpOpt < 0 or temporalInterpOpt > 2: - err_out_screen( - "Invalid ForcingTemporalInterpolation chosen in the configuration file. " - "Please choose a value of 0-2 for each corresponding input forcing." - ) - - # Read in the temperature downscaling options. - try: - self.t2dDownscaleOpt = cfg_bmi["TemperatureDownscaling"] - except KeyError as e: - err_out_screen( - "Unable to locate TemperatureDownscaling under the Downscaling section of the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate TemperatureDownscaling under the Downscaling section of the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper TemperatureDownscaling options specified in the configuration file.", - e, - ) - if len(self.t2dDownscaleOpt) != self.number_inputs: - err_out_screen( - "Please specify TemperatureDownscaling values for each corresponding input forcings in the configuration file." - ) - # Ensure the downscaling options chosen make sense. - count_tmp = 0 - for optTmp in self.t2dDownscaleOpt: - if optTmp < 0 or optTmp > 2: - err_out_screen( - "Invalid TemperatureDownscaling options specified in the configuration file." - ) - if optTmp == 2: - param_flag[count_tmp] = 1 - count_tmp = count_tmp + 1 - - # Read in the pressure downscaling options. - try: - self.psfcDownscaleOpt = cfg_bmi["PressureDownscaling"] - except KeyError as e: - err_out_screen( - "Unable to locate PressureDownscaling under the Downscaling section of the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate PressureDownscaling under the Downscaling section of the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper PressureDownscaling options specified in the configuration file." - ) - if len(self.psfcDownscaleOpt) != self.number_inputs: - err_out_screen( - "Please specify PressureDownscaling values for each corresponding input forcings in the configuration file." - ) - # Ensure the downscaling options chosen make sense. - for optTmp in self.psfcDownscaleOpt: - if optTmp < 0 or optTmp > 1: - err_out_screen( - "Invalid PressureDownscaling options specified in the configuration file." - ) - - # Read in the shortwave downscaling options - try: - self.swDownscaleOpt = cfg_bmi["ShortwaveDownscaling"] - except KeyError as e: - err_out_screen( - "Unable to locate ShortwaveDownscaling under the Downscaling section of the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate ShortwaveDownscaling under the Downscaling section of the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper ShortwaveDownscaling options specified in the configuration file.", - e, - ) - if len(self.swDownscaleOpt) != self.number_inputs: - err_out_screen( - "Please specify ShortwaveDownscaling values for each corresponding input forcings in the configuration file." - ) - # Ensure the downscaling options chosen make sense. - for optTmp in self.swDownscaleOpt: - if optTmp < 0 or optTmp > 1: - err_out_screen( - "Invalid ShortwaveDownscaling options specified in the configuration file." - ) - - # Read in humidity downscaling options. - try: - self.q2dDownscaleOpt = cfg_bmi["HumidityDownscaling"] - except KeyError as e: - err_out_screen( - "Unable to locate HumidityDownscaling under the Downscaling section of the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate HumidityDownscaling under the Downscaling section of the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper HumidityDownscaling options specified in the configuration file.", - e, - ) - if len(self.q2dDownscaleOpt) != self.number_inputs: - err_out_screen( - "Please specify HumidityDownscaling values for each corresponding " - "input forcings in the configuration file." - ) - # Ensure the downscaling options chosen make sense. - for optTmp in self.q2dDownscaleOpt: - if optTmp < 0 or optTmp > 1: - err_out_screen( - "Invalid HumidityDownscaling options specified in the configuration file." - ) - - # Read in the precipitation downscaling options - try: - self.precipDownscaleOpt = cfg_bmi["PrecipDownscaling"] - except KeyError as e: - err_out_screen( - "Unable to locate PrecipDownscaling under the Downscaling section of the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate PrecipDownscaling under the Downscaling section of the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper PrecipDownscaling options specified in the configuration file.", - e, + self.check_number_of_inputs_forcings(value, "ShortwaveBiasCorrection") + self.check_input_values_in_range( + value, "ShortwaveBiasCorrection", [0, 1, 2] ) - if not self.precip_only_flag: - if len(self.precipDownscaleOpt) != self.number_inputs: - err_out_screen( - "Please specify PrecipDownscaling values for each corresponding " - "input forcings in the configuration file." - ) - # Ensure the downscaling options chosen make sense. - count_tmp = 0 - for optTmp in self.precipDownscaleOpt: - if optTmp < 0 or optTmp > 1: - err_out_screen( - "Invalid PrecipDownscaling options specified in the configuration file." - ) - if optTmp == 1: - param_flag[count_tmp] = 1 - count_tmp = count_tmp + 1 + self._swBiasCorrectOpt = value - # Read in the downscaling parameter directory. - try: - self.dScaleParamDirs = cfg_bmi["DownscalingParamDirs"] - except KeyError as e: - err_out_screen( - "Unable to locate DownscalingParamDirs in the configuration file.", e - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate DownscalingParamDirs in the configuration file.", e - ) - if len(self.dScaleParamDirs) != len(self.input_forcings): - err_out_screen( - "Please specify a downscaling parameter directory for each " - "corresponding downscaling option that requires one." + @property + def lwBiasCorrectOpt(self) -> list: + """Get the list of longwave radiation bias correction options specified by the user in the configuration file. This is used to control how longwave radiation input forcings are bias corrected based on the longwave radiation bias correction option specified for each input forcing in the configuration file.""" + return self._lwBiasCorrectOpt + + @lwBiasCorrectOpt.setter + def lwBiasCorrectOpt(self, value: list) -> None: + """Set the list of longwave radiation bias correction options specified by the user in the configuration file. This is used to control how longwave radiation input forcings are bias corrected based on the longwave radiation bias correction option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("LongwaveBiasCorrection") + if not self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "LongwaveBiasCorrection") + self.check_input_values_in_range( + value, "LongwaveBiasCorrection", [0, 1, 2, 4] ) - # Loop through each downscaling parameter directory and make sure they exist. - for dirTmp in range(0, len(self.dScaleParamDirs)): - if not os.path.isdir(self.dScaleParamDirs[dirTmp]): - err_out_screen( - "Unable to locate parameter directory: " - + os.path.abspath(self.dScaleParamDirs[dirTmp]) - ) + self._lwBiasCorrectOpt = value - if ( - [1] in self.q2dDownscaleOpt - or [1] in self.swDownscaleOpt - or [1] in self.psfcDownscaleOpt - or [1, 2] in self.t2dDownscaleOpt - ): - # Process the geogrid information for downscaling - try: - self.sinalpha_var = cfg_bmi["SINALPHA"] - except Exception: - self.sinalpha_var = None - try: - self.cosalpha_var = cfg_bmi["COSALPHA"] - except Exception: - self.cosalpha_var = None - if self.grid_type.lower() == "hydrofabric": - try: - self.slope_var = cfg_bmi["SLOPE"] - except KeyError as e: - err_out_screen( - "Unable to locate SLOPE variable in the hydrofabric configuration file. Required variable since user turned on a downscaling option.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate SLOPE variable in the hydrofabric configuration file. Required variable since user turned on a downscaling option.", - e, - ) - try: - self.slope_azimuth_var = cfg_bmi["SLOPE_AZIMUTH"] - except KeyError as e: - err_out_screen( - "Unable to locate SLOPE_AZIMUTH variable in the hydrofabric configuration file. Required variable since user turned on a downscaling option.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate SLOPE_AZIMUTH variable in the hydrofabric configuration file. Required variable since user turned on a downscaling option.", - e, - ) - else: - try: - self.slope_var = cfg_bmi["SLOPE"] - except Exception: - self.slope_var = None - try: - self.slope_azimuth_var = cfg_bmi["SLOPE_AZIMUTH"] - except Exception: - self.slope_azimuth_var = None - if self.grid_type.lower() == "unstructured": - try: - self.slope_var_elem = cfg_bmi["SLOPE_ELEM"] - except Exception: - self.slope_var_elem = None - try: - self.slope_azimuth_var_elem = cfg_bmi["SLOPE_AZIMUTH_ELEM"] - except Exception: - self.slope_azimuth_var_elem = None - - if self.grid_type.lower() == "unstructured": - try: - self.hgt_elem_var = cfg_bmi["HGTVAR_ELEM"] - except KeyError as e: - err_out_screen( - "Unable to locate HGTVAR_ELEM in the configuration file. Required variable since user turned on a downscaling option.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate HGTVAR_ELEM in the configuration file. Required variable since user turned on a downscaling option.", - e, - ) - - try: - self.hgt_var = cfg_bmi["HGTVAR"] - except KeyError as e: - err_out_screen( - "Unable to locate HGTVAR in the configuration file. Required variable since user turned on a downscaling option.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate HGTVAR in the configuration file. Required variable since user turned on a downscaling option.", - e, - ) - - # * Bias Correction Options * + @property + def precipBiasCorrectOpt(self): + """Get the list of precipitation bias correction options specified by the user in the configuration file. This is used to control how precipitation input forcings are bias corrected based on the precipitation bias correction option specified for each input forcing in the configuration file.""" + return self._precipBiasCorrectOpt + + @precipBiasCorrectOpt.setter + def precipBiasCorrectOpt(self, value): + """Set the list of precipitation bias correction options specified by the user in the configuration file. This is used to control how precipitation input forcings are bias corrected based on the precipitation bias correction option specified for each input forcing in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("PrecipBiasCorrection") if not self.precip_only_flag: - # Read in temperature bias correction options - try: - self.t2BiasCorrectOpt = cfg_bmi["TemperatureBiasCorrection"] - except KeyError as e: - err_out_screen( - "Unable to locate TemperatureBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate TemperatureBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except json.JSONDecodeError as e: - err_out_screen( - "Improper TemperatureBiasCorrection options specified in the configuration file.", - e, - ) - if len(self.t2BiasCorrectOpt) != self.number_inputs: - err_out_screen( - "Please specify TemperatureBiasCorrection values for each corresponding input forcings in the configuration file." - ) - # Ensure the bias correction options chosen make sense. - for optTmp in self.t2BiasCorrectOpt: - if optTmp < 0 or optTmp > 4: - err_out_screen( - "Invalid TemperatureBiasCorrection options specified in the configuration file." - ) - - # Read in surface pressure bias correction options. - try: - self.psfcBiasCorrectOpt = cfg_bmi["PressureBiasCorrection"] - except KeyError as e: - err_out_screen( - "Unable to locate PressureBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate PressureBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except json.JSONDecodeError as e: - err_out_screen( - "Improper PressureBiasCorrection options specified in the configuration file.", - e, - ) - if len(self.psfcDownscaleOpt) != self.number_inputs: - err_out_screen( - "Please specify PressureBiasCorrection values for each corresponding input forcings in the configuration file." - ) - # Ensure the bias correction options chosen make sense. - for optTmp in self.psfcBiasCorrectOpt: - if optTmp < 0 or optTmp > 1: - err_out_screen( - "Invalid PressureBiasCorrection options specified in the configuration file." - ) - if optTmp == 1: - # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. - self.runCfsNldasBiasCorrect = True + self.check_number_of_inputs_forcings(value, "PrecipBiasCorrection") + self.check_input_values_in_range(value, "PrecipBiasCorrection", [0, 1]) + self._precipBiasCorrectOpt = value - # Read in humidity bias correction options. - try: - self.q2BiasCorrectOpt = cfg_bmi["HumidityBiasCorrection"] - except KeyError as e: - err_out_screen( - "Unable to locate HumidityBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate HumidityBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except json.JSONDecodeError as e: - err_out_screen( - "Improper HumdityBiasCorrection options specified in the configuration file.", - e, - ) - if len(self.q2BiasCorrectOpt) != self.number_inputs: - err_out_screen( - "Please specify HumidityBiasCorrection values for each corresponding input forcings in the configuration file." - ) - # Ensure the bias correction options chosen make sense. - for optTmp in self.q2BiasCorrectOpt: - if optTmp < 0 or optTmp > 2: - err_out_screen( - "Invalid HumidityBiasCorrection options specified in the configuration file." - ) - if optTmp == 1: - # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. - self.runCfsNldasBiasCorrect = True - - # Read in wind bias correction options. - try: - self.windBiasCorrect = cfg_bmi["WindBiasCorrection"] - except KeyError as e: - err_out_screen( - "Unable to locate WindBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate WindBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except json.JSONDecodeError as e: - err_out_screen( - "Improper WindBiasCorrection options specified in the configuration file.", - e, - ) - if len(self.windBiasCorrect) != self.number_inputs: - err_out_screen( - "Please specify WindBiasCorrection values for each corresponding input forcings in the configuration file." - ) - # Ensure the bias correction options chosen make sense. - for optTmp in self.windBiasCorrect: - if optTmp < 0 or optTmp > 4: - err_out_screen( - "Invalid WindBiasCorrection options specified in the configuration file." - ) - if optTmp == 1: - # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. - self.runCfsNldasBiasCorrect = True - - # Read in shortwave radiation bias correction options. - try: - self.swBiasCorrectOpt = cfg_bmi["SwBiasCorrection"] - except KeyError as e: - err_out_screen( - "Unable to locate SwBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate SwBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except json.JSONDecodeError as e: - err_out_screen( - "Improper SwBiasCorrection options specified in the configuration file.", - e, - ) - if len(self.swBiasCorrectOpt) != self.number_inputs: - err_out_screen( - "Please specify SwBiasCorrection values for each corresponding input forcings in the configuration file." - ) - # Ensure the bias correction options chosen make sense. - for optTmp in self.swBiasCorrectOpt: - if optTmp < 0 or optTmp > 2: - err_out_screen( - "Invalid SwBiasCorrection options specified in the configuration file." - ) - if optTmp == 1: - # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. - self.runCfsNldasBiasCorrect = True - - # Read in longwave radiation bias correction options. - try: - self.lwBiasCorrectOpt = cfg_bmi["LwBiasCorrection"] - except KeyError as e: - err_out_screen( - "Unable to locate LwBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate LwBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except json.JSONDecodeError as e: - err_out_screen( - "Improper LwBiasCorrection options specified in the configuration file.", - e, - ) - if len(self.lwBiasCorrectOpt) != self.number_inputs: - err_out_screen( - "Please specify LwBiasCorrection values for each corresponding input forcings in the configuration file." - ) - # Ensure the bias correction options chosen make sense. - for optTmp in self.lwBiasCorrectOpt: - if optTmp < 0 or optTmp > 4: - err_out_screen( - "Invalid LwBiasCorrection options specified in the configuration file." - ) - if optTmp == 1: - # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. - self.runCfsNldasBiasCorrect = True + @property + def bias_correction_properties(self) -> dict: + """Get the dictionary of bias correction properties specified by the user in the configuration file. This is used to control how input forcings are bias corrected based on the bias correction options specified for each input forcing in the configuration file.""" + bias_correction_properties = { + "surface temperature": self.t2BiasCorrectOpt, + "surface pressure": self.psfcBiasCorrectOpt, + "specific humidity": self.q2BiasCorrectOpt, + "wind forcings": self.windBiasCorrect, + "short-wave radiation": self.swBiasCorrectOpt, + "long-wave radiation": self.lwBiasCorrectOpt, + "Precipitation": self.precipBiasCorrectOpt, + } + return bias_correction_properties - # Read in precipitation bias correction options. - try: - self.precipBiasCorrectOpt = cfg_bmi["PrecipBiasCorrection"] - except KeyError as e: - err_out_screen( - "Unable to locate PrecipBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate PrecipBiasCorrection under the BiasCorrection section of the configuration file.", - e, - ) - except json.JSONDecodeError as e: - err_out_screen( - "Improper PrecipBiasCorrection options specified in the configuration file.", - e, - ) - if not self.precip_only_flag: - if len(self.precipBiasCorrectOpt) != self.number_inputs: - err_out_screen( - "Please specify PrecipBiasCorrection values for each corresponding input forcings in the configuration file." - ) - # Ensure the bias correction options chosen make sense. - for optTmp in self.precipBiasCorrectOpt: - if optTmp < 0 or optTmp > 1: - err_out_screen( - "Invalid PrecipBiasCorrection options specified in the configuration file." - ) - if optTmp == 1: - # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. - self.runCfsNldasBiasCorrect = True - - # Putting a constraint here that CFSv2-NLDAS bias correction (NWM only) is chosen, it must be turned on - # for ALL variables. - if self.runCfsNldasBiasCorrect: - if ( - min(self.precipBiasCorrectOpt) != 1 - and max(self.precipBiasCorrectOpt) != 1 - ): - err_out_screen( - "CFSv2-NLDAS NWM bias correction must be activated for Precipitation under this configuration." - ) - if min(self.lwBiasCorrectOpt) != 1 and max(self.lwBiasCorrectOpt) != 1: - err_out_screen( - "CFSv2-NLDAS NWM bias correction must be activated for long-wave radiation under this configuration." - ) - if min(self.swBiasCorrectOpt) != 1 and max(self.swBiasCorrectOpt) != 1: - err_out_screen( - "CFSv2-NLDAS NWM bias correction must be activated for short-wave radiation under this configuration." - ) - if min(self.t2BiasCorrectOpt) != 1 and max(self.t2BiasCorrectOpt) != 1: - err_out_screen( - "CFSv2-NLDAS NWM bias correction must be activated for surface temperature under this configuration." - ) - if min(self.windBiasCorrect) != 1 and max(self.windBiasCorrect) != 1: - err_out_screen( - "CFSv2-NLDAS NWM bias correction must be activated for wind forcings under this configuration." - ) - if min(self.q2BiasCorrectOpt) != 1 and max(self.q2BiasCorrectOpt) != 1: - err_out_screen( - "CFSv2-NLDAS NWM bias correction must be activated for specific humidity under this configuration." - ) - if ( - min(self.psfcBiasCorrectOpt) != 1 - and max(self.psfcBiasCorrectOpt) != 1 - ): + @property + def runCfsNldasBiasCorrect(self) -> bool: + """Get the flag for whether to run the NWM-specific bias correction of CFSv2 input forcings specified by the user in the configuration file. This is used to control whether the NWM-specific bias correction of CFSv2 input forcings is run based on whether the user has chosen to run this bias correction in the configuration file.""" + for optTmp in self.bias_correction_properties.values(): + if optTmp == 1: + runCfsNldasBiasCorrect = True + break + if runCfsNldasBiasCorrect: + for ( + bias_correct_name, + bias_correct, + ) in self.bias_correction_properties.items(): + if min(bias_correct) != 1 and max(bias_correct) != 1: err_out_screen( - "CFSv2-NLDAS NWM bias correction must be activated for surface pressure under this configuration." + f"CFSv2-NLDAS NWM bias correction must be activated for {bias_correct_name} under this configuration." ) # Make sure we don't have any other forcings activated. This can only be ran for CFSv2. for opt_tmp in self.input_forcings: @@ -1701,168 +1044,229 @@ def validate_config(self, cfg_bmi: dict) -> None: "CFSv2-NLDAS NWM bias correction can only be used in CFSv2-only configurations" ) - # Read in supplemental precipitation options as an array of values to map. - try: - self.supp_precip_forcings = cfg_bmi["SuppPcp"] - except KeyError as e: - err_out_screen( - "Unable to locate SuppPcp under SuppForcing section in configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate SuppPcp under SuppForcing section in configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen("Improper SuppPcp option specified in configuration file", e) - self.number_supp_pcp = len(self.supp_precip_forcings) + def number_supp_pcp(self) -> int: + """Get the number of supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how many supplemental precipitation input forcings are processed based on the number of supplemental precipitation input forcings specified in the configuration file.""" + return len(self.supp_precip_forcings) - # Read in the supp pcp types (GRIB[1|2], NETCDF) - try: - self.supp_precip_file_types = cfg_bmi["SuppPcpForcingTypes"] - self.supp_precip_file_types = [ - stype.strip() for stype in self.supp_precip_file_types - ] - if self.supp_precip_file_types == [""]: - self.supp_precip_file_types = [] - except KeyError as e: - err_out_screen( - "Unable to locate SuppPcpForcingTypes in SuppForcing section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate SuppPcpForcingTypes in SuppForcing section in the configuration file.", - e, - ) - if len(self.supp_precip_file_types) != self.number_supp_pcp: - err_out_screen( - "Number of SuppPcpForcingTypes ({}) must match the number " - "of SuppPcp inputs ({}) in the configuration file.".format( - len(self.supp_precip_file_types), self.number_supp_pcp - ) - ) - for file_type in self.supp_precip_file_types: - if file_type not in ["GRIB1", "GRIB2", "NETCDF"]: - err_out_screen( - 'Invalid SuppForcing file type "{}" specified. ' - "Only GRIB1, GRIB2, and NETCDF are supported".format(file_type) - ) + @property + def supp_precip_file_types(self) -> list: + """Get the list of supplemental precipitation input forcing file types specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are read in and processed based on the file types specified for each supplemental precipitation input forcing in the configuration file.""" + return self._supp_precip_file_types + + @supp_precip_file_types.setter + def supp_precip_file_types(self, value: list) -> None: + """Set the list of supplemental precipitation input forcing file types specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are read in and processed based on the file types specified for each supplemental precipitation input forcing in the configuration file.""" + if value is None: + value = self.try_config_get("SuppPcpForcingTypes") + if value is not None: + value = [stype.strip() for stype in value] + if value == [""]: + value = [] + + self.check_number_of_inputs_supp_pcp(value, "SuppPcpForcingTypes") + self.check_input_values_in_range( + value, + "SuppPcpForcingTypes", + self.supplemental_precip_file_type_options, + ) + self._supp_precip_file_types = value + @property + def supplemental_precip_file_type_options(self) -> list: + """Get the list of valid supplemental precipitation input forcing file types that can be specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are read in and processed based on the file types specified for each supplemental precipitation input forcing in the configuration file.""" + return ["GRIB1", "GRIB2", "NETCDF"] + + @property + def rqiMethod(self) -> list: + """Get the list of radar quality index (RQI) thresholding methods specified by the user in the configuration file. This is used to control how radar-based supplemental precipitation input forcings are processed based on the RQI thresholding method specified for each radar-based supplemental precipitation input forcing in the configuration file.""" if self.number_supp_pcp > 0: - # Check to make sure supplemental precip options make sense. Also read in the RQI threshold - # if any radar products where chosen. for suppOpt in self.supp_precip_forcings: - if suppOpt < 0 or suppOpt > 16: - err_out_screen( - "Please specify SuppForcing values between 1 and 16." - ) - # Read in RQI threshold to apply to radar products. - if suppOpt in (1, 2, 7, 10, 11, 12): - try: - self.rqiMethod = cfg_bmi["RqiMethod"] - except KeyError as e: - err_out_screen( - "Unable to locate RqiMethod under SuppForcing section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate RqiMethod under SuppForcing section in the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper RqiMethod option in the configuration file.", e - ) + # Read in RQI threshold to apply to radar products. + if suppOpt in (1, 2, 7, 10, 11, 12): + rqiMethod = self.extract_input_variable("RqiMethod") # Check that if we have more than one RqiMethod, it's the correct number - if type(self.rqiMethod) is list: - if len(self.rqiMethod) != self.number_supp_pcp: - err_out_screen( - "Number of RqiMethods ({}) must match the number " - "of SuppPcp inputs ({}) in the configuration file, or " - "supply a single method for all inputs".format( - len(self.rqiMethod), self.number_supp_pcp - ) - ) - elif type(self.rqiMethod) is int: + if type(rqiMethod) is list: + self.check_number_of_inputs_supp_pcp(rqiMethod, "RqiMethod") + elif type(rqiMethod) is int: # Support 'classic' mode of single method - self.rqiMethod = [self.rqiMethod] * self.number_supp_pcp + rqiMethod = [rqiMethod] * self.number_supp_pcp # Make sure the RqiMethod(s) makes sense. - for method in self.rqiMethod: - if method < 0 or method > 2: - err_out_screen( - "Please specify RqiMethods of either 0, 1, or 2." - ) + for method in rqiMethod: + self.check_input_values_in_range(method, "RqiMethod", [0, 1, 2]) + return rqiMethod - try: - self.rqiThresh = cfg_bmi["RqiThreshold"] - except KeyError as e: - err_out_screen( - "Unable to locate RqiThreshold under SuppForcing section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate RqiThreshold under SuppForcing section in the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper RqiThreshold option in the configuration file.", e - ) + @property + def rqiThresh(self): + """Get the radar quality index (RQI) threshold value specified by the user in the configuration file. This is used to control how radar-based supplemental precipitation input forcings are processed based on the RQI threshold value specified in the configuration file.""" + if self.number_supp_pcp > 0: + for suppOpt in self.supp_precip_forcings: + # Read in RQI threshold to apply to radar products. + if suppOpt in (1, 2, 7, 10, 11, 12): + rqiThresh = self.extract_input_variable("RqiThresh") - # Check that if we have more than one RqiThreshold, it's the correct number - if type(self.rqiThresh) is list: - if len(self.rqiThresh) != self.number_supp_pcp: - err_out_screen( - "Number of RqiThresholds ({}) must match the number " - "of SuppPcp inputs ({}) in the configuration file, or " - "supply a single threshold for all inputs".format( - len(self.rqiThresh), self.number_supp_pcp - ) - ) - elif type(self.rqiThresh) is float: + # Check that if we have more than one RqiThresh, it's the correct number + if type(rqiThresh) is list: + self.check_number_of_inputs_supp_pcp(rqiThresh, "RqiThresh") + elif type(rqiThresh) is (int, float): # Support 'classic' mode of single threshold - self.rqiThresh = [self.rqiThresh] * self.number_supp_pcp + rqiThresh = [rqiThresh] * self.number_supp_pcp - # Make sure the RQI threshold makes sense. + # Make sure the RqiThresh(es) makes sense. for threshold in self.rqiThresh: if threshold < 0.0 or threshold > 1.0: err_out_screen( "Please specify RqiThresholds between 0.0 and 1.0." ) + return threshold - # Read in the input directories for each supplemental precipitation product. - try: - self.supp_precip_dirs = cfg_bmi["SuppPcpDirectories"] - except KeyError as e: - err_out_screen( - "Unable to locate SuppPcpDirectories in SuppForcing section in the configuration file.", - e, + @property + def supp_precip_dirs(self): + """Get the list of pathways to the supplemental precipitation input forcing directories specified by the user in the configuration file. This is used to control where the program looks for supplemental precipitation input forcing files for each supplemental precipitation input forcing based on the directory specified for each supplemental precipitation input forcing in the configuration file.""" + if self.number_supp_pcp > 0: + return self._supp_precip_dirs + + @supp_precip_dirs.setter + def supp_precip_dirs(self, value): + """Set the list of pathways to the supplemental precipitation input forcing directories specified by the user in the configuration file. This is used to control where the program looks for supplemental precipitation input forcing files for each supplemental precipitation input forcing based on the directory specified for each supplemental precipitation input forcing in the configuration file.""" + if value is None and self.number_supp_pcp > 0: + value = self.extract_input_variable("SuppPcpDirectories") + if value > 0: + self.check_number_of_inputs_supp_pcp(value, "SuppPcpDirectories") + for dirTmp in range(0, len(value)): + value[dirTmp] = value[dirTmp].strip() + if not os.path.isdir(value[dirTmp]): + try: + os.makedirs(value[dirTmp], exist_ok=True) + LOG.debug(f"Created supp pcp directory: {value[dirTmp]}") + except OSError as e: + err_out_screen( + f"Unable to create supp pcp directory: {value[dirTmp]}. Error: {e}" + ) + self._supp_precip_dirs = value + + @property + def supp_precip_mandatory(self): + """Get the list of flags for whether each supplemental precipitation input forcing specified by the user in the configuration file is mandatory or optional. This is used to control whether an error is raised if supplemental precipitation input forcing files are not found for each supplemental precipitation input forcing based on whether the user has specified each supplemental precipitation input forcing as mandatory or optional in the configuration file.""" + return self._supp_precip_mandatory + + @supp_precip_mandatory.setter + def supp_precip_mandatory(self, value): + """Set the list of flags for whether each supplemental precipitation input forcing specified by the user in the configuration file is mandatory or optional. This is used to control whether an error is raised if supplemental precipitation input forcing files are not found for each supplemental precipitation input forcing based on whether the user has specified each supplemental precipitation input forcing as mandatory or optional in the configuration file.""" + if value is None and self.number_supp_pcp > 0: + value = self.extract_input_variable("SuppPcpMandatory") + if self.number_supp_pcp > 0: + for enforceOpt in value: + self.check_input_values_in_range(enforceOpt, "SuppPcpMandatory", [0, 1]) + self._supp_precip_mandatory = value + + @property + def regrid_opt_supp_pcp(self): + """Get the list of regridding options for supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how supplemental precipitation input forcings are regridded based on the regridding option specified for each supplemental precipitation input forcing in the configuration file.""" + return self._regrid_opt_supp_pcp + + @regrid_opt_supp_pcp.setter + def regrid_opt_supp_pcp(self, value): + """Set the list of regridding options for supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how supplemental precipitation input forcings are regridded based on the regridding option specified for each supplemental precipitation input forcing in the configuration file.""" + if value is None and self.number_supp_pcp > 0: + value = self.extract_input_variable("RegridOptSuppPcp") + if self.number_supp_pcp > 0: + for optTmp in value: + self.check_input_values_in_range(optTmp, "RegridOptSuppPcp", [1, 2, 3]) + self._regrid_opt_supp_pcp = value + + @property + def suppTemporalInterp(self): + """Get the list of flags for whether temporal interpolation of supplemental precipitation input forcings specified by the user in the configuration file is performed or not. This is used to control whether temporal interpolation of supplemental precipitation input forcings is performed based on whether the user has chosen to perform temporal interpolation for each supplemental precipitation input forcing in the configuration file.""" + if self.number_supp_pcp > 0: + return self._suppTemporalInterp + + @suppTemporalInterp.setter + def suppTemporalInterp(self, value): + """Set the list of flags for whether temporal interpolation of supplemental precipitation input forcings specified by the user in the configuration file is performed or not. This is used to control whether temporal interpolation of supplemental precipitation input forcings is performed based on whether the user has chosen to perform temporal interpolation for each supplemental precipitation input forcing in the configuration file.""" + if value is None and self.number_supp_pcp > 0: + value = self.extract_input_variable("SuppPcpTemporalInterpolation") + if self.number_supp_pcp > 0: + for optTmp in value: + self.check_input_values_in_range( + optTmp, "SuppPcpTemporalInterpolation", [0, 1, 2] ) - except configparser.NoOptionError as e: + self._suppTemporalInterp = value + + @property + def supp_pcp_max_hours(self): + """Get the list of maximum forecast hours for supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how supplemental precipitation input forcings are processed based on the maximum forecast hour specified for each supplemental precipitation input forcing in the configuration file.""" + if self.number_supp_pcp > 0: + return self._supp_pcp_max_hours + + @supp_pcp_max_hours.setter + def supp_pcp_max_hours(self, value): + """Set the list of maximum forecast hours for supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how supplemental precipitation input forcings are processed based on the maximum forecast hour specified for each supplemental precipitation input forcing in the configuration file.""" + if value is None and self.number_supp_pcp > 0: + value = self.extract_input_variable("SuppPcpMaxHours") + if self.number_supp_pcp > 0: + if isinstance(value, list): + self.check_input_values_positive(value, "SuppPcpMaxHours") + elif isinstance(value, float) or isinstance(value, int): + self.check_input_values_positive(value, "SuppPcpMaxHours") + value = [value] * self.number_supp_pcp + self._supp_pcp_max_hours = value + + @property + def supp_input_offsets(self): + """Get the list of time offsets to apply to supplemental precipitation input forcing files specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are processed based on the time offset specified for each supplemental precipitation input forcing in the configuration file.""" + return self._supp_input_offsets + + @supp_input_offsets.setter + def supp_input_offsets(self, value): + """Set the list of time offsets to apply to supplemental precipitation input forcing files specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are processed based on the time offset specified for each supplemental precipitation input forcing in the configuration file.""" + if value is None and self.number_supp_pcp > 0: + value = self.extract_input_variable("SuppPcpInputOffsets") + if self.number_supp_pcp > 0: + self.check_number_of_inputs_supp_pcp(value, "SuppPcpInputOffsets") + + @property + def supp_precip_param_dir(self): + """Get the directory where downscaling parameters for supplemental precipitation input forcings are stored specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for supplemental precipitation input forcings based on the directory specified for supplemental precipitation input forcings in the configuration file.""" + if self.number_supp_pcp > 0: + return self._supp_precip_param_dir + + @supp_precip_param_dir.setter + def supp_precip_param_dir(self, value): + """Set the directory where downscaling parameters for supplemental precipitation input forcings are stored specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for supplemental precipitation input forcings based on the directory specified for supplemental precipitation input forcings in the configuration file.""" + if value is None and self.number_supp_pcp > 0: + value = self.extract_input_variable("SuppPcpDownscalingParamDir") + if self.number_supp_pcp > 0: + if not os.path.isdir(value): err_out_screen( - "Unable to locate SuppPcpDirectories in SuppForcing section in the configuration file.", - e, + f"Unable to locate parameter directory: {os.path.abspath(value)}" ) + self._supp_precip_param_dir = value + + @property + def supp_precip_dirs(self): + """Get the list of pathways to the supplemental precipitation input forcing directories specified by the user in the configuration file. This is used to control where the program looks for supplemental precipitation input forcing files for each supplemental precipitation input forcing based on the directory specified for each supplemental precipitation input forcing in the configuration file.""" + if self.number_supp_pcp > 0: + return self._supp_precip_dirs + @supp_precip_dirs.setter + def supp_precip_dirs(self, value): + """Set the list of pathways to the supplemental precipitation input forcing directories specified by the user in the configuration file. This is used to control where the program looks for supplemental precipitation input forcing files for each supplemental precipitation input forcing based on the directory specified for each supplemental precipitation input forcing in the configuration file.""" + if value is None and self.number_supp_pcp > 0: + value = self.extract_input_variable("SuppPcpDirectories") + if self.number_supp_pcp > 0: # Loop through and ensure all supp pcp directories exist. Also strip out any whitespace # or new line characters. - for dirTmp in range(0, len(self.supp_precip_dirs)): - self.supp_precip_dirs[dirTmp] = self.supp_precip_dirs[dirTmp].strip() - if not os.path.isdir(self.supp_precip_dirs[dirTmp]): + for dirTmp in range(0, len(value)): + value[dirTmp] = value[dirTmp].strip() + if not os.path.isdir(value[dirTmp]): try: - os.makedirs(self.supp_precip_dirs[dirTmp], exist_ok=True) - LOG.debug( - f"Created supp pcp directory: {self.supp_precip_dirs[dirTmp]}" - ) + os.makedirs(value[dirTmp], exist_ok=True) + LOG.debug(f"Created supp pcp directory: {value[dirTmp]}") except OSError as e: err_out_screen( - f"Unable to create supp pcp directory: {self.supp_precip_dirs[dirTmp]}. Error: {e}" + f"Unable to create supp pcp directory: {value[dirTmp]}. Error: {e}" ) # Special case for ExtAnA where we treat comma separated stage IV, MRMS data as one SuppPcp input @@ -1871,254 +1275,172 @@ def validate_config(self, cfg_bmi: dict) -> None: err_out_screen( "CONUS or Alaska Stage IV/MRMS SuppPcp option is only supported as a standalone option" ) - self.supp_precip_dirs = [",".join(self.supp_precip_dirs)] + value = [",".join(value)] + self._supp_precip_dirs = value - if len(self.supp_precip_dirs) != self.number_supp_pcp: - err_out_screen( - "Number of SuppPcpDirectories must match the number of SuppForcing in the configuration file." - ) + @property + def supp_precip_param_dir(self): + """Get the directory where downscaling parameters for supplemental precipitation input forcings are stored specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for supplemental precipitation input forcings based on the directory specified for supplemental precipitation input forcings in the configuration file.""" + if self.number_supp_pcp > 0: + return self._supp_precip_param_dir - # Process supplemental precipitation enforcement options + @supp_precip_param_dir.setter + def supp_precip_param_dir(self, value): + """Set the directory where downscaling parameters for supplemental precipitation input forcings are stored specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for supplemental precipitation input forcings based on the directory specified for supplemental precipitation input forcings in the configuration file.""" + if value is None and self.number_supp_pcp > 0: + value = self.extract_input_variable("SuppPcpDownscalingParamDir") + if self.number_supp_pcp > 0: try: - self.supp_precip_mandatory = cfg_bmi["SuppPcpMandatory"] - except KeyError as e: - err_out_screen( - "Unable to locate SuppPcpMandatory under the SuppForcing section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate SuppPcpMandatory under the SuppForcing section in the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper SuppPcpMandatory options specified in the configuration file.", - e, - ) - if len(self.supp_precip_mandatory) != self.number_supp_pcp: - err_out_screen( - "Please specify SuppPcpMandatory values for each corresponding " - "supplemental precipitation options in the configuration file." - ) - # Check to make sure enforcement options makes sense. - for enforceOpt in self.supp_precip_mandatory: - if enforceOpt < 0 or enforceOpt > 1: - err_out_screen( - "Invalid SuppPcpMandatory chosen in the configuration file. " - "Please choose a value of 0 or 1 for each corresponding " - "supplemental precipitation product." - ) + os.makedirs(value, exist_ok=True) + LOG.debug(f"Created missing SuppPcpParamDir: {value}") + except OSError as e: + err_out_screen(f"Unable to locate SuppPcpParamDir: {value}. Error: {e}") - # Read in the regridding options. - try: - self.regrid_opt_supp_pcp = cfg_bmi["RegridOptSuppPcp"] - except KeyError as e: - err_out_screen( - "Unable to locate RegridOptSuppPcp under the SuppForcing section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate RegridOptSuppPcp under the SuppForcing section in the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper RegridOptSuppPcp options specified in the configuration file.", - e, - ) - if len(self.regrid_opt_supp_pcp) != self.number_supp_pcp: - err_out_screen( - "Please specify RegridOptSuppPcp values for each corresponding supplemental " - "precipitation product in the configuration file." - ) - # Check to make sure regridding options makes sense. - for regridOpt in self.regrid_opt_supp_pcp: - if regridOpt < 1 or regridOpt > 3: - err_out_screen( - "Invalid RegridOptSuppPcp chosen in the configuration file. " - "Please choose a value of 1-3 for each corresponding " - "supplemental precipitation product." + @property + def cfsv2EnsMember(self): + """Get the CFSv2 ensemble member to process specified by the user in the configuration file. This is used to control which CFSv2 ensemble member is processed for CFSv2 input forcings based on the ensemble member specified in the configuration file.""" + return self._cfsv2EnsMember + + @cfsv2EnsMember.setter + def cfsv2EnsMember(self, value): + """Set the CFSv2 ensemble member to process specified by the user in the configuration file. This is used to control which CFSv2 ensemble member is processed for CFSv2 input forcings based on the ensemble member specified in the configuration file.""" + if value is None and not self.precip_only_flag: + # Read in Ensemble information + # Read in CFS ensemble member information IF we have chosen CFSv2 as an input + # forcing. + for opt_tmp in self.input_forcings: + if opt_tmp == 7: + value = self.extract_input_variable("cfsEnsNumber") + self.check_input_values_in_range( + value, "cfsEnsNumber", [1, 2, 3, 4] ) - # Read in temporal interpolation options. - try: - self.suppTemporalInterp = cfg_bmi["SuppPcpTemporalInterpolation"] - except KeyError as e: - err_out_screen( - "Unable to locate SuppPcpTemporalInterpolation under the SuppForcing section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate SuppPcpTemporalInterpolation under the SuppForcing section in the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper SuppPcpTemporalInterpolation options specified in the configuration file.", - e, - ) - if len(self.suppTemporalInterp) != self.number_supp_pcp: + @property + def customFcstFreq(self): + """Get the custom forecast frequency in minutes specified by the user in the configuration file. This is used to control how often forecasts are issued based on the custom forecast frequency specified in the configuration file.""" + return self._customFcstFreq + + @customFcstFreq.setter + def customFcstFreq(self, value): + """Set the custom forecast frequency in minutes specified by the user in the configuration file. This is used to control how often forecasts are issued based on the custom forecast frequency specified in the configuration file.""" + if value is None and not self.precip_only_flag: + value = self.extract_input_variable("CustomFcstFreq") + if len(self.customFcstFreq) != self.number_custom_inputs: err_out_screen( - "Please specify SuppPcpTemporalInterpolation values for each " - "corresponding supplemental precip products in the configuration file." + f"Improper custom_input fcst_freq specified. This number ({len(self.customFcstFreq)}) must match the frequency of custom input forcings selected ({self.number_custom_inputs})." ) - # Ensure the SuppPcpTemporalInterpolation values make sense. - for temporalInterpOpt in self.suppTemporalInterp: - if temporalInterpOpt < 0 or temporalInterpOpt > 2: - err_out_screen( - "Invalid SuppPcpTemporalInterpolation chosen in the configuration file. " - "Please choose a value of 0-2 for each corresponding input forcing" - ) + self._customFcstFreq = value - # Read in max time option - try: - self.supp_pcp_max_hours = cfg_bmi["SuppPcpMaxHours"] - except (KeyError, configparser.NoOptionError): - self.supp_pcp_max_hours = ( - None # if missing, don't care, just assume all time - ) + def _validate_config(self) -> None: + """Validate in options from the configuration file and check that proper options were provided.""" + self.b_date_proc - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper SuppPcpMaxHours options specified in the configuration file.", - e, - ) + # if not self.precip_only_flag: - if type(self.supp_pcp_max_hours) is list: - if len(self.supp_pcp_max_hours) != self.number_supp_pcp: - err_out_screen( - "Number of SuppPcpMaxHours ({}) must match the number " - "of SuppPcp inputs ({}) in the configuration file, or " - "supply a single threshold for all inputs".format( - len(self.supp_pcp_max_hours), self.number_supp_pcp - ) - ) - elif type(self.supp_pcp_max_hours) is float: - # Support 'classic' mode of single threshold - self.supp_pcp_max_hours = [ - self.supp_pcp_max_hours - ] * self.number_supp_pcp + if self.output_freq <= 0: + err_out_screen( + "Please specify an OutputFrequency that is greater than zero minutes." + ) - # Read in the SuppPcpInputOffsets options. - try: - self.supp_input_offsets = cfg_bmi["SuppPcpInputOffsets"] - except KeyError as e: - err_out_screen( - "Unable to locate SuppPcpInputOffsets under SuppForcing section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate SuppPcpInputOffsets under SuppForcing section in the configuration file.", - e, - ) - except json.decoder.JSONDecodeError as e: - err_out_screen( - "Improper SuppPcpInputOffsets option specified in the configuration file.", - e, - ) - if len(self.supp_input_offsets) != self.number_supp_pcp: - err_out_screen( - "Please specify SuppPcpInputOffsets values for each " - "corresponding input forcings for SuppForcing." - ) - # Check to make sure the input offset options make sense. There will be additional - # checking later when input choices are mapped to input products. - for inputOffset in self.supp_input_offsets: - if inputOffset < 0: - err_out_screen( - "Please specify SuppPcpInputOffsets values greater than or equal to zero." - ) + if self.sub_output_hour < 0: + err_out_screen( + "Please specify an SubOutputHour that is greater than zero minutes." + ) + if self.sub_output_hour == 0: + self.sub_output_hour = None - # Read in the optional parameter directory for supplemental precipitation. - try: - self.supp_precip_param_dir = cfg_bmi["SuppPcpParamDir"] - except KeyError as e: - err_out_screen( - "Unable to locate SuppPcpParamDir under the SuppForcing section in the configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate SuppPcpParamDir under the SuppForcing section in the configuration file.", - e, - ) - except ValueError as e: - err_out_screen( - "Improper SuppPcpParamDir option specified in the configuration file.", - e, - ) - if not os.path.isdir(self.supp_precip_param_dir): - try: - os.makedirs(self.supp_precip_param_dir, exist_ok=True) - LOG.debug( - f"Created missing SuppPcpParamDir: {self.supp_precip_param_dir}" - ) - except OSError as e: - err_out_screen( - f"Unable to locate SuppPcpParamDir: {self.supp_precip_param_dir}. Error: {e}" - ) + if self.sub_output_freq < 0: + err_out_screen( + "Please specify an SubOutFreq that is greater than zero minutes." + ) + if self._sub_output_freq == 0: + self.sub_output_freq = None - if not self.precip_only_flag: - # Read in Ensemble information - # Read in CFS ensemble member information IF we have chosen CFSv2 as an input - # forcing. - for opt_tmp in self.input_forcings: - if opt_tmp == 7: - try: - self.cfsv2EnsMember = cfg_bmi["cfsEnsNumber"] - LOG.debug(f"ens mem: {self.cfsv2EnsMember}") - LOG.debug(f"cfg ens mem: {cfg_bmi['cfsEnsNumber']}") - except KeyError as e: - err_out_screen( - "Unable to locate cfsEnsNumber under the Ensembles section of the configuration file", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate cfsEnsNumber under the Ensembles section of the configuration file", - e, - ) - except json.JSONDecodeError as e: - err_out_screen( - "Improper cfsEnsNumber options specified in the configuration file", - e, - ) - if int(self.cfsv2EnsMember) < 1 or int(self.cfsv2EnsMember) > 4: - err_out_screen( - "Please chose an cfsEnsNumber value of 1,2,3 or 4." - ) + # TODO Can this be a /tmp directory? + self.make_scratch_dir() - # Read in information for the custom input NetCDF files that are to be processed. - # Read in the ForecastInputHorizons options. - try: - self.customFcstFreq = cfg_bmi["custom_input_fcst_freq"] - except KeyError as e: - err_out_screen( - "Unable to locate custom_input_fcst_freq under Custom section in configuration file.", - e, - ) - except configparser.NoOptionError as e: - err_out_screen( - "Unable to locate custom_input_fcst_freq under Custom section in configuration file.", - e, - ) - except json.decoder.JSONDecodeError as je: + if self.useCompression not in [0, 1]: + err_out_screen("Please choose a compressOut value of 0 or 1.") + + if self.ana_flag in [0, 1]: + err_out_screen("Please choose a AnAFlag value of 0 or 1.") + + if self.look_back <= 0 and self.look_back != -9999: + err_out_screen("Please specify a positive LookBack or -9999 for realtime.") + + if self.fcst_freq <= 0: + err_out_screen( + "Please specify a ForecastFrequency in the configuration file greater than zero." + ) + # Currently, we only support daily or sub-daily forecasts. Any other iterations should + # be done using custom config files for each forecast cycle. + if self.fcst_freq > 1440: + err_out_screen( + "Only forecast cycles of daily or sub-daily are supported at this time" + ) + + # Read in the ForecastShift option. This is ONLY done for the realtime instance as + # it's used to calculate the beginning of the processing window. + if True: # was: self.realtime_flag: + self.fcst_shift = self.extract_input_variable("ForecastShift") + if self.fcst_shift < 0: err_out_screen( - "Improper custom_input_fcst_freq option specified in configuration file: " - + str(je) + "Please specify a ForecastShift in the configuration file greater than or equal to zero." ) - if len(self.customFcstFreq) != self.number_custom_inputs: + + # Calculate the beginning/ending processing dates if we are running realtime + if self.realtime_flag: + calculate_lookback_window(self) + + # if self.refcst_flag: + # Calculate the number of forecasts to issue, and verify the user has chosen a + # correct divider based on the dates + # dt_tmp = self.e_date_proc - self.b_date_proc + # if (dt_tmp.days * 1440 + dt_tmp.seconds / 60.0) % self.fcst_freq != 0: + # err_out_screen('Please choose an equal divider forecast frequency for your ' + # 'specified reforecast range.') + # self.nFcsts = int((dt_tmp.days * 1440 + dt_tmp.seconds / 60.0) / self.fcst_freq) + + # Flag to constrain AORC forcing data cycle output + # for optTmp in self.input_forcings: + # if optTmp == 12: + # self.nFcsts = 1 + self.nFcsts = 1 + + if self.look_back != -9999: + calculate_lookback_window(self) + + # Process geospatial information + + if len(self.spatial_meta) == 0: + # No spatial metadata file found. + self.spatial_meta = None + else: + if not os.path.isfile(self.spatial_meta): err_out_screen( - f"Improper custom_input fcst_freq specified. " - f"This number ({len(self.customFcstFreq)}) must " - f"match the frequency of custom input forcings selected " - f"({self.number_custom_inputs})." + "Unable to locate optional spatial metadata file: " + + self.spatial_meta ) + # Calculate the beginning/ending processing dates if we are running realtime + if self.realtime_flag: + calculate_lookback_window(self) + + # Create temporary array to hold flags if we need input parameter files. + param_flag = np.zeros([len(self.input_forcings)], int) + + count_tmp = 0 + for optTmp in self.precipDownscaleOpt: + if optTmp == 1: + param_flag[count_tmp] = 1 + count_tmp = count_tmp + 1 + + for suppOpt in self.supp_precip_forcings: + if suppOpt not in list(range(1, self.supp_precip_count + 1)): + err_out_screen( + f"Please specify SuppForcing values between 1 and {self.supp_precip_count}." + ) + @property def nwm_domain(self) -> str: """Extract NWM domain from the geogrid filename, using regex pattern.""" From 3bbfe2acc06250551e6185f3aec9f9236aca8629 Mon Sep 17 00:00:00 2001 From: Matthew Deshotel Date: Wed, 13 May 2026 11:51:47 -0400 Subject: [PATCH 032/132] fix setter logic; update doc strings --- .../NextGen_Forcings_Engine/core/config.py | 1084 +++++++++-------- .../NextGen_Forcings_Engine/core/consts.py | 65 +- 2 files changed, 636 insertions(+), 513 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 4c7b3add..69cb32f7 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -5,7 +5,6 @@ import re import uuid from datetime import datetime, timedelta, timezone -from functools import cached_property # Use the Error, Warning, and Trapping System Package for logging import numpy as np @@ -39,15 +38,17 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No geogrid (str, optional): The filepath to the geogrid file to be used for processing. This is used to specify the grid information for regridding input forcings. If not provided, it will be read from the configuration file. """ + if geogrid is not None: + self.user_provided_geogrid_flag = True + else: + self.user_provided_geogrid_flag = False + + self.b_date_proc = b_date + self._cfg_bmi = cfg_bmi + self._geogrid = geogrid + self.bmi_time_index = 0 self.precip_only_flag = False - self.number_custom_inputs = 0 - self.useCompression = 0 - self.useFloats = 0 - self._b_date_proc = b_date - self._cfg_bmi = cfg_bmi - self.runCfsNldasBiasCorrect = False - self.rqiThresh = 1.0 self.globalNdv = -9999.0 self.d_program_init = datetime.now(timezone.utc) self.errFlag = 0 @@ -58,16 +59,56 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4" ) self.nwm_source = "s3://noaa-nwm-retrospective-3-0-pds" - self._geogrid = geogrid + self.broadcast_new_64bit_uid() self._scratch_dir_has_been_uniquefied = False + # Create temporary array to hold flags if we need input parameter files. + self.param_flag = np.zeros([len(self.input_forcings)], int) # set list of attibutes from consts.py to None. # These are indexed from the consts dictionary using the class name for attr in CONFIGOPTIONS[self.__class__.__name__]: setattr(self, attr, None) - self._validate_config() + + self.set_attrs(self.try_config_get_except_attr_map) + self.supp_precip_forcings = self.extract_input_variable("SuppPcp") + self.set_attrs(CONFIGOPTIONS["extract_input_variable_attrs_map"]) + + if self.precip_only_flag: + self.set_attrs( + CONFIGOPTIONS["extract_input_variable_attrs_map_precip_only"] + ) + else: + self.set_attrs( + CONFIGOPTIONS["extract_input_variable_attrs_map_not_precip_only"] + ) + if 27 in self.input_forcings: + self.nwm_geogrid = self.extract_input_variable("NWMGeogridIn") + + if self.perform_downscaling: + self.set_attrs(CONFIGOPTIONS["downscaling_attrs_map"]) + if self.grid_type == "unstructured": + self.set_attrs(CONFIGOPTIONS["downscaling_unstructred_attrs_map"]) + + for cfg_bmi_attr, config_options_attr in CONFIGOPTIONS[ + "extract_input_variable_set_default_attrs_map" + ].items(): + setattr( + self, + config_options_attr, + self.extract_input_variable_set_default(cfg_bmi_attr), + ) + + @property + def try_config_get_except_attr_map(self) -> dict: + """Get the mapping of configuration variable names to class attribute names for variables that are extracted directly from the configuration file without any additional processing. This is used to control how variables are extracted from the configuration file and assigned to class attributes in a consistent way based on the mapping specified in the consts.py file.""" + dict_map = CONFIGOPTIONS["try_config_get_except_attr_map"] + if self._b_date_proc is not None: + dict_map.pop("b_date_proc") + if self._geogrid is not None: + dict_map.pop("geogrid") + return dict_map @property def cfg_bmi(self) -> dict: @@ -81,7 +122,6 @@ def cfg_bmi(self, value: dict) -> None: raise TypeError( f"Expected dict, got {type(value)} for type of cfg_bmi: {value}" ) - self._validate_config() self._cfg_bmi = value @property @@ -108,20 +148,13 @@ def precip_only_flag(self) -> bool: if int(self.supp_precip_forcings[0]) == 14: return True - def set_attrs(self): + def set_attrs(self, attrs_dict: dict): """Set the attributes of the class based on the configuration file. This is used to populate the attributes of the class after they have been read in and validated from the configuration file.""" - for cfg_bmi_attr, config_options_attr in CONFIGOPTIONS[ - "cfg_bmi_to_attrs_map" - ].items(): + for cfg_bmi_attr, config_options_attr in attrs_dict.items(): setattr( self, config_options_attr, self.extract_input_variable(cfg_bmi_attr) ) - if self.output_freq <= 0: - err_out_screen( - "Please specify an OutputFrequency that is greater than zero minutes." - ) - def extract_input_variable(self, variable_name: str) -> str: """Extract the variable name from the configuration file for a given variable.""" try: @@ -154,10 +187,10 @@ def extract_input_variable_set_default(self, variable_name: str, default=0) -> s err_out_screen(f"Please choose a {variable_name} value of 0 or 1.") return variable - def try_config_get(self, variable_name: str, default=None) -> str: + def try_config_get(self, variable_name: str) -> str: """Try to get a variable from the configuration file, and return a default value if it is not found.""" try: - var = self.cfg_bmi.get(variable_name, default) + var = self.cfg_bmi.get(variable_name) if var is None: err_out_screen( f"Unable to locate {variable_name} in the configuration file." @@ -184,7 +217,7 @@ def check_number_of_inputs_forcings(self, value: list, variable_name: str) -> No def check_number_of_inputs_supp_pcp(self, value: list, variable_name: str) -> None: """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable, specifically for supplemental precip forcing variables which should match the number of supplemental precip forcing options specified by the user in the configuration file.""" return self.check_number_of_inputs( - value, variable_name, " supplemental precip forcings" + value, variable_name, " SupplementalPrecipForcings" ) def check_input_values_in_range( @@ -222,12 +255,11 @@ def uniquefy_scratch_dir_as_child(self, uid: str) -> None: ) self.scratch_dir = os.path.join(self.scratch_dir, uid) self._scratch_dir_has_been_uniquefied = True - self.make_scratch_dir() - def make_scratch_dir(self) -> None: + def make_scratch_dir(self, scratch_dir: str) -> None: """Make the scratch dir and its parents.""" - os.makedirs(self.scratch_dir, exist_ok=True) - LOG.debug(f"Scratch dir: {self.scratch_dir}") + os.makedirs(scratch_dir, exist_ok=True) + LOG.debug(f"Scratch dir: {scratch_dir}") def broadcast_new_64bit_uid(self) -> None: """Broadcast a random uint64 then save the hash of that to self.uid64, which effectively broadcasts the same unique string to all ranks. @@ -238,16 +270,209 @@ def broadcast_new_64bit_uid(self) -> None: raise RuntimeError("self.uid64 has already been initialized.") self.uid64 = mpi_utils.get_new_broadcasted_uid() + @property + def supp_precip_forcings(self): + """Choose a set of supplemental precipitation file(s) to layer into the final LDASIN forcing files processed from the options above. The following is a mapping of numeric values to external input native forcing files. + + 1. MRMS GRIB2 hourly radar-only QPE + 2. MRMS GRIB2 hourly gage-corrected radar QPE + 3. WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. + 4. WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. + 5. CONUS MRMS GRIB2 hourly MultiSensor QPE (Pass 2 or Pass 1) + 6. Hawaii MRMS GRIB2 hourly MultiSensor QPE (Pass 2 or Pass 1) + 7. MRMS SBCv2 Liquid Water Fraction (netCDF only) + 8. NBM Conus MR + 9. NBM Alaska MR + 10. Alaska MRMS (no liquid water fraction) + 11. Alaska Stage IV NWS Precip + 12. CONUS Stage IV NWS Precip + 13. MRMS PrecipFlag precipitation classification file + 14. Custom Frequency Supplementary Precipitation product (sub-hourly precip) + 15. NBM Puerto Rico + 16. NBM Hawaii + - Example- SuppPcp: [1, 5, 13] + """ + return self._supp_precip_forcings + + @supp_precip_forcings.setter + def supp_precip_forcings(self, value: list) -> None: + """Set the list of supplemental precip forcing options specified by the user in the configuration file. This is used to control which supplemental precip forcings are processed and how they are processed based on the other configuration options specified for each supplemental precip forcing.""" + self.check_input_values_in_range( + value, + "SuppPcp", + list(range(1, self.supp_precip_count + 1)), + ) + self._supp_precip_forcings = value + + @property + def output_freq(self) -> int: + """Get the output frequency in minutes specified by the user in the configuration file. This is used to control the output frequency of the processed forcings, and is necessary for both realtime and reforecast simulations.""" + return self._output_freq + + @output_freq.setter + def output_freq(self, value: int) -> None: + """Specify the output frequency in minutes. Note that any frequencies at higher intervals than what if provided as input will entail input forcing data being temporally interpolated. + + Example- OutputFrequency: 60 + """ + self.check_input_values_positive([value], "OutputFrequency") + self._output_freq = value + + @property + def sub_output_hour(self) -> int: + """Get the sub-daily output hour specified by the user in the configuration file. This is used to control the output frequency of the processed forcings for sub-daily output frequencies, and is only necessary if the user has chosen a sub-daily output frequency in the configuration file.""" + return self._sub_output_hour + + @sub_output_hour.setter + def sub_output_hour(self, value: int) -> None: + """Sub output hour. + + New variable currently for NWMv3.1 operations to properly ingest GFS 13km forecast data that outputs various frequencies throughout the forecast cycle lifetime. This variable will properly account for reading time slices of the forecast cycle. Currently only needed for GFS 13km operational configuration. Otherwise, set this value to 0. + + Example- SubOutputHour: 0 + """ + self.check_input_values_positive([value], "SubOutputHour") + if value < 0: + err_out_screen( + "Please specify an SubOutputHour that is greater than zero minutes." + ) + if value == 0: + value = None + self._sub_output_hour = value + + @property + def sub_output_freq(self) -> int: + """Calculate the sub-daily output frequency in minutes based on the output frequency and sub-daily output hour specified by the user in the configuration file. This is used to control the output frequency of the processed forcings for sub-daily output frequencies, and is only necessary if the user has chosen a sub-daily output frequency in the configuration file.""" + return self._sub_output_freq + + @sub_output_freq.setter + def sub_output_freq(self, value: int) -> None: + """Sub output frequency. + + New variable currently for NWMv3.1 operations to properly ingest GFS 13km forecast data that outputs various frequencies throughout the forecast cycle lifetime. This variable will properly account for reading time slices of the forecast cycle. Currently only needed for GFS 13km operational configuration. Otherwise, set this value to 0. + + Example- SubOutputFreq: 0 + """ + if value < 0: + err_out_screen( + "Please specify an SubOutFreq that is greater than zero minutes." + ) + if value == 0: + value = None + self._sub_output_freq = value + + @property + def scratch_dir(self) -> str: + """Specify a scratch directory that will be used for storage of temporary files. These files will be removed automatically by the program. at the end of the BMI instance. However, this directory will also store the output forcing file if requested by the user as well (will not be deleted in this instance). + + Example- ScratchDir: "./ScratchDir + """ + return self._scratch_dir + + @scratch_dir.setter + def scratch_dir(self, value: str) -> None: + """Set the pathway to the scratch directory specified by the user in the configuration file. This is used to control where intermediate files are written during processing, and is necessary for both realtime and reforecast simulations.""" + self.make_scratch_dir(value) + self._scratch_dir = value + + @property + def useCompression(self) -> int: + """Flag to activate scale_factor / add_offset byte packing in the output files. 0 - Deactivate compression 1 - Activate compression, Only applicable in this instance when you request a netcdf output forcing file (Output: 1). Otherwise, just set to 0. + + Example- compressOutput: 0 + """ + return self._useCompression + + @useCompression.setter + def useCompression(self, value: int) -> None: + """Set the flag for whether to use compression when writing output files specified by the user in the configuration file. This is used to control whether output files are compressed, which can save disk space but may increase processing time.""" + if value is None: + value = 0 + self.check_input_values_in_range([value], "compressOutput", [0, 1]) + self._useCompression = value + + @property + def ana_flag(self) -> int: + """If this is AnA run, set AnAFlag to 1, otherwise 0. Setting this flag will change the behavior of some Bias Correction routines as the ForecastInputOffsets options. + + Example- AnAFlag: 1 + """ + return self._ana_flag + + @ana_flag.setter + def ana_flag(self, value: int) -> None: + """Set the flag for whether to include the analysis time step in the output files specified by the user in the configuration file. This is used to control whether the analysis time step is included in the output files, which can be useful for certain applications but may not be necessary for all users.""" + value = int(value) + self.check_input_values_in_range([value], "AnAFlag", [0, 1]) + self._ana_flag = value + + @property + def look_back(self) -> int: + """Specify a lookback period in minutes to process data. This is required if you are only processing an AnA operational configuration. This value should specify how far back you need to look in time from your "RefcstBDateProc" start date that you specified. In this instance, that start date will be your actual end date. If no LookBack specified, please specify -9999. + + Example- LookBack: 180 + """ + return self._look_back + + @look_back.setter + def look_back(self, value: int) -> None: + """Set the look back window in hours specified by the user in the configuration file. This is used to calculate the processing window for reforecast simulations, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" + if value <= 0 and value != -9999: + err_out_screen("Please specify a positive LookBack or -9999 for realtime.") + if value != -9999: + calculate_lookback_window(self) + self._look_back = value + + @property + def fcst_freq(self) -> int: + """Specify a forecast frequency in minutes. This value specifies how often to generate a set of forecast forcings. If generating hourly retrospective forcings, specify this value to be 60. + + Example- ForecastFrequency: 60 + """ + return self._fcst_freq + + @fcst_freq.setter + def fcst_freq(self, value: int) -> None: + """Set the forecast frequency in hours specified by the user in the configuration file. This is used to calculate the processing window for reforecast simulations, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" + self.check_input_values_positive([value], "ForecastFrequency") + if value > 1440: + err_out_screen( + "Only forecast cycles of daily or sub-daily are supported at this time" + ) + self._fcst_freq = value + + @property + def spatial_meta(self): + """Specify the optional land spatial metadata file. If found, coordinate projection information and coordinate will be translated from to the final output file. This variable is only a special case if the user is specifying the original WRF-Hydro domain from earlier NWM versions. Otherwise, just leave the one blank (''). + + Example- SpatialMetaIn: ./GEOGRID_LDASOUT_Spatial_Metadata_CONUS.nc + """ + return self._spatial_meta + + @spatial_meta.setter + def spatial_meta(self, value: str) -> None: + """Set the spatial metadata options specified by the user in the configuration file. This is used to control how spatial metadata is handled during processing, and is necessary for both realtime and reforecast simulations.""" + if len(value) == 0: + # No spatial metadata file found. + value = None + else: + if not os.path.isfile(value): + err_out_screen( + f"Unable to locate optional spatial metadata file: {value}." + ) + self._spatial_meta = value + @property def b_date_proc(self) -> str: - """Get the beginning date of processing for reforecast simulations. This is used to calculate the processing window for reforecast simulations, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" + """If running an operational configuration in realtime or just using a retrospective dataset (NWM, AORC, ERA5), this will be the defined start date for the NextGen Forcing Engine BMI which is assumed to be the beginning of the forecast cycle (i.e. hour 0) or just the start date of the retrospective dataset. From there the first time step will be hour 1 from the start date specified here. If you're running an AnA configuration however, this variable becomes the end date of the simulation and the "LookBack" value specified above will be how far back you look in time for the AnA operational configuration. + + Example- RefcstBDateProc: 202210071400 + """ return self._bdate_proc @b_date_proc.setter def b_date_proc(self, value: str | datetime) -> None: - """Set the beginning date of processing for reforecast simulations. This is used to calculate the processing window for reforecast simulations, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" - if value is None: - value = self.try_config_get("RefcstBDateProc") + """Set the beginning date of processing for reforecast simulations. This is used to calculate the processing window for reforecast simulations.""" if isinstance(value, datetime): self._b_date_proc = value if value != -9999: @@ -270,11 +495,15 @@ def b_date_proc(self, value: str | datetime) -> None: def realtime_flag(self) -> bool: """Flag to indicate whether the user has chosen to run a realtime simulation, which will trigger some different processing pathways and error checking for certain configuration options, and will also control how the processing window is calculated.""" if self.look_back == -9999: - return False + value = False elif self.b_date_proc == -9999: - return True + value = True else: - return False + value = False + # Calculate the beginning/ending processing dates if we are running realtime + if value: + calculate_lookback_window(self) + return value @property def refcst_flag(self) -> bool: @@ -294,34 +523,31 @@ def geopackage(self) -> str: @geopackage.setter def geopackage(self, value: str) -> None: """Set the pathway to the geopackage file to be used for processing. This is used to specify the grid information for regridding input forcings, and is only necessary if the user is running a simulation that requires regridding of input forcings.""" - if value is not None: - self._geopackage = value - else: - self._geopackage = self.try_config_get("Geopackage") + self._geopackage = value @property def geogrid(self) -> str: - """Get the pathway to the geogrid file to be used for processing. This is used to specify the grid information for regridding input forcings, and is only necessary if the user is running a simulation that requires regridding of input forcings.""" + """Specify a geogrid file (e.g. latitude, longitude, mesh connectivity, elevation, slope) that defines domain to which the forcings are being processed to. + + Example- GeogridIn: ./geo_em_CONUS.nc + """ return self._geogrid @geogrid.setter def geogrid(self, value: str) -> None: """Set the pathway to the geogrid file to be used for processing. This is used to specify the grid information for regridding input forcings, and is only necessary if the user is running a simulation that requires regridding of input forcings.""" - if value is not None: + if self.user_provided_geogrid_flag: self._geogrid = value + if value is None: + err_out_screen("Unable to locate GeogridIn in the configuration file.") else: - geogrid_base = self.try_config_get("GeogridIn") - if geogrid_base is None: - err_out_screen("Unable to locate GeogridIn in the configuration file.") - self.geogrid = None - else: - geogrid_parent = os.path.dirname(geogrid_base) - geogrid_filename = os.path.basename(geogrid_base) - if self.uid64 is None: - raise ValueError("self.uid64 cannot be None, please initialize it.") - self._geogrid = os.path.join( - geogrid_parent, f"{self.uid64}_{geogrid_filename}" - ) + geogrid_parent = os.path.dirname(value) + geogrid_filename = os.path.basename(value) + if self.uid64 is None: + raise ValueError("self.uid64 cannot be None, please initialize it.") + self._geogrid = os.path.join( + geogrid_parent, f"{self.uid64}_{geogrid_filename}" + ) self.try_make_dir(geogrid_parent, " esmf_mesh") def try_make_dir(self, directory: str, optional_str: str = "") -> None: @@ -336,21 +562,18 @@ def try_make_dir(self, directory: str, optional_str: str = "") -> None: ) @property - def input_forcing_options(self) -> list: + def input_forcings(self) -> list: """Get the list of input forcing options specified by the user in the configuration file. This is used to control which input forcings are processed and how they are processed based on the other configuration options specified for each input forcing.""" - return self._input_forcing_options + return self._input_forcings - @input_forcing_options.setter - def input_forcing_options(self, value: list) -> None: + @input_forcings.setter + def input_forcings(self, value: list) -> None: """Set the list of input forcing options specified by the user in the configuration file. This is used to control which input forcings are processed and how they are processed based on the other configuration options specified for each input forcing.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("InputForcings") if not self.precip_only_flag: - for force_opt in value: - self.check_input_values_in_range( - value, "InputForcings", list(range(1, self.force_count + 1)) - ) - self._input_forcing_options = value + self.check_input_values_in_range( + value, "InputForcings", list(range(1, self.force_count + 1)) + ) + self._input_forcings = value @property def number_inputs(self) -> int: @@ -360,30 +583,32 @@ def number_inputs(self) -> int: err_out_screen( "Please choose at least one InputForcings dataset to process" ) - return len(self.input_forcing_options) + return len(self.input_forcings) @property def number_custom_inputs(self) -> int: """Calculate the number of custom input forcing options specified by the user in the configuration file. This is used to control the flow of the program based on how many custom input forcings are being processed, since custom input forcings require some different processing pathways.""" if not self.precip_only_flag: count = 0 - for force_opt in self.input_forcing_options: + for force_opt in self.input_forcings: if force_opt == 10: count += 1 return count + else: + return 0 @property def nwm_geogrid(self) -> str: - """Get the pathway to the NWM geogrid file specified by the user in the configuration file. This is used to specify the grid information for regridding NWM input forcings, and is only necessary if the user has chosen to regrid NWM input forcings in the configuration file.""" + """Only for the NWM v3 retorspective forcing module option (27) that requires the geo_em_NWM_DOMAIN.nc file as input for the NextGen Forcings Engine to properly setup up the ESMF grid object for the NWM forcing files since that information is not readily available in the NWM v3 retrospective forcing files.""" return self._nwm_geogrid @nwm_geogrid.setter def nwm_geogrid(self, value: str) -> None: """Set the pathway to the NWM geogrid file specified by the user in the configuration file. This is used to specify the grid information for regridding NWM input forcings, and is only necessary if the user has chosen to regrid NWM input forcings in the configuration file.""" - if value is None and not self.precip_only_flag: - if 27 in self.input_forcing_options: - value = self.extract_input_variable("NWM_Geogrid") - self._nwm_geogrid = value + if not self.precip_only_flag and 27 in self.input_forcings: + self._nwm_geogrid = value + else: + self._nwm_geogrid = None @property def input_force_types(self) -> list: @@ -392,9 +617,10 @@ def input_force_types(self) -> list: @input_force_types.setter def input_force_types(self, value: list) -> None: - """Set the list of input forcing file types specified by the user in the configuration file. This is used to control how input forcings are read in and processed based on the file type specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("InputForcingTypes") + """Specify the file type for each forcing (comma separated). Valid types are GRIB1, GRIB2, NETCDF, and NETCDF4. + + Example- InputForcingTypes: [GRIB2,GRIB2]\ + """ if not self.precip_only_flag: if value == [""]: value = [] @@ -403,6 +629,8 @@ def input_force_types(self, value: list) -> None: value, "InputForcingTypes", self.file_types ) self._input_force_types = value + else: + self._input_force_types = None @property def file_types(self): @@ -416,9 +644,10 @@ def input_force_dirs(self) -> list: @input_force_dirs.setter def input_force_dirs(self, value: list) -> None: - """Set the list of input forcing directories specified by the user in the configuration file. This is used to control where input forcings are read in from for each input forcing specified by the user in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("InputForcingDirectories") + """Specify the input directories for each forcing product. If a user has the ability to connect to the AWS servers and they specify configuration #12 (CONUS AORC data) or configuration #27 (NWM retrospective forcing data) then this specific configuration input can be left as a blank string (""). + + Example- InputForcingDirectories: [./GFS,./NDFD] + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "InputForcingDirectories") # Loop through and ensure all input directories exist. Also strip out any whitespace @@ -426,7 +655,7 @@ def input_force_dirs(self, value: list) -> None: for dir_tmp in range(0, len(value)): value[dir_tmp] = value[dir_tmp].strip() dir_path = value[dir_tmp] - forcing_type = self.input_forcing_options[dir_tmp] + forcing_type = self.input_forcings[dir_tmp] is_aws_forcing = forcing_type in [12, 21, 27] if not os.path.isdir(dir_path): @@ -435,6 +664,8 @@ def input_force_dirs(self, value: list) -> None: else: self.try_make_dir(dir_path, " forcing") self._input_force_dirs = value + else: + self._input_force_dirs = None def input_force_mandatory(self) -> list: """Get the list of input forcing mandatory flags specified by the user in the configuration file. This is used to control whether the program should raise an error if input forcings for a given forecast cycle are not found for each input forcing specified by the user in the configuration file.""" @@ -442,12 +673,16 @@ def input_force_mandatory(self) -> list: @input_force_mandatory.setter def input_force_mandatory(self, value: list) -> None: - """Set the list of input forcing mandatory flags specified by the user in the configuration file. This is used to control whether the program should raise an error if input forcings for a given forecast cycle are not found for each input forcing specified by the user in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("InputMandatory") + """Specify whether the input forcings listed above are mandatory, or optional. This is important for layering contingencies if a product is missing, but forcing files are still desired. 0 - Not mandatory, 1 - Mandatory. NOTE!!! If no files are found for any products, code will error out indicating the final field is all missing values. + + Example- InputMandatory: [1,1] + """ + if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "InputMandatory") self.check_input_values_in_range(value, "InputMandatory", [0, 1]) - self._input_force_mandatory = value + self._input_force_mandatory = value + else: + self._input_force_mandatory = None def customSuppPcpFreq(self) -> int: """Get the custom supplemental precip output frequency specified by the user in the configuration file. This is used to control the output frequency of supplemental precip forcings if the user has chosen to run the supplemental precip forcings module only.""" @@ -456,81 +691,85 @@ def customSuppPcpFreq(self) -> int: @customSuppPcpFreq.setter def customSuppPcpFreq(self, value: int) -> None: """Set the custom supplemental precip output frequency specified by the user in the configuration file. This is used to control the output frequency of supplemental precip forcings if the user has chosen to run the supplemental precip forcings module only.""" - if value is None and self.precip_only_flag: - value = self.extract_input_variable("customSuppPcpFreq") + if self.precip_only_flag: self.check_input_values_positive([value], "customSuppPcpFreq") - self._customSuppPcpFreq = value - - @property - def include_lqfrac(self): - """Get the flag for whether to include the liquid/solid precipitation fraction variable in the output files specified by the user in the configuration file. This is used to control whether the liquid/solid precipitation fraction variable is included in the output files.""" - return self._include_lqfrac - - @include_lqfrac.setter - def include_lqfrac(self, value): - """Set the flag for whether to include the liquid/solid precipitation fraction variable in the output files specified by the user in the configuration file. This is used to control whether the liquid/solid precipitation fraction variable is included in the output files.""" - if value is None: - value = self.extract_input_variable_set_default("includeLQFrac", default=0) - - @property - def include_lqfrac(self): - """Get the flag for whether to include the liquid/solid precipitation fraction variable in the output files specified by the user in the configuration file. This is used to control whether the liquid/solid precipitation fraction variable is included in the output files.""" - return self._include_lqfrac - - @include_lqfrac.setter - def include_lqfrac(self, value): - if value is None: - value = self.extract_input_variable_set_default("includeLQFrac", default=0) - self._include_lqfrac = value - - @property - def forcing_output(self) -> int: - """Get the flag for whether to output the input forcings specified by the user in the configuration file. This is used to control whether the input forcings are output in addition to the processed forcings.""" - return self._forcing_output - - @forcing_output.setter - def forcing_output(self, value: int) -> None: - if value is None: - value = self.extract_input_variable_set_default("Output", default=0) - self._forcing_output = value + self._customSuppPcpFreq = value + else: + self._customSuppPcpFreq = None def fcst_shift(self) -> int: - """Get the forecast shift specified by the user in the configuration file. This is used to control the calculation of the processing window for realtime simulations.""" + """Forecast cycles are determined by splitting up a day by equal ForecastFrequency interval. If there is a desire to shift the cycles to a different time step, ForecastShift will shift forecast cycles ahead by a determined set of minutes. For example, ForecastFrequency of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, a ForecastShift of 1 hour will produce forecast cycles at 01, 07, 13, and 18 UTC. NOTE - This is only used by the realtime instance to calculate forecast cycles accordingly. Re-forecasts will use the beginning and ending dates specified in conjunction with the forecast frequency to determine forecast cycle dates. + + Example- ForecastShift: 0 + """ return self._fcst_shift @fcst_shift.setter def fcst_shift(self, value: int) -> None: + if True: # was: self.realtime_flag: + self.check_input_values_positive([value], "ForecastShift") + # Calculate the beginning/ending processing dates if we are running realtime + if self.realtime_flag: + calculate_lookback_window(self) + self._fcst_shift = value + + # NOTE this commented out code copied from pre-refactored code on 5/6/2026 + # if self.refcst_flag: + # Calculate the number of forecasts to issue, and verify the user has chosen a + # correct divider based on the dates + # dt_tmp = self.e_date_proc - self.b_date_proc + # if (dt_tmp.days * 1440 + dt_tmp.seconds / 60.0) % self.fcst_freq != 0: + # err_out_screen('Please choose an equal divider forecast frequency for your ' + # 'specified reforecast range.') + # self.nFcsts = int((dt_tmp.days * 1440 + dt_tmp.seconds / 60.0) / self.fcst_freq) + + # Flag to constrain AORC forcing data cycle output + # for optTmp in self.input_forcings: + # if optTmp == 12: + # self.nFcsts = 1 + + @property + def nFcsts(self): + """Get the number of forecasts to issue for a reforecast simulation based on the forecast shift and the processing window specified by the user in the configuration file. This is used to control how many forecast time steps are output for a reforecast simulation, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" + return self._nFcsts + + @nFcsts.setter + def nFcsts(self, value: int) -> None: + """Set the number of forecasts to issue for a reforecast simulation based on the forecast shift and the processing window specified by the user in the configuration file. This is used to control how many forecast time steps are output for a reforecast simulation, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" if value is None: - value = self.extract_input_variable("ForecastShift") - self.check_input_values_positive([value], "ForecastShift") - self._fcst_shift = value + value = 1 + self._nFcsts = value @property def fcst_input_horizons(self) -> list: - """Get the list of forecast input horizons specified by the user in the configuration file. This is used to control the calculation of the forecast cycle length and the processing of input forcings based on the forecast time horizons specified for each input forcing.""" + """Specify how much (in minutes) of each input forcing is desires for each forecast cycle. See documentation for examples. The length of this array must match the input forcing choices. + + - Example- ForecastInputHorizons: [60, 60] + """ return self._fcst_input_horizons @fcst_input_horizons.setter def fcst_input_horizons(self, value: list) -> None: - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("ForecastInputHorizons") if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "ForecastInputHorizons") self.check_input_values_positive(value, "ForecastInputHorizons") else: - if value is None: - value = self.extract_input_variable("ForecastInputHorizons") + if len(self.fcst_input_horizons) != 1: + err_out_screen( + "Please specify ForecastInputHorizon values for each corresponding input forcings for forecasts." + ) self._fcst_input_horizons = value @property def fcst_input_offsets(self): - """Get the list of forecast input offsets specified by the user in the configuration file. This is used to control the calculation of the processing window for both realtime and reforecast simulations based on the forecast time horizons and input offsets specified for each input forcing.""" + """Option for applying an offset to input forcings to use a different forecasted interval. For example, a user may wish to use 4-5 hour forecasted fields from an NWP grid from one of their input forcings. In that instance the offset would be 4 hours, but 0 for other remaining forcings. + + Example- ForecastInputOffsets: [0, 0] + """ return self._fcst_input_offsets @fcst_input_offsets.setter def fcst_input_offsets(self, value: list) -> None: - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("ForecastInputOffsets") if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "ForecastInputOffsets") self.check_input_values_positive(value, "ForecastInputOffsets") @@ -549,6 +788,7 @@ def cycle_length_minutes(self) -> int: ) return cycle_len + @property def num_output_steps(self) -> int: """Calculate the number of output time steps per forecast cycle based on the forecast cycle length and the output frequency specified by the user in the configuration file.""" if self.sub_output_hour is None: @@ -564,11 +804,13 @@ def num_output_steps(self) -> int: ) return num_steps + @property def num_supp_output_steps(self) -> int: """Calculate the number of supplemental precip output time steps per forecast cycle based on the forecast cycle length and the custom supplemental precip output frequency specified by the user in the configuration file.""" if self.precip_only_flag: return int(self.cycle_length_minutes / self.customSuppPcpFreq) + @property def actual_output_steps(self) -> int: """Calculate the actual number of output time steps per forecast cycle based on whether the user has chosen to run a reforecast simulation with a specified processing window, which will only output time steps for which input forcings are available based on the processing window and forecast time horizons specified by the user in the configuration file.""" if self.ana_flag: @@ -578,16 +820,17 @@ def actual_output_steps(self) -> int: @property def grid_type(self) -> str: - """Get the grid type specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings based on the grid type specified by the user in the configuration file.""" + """Tells the NextGen Forcings Engine BMI which grid type the engine is initalizing as a BMI instance. This is a required field and the proper string values should be "gridded", "hydrofabric", or "unstructured". + + Example- GRID_TYPE: "gridded" + """ return self._grid_type @grid_type.setter def grid_type(self, value: str) -> None: """Set the grid type specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings based on the grid type specified by the user in the configuration file.""" - if value is None: - value = self.extract_input_variable("GRID_TYPE") self.check_input_values_in_range( - [value], "GRID_TYPE", ["gridded", "unstructured", "hydrofabric"] + [value.lower()], "GRID_TYPE", ["gridded", "unstructured", "hydrofabric"] ) self._grid_type = value.lower() @@ -599,7 +842,10 @@ def raise_grid_type_error(self, grid_type: str, variable_name: str) -> None: @property def lon_var(self) -> str: - """Get the longitude variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen a gridded grid type in the configuration file.""" + """Naming convention of the longitude variable within the "GeogridIn" file the user has specified. Variable naming convention ONLY for gridded domain configurations. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. In the case for "gridded" domain configuration options and a user specifying downscaling options while only specifying a height variable feature on the grid, this netcdf variable (LONVAR) is then EXPECTED to contain a netcdf metadata attribute called "dx" that specifies the grid spacing in the longtiudinal direction. Otherwise, it will throw an error and not be able to calculate the slope and tilt of each grid cell. + + Example- LONVAR: "XLONG_M" + """ if self.grid_type == "gridded": return self.extract_input_variable("LONVAR") else: @@ -607,7 +853,10 @@ def lon_var(self) -> str: @property def lat_var(self) -> str: - """Get the latitude variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen a gridded grid type in the configuration file.""" + """Naming convention of the latitude variable within the "GeogridIn" file the user has specified. Variable naming convention ONLY for gridded domain configurations. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. In the case for "gridded" domain configuration options and a user specifying downscaling options while only specifying a height variable feature on the grid, this netcdf variable (LATVAR) is then EXPECTED to contain a netcdf metadata attribute called "dy" that specifies the grid spacing in the latitudinal direction. Otherwise, it will throw an error and not be able to calculate the slope and tilt of each grid cell. + + Example- LATVAR: "XLAT_M" + """ if self.grid_type == "gridded": return self.extract_input_variable("LATVAR") else: @@ -615,7 +864,10 @@ def lat_var(self) -> str: @property def nodecoords_var(self) -> str: - """Get the node coordinates variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen an unstructured or hydrofabric grid type in the configuration file.""" + """Naming convention of the node coordinates variable within the "GeogridIn" file the user has specified for ONLY an unstructured mesh or the NextGen hydrofabric. This is a 2-D array stating the latitude and longitude coordinates for all the nodes in the mesh. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. + + Example- NodeCoods: "nodecoords" + """ if self.grid_type in ["unstructured", "hydrofabric"]: return self.extract_input_variable("NodeCoords") else: @@ -623,7 +875,10 @@ def nodecoords_var(self) -> str: @property def elemcoords_var(self) -> str: - """Get the element coordinates variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen an unstructured or hydrofabric grid type in the configuration file.""" + """Naming convention of the element coordinates variable within the "GeogridIn" file the user has specified for ONLY an unstructured mesh or the NextGen hydrofabric. This is a 2-D array stating the latitude and longitude coordinates for all the elements in the mesh. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. + + Example- ElemCoods: "elemcoords" + """ if self.grid_type in ["unstructured", "hydrofabric"]: return self.extract_input_variable("ElemCoords") else: @@ -631,7 +886,10 @@ def elemcoords_var(self) -> str: @property def elemconn_var(self) -> str: - """Get the element connectivity variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen an unstructured or hydrofabric grid type in the configuration file.""" + """Naming convention of the element connectivity variable within the "GeogridIn" file the user has specified for ONLY an unstructured mesh or the NextGen hydrofabric. This is a 2-D array stating the node ids for each element connecting the entire mesh structure. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. + + Example- ElemConn: "elemconn" + """ if self.grid_type in ["unstructured", "hydrofabric"]: return self.extract_input_variable("ElemConn") else: @@ -639,7 +897,10 @@ def elemconn_var(self) -> str: @property def numelemconn_var(self) -> str: - """Get the number of element connectivity variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen an unstructured or hydrofabric grid type in the configuration file.""" + """Naming convention of the number of nodes per element variable within the "GeogridIn" file the user has specified for ONLY an unstructured mesh or the NextGen hydrofabric. This is a 1-D array stating the how many nodes are connecting each element within the unstructured mesh. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. + + Example- NumElemConn: "numelemconn" + """ if self.grid_type in ["unstructured", "hydrofabric"]: return self.extract_input_variable("NumElemConn") else: @@ -647,7 +908,10 @@ def numelemconn_var(self) -> str: @property def element_id_var(self) -> str: - """Get the element ID variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings if the user has chosen a hydrofabric grid type in the configuration file.""" + """Naming convention of the element id variable within the "GeogridIn" file the user has specified for ONLY the NextGen hydrofabric. This is a 1-D array stating the catchment id numeric naming convention within the "divides" geopackage layer of a given NextGen hydrofabric file. This variable is required in order for the NextGen Forcings Engine to properly advertise the element ids of the unstructured mesh linked to the NextGen hydrofabric catchment ids. + + Example- ElemID: "element_ids" + """ if self.grid_type == "hydrofabric": return self.extract_input_variable("ElemID") else: @@ -655,29 +919,31 @@ def element_id_var(self) -> str: @property def ignored_border_widths(self) -> list: - """Get the list of ignored border widths specified by the user in the configuration file. This is used to control how the program processes input forcings based on the ignored border widths specified for each input forcing in the configuration file.""" + """Border width (in grid cells) to ignore for each input dataset. NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output. + + Example- IgnoredBorderWidths: [0,10] + """ return self._ignored_border_widths @ignored_border_widths.setter def ignored_border_widths(self, value: list) -> None: """Set the list of ignored border widths specified by the user in the configuration file. This is used to control how the program processes input forcings based on the ignored border widths specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("IgnoredBorderWidths") - if self.precip_only_flag: + if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "IgnoredBorderWidths") self.check_input_values_positive(value, "IgnoredBorderWidths") self._ignored_border_widths = value @property def regrid_opt(self): - """Get the list of regridding options specified by the user in the configuration file. This is used to control how input forcings are regridded based on the regridding option specified for each input forcing in the configuration file.""" + """Choose regridding options for each input forcing files being used. Options available are: 1 - ESMF Bilinear, 2 - ESMF Nearest Neighbor, 3 - ESMF Conservative Bilinear. + + Example- RegridOpt: [1,1] + """ return self._regrid_opt @regrid_opt.setter def regrid_opt(self, value: list) -> None: """Set the list of regridding options specified by the user in the configuration file. This is used to control how input forcings are regridded based on the regridding option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("RegridOpt") if self.precip_only_flag: self.check_number_of_inputs_forcings(value, "RegridOpt") self.check_input_values_in_range(value, "RegridOpt", [1, 2, 3]) @@ -691,9 +957,7 @@ def weightsDir(self) -> str: @weightsDir.setter def weightsDir(self, value: str) -> None: """Set the pathway to the ESMF weights directory specified by the user in the configuration file. This is used to control where the program looks for ESMF weights files if the user has chosen to use pre-generated ESMF weights files for regridding input forcings in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.try_config_get("RegridWeightsDir") - if self.precip_only_flag: + if not self.precip_only_flag: if value is not None and not os.path.exists(value): err_out_screen( f"ESMF Weights file directory specified ({value}) but does not exist" @@ -707,26 +971,10 @@ def forceTemoralInterp(self) -> list: @forceTemoralInterp.setter def forceTemoralInterp(self, value: list) -> None: - """Set the list of forcing temporal interpolation options specified by the user in the configuration file. This is used to control how input forcings are temporally interpolated based on the temporal interpolation option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("ForcingTemporalInterpolation") - if not self.precip_only_flag: - self.check_number_of_inputs_forcings(value, "ForcingTemporalInterpolation") - self.check_input_values_in_range( - value, "ForcingTemporalInterpolation", [0, 1, 2] - ) - self._forceTemoralInterp = value - - @property - def forceTemoralInterp(self): - """Get the list of forcing temporal interpolation options specified by the user in the configuration file. This is used to control how input forcings are temporally interpolated based on the temporal interpolation option specified for each input forcing in the configuration file.""" - return self._forceTemoralInterp + """Specify an temporal interpolation for the forcing variables. Interpolation will be done between the two neighboring input forcing states that exist. If only one nearest state exist (I.E. only a state forward in time, or behind), then that state will be used as a "nearest neighbor". NOTE - All input options here must be of the same length of the input forcing number. Also note all temporal interpolation occurs BEFORE downscaling and bias correction. 0 - No temporal interpolation. 1 - Nearest Neighbor, 2 - Linear weighted, average. - @forceTemoralInterp.setter - def forceTemoralInterp(self, value): - """Set the list of forcing temporal interpolation options specified by the user in the configuration file. This is used to control how input forcings are temporally interpolated based on the temporal interpolation option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("ForcingTemporalInterpolation") + Example- ForcingTemporalInterpolation: [0,0] + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "ForcingTemporalInterpolation") self.check_input_values_in_range( @@ -736,29 +984,36 @@ def forceTemoralInterp(self, value): @property def t2dDownscaleOpt(self) -> list: - """Get the list of temperature downscaling options specified by the user in the configuration file. This is used to control how temperature input forcings are downscaled based on the temperature downscaling option specified for each input forcing in the configuration file.""" + """Specify a temperature downscaling method: 0 - No downscaling, 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation to the WRF-Hydro elevation, 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain (only NWM), 3 - Use a dynamic lapse rate calculated at each timstep. + + Example- TemperatureDownscaling: [3, 3] + """ return self._t2dDownscaleOpt @t2dDownscaleOpt.setter def t2dDownscaleOpt(self, value: list) -> None: """Set the list of temperature downscaling options specified by the user in the configuration file. This is used to control how temperature input forcings are downscaled based on the temperature downscaling option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("TemperatureDownscaling") if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "TemperatureDownscaling") self.check_input_values_in_range(value, "TemperatureDownscaling", [0, 1, 2]) + count = 0 + for opt in value: + if opt == 2: + self.param_flag[count] = 1 + count += 1 self._t2dDownscaleOpt = value @property def psfcDownscaleOpt(self) -> list: - """Get the list of pressure downscaling options specified by the user in the configuration file. This is used to control how pressure input forcings are downscaled based on the pressure downscaling option specified for each input forcing in the configuration file.""" + """Specify a surface pressure downscaling method: 0 - No downscaling, 1 - Use input elevation and WRF-Hydro elevation to downscale surface pressure. + + Example- PressureDownscaling: [1, 1] + """ return self._psfcDownscaleOpt @psfcDownscaleOpt.setter def psfcDownscaleOpt(self, value: list) -> None: """Set the list of pressure downscaling options specified by the user in the configuration file. This is used to control how pressure input forcings are downscaled based on the pressure downscaling option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("PressureDownscaling") if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "PressureDownscaling") self.check_input_values_in_range(value, "PressureDownscaling", [0, 1]) @@ -766,14 +1021,15 @@ def psfcDownscaleOpt(self, value: list) -> None: @property def swDownscaleOpt(self) -> list: - """Get the list of shortwave downscaling options specified by the user in the configuration file. This is used to control how shortwave radiation input forcings are downscaled based on the shortwave downscaling option specified for each input forcing in the configuration file.""" + """Specify a shortwave radiation downscaling routine. 0 - No downscaling, 1 - Run a topographic adjustment using the WRF-Hydro elevation. + + Example- ShortwaveDownscaling: [1, 1] + """ return self._swDownscaleOpt @swDownscaleOpt.setter def swDownscaleOpt(self, value: list) -> None: """Set the list of shortwave downscaling options specified by the user in the configuration file. This is used to control how shortwave radiation input forcings are downscaled based on the shortwave downscaling option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("ShortwaveDownscaling") if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "ShortwaveDownscaling") self.check_input_values_in_range(value, "ShortwaveDownscaling", [0, 1]) @@ -781,14 +1037,15 @@ def swDownscaleOpt(self, value: list) -> None: @property def q2dDownscaleOpt(self) -> list: - """Get the list of humidity downscaling options specified by the user in the configuration file. This is used to control how humidity input forcings are downscaled based on the humidity downscaling option specified for each input forcing in the configuration file.""" + """Specify a specific humidity downscaling routine. 0 - No downscaling, 1 - Use regridded humidity, along with downscaled temperature/pressure to extrapolate a downscaled surface specific humidty. + + Example- HumidityDownscaling: [1, 1] + """ return self._q2dDownscaleOpt @q2dDownscaleOpt.setter def q2dDownscaleOpt(self, value: list) -> None: """Set the list of humidity downscaling options specified by the user in the configuration file. This is used to control how humidity input forcings are downscaled based on the humidity downscaling option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("HumidityDownscaling") if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "HumidityDownscaling") self.check_input_values_in_range(value, "HumidityDownscaling", [0, 1]) @@ -796,29 +1053,36 @@ def q2dDownscaleOpt(self, value: list) -> None: @property def precipDownscaleOpt(self) -> list: - """Get the list of precipitation downscaling options specified by the user in the configuration file. This is used to control how precipitation input forcings are downscaled based on the precipitation downscaling option specified for each input forcing in the configuration file.""" + """Specify a precipitation downscaling routine. 0 - No downscaling, 1 - NWM mountain mapper downscaling using monthly PRISM climo. + + Example- PrecipDownscaling: [0, 0] + """ return self._precipDownscaleOpt @precipDownscaleOpt.setter def precipDownscaleOpt(self, value: list) -> None: """Set the list of precipitation downscaling options specified by the user in the configuration file. This is used to control how precipitation input forcings are downscaled based on the precipitation downscaling option specified for each input forcing in the configuration file.""" - if value is None: - value = self.extract_input_variable("PrecipDownscaling") - self.check_number_of_inputs_forcings(value, "PrecipDownscaling") + if not self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "PrecipDownscaling") self.check_input_values_in_range(value, "PrecipDownscaling", [0, 1]) - + count = 0 + for opt in value: + if opt == 1: + self.param_flag[count] = 1 + count += 1 self._precipDownscaleOpt = value @property def dScaleParamDirs(self) -> list: - """Get the list of downscaling parameter directories specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for each input forcing based on the downscaling parameter directory specified for each input forcing in the configuration file.""" + """Specify the input parameter directory containing necessary downscaling grids. This is ONLY needed for the original NWM WRF-Hydro domain. Otherwise, just point it to a random directory and it will be ignored. + + Example- DownscalingParamDirs: ["./forcingParam/AnA", "./forcingParam/AnA"] + """ return self._dScaleParamDirs @dScaleParamDirs.setter def dScaleParamDirs(self, value: list) -> None: """Set the list of downscaling parameter directories specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for each input forcing based on the downscaling parameter directory specified for each input forcing in the configuration file.""" - if value is None: - value = self.extract_input_variable("DownscalingParamDirs") self.check_number_of_inputs_forcings(value, "DownscalingParamDirs") for dirTmp in range(0, len(value)): dir_path = value[dirTmp] @@ -839,73 +1103,17 @@ def perform_downscaling(self) -> bool: ): return True - @property - def sinalpha_var(self) -> str: - """Get the sine of the grid orientation variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the grid orientation variable specified for each input forcing in the configuration file.""" - if self.perform_downscaling: - return self.extract_input_variable("SINALPHA") - - @property - def cosalpha_var(self) -> str: - """Get the cosine of the grid orientation variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the grid orientation variable specified for each input forcing in the configuration file.""" - if self.perform_downscaling: - return self.extract_input_variable("COSALPHA") - - @property - def slope_var(self) -> str: - """Get the slope variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the slope variable specified for each input forcing in the configuration file.""" - if self.perform_downscaling: - return self.extract_input_variable("SLOPE") - - @property - def slope_azimuth_var(self) -> str: - """Get the slope azimuth variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the slope azimuth variable specified for each input forcing in the configuration file.""" - if self.perform_downscaling: - return self.extract_input_variable("SLOPE_AZIMUTH") - - @property - def slope_var_elem(self) -> str: - """Get the slope variable name specified by the user in the configuration file for element-based grids. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the slope variable specified for each input forcing in the configuration file for element-based grids.""" - if self.perform_downscaling: - if self.grid_type == "unstructured": - return self.extract_input_variable("SLOPE_ELEM") - else: - self.raise_grid_type_error(self.grid_type, "SLOPE_ELEM") - - @property - def slope_azimuth_var_elem(self) -> str: - """Get the slope azimuth variable name specified by the user in the configuration file for element-based grids. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the slope azimuth variable specified for each input forcing in the configuration file for element-based grids.""" - if self.perform_downscaling: - if self.grid_type == "unstructured": - return self.extract_input_variable("SLOPE_AZIMUTH_ELEM") - else: - self.raise_grid_type_error(self.grid_type, "SLOPE_AZIMUTH_ELEM") - - @property - def hgt_elem_var(self) -> str: - """Get the height variable name specified by the user in the configuration file for element-based grids. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the height variable specified for each input forcing in the configuration file for element-based grids.""" - if self.perform_downscaling: - if self.grid_type == "unstructured": - return self.extract_input_variable("HGT_ELEM") - else: - self.raise_grid_type_error(self.grid_type, "HGT_ELEM") - - @property - def hgt_var(self) -> str: - """Get the height variable name specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for downscaling input forcings based on the height variable specified for each input forcing in the configuration file.""" - if self.perform_downscaling: - return self.extract_input_variable("HGT") - @property def t2BiasCorrectOpt(self) -> list: - """Get the list of temperature bias correction options specified by the user in the configuration file. This is used to control how temperature input forcings are bias corrected based on the temperature bias correction option specified for each input forcing in the configuration file.""" + """Specify a temperature bias correction method. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION), 3 - NCAR parametric GFS bias correction, 4 - NCAR parametric HRRR bias correction. + + Example- TemperatureBiasCorrection: [0, 4] + """ return self._t2BiasCorrectOpt @t2BiasCorrectOpt.setter def t2BiasCorrectOpt(self, value: list) -> None: """Set the list of temperature bias correction options specified by the user in the configuration file. This is used to control how temperature input forcings are bias corrected based on the temperature bias correction option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("TemperatureBiasCorrection") if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "TemperatureBiasCorrection") self.check_input_values_in_range( @@ -915,14 +1123,15 @@ def t2BiasCorrectOpt(self, value: list) -> None: @property def psfcBiasCorrectOpt(self) -> list: - """Get the list of pressure bias correction options specified by the user in the configuration file. This is used to control how pressure input forcings are bias corrected based on the pressure bias correction option specified for each input forcing in the configuration file.""" + """Specify a surface pressure bias correction method. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY. + + Example- PressureBiasCorrection: [0,0] + """ return self._psfcBiasCorrectOpt @psfcBiasCorrectOpt.setter def psfcBiasCorrectOpt(self, value: list) -> None: """Set the list of pressure bias correction options specified by the user in the configuration file. This is used to control how pressure input forcings are bias corrected based on the pressure bias correction option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("PressureBiasCorrection") if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "PressureBiasCorrection") self.check_input_values_in_range(value, "PressureBiasCorrection", [0, 1]) @@ -930,14 +1139,15 @@ def psfcBiasCorrectOpt(self, value: list) -> None: @property def q2BiasCorrectOpt(self): - """Get the list of humidity bias correction options specified by the user in the configuration file. This is used to control how humidity input forcings are bias corrected based on the humidity bias correction option specified for each input forcing in the configuration file.""" + """Specify a specific humidity bias correction method. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). + + Example- HumidityBiasCorrection: [0,0] + """ return self._q2BiasCorrectOpt @q2BiasCorrectOpt.setter def q2BiasCorrectOpt(self, value): """Set the list of humidity bias correction options specified by the user in the configuration file. This is used to control how humidity input forcings are bias corrected based on the humidity bias correction option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("HumidityBiasCorrection") if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "HumidityBiasCorrection") self.check_input_values_in_range(value, "HumidityBiasCorrection", [0, 1, 2]) @@ -945,63 +1155,65 @@ def q2BiasCorrectOpt(self, value): @property def windBiasCorrect(self): - """Get the list of wind bias correction options specified by the user in the configuration file. This is used to control how wind input forcings are bias corrected based on the wind bias correction option specified for each input forcing in the configuration file.""" + """Specify a wind bias correction. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION), 3 - NCAR parametric GFS bias correction, 4 - NCAR parametric HRRR bias correction. + + Example- WindBiasCorrection: [0, 4] + """ return self._windBiasCorrect @windBiasCorrect.setter def windBiasCorrect(self, value): """Set the list of wind bias correction options specified by the user in the configuration file. This is used to control how wind input forcings are bias corrected based on the wind bias correction option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("WindBiasCorrection") if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "WindBiasCorrection") - self.check_input_values_in_range(value, "WindBiasCorrection", [0, 4]) + self.check_input_values_in_range( + value, "WindBiasCorrection", [0, 1, 2, 3, 4] + ) self._windBiasCorrect = value @property def swBiasCorrectOpt(self) -> list: - """Get the list of shortwave radiation bias correction options specified by the user in the configuration file. This is used to control how shortwave radiation input forcings are bias corrected based on the shortwave radiation bias correction option specified for each input forcing in the configuration file.""" + """Specify a bias correction for incoming short wave radiation flux. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). + + Example- SwBiasCorrection: [0, 2] + """ return self._swBiasCorrectOpt @swBiasCorrectOpt.setter def swBiasCorrectOpt(self, value: list) -> None: """Set the list of shortwave radiation bias correction options specified by the user in the configuration file. This is used to control how shortwave radiation input forcings are bias corrected based on the shortwave radiation bias correction option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("ShortwaveBiasCorrection") if not self.precip_only_flag: - self.check_number_of_inputs_forcings(value, "ShortwaveBiasCorrection") - self.check_input_values_in_range( - value, "ShortwaveBiasCorrection", [0, 1, 2] - ) + self.check_number_of_inputs_forcings(value, "SwBiasCorrection") + self.check_input_values_in_range(value, "SwBiasCorrection", [0, 1, 2]) self._swBiasCorrectOpt = value @property def lwBiasCorrectOpt(self) -> list: - """Get the list of longwave radiation bias correction options specified by the user in the configuration file. This is used to control how longwave radiation input forcings are bias corrected based on the longwave radiation bias correction option specified for each input forcing in the configuration file.""" + """Specify a bias correction for incoming long wave radiation flux. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION), 3 - NCAR parametric GFS bias correction. + + Example- LwBiasCorrection: [0, 2] + """ return self._lwBiasCorrectOpt @lwBiasCorrectOpt.setter def lwBiasCorrectOpt(self, value: list) -> None: """Set the list of longwave radiation bias correction options specified by the user in the configuration file. This is used to control how longwave radiation input forcings are bias corrected based on the longwave radiation bias correction option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("LongwaveBiasCorrection") if not self.precip_only_flag: - self.check_number_of_inputs_forcings(value, "LongwaveBiasCorrection") - self.check_input_values_in_range( - value, "LongwaveBiasCorrection", [0, 1, 2, 4] - ) + self.check_number_of_inputs_forcings(value, "LwBiasCorrection") + self.check_input_values_in_range(value, "LwBiasCorrection", [0, 1, 2, 3, 4]) self._lwBiasCorrectOpt = value @property def precipBiasCorrectOpt(self): - """Get the list of precipitation bias correction options specified by the user in the configuration file. This is used to control how precipitation input forcings are bias corrected based on the precipitation bias correction option specified for each input forcing in the configuration file.""" + """Specify a bias correction for precipitation. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY. + + Example- PrecipBiasCorrection: [0, 0] + """ return self._precipBiasCorrectOpt @precipBiasCorrectOpt.setter def precipBiasCorrectOpt(self, value): """Set the list of precipitation bias correction options specified by the user in the configuration file. This is used to control how precipitation input forcings are bias corrected based on the precipitation bias correction option specified for each input forcing in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("PrecipBiasCorrection") if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "PrecipBiasCorrection") self.check_input_values_in_range(value, "PrecipBiasCorrection", [0, 1]) @@ -1010,8 +1222,8 @@ def precipBiasCorrectOpt(self, value): @property def bias_correction_properties(self) -> dict: """Get the dictionary of bias correction properties specified by the user in the configuration file. This is used to control how input forcings are bias corrected based on the bias correction options specified for each input forcing in the configuration file.""" - bias_correction_properties = { - "surface temperature": self.t2BiasCorrectOpt, + return { + # "surface temperature": self.t2BiasCorrectOpt, #NOTE surface temperature was excluded from this consideration in the orignal code (5/7/2026 pre-refactor). Should it actually be included? "surface pressure": self.psfcBiasCorrectOpt, "specific humidity": self.q2BiasCorrectOpt, "wind forcings": self.windBiasCorrect, @@ -1019,16 +1231,17 @@ def bias_correction_properties(self) -> dict: "long-wave radiation": self.lwBiasCorrectOpt, "Precipitation": self.precipBiasCorrectOpt, } - return bias_correction_properties @property def runCfsNldasBiasCorrect(self) -> bool: """Get the flag for whether to run the NWM-specific bias correction of CFSv2 input forcings specified by the user in the configuration file. This is used to control whether the NWM-specific bias correction of CFSv2 input forcings is run based on whether the user has chosen to run this bias correction in the configuration file.""" - for optTmp in self.bias_correction_properties.values(): - if optTmp == 1: - runCfsNldasBiasCorrect = True - break - if runCfsNldasBiasCorrect: + run_cfs_nldas_bias_correct = False + for bias_option in self.bias_correction_properties.values(): + for opt in bias_option: + if opt == 1: + run_cfs_nldas_bias_correct = True + break + if run_cfs_nldas_bias_correct: for ( bias_correct_name, bias_correct, @@ -1043,7 +1256,9 @@ def runCfsNldasBiasCorrect(self) -> bool: err_out_screen( "CFSv2-NLDAS NWM bias correction can only be used in CFSv2-only configurations" ) + return run_cfs_nldas_bias_correct + @property def number_supp_pcp(self) -> int: """Get the number of supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how many supplemental precipitation input forcings are processed based on the number of supplemental precipitation input forcings specified in the configuration file.""" return len(self.supp_precip_forcings) @@ -1056,13 +1271,10 @@ def supp_precip_file_types(self) -> list: @supp_precip_file_types.setter def supp_precip_file_types(self, value: list) -> None: """Set the list of supplemental precipitation input forcing file types specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are read in and processed based on the file types specified for each supplemental precipitation input forcing in the configuration file.""" - if value is None: - value = self.try_config_get("SuppPcpForcingTypes") if value is not None: value = [stype.strip() for stype in value] if value == [""]: value = [] - self.check_number_of_inputs_supp_pcp(value, "SuppPcpForcingTypes") self.check_input_values_in_range( value, @@ -1078,120 +1290,103 @@ def supplemental_precip_file_type_options(self) -> list: @property def rqiMethod(self) -> list: - """Get the list of radar quality index (RQI) thresholding methods specified by the user in the configuration file. This is used to control how radar-based supplemental precipitation input forcings are processed based on the RQI thresholding method specified for each radar-based supplemental precipitation input forcing in the configuration file.""" + """Optional RQI method for radar-based data. 0 - Do not use any RQI filtering. Use all radar-based estimates. 1 - Use hourly MRMS Radar Quality Index grids, 2 - Use NWM monthly climatology grids (NWM only!!!!). + + Example- RqiMethod: 2 + """ + value = None if self.number_supp_pcp > 0: for suppOpt in self.supp_precip_forcings: # Read in RQI threshold to apply to radar products. if suppOpt in (1, 2, 7, 10, 11, 12): - rqiMethod = self.extract_input_variable("RqiMethod") + value = self.extract_input_variable("RqiMethod") # Check that if we have more than one RqiMethod, it's the correct number - if type(rqiMethod) is list: - self.check_number_of_inputs_supp_pcp(rqiMethod, "RqiMethod") - elif type(rqiMethod) is int: + if type(value) is list: + self.check_number_of_inputs_supp_pcp(value, "RqiMethod") + elif type(value) is int: # Support 'classic' mode of single method - rqiMethod = [rqiMethod] * self.number_supp_pcp + value = [value] * self.number_supp_pcp # Make sure the RqiMethod(s) makes sense. - for method in rqiMethod: + for method in value: self.check_input_values_in_range(method, "RqiMethod", [0, 1, 2]) - return rqiMethod + return value @property def rqiThresh(self): - """Get the radar quality index (RQI) threshold value specified by the user in the configuration file. This is used to control how radar-based supplemental precipitation input forcings are processed based on the RQI threshold value specified in the configuration file.""" + """Optional RQI threshold to be used to mask out. Currently used for MRMS products. Please choose a value from 0.0-1.0. Associated radar quality index files will be expected from MRMS data. + + Example- RqiThreshold: 0.9 + """ + value = 1.0 if self.number_supp_pcp > 0: - for suppOpt in self.supp_precip_forcings: + for supp_opt in self.supp_precip_forcings: # Read in RQI threshold to apply to radar products. - if suppOpt in (1, 2, 7, 10, 11, 12): - rqiThresh = self.extract_input_variable("RqiThresh") + if supp_opt in (1, 2, 7, 10, 11, 12): + value = self.extract_input_variable("RqiThresh") # Check that if we have more than one RqiThresh, it's the correct number - if type(rqiThresh) is list: - self.check_number_of_inputs_supp_pcp(rqiThresh, "RqiThresh") - elif type(rqiThresh) is (int, float): + if type(value) is list: + self.check_number_of_inputs_supp_pcp(value, "RqiThresh") + elif type(value) is (int, float): # Support 'classic' mode of single threshold - rqiThresh = [rqiThresh] * self.number_supp_pcp + value = [value] * self.number_supp_pcp # Make sure the RqiThresh(es) makes sense. - for threshold in self.rqiThresh: + for threshold in value: if threshold < 0.0 or threshold > 1.0: err_out_screen( "Please specify RqiThresholds between 0.0 and 1.0." ) - return threshold - - @property - def supp_precip_dirs(self): - """Get the list of pathways to the supplemental precipitation input forcing directories specified by the user in the configuration file. This is used to control where the program looks for supplemental precipitation input forcing files for each supplemental precipitation input forcing based on the directory specified for each supplemental precipitation input forcing in the configuration file.""" - if self.number_supp_pcp > 0: - return self._supp_precip_dirs - - @supp_precip_dirs.setter - def supp_precip_dirs(self, value): - """Set the list of pathways to the supplemental precipitation input forcing directories specified by the user in the configuration file. This is used to control where the program looks for supplemental precipitation input forcing files for each supplemental precipitation input forcing based on the directory specified for each supplemental precipitation input forcing in the configuration file.""" - if value is None and self.number_supp_pcp > 0: - value = self.extract_input_variable("SuppPcpDirectories") - if value > 0: - self.check_number_of_inputs_supp_pcp(value, "SuppPcpDirectories") - for dirTmp in range(0, len(value)): - value[dirTmp] = value[dirTmp].strip() - if not os.path.isdir(value[dirTmp]): - try: - os.makedirs(value[dirTmp], exist_ok=True) - LOG.debug(f"Created supp pcp directory: {value[dirTmp]}") - except OSError as e: - err_out_screen( - f"Unable to create supp pcp directory: {value[dirTmp]}. Error: {e}" - ) - self._supp_precip_dirs = value + return value @property def supp_precip_mandatory(self): - """Get the list of flags for whether each supplemental precipitation input forcing specified by the user in the configuration file is mandatory or optional. This is used to control whether an error is raised if supplemental precipitation input forcing files are not found for each supplemental precipitation input forcing based on whether the user has specified each supplemental precipitation input forcing as mandatory or optional in the configuration file.""" + """Specify whether the Supplemental Precips listed above are mandatory, or optional. This is important for layering contingencies if a product is missing, but forcing files are still desired. 0 - Not mandatory, 1 - Mandatory. + + Example- SuppPcpMandatory: [0, 0, 0] + """ return self._supp_precip_mandatory @supp_precip_mandatory.setter def supp_precip_mandatory(self, value): """Set the list of flags for whether each supplemental precipitation input forcing specified by the user in the configuration file is mandatory or optional. This is used to control whether an error is raised if supplemental precipitation input forcing files are not found for each supplemental precipitation input forcing based on whether the user has specified each supplemental precipitation input forcing as mandatory or optional in the configuration file.""" - if value is None and self.number_supp_pcp > 0: - value = self.extract_input_variable("SuppPcpMandatory") if self.number_supp_pcp > 0: - for enforceOpt in value: - self.check_input_values_in_range(enforceOpt, "SuppPcpMandatory", [0, 1]) + self.check_input_values_in_range(value, "SuppPcpMandatory", [0, 1]) self._supp_precip_mandatory = value @property def regrid_opt_supp_pcp(self): - """Get the list of regridding options for supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how supplemental precipitation input forcings are regridded based on the regridding option specified for each supplemental precipitation input forcing in the configuration file.""" + """Specify regridding options for the supplemental precipitation products. Options available are: 1 - ESMF Bilinear, 2 - ESMF Nearest Neighbor, 3 - ESMF Conservative Bilinear. + + Example- RegridOptSuppPcp: [1, 1, 1] + """ return self._regrid_opt_supp_pcp @regrid_opt_supp_pcp.setter def regrid_opt_supp_pcp(self, value): """Set the list of regridding options for supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how supplemental precipitation input forcings are regridded based on the regridding option specified for each supplemental precipitation input forcing in the configuration file.""" - if value is None and self.number_supp_pcp > 0: - value = self.extract_input_variable("RegridOptSuppPcp") if self.number_supp_pcp > 0: - for optTmp in value: - self.check_input_values_in_range(optTmp, "RegridOptSuppPcp", [1, 2, 3]) + self.check_input_values_in_range(value, "RegridOptSuppPcp", [1, 2, 3]) self._regrid_opt_supp_pcp = value @property def suppTemporalInterp(self): - """Get the list of flags for whether temporal interpolation of supplemental precipitation input forcings specified by the user in the configuration file is performed or not. This is used to control whether temporal interpolation of supplemental precipitation input forcings is performed based on whether the user has chosen to perform temporal interpolation for each supplemental precipitation input forcing in the configuration file.""" + """Specify the time interpretation methods for the supplemental precipitation products. + + Example- SuppPcpTemporalInterpolation: [0, 0, 0] + """ if self.number_supp_pcp > 0: return self._suppTemporalInterp @suppTemporalInterp.setter def suppTemporalInterp(self, value): """Set the list of flags for whether temporal interpolation of supplemental precipitation input forcings specified by the user in the configuration file is performed or not. This is used to control whether temporal interpolation of supplemental precipitation input forcings is performed based on whether the user has chosen to perform temporal interpolation for each supplemental precipitation input forcing in the configuration file.""" - if value is None and self.number_supp_pcp > 0: - value = self.extract_input_variable("SuppPcpTemporalInterpolation") if self.number_supp_pcp > 0: - for optTmp in value: - self.check_input_values_in_range( - optTmp, "SuppPcpTemporalInterpolation", [0, 1, 2] - ) + self.check_input_values_in_range( + value, "SuppPcpTemporalInterpolation", [0, 1, 2] + ) self._suppTemporalInterp = value @property @@ -1203,8 +1398,6 @@ def supp_pcp_max_hours(self): @supp_pcp_max_hours.setter def supp_pcp_max_hours(self, value): """Set the list of maximum forecast hours for supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how supplemental precipitation input forcings are processed based on the maximum forecast hour specified for each supplemental precipitation input forcing in the configuration file.""" - if value is None and self.number_supp_pcp > 0: - value = self.extract_input_variable("SuppPcpMaxHours") if self.number_supp_pcp > 0: if isinstance(value, list): self.check_input_values_positive(value, "SuppPcpMaxHours") @@ -1215,59 +1408,36 @@ def supp_pcp_max_hours(self, value): @property def supp_input_offsets(self): - """Get the list of time offsets to apply to supplemental precipitation input forcing files specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are processed based on the time offset specified for each supplemental precipitation input forcing in the configuration file.""" + """In AnA runs, this value is the offset from the available forecast and 00z. For example, if forecast are available at 06z and 18z, set this value to 6. + + Example- SuppPcpInputOffsets = [0, 0, 0] + """ return self._supp_input_offsets @supp_input_offsets.setter def supp_input_offsets(self, value): """Set the list of time offsets to apply to supplemental precipitation input forcing files specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are processed based on the time offset specified for each supplemental precipitation input forcing in the configuration file.""" - if value is None and self.number_supp_pcp > 0: - value = self.extract_input_variable("SuppPcpInputOffsets") if self.number_supp_pcp > 0: self.check_number_of_inputs_supp_pcp(value, "SuppPcpInputOffsets") - @property - def supp_precip_param_dir(self): - """Get the directory where downscaling parameters for supplemental precipitation input forcings are stored specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for supplemental precipitation input forcings based on the directory specified for supplemental precipitation input forcings in the configuration file.""" - if self.number_supp_pcp > 0: - return self._supp_precip_param_dir - - @supp_precip_param_dir.setter - def supp_precip_param_dir(self, value): - """Set the directory where downscaling parameters for supplemental precipitation input forcings are stored specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for supplemental precipitation input forcings based on the directory specified for supplemental precipitation input forcings in the configuration file.""" - if value is None and self.number_supp_pcp > 0: - value = self.extract_input_variable("SuppPcpDownscalingParamDir") - if self.number_supp_pcp > 0: - if not os.path.isdir(value): - err_out_screen( - f"Unable to locate parameter directory: {os.path.abspath(value)}" - ) - self._supp_precip_param_dir = value - @property def supp_precip_dirs(self): - """Get the list of pathways to the supplemental precipitation input forcing directories specified by the user in the configuration file. This is used to control where the program looks for supplemental precipitation input forcing files for each supplemental precipitation input forcing based on the directory specified for each supplemental precipitation input forcing in the configuration file.""" - if self.number_supp_pcp > 0: - return self._supp_precip_dirs + """Specify the correponding supplemental precipitation directories that will be searched for input files. + + Example- SuppPcpDirectories: ['./MRMS_CONUS_GAUGE', './MRMS_CONUS_MULTISENSOR', './MRMS_CLASSIFICATION'] + """ + return self._supp_precip_dirs @supp_precip_dirs.setter def supp_precip_dirs(self, value): """Set the list of pathways to the supplemental precipitation input forcing directories specified by the user in the configuration file. This is used to control where the program looks for supplemental precipitation input forcing files for each supplemental precipitation input forcing based on the directory specified for each supplemental precipitation input forcing in the configuration file.""" - if value is None and self.number_supp_pcp > 0: - value = self.extract_input_variable("SuppPcpDirectories") if self.number_supp_pcp > 0: + self.check_number_of_inputs_supp_pcp(value, "SuppPcpDirectories") # Loop through and ensure all supp pcp directories exist. Also strip out any whitespace # or new line characters. for dirTmp in range(0, len(value)): value[dirTmp] = value[dirTmp].strip() - if not os.path.isdir(value[dirTmp]): - try: - os.makedirs(value[dirTmp], exist_ok=True) - LOG.debug(f"Created supp pcp directory: {value[dirTmp]}") - except OSError as e: - err_out_screen( - f"Unable to create supp pcp directory: {value[dirTmp]}. Error: {e}" - ) + self.try_make_dir(value[dirTmp], " supp pcp") # Special case for ExtAnA where we treat comma separated stage IV, MRMS data as one SuppPcp input if 11 in self.supp_precip_forcings or 12 in self.supp_precip_forcings: @@ -1277,34 +1447,31 @@ def supp_precip_dirs(self, value): ) value = [",".join(value)] self._supp_precip_dirs = value + else: + self._supp_precip_dirs = None @property def supp_precip_param_dir(self): - """Get the directory where downscaling parameters for supplemental precipitation input forcings are stored specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for supplemental precipitation input forcings based on the directory specified for supplemental precipitation input forcings in the configuration file.""" + """Specify an optional directory that contains supplemental precipitation parameter fields, I.E monthly RQI climatology. This is ONLY needed for the original NWM WRF-Hydro domain. Otherwise, just point it to a random directory and it will be ignored. + + Example- SuppPcpParamDir: ['./forcingParam/AnA','./forcingParam/AnA','./forcingParam/AnA'] + """ if self.number_supp_pcp > 0: return self._supp_precip_param_dir @supp_precip_param_dir.setter def supp_precip_param_dir(self, value): """Set the directory where downscaling parameters for supplemental precipitation input forcings are stored specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for supplemental precipitation input forcings based on the directory specified for supplemental precipitation input forcings in the configuration file.""" - if value is None and self.number_supp_pcp > 0: - value = self.extract_input_variable("SuppPcpDownscalingParamDir") if self.number_supp_pcp > 0: - try: - os.makedirs(value, exist_ok=True) - LOG.debug(f"Created missing SuppPcpParamDir: {value}") - except OSError as e: - err_out_screen(f"Unable to locate SuppPcpParamDir: {value}. Error: {e}") + self.try_make_dir(value, " SuppPcpParamDir") + self._supp_precip_param_dir = value + else: + self._supp_precip_param_dir = None @property def cfsv2EnsMember(self): - """Get the CFSv2 ensemble member to process specified by the user in the configuration file. This is used to control which CFSv2 ensemble member is processed for CFSv2 input forcings based on the ensemble member specified in the configuration file.""" - return self._cfsv2EnsMember - - @cfsv2EnsMember.setter - def cfsv2EnsMember(self, value): """Set the CFSv2 ensemble member to process specified by the user in the configuration file. This is used to control which CFSv2 ensemble member is processed for CFSv2 input forcings based on the ensemble member specified in the configuration file.""" - if value is None and not self.precip_only_flag: + if not self.precip_only_flag: # Read in Ensemble information # Read in CFS ensemble member information IF we have chosen CFSv2 as an input # forcing. @@ -1314,6 +1481,7 @@ def cfsv2EnsMember(self, value): self.check_input_values_in_range( value, "cfsEnsNumber", [1, 2, 3, 4] ) + return value @property def customFcstFreq(self): @@ -1322,124 +1490,18 @@ def customFcstFreq(self): @customFcstFreq.setter def customFcstFreq(self, value): - """Set the custom forecast frequency in minutes specified by the user in the configuration file. This is used to control how often forecasts are issued based on the custom forecast frequency specified in the configuration file.""" - if value is None and not self.precip_only_flag: - value = self.extract_input_variable("CustomFcstFreq") - if len(self.customFcstFreq) != self.number_custom_inputs: - err_out_screen( - f"Improper custom_input fcst_freq specified. This number ({len(self.customFcstFreq)}) must match the frequency of custom input forcings selected ({self.number_custom_inputs})." - ) - self._customFcstFreq = value - - def _validate_config(self) -> None: - """Validate in options from the configuration file and check that proper options were provided.""" - self.b_date_proc + """Options for specifying custom input NetCDF forcing files (in minutes). Choose the input frequency of files that are being processed. I.E., are the input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the length of custom input frequencies to match the number of custom NetCDF inputs selected above in the Logistics section. - # if not self.precip_only_flag: - - if self.output_freq <= 0: - err_out_screen( - "Please specify an OutputFrequency that is greater than zero minutes." - ) - - if self.sub_output_hour < 0: - err_out_screen( - "Please specify an SubOutputHour that is greater than zero minutes." - ) - if self.sub_output_hour == 0: - self.sub_output_hour = None - - if self.sub_output_freq < 0: - err_out_screen( - "Please specify an SubOutFreq that is greater than zero minutes." - ) - if self._sub_output_freq == 0: - self.sub_output_freq = None - - # TODO Can this be a /tmp directory? - self.make_scratch_dir() - - if self.useCompression not in [0, 1]: - err_out_screen("Please choose a compressOut value of 0 or 1.") - - if self.ana_flag in [0, 1]: - err_out_screen("Please choose a AnAFlag value of 0 or 1.") - - if self.look_back <= 0 and self.look_back != -9999: - err_out_screen("Please specify a positive LookBack or -9999 for realtime.") - - if self.fcst_freq <= 0: - err_out_screen( - "Please specify a ForecastFrequency in the configuration file greater than zero." - ) - # Currently, we only support daily or sub-daily forecasts. Any other iterations should - # be done using custom config files for each forecast cycle. - if self.fcst_freq > 1440: - err_out_screen( - "Only forecast cycles of daily or sub-daily are supported at this time" - ) - - # Read in the ForecastShift option. This is ONLY done for the realtime instance as - # it's used to calculate the beginning of the processing window. - if True: # was: self.realtime_flag: - self.fcst_shift = self.extract_input_variable("ForecastShift") - if self.fcst_shift < 0: + Example- custom_input_fcst_freq: [] + """ + if not self.precip_only_flag: + if len(value) != self.number_custom_inputs: err_out_screen( - "Please specify a ForecastShift in the configuration file greater than or equal to zero." + f"Improper custom_input fcst_freq specified. This number ({len(value)}) must match the frequency of custom input forcings selected ({self.number_custom_inputs})." ) - - # Calculate the beginning/ending processing dates if we are running realtime - if self.realtime_flag: - calculate_lookback_window(self) - - # if self.refcst_flag: - # Calculate the number of forecasts to issue, and verify the user has chosen a - # correct divider based on the dates - # dt_tmp = self.e_date_proc - self.b_date_proc - # if (dt_tmp.days * 1440 + dt_tmp.seconds / 60.0) % self.fcst_freq != 0: - # err_out_screen('Please choose an equal divider forecast frequency for your ' - # 'specified reforecast range.') - # self.nFcsts = int((dt_tmp.days * 1440 + dt_tmp.seconds / 60.0) / self.fcst_freq) - - # Flag to constrain AORC forcing data cycle output - # for optTmp in self.input_forcings: - # if optTmp == 12: - # self.nFcsts = 1 - self.nFcsts = 1 - - if self.look_back != -9999: - calculate_lookback_window(self) - - # Process geospatial information - - if len(self.spatial_meta) == 0: - # No spatial metadata file found. - self.spatial_meta = None + self._customFcstFreq = value else: - if not os.path.isfile(self.spatial_meta): - err_out_screen( - "Unable to locate optional spatial metadata file: " - + self.spatial_meta - ) - - # Calculate the beginning/ending processing dates if we are running realtime - if self.realtime_flag: - calculate_lookback_window(self) - - # Create temporary array to hold flags if we need input parameter files. - param_flag = np.zeros([len(self.input_forcings)], int) - - count_tmp = 0 - for optTmp in self.precipDownscaleOpt: - if optTmp == 1: - param_flag[count_tmp] = 1 - count_tmp = count_tmp + 1 - - for suppOpt in self.supp_precip_forcings: - if suppOpt not in list(range(1, self.supp_precip_count + 1)): - err_out_screen( - f"Please specify SuppForcing values between 1 and {self.supp_precip_count}." - ) + self._customFcstFreq = None @property def nwm_domain(self) -> str: diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py index b442aecd..16e399d6 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py @@ -1359,17 +1359,78 @@ "uid64", ], "var_rename_map": {"config_path": "cfg_bmi"}, - "cfg_bmi_to_attrs_map": { + "extract_input_variable_attrs_map": { "SuppPcp": "supp_precip_forcings", "OutputFrequency": "output_freq", "SubOutputHour": "sub_output_hour", "SubOutFreq": "sub_output_freq", "ScratchDir": "scratch_dir", - "compressOutput": "useCompression", + "compressOutput": "useCompression", # 0 "AnAFlag": "ana_flag", "LookBack": "look_back", "ForecastFrequency": "fcst_freq", + "ForecastShift": "fcst_shift", + "GRID_TYPE": "grid_type", + "DownscalingParamDirs": "dScaleParamDirs", + "SuppPcpDirectories": "supp_precip_dirs", + "SuppPcpMandatory": "supp_precip_mandatory", + "RegridOptSuppPcp": "regrid_opt_supp_pcp", + "SuppPcpTemporalInterpolation": "suppTemporalInterp", + "SuppPcpMaxHours": "supp_pcp_max_hours", + "SuppPcpInputOffsets": "supp_input_offsets", + "SuppPcpParamDirs": "supp_precip_param_dir", + }, + "extract_input_variable_attrs_map_precip_only": { + "customSuppPcpFreq": "customSuppPcpFreq", + }, + "extract_input_variable_attrs_map_not_precip_only": { + "ForecastInputHorizons": "fcst_input_horizons", # np + "ForecastInputOffsets": "fcst_input_offsets", # np + "IgnoredBorderWidths": "ignored_border_widths", # np + "RegridOpt": "regrid_opt", # np + "RegridWeightsDir": "weightsDir", # np + "ForcingTemporalInterpolation": "forceTemoralInterp", # np + "TemperatureDownscaling": "t2dDownscaleOpt", # np + "PressureDownscaling": "psfcDownscaleOpt", # np + "ShortwaveDownscaling": "swDownscaleOpt", # np + "HumidityDownscaling": "q2dDownscaleOpt", # np + "PrecipDownscaling": "precipDownscaleOpt", # np -complicated partial np + "TemperatureBiasCorrection": "t2BiasCorrectOpt", # np #no + "PressureBiasCorrection": "psfcBiasCorrectOpt", # np #yes + "HumidityBiasCorrection": "q2BiasCorrectOpt", # np #yes + "WindBiasCorrection": "windBiasCorrect", # np #yes + "SwBiasCorrection": "swBiasCorrectOpt", # np #yes + "LwBiasCorrection": "lwBiasCorrectOpt", # np #yes + "PrecipBiasCorrection": "precipBiasCorrectOpt", # np #yes + "InputForcings": "input_forcings", # np + "InputForcingTypes": "input_force_types", # np + "InputForcingDirectories": "input_force_dirs", # np + "InputMandatory": "input_force_mandatory", # np + "custom_input_fcst_freq": "customFcstFreq", # np + }, + "downscaling_attrs_map": { + "SINALPHA": "sinalpha_var", + "COSALPHA": "cosalpha_var", + "SLOPE": "slope_var", + "SLOPE_AZIMUTH": "slope_azimuth_var", + "HGT": "hgt_var", + }, + "downscaling_unstructred_attrs_map": { + "SLOPE_ELEM": "slope_var_elem", + "SLOPE_AZIMUTH_ELEM": "slope_azimuth_var_elem", + "HGT_ELEM": "hgt_elem_var", + }, + "extract_input_variable_set_default_attrs_map": { + "includeLQFrac": "include_lqfrac", + "floatOutput": "useFloats", + "Output": "forcing_output", + }, + "try_config_get_except_attr_map": { + "RefcstBDateProc": "b_date_proc", + "Geopackage": "geopackage", + "GeogridIn": "geogrid", "SpatialMetaIn": "spatial_meta", + "SuppPcpForcingTypes": "supp_precip_file_types", }, "file_types": ["GRIB1", "GRIB2", "NETCDF", "NETCDF4", "NWM", "ZARR"], } From 8cb6a8d589ec02c9965c261660c7def42ae6a1c3 Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Mon, 18 May 2026 09:42:58 -0500 Subject: [PATCH 033/132] fix circular imports --- .../NextGen_Forcings_Engine/core/parallel.py | 16 ++++++++++------ .../NextGen_Forcings_Engine/esmf_utils.py | 8 ++++++-- .../NextGen_Forcings_Engine/os_utils.py | 16 +++++++++++++--- .../NextGen_Forcings_Engine/retry_utils.py | 9 ++++++++- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index 742e4511..dee8126c 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -1,11 +1,12 @@ from __future__ import annotations + import atexit -from functools import partial import os import signal import sys -import typing -from typing import TypeVar +from functools import partial +from typing import TYPE_CHECKING + import mpi4py import numpy as np @@ -13,8 +14,11 @@ from mpi4py import MPI # noqa: E402 -from . import err_handler -from . import mpi_utils +if TYPE_CHECKING: + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( + ConfigOptions, + ) +from . import err_handler, mpi_utils # If MPI was initialized outside of python, # disable initialization/finalization behavior @@ -23,8 +27,8 @@ mpi4py.rc.finalize = False if typing.TYPE_CHECKING: - from .geoMod import GriddedGeoMeta from .config import ConfigOptions + from .geoMod import GriddedGeoMeta _T = TypeVar("_T") diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/esmf_utils.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/esmf_utils.py index 2076278a..f49b8960 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/esmf_utils.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/esmf_utils.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any import types import esmpy as ESMF @@ -7,8 +10,9 @@ import shapely from . import retry_utils -from .core.config import ConfigOptions -from .core.parallel import MpiConfig +if TYPE_CHECKING: + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ConfigOptions + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import MpiConfig @retry_utils.retry_w_mpi_context( diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/os_utils.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/os_utils.py index 3d823c60..67827d49 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/os_utils.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/os_utils.py @@ -1,9 +1,19 @@ -from . import retry_utils +from __future__ import annotations + import traceback import types import typing -from .core.parallel import MpiConfig -from .core.config import ConfigOptions +from typing import TYPE_CHECKING + +from . import retry_utils + +if TYPE_CHECKING: + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( + ConfigOptions, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import ( + MpiConfig, + ) import os diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/retry_utils.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/retry_utils.py index 779dd7dd..f542c3ad 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/retry_utils.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/retry_utils.py @@ -1,10 +1,17 @@ +from __future__ import annotations + import functools import time import traceback import types +from typing import TYPE_CHECKING from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import MpiConfig -from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ConfigOptions + +if TYPE_CHECKING: + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( + ConfigOptions, + ) def retry_w_mpi_context( From 38b2cf6fcb7f3014336a57be7036030d7f8ab15d Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Mon, 18 May 2026 09:43:45 -0500 Subject: [PATCH 034/132] updat for NHF --- .../NextGen_hyfab_to_ESMF_Mesh.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ESMF_Mesh_Domain_Configuration_Production/NextGen_hyfab_to_ESMF_Mesh.py b/ESMF_Mesh_Domain_Configuration_Production/NextGen_hyfab_to_ESMF_Mesh.py index 1783ee34..c9426a94 100644 --- a/ESMF_Mesh_Domain_Configuration_Production/NextGen_hyfab_to_ESMF_Mesh.py +++ b/ESMF_Mesh_Domain_Configuration_Production/NextGen_hyfab_to_ESMF_Mesh.py @@ -1,11 +1,12 @@ +import argparse +import os +import pathlib +import uuid + import geopandas as gpd import netCDF4 import numpy as np import pandas as pd -import argparse -import pathlib -import os -import uuid gpd.options.display_precision = 16 np.set_printoptions(precision=128) From 72503269f29aabcfae6fb8ecffb2e6420b2ccd97 Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Mon, 18 May 2026 09:44:24 -0500 Subject: [PATCH 035/132] remove calls to validate_config and initialize --- .../NextGen_Forcings_Engine/bmi_model.py | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 90bae785..06fe6d13 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -267,16 +267,16 @@ def initialize(self, config_file: str, output_path: str | None = None) -> None: self._job_meta = ConfigOptions(self.cfg_bmi) # Parse the configuration options - try: - self._job_meta.validate_config(self.cfg_bmi) - except KeyboardInterrupt as e: - err_handler.err_out_screen("User keyboard interrupt", e) - except ImportError as e: - err_handler.err_out_screen("Missing Python packages", e) - except InterruptedError as e: - err_handler.err_out_screen("External kill signal detected", e) - except Exception as e: - err_handler.err_out_screen("Unhandled exception", e) + # try: + # self._job_meta.validate_config(self.cfg_bmi) + # except KeyboardInterrupt as e: + # err_handler.err_out_screen("User keyboard interrupt", e) + # except ImportError as e: + # err_handler.err_out_screen("Missing Python packages", e) + # except InterruptedError as e: + # err_handler.err_out_screen("External kill signal detected", e) + # except Exception as e: + # err_handler.err_out_screen("Unhandled exception", e) # Set NWM version and config, if provided in the config if self.cfg_bmi.get("NWM_VERSION") is not None: @@ -445,8 +445,24 @@ def initialize_with_params( :param output_path: The output path for model results. If omitted, a default path will be generated. :raises ValueError: If an invalid grid type is specified, an exception is raised. """ + # This is required prior to the first log message. + LOG.bind() + + bmi_cfg_file = Path(config_file).resolve() + if not bmi_cfg_file.is_file(): + LOG.critical(f"Config file {bmi_cfg_file} not found, nothing to do...") + raise RuntimeError( + f"Config file {bmi_cfg_file} not found, nothing to do..." + ) + + LOG.info(f"Reading config file: {bmi_cfg_file}") + with bmi_cfg_file.open("r") as fp: + cfg = yaml.safe_load(fp) + + self.cfg_bmi = parse_config(cfg) # Set the job metadata parameters (b_date, geogrid) using config_options - self._job_meta = ConfigOptions(self.cfg_bmi, b_date=b_date, geogrid_arg=geogrid) + self.cfg_bmi = parse_config(cfg) + self._job_meta = ConfigOptions(self.cfg_bmi, b_date=b_date, geogrid=geogrid) # Now that _job_meta is set, call initialize() to set up the core model self.initialize(config_file, output_path=output_path) From 6c1a680eceb6db10e2f6b95713978f7cceb7fcdb Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Mon, 18 May 2026 09:44:58 -0500 Subject: [PATCH 036/132] debugging changes --- .../NextGen_Forcings_Engine/core/config.py | 143 +++++++++-------- .../NextGen_Forcings_Engine/core/consts.py | 144 +++++++++--------- 2 files changed, 154 insertions(+), 133 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 69cb32f7..f1b6c4d7 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -44,11 +44,10 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No self.user_provided_geogrid_flag = False self.b_date_proc = b_date - self._cfg_bmi = cfg_bmi - self._geogrid = geogrid + self.cfg_bmi = cfg_bmi + self.geogrid = geogrid self.bmi_time_index = 0 - self.precip_only_flag = False self.globalNdv = -9999.0 self.d_program_init = datetime.now(timezone.utc) self.errFlag = 0 @@ -60,9 +59,13 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No ) self.nwm_source = "s3://noaa-nwm-retrospective-3-0-pds" - self.broadcast_new_64bit_uid() - + self._scratch_dir_has_been_uniquefied = False + self.supp_precip_forcings = self.extract_input_variable("SuppPcp") + if not self.precip_only_flag: + self.input_forcings = self.extract_input_variable("InputForcings") + + # Create temporary array to hold flags if we need input parameter files. self.param_flag = np.zeros([len(self.input_forcings)], int) @@ -70,9 +73,11 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No # These are indexed from the consts dictionary using the class name for attr in CONFIGOPTIONS[self.__class__.__name__]: setattr(self, attr, None) + self.broadcast_new_64bit_uid() - self.set_attrs(self.try_config_get_except_attr_map) - self.supp_precip_forcings = self.extract_input_variable("SuppPcp") + for cfg_bmi_attr, config_options_attr in self.try_config_get_except_attr_map.items(): + setattr(self,config_options_attr,self.try_config_get(cfg_bmi_attr)) + self.set_attrs(CONFIGOPTIONS["extract_input_variable_attrs_map"]) if self.precip_only_flag: @@ -94,20 +99,24 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No for cfg_bmi_attr, config_options_attr in CONFIGOPTIONS[ "extract_input_variable_set_default_attrs_map" ].items(): + if config_options_attr=="supp_pcp_max_hours": + default=None + else: + default=0 setattr( self, config_options_attr, - self.extract_input_variable_set_default(cfg_bmi_attr), + self.extract_input_variable_set_default(cfg_bmi_attr,default), ) @property def try_config_get_except_attr_map(self) -> dict: """Get the mapping of configuration variable names to class attribute names for variables that are extracted directly from the configuration file without any additional processing. This is used to control how variables are extracted from the configuration file and assigned to class attributes in a consistent way based on the mapping specified in the consts.py file.""" dict_map = CONFIGOPTIONS["try_config_get_except_attr_map"] - if self._b_date_proc is not None: - dict_map.pop("b_date_proc") - if self._geogrid is not None: - dict_map.pop("geogrid") + if self._b_date_proc is not None and "RefcstBDateProc" in dict_map: + dict_map.pop("RefcstBDateProc") + if self.geogrid is not None and "GeogridIn" in dict_map: + dict_map.pop("GeogridIn") return dict_map @property @@ -127,7 +136,7 @@ def cfg_bmi(self, value: dict) -> None: @property def force_count(self) -> int: """Calculate the number of total possible input forcing options based on the length of the InputForcings list in the consts.py file. This is used for error checking to ensure users specify valid input forcing options in the configuration file.""" - return len(FORCINGINPUTMOD["InputForcings"]["PRODUCT_NAME"]) + return len(FORCINGINPUTMOD["PRODUCT_NAME"]) @property def supp_precip_count(self) -> int: @@ -144,9 +153,11 @@ def number_supp_pcp(self) -> int: @property def precip_only_flag(self) -> bool: """Flag to indicate whether the user has chosen to run the supplemental precip forcings module only, which will trigger some different processing pathways and error checking for certain configuration options.""" + precip_only = False if self.number_supp_pcp == 1: if int(self.supp_precip_forcings[0]) == 14: - return True + precip_only = True + return precip_only def set_attrs(self, attrs_dict: dict): """Set the attributes of the class based on the configuration file. This is used to populate the attributes of the class after they have been read in and validated from the configuration file.""" @@ -154,7 +165,12 @@ def set_attrs(self, attrs_dict: dict): setattr( self, config_options_attr, self.extract_input_variable(cfg_bmi_attr) ) - + def set_attrs_use_default(self,attrs_dict:dict): + """Set the attributes of the class based on the configuration file. Set default value to default if not found in config file.""" + for cfg_bmi_attr, config_options_attr in attrs_dict.items(): + setattr( + self, config_options_attr, self.extract_input_variable_set_default(cfg_bmi_attr) + ) def extract_input_variable(self, variable_name: str) -> str: """Extract the variable name from the configuration file for a given variable.""" try: @@ -183,8 +199,9 @@ def extract_input_variable_set_default(self, variable_name: str, default=0) -> s err_out_screen( f"Improper {variable_name} value: {self.cfg_bmi[variable_name]}", e ) - if variable not in [0, 1]: - err_out_screen(f"Please choose a {variable_name} value of 0 or 1.") + if default==0: + if variable not in [0, 1]: + err_out_screen(f"Please choose a {variable_name} value of 0 or 1.") return variable def try_config_get(self, variable_name: str) -> str: @@ -202,22 +219,22 @@ def try_config_get(self, variable_name: str) -> str: ) def check_number_of_inputs( - self, value: list, variable_name: str, input_type: str + self, value: list, variable_name: str, input_type: str,number_inputs:int ) -> None: """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable.""" - if len(value) != self.number_inputs: + if len(value) != number_inputs: err_out_screen( f"Number of {variable_name} values must match the number of {input_type} in the configuration file." ) def check_number_of_inputs_forcings(self, value: list, variable_name: str) -> None: """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable, specifically for input forcings variables which should match the number of input forcing options specified by the user in the configuration file.""" - return self.check_number_of_inputs(value, variable_name, " InputForcings") + return self.check_number_of_inputs(value, variable_name, " InputForcings",self.number_inputs) def check_number_of_inputs_supp_pcp(self, value: list, variable_name: str) -> None: """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable, specifically for supplemental precip forcing variables which should match the number of supplemental precip forcing options specified by the user in the configuration file.""" return self.check_number_of_inputs( - value, variable_name, " SupplementalPrecipForcings" + value, variable_name, " SupplementalPrecipForcings",self.number_supp_pcp ) def check_input_values_in_range( @@ -225,11 +242,19 @@ def check_input_values_in_range( ) -> None: """Check that the input values specified by the user in the configuration file are within a valid range for a given variable.""" for val in value: - if val in valid_input_options: + if val not in valid_input_options: err_out_screen( f"Invalid {variable_name} value '{val}' specified in configuration file. Please specify valid values: {valid_input_options}." ) + def check_input_values_non_negative(self, value: list, variable_name: str) -> None: + """Check that the input values specified by the user in the configuration file are positive for a given variable.""" + for val in value: + if float(val) < 0: + err_out_screen( + f"Invalid {variable_name} value '{val}' specified in configuration file. Please specify values greater than or equal to zero." + ) + def check_input_values_positive(self, value: list, variable_name: str) -> None: """Check that the input values specified by the user in the configuration file are positive for a given variable.""" for val in value: @@ -237,7 +262,6 @@ def check_input_values_positive(self, value: list, variable_name: str) -> None: err_out_screen( f"Invalid {variable_name} value '{val}' specified in configuration file. Please specify values greater than zero." ) - def uniquefy_scratch_dir_as_child(self, uid: str) -> None: """Modify the existing scratch dir by adding the UID string available to all ranks from the MpiConfig class. @@ -297,11 +321,12 @@ def supp_precip_forcings(self): @supp_precip_forcings.setter def supp_precip_forcings(self, value: list) -> None: """Set the list of supplemental precip forcing options specified by the user in the configuration file. This is used to control which supplemental precip forcings are processed and how they are processed based on the other configuration options specified for each supplemental precip forcing.""" - self.check_input_values_in_range( - value, - "SuppPcp", - list(range(1, self.supp_precip_count + 1)), - ) + if len(value)>0: + self.check_input_values_in_range( + [int(i) for i in value], + "SuppPcp", + list(range(1, self.supp_precip_count + 1)), + ) self._supp_precip_forcings = value @property @@ -315,7 +340,7 @@ def output_freq(self, value: int) -> None: Example- OutputFrequency: 60 """ - self.check_input_values_positive([value], "OutputFrequency") + self.check_input_values_non_negative([value], "OutputFrequency") self._output_freq = value @property @@ -331,11 +356,7 @@ def sub_output_hour(self, value: int) -> None: Example- SubOutputHour: 0 """ - self.check_input_values_positive([value], "SubOutputHour") - if value < 0: - err_out_screen( - "Please specify an SubOutputHour that is greater than zero minutes." - ) + self.check_input_values_non_negative([value], "SubOutputHour") if value == 0: value = None self._sub_output_hour = value @@ -434,7 +455,7 @@ def fcst_freq(self) -> int: @fcst_freq.setter def fcst_freq(self, value: int) -> None: """Set the forecast frequency in hours specified by the user in the configuration file. This is used to calculate the processing window for reforecast simulations, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" - self.check_input_values_positive([value], "ForecastFrequency") + self.check_input_values_non_negative([value], "ForecastFrequency") if value > 1440: err_out_screen( "Only forecast cycles of daily or sub-daily are supported at this time" @@ -468,7 +489,7 @@ def b_date_proc(self) -> str: Example- RefcstBDateProc: 202210071400 """ - return self._bdate_proc + return self._b_date_proc @b_date_proc.setter def b_date_proc(self, value: str | datetime) -> None: @@ -539,7 +560,8 @@ def geogrid(self, value: str) -> None: if self.user_provided_geogrid_flag: self._geogrid = value if value is None: - err_out_screen("Unable to locate GeogridIn in the configuration file.") + self._geogrid = value + # err_out_screen("Unable to locate GeogridIn in the configuration file.") else: geogrid_parent = os.path.dirname(value) geogrid_filename = os.path.basename(value) @@ -548,7 +570,8 @@ def geogrid(self, value: str) -> None: self._geogrid = os.path.join( geogrid_parent, f"{self.uid64}_{geogrid_filename}" ) - self.try_make_dir(geogrid_parent, " esmf_mesh") + self.try_make_dir(geogrid_parent, " esmf_mesh") + def try_make_dir(self, directory: str, optional_str: str = "") -> None: """Try to make a directory, and catch any errors.""" @@ -573,7 +596,7 @@ def input_forcings(self, value: list) -> None: self.check_input_values_in_range( value, "InputForcings", list(range(1, self.force_count + 1)) ) - self._input_forcings = value + self._input_forcings = value @property def number_inputs(self) -> int: @@ -628,14 +651,13 @@ def input_force_types(self, value: list) -> None: self.check_input_values_in_range( value, "InputForcingTypes", self.file_types ) - self._input_force_types = value - else: - self._input_force_types = None + self._input_force_types = value + @property def file_types(self): """Get the list of input forcing file types specified by the user in the configuration file. This is used to control how input forcings are read in and processed based on the file type specified for each input forcing in the configuration file.""" - return self.CONFIGOPTIONS["file_types"] + return CONFIGOPTIONS["file_types"] @property def input_force_dirs(self) -> list: @@ -663,10 +685,9 @@ def input_force_dirs(self, value: list) -> None: self.aws = True else: self.try_make_dir(dir_path, " forcing") - self._input_force_dirs = value - else: - self._input_force_dirs = None + self._input_force_dirs = value + @property def input_force_mandatory(self) -> list: """Get the list of input forcing mandatory flags specified by the user in the configuration file. This is used to control whether the program should raise an error if input forcings for a given forecast cycle are not found for each input forcing specified by the user in the configuration file.""" return self._input_force_mandatory @@ -680,10 +701,10 @@ def input_force_mandatory(self, value: list) -> None: if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "InputMandatory") self.check_input_values_in_range(value, "InputMandatory", [0, 1]) - self._input_force_mandatory = value - else: - self._input_force_mandatory = None + self._input_force_mandatory = value + + @property def customSuppPcpFreq(self) -> int: """Get the custom supplemental precip output frequency specified by the user in the configuration file. This is used to control the output frequency of supplemental precip forcings if the user has chosen to run the supplemental precip forcings module only.""" return self._customSuppPcpFreq @@ -692,11 +713,12 @@ def customSuppPcpFreq(self) -> int: def customSuppPcpFreq(self, value: int) -> None: """Set the custom supplemental precip output frequency specified by the user in the configuration file. This is used to control the output frequency of supplemental precip forcings if the user has chosen to run the supplemental precip forcings module only.""" if self.precip_only_flag: - self.check_input_values_positive([value], "customSuppPcpFreq") + self.check_input_values_non_negative([value], "customSuppPcpFreq") self._customSuppPcpFreq = value else: self._customSuppPcpFreq = None + @property def fcst_shift(self) -> int: """Forecast cycles are determined by splitting up a day by equal ForecastFrequency interval. If there is a desire to shift the cycles to a different time step, ForecastShift will shift forecast cycles ahead by a determined set of minutes. For example, ForecastFrequency of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, a ForecastShift of 1 hour will produce forecast cycles at 01, 07, 13, and 18 UTC. NOTE - This is only used by the realtime instance to calculate forecast cycles accordingly. Re-forecasts will use the beginning and ending dates specified in conjunction with the forecast frequency to determine forecast cycle dates. @@ -707,7 +729,7 @@ def fcst_shift(self) -> int: @fcst_shift.setter def fcst_shift(self, value: int) -> None: if True: # was: self.realtime_flag: - self.check_input_values_positive([value], "ForecastShift") + self.check_input_values_non_negative([value], "ForecastShift") # Calculate the beginning/ending processing dates if we are running realtime if self.realtime_flag: calculate_lookback_window(self) @@ -752,7 +774,7 @@ def fcst_input_horizons(self) -> list: def fcst_input_horizons(self, value: list) -> None: if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "ForecastInputHorizons") - self.check_input_values_positive(value, "ForecastInputHorizons") + self.check_input_values_non_negative(value, "ForecastInputHorizons") else: if len(self.fcst_input_horizons) != 1: err_out_screen( @@ -772,7 +794,7 @@ def fcst_input_offsets(self): def fcst_input_offsets(self, value: list) -> None: if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "ForecastInputOffsets") - self.check_input_values_positive(value, "ForecastInputOffsets") + self.check_input_values_non_negative(value, "ForecastInputOffsets") self._fcst_input_offsets = value @property @@ -930,7 +952,7 @@ def ignored_border_widths(self, value: list) -> None: """Set the list of ignored border widths specified by the user in the configuration file. This is used to control how the program processes input forcings based on the ignored border widths specified for each input forcing in the configuration file.""" if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "IgnoredBorderWidths") - self.check_input_values_positive(value, "IgnoredBorderWidths") + self.check_input_values_non_negative(value, "IgnoredBorderWidths") self._ignored_border_widths = value @property @@ -1092,6 +1114,7 @@ def dScaleParamDirs(self, value: list) -> None: ) self._dScaleParamDirs = value + @property def perform_downscaling(self) -> bool: """Determine whether downscaling of input forcings is necessary based on the downscaling options specified by the user for each input forcing in the configuration file.""" if ( @@ -1377,8 +1400,7 @@ def suppTemporalInterp(self): Example- SuppPcpTemporalInterpolation: [0, 0, 0] """ - if self.number_supp_pcp > 0: - return self._suppTemporalInterp + return self._suppTemporalInterp @suppTemporalInterp.setter def suppTemporalInterp(self, value): @@ -1392,17 +1414,15 @@ def suppTemporalInterp(self, value): @property def supp_pcp_max_hours(self): """Get the list of maximum forecast hours for supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how supplemental precipitation input forcings are processed based on the maximum forecast hour specified for each supplemental precipitation input forcing in the configuration file.""" - if self.number_supp_pcp > 0: - return self._supp_pcp_max_hours + return self._supp_pcp_max_hours @supp_pcp_max_hours.setter def supp_pcp_max_hours(self, value): """Set the list of maximum forecast hours for supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how supplemental precipitation input forcings are processed based on the maximum forecast hour specified for each supplemental precipitation input forcing in the configuration file.""" if self.number_supp_pcp > 0: if isinstance(value, list): - self.check_input_values_positive(value, "SuppPcpMaxHours") + self.check_number_of_inputs_supp_pcp(value, "SuppPcpMaxHours") elif isinstance(value, float) or isinstance(value, int): - self.check_input_values_positive(value, "SuppPcpMaxHours") value = [value] * self.number_supp_pcp self._supp_pcp_max_hours = value @@ -1456,8 +1476,7 @@ def supp_precip_param_dir(self): Example- SuppPcpParamDir: ['./forcingParam/AnA','./forcingParam/AnA','./forcingParam/AnA'] """ - if self.number_supp_pcp > 0: - return self._supp_precip_param_dir + return self._supp_precip_param_dir @supp_precip_param_dir.setter def supp_precip_param_dir(self, value): diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py index 16e399d6..5a4490f2 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py @@ -1264,90 +1264,90 @@ "ConfigOptions": [ "bmi_time", "current_time", - "input_forcings", - "supp_precip_forcings", - "input_force_dirs", - "input_force_types", + # "input_forcings", + # "supp_precip_forcings", + # "input_force_dirs", + # "input_force_types", "supp_precip_dirs", - "supp_precip_file_types", + # "supp_precip_file_types", "supp_precip_param_dir", - "input_force_mandatory", + # "input_force_mandatory", "supp_precip_mandatory", - "supp_pcp_max_hours", - "number_inputs", - "number_supp_pcp", - "output_freq", - "sub_output_hour", - "sub_output_freq", - "scratch_dir", - "num_output_steps", - "num_supp_output_steps", - "actual_output_steps", - "realtime_flag", - "refcst_flag", - "ana_flag", + # "supp_pcp_max_hours", + # "number_inputs", + # "number_supp_pcp", + # "output_freq", + # "sub_output_hour", + # "sub_output_freq", + # "scratch_dir", + # "num_output_steps", + # "num_supp_output_steps", + # "actual_output_steps", + # "realtime_flag", + # "refcst_flag", + # "ana_flag", "e_date_proc", "first_fcst_cycle", "current_fcst_cycle", "current_output_step", - "cycle_length_minutes", + # "cycle_length_minutes", "prev_output_date", "current_output_date", - "look_back", + # "look_back", "future_time", - "fcst_freq", + # "fcst_freq", "nFcsts", - "fcst_shift", - "fcst_input_horizons", - "fcst_input_offsets", + # "fcst_shift", + # "fcst_input_horizons", + # "fcst_input_offsets", "process_window", - "spatial_meta", - "grid_type", + # "spatial_meta", + # "grid_type", "grid_meta", "ExactExtract", - "lat_var", - "lon_var", - "hgt_var", - "cosalpha_var", - "sinalpha_var", - "slope_var", - "slope_azimuth_var", - "slope_var_elem", - "slope_azimuth_var_elem", - "nodecoords_var", - "elemcoords_var", - "elemconn_var", - "numelemconn_var", - "element_id_var", - "hgt_elem_var", - "ignored_border_widths", - "regrid_opt", - "weightsDir", - "regrid_opt_supp_pcp", + # "lat_var", + # "lon_var", + # "hgt_var", + # "cosalpha_var", + # "sinalpha_var", + # "slope_var", + # "slope_azimuth_var", + # "slope_var_elem", + # "slope_azimuth_var_elem", + # "nodecoords_var", + # "elemcoords_var", + # "elemconn_var", + # "numelemconn_var", + # "element_id_var", + # "hgt_elem_var", + # "ignored_border_widths", + # "regrid_opt", + # "weightsDir", + # "regrid_opt_supp_pcp", "errMsg", "statusMsg", "logFile", "logHandle", - "dScaleParamDirs", + # "dScaleParamDirs", "paramFlagArray", - "forceTemoralInterp", - "suppTemporalInterp", - "t2dDownscaleOpt", - "swDownscaleOpt", - "psfcDownscaleOpt", - "precipDownscaleOpt", - "q2dDownscaleOpt", - "t2BiasCorrectOpt", - "psfcBiasCorrectOpt", - "q2BiasCorrectOpt", - "windBiasCorrect", - "swBiasCorrectOpt", - "lwBiasCorrectOpt", - "precipBiasCorrectOpt", - "cfsv2EnsMember", - "customSuppPcpFreq", - "customFcstFreq", - "rqiMethod", + # "forceTemoralInterp", + # "suppTemporalInterp", + # "t2dDownscaleOpt", + # "swDownscaleOpt", + # "psfcDownscaleOpt", + # "precipDownscaleOpt", + # "q2dDownscaleOpt", + # "t2BiasCorrectOpt", + # "psfcBiasCorrectOpt", + # "q2BiasCorrectOpt", + # "windBiasCorrect", + # "swBiasCorrectOpt", + # "lwBiasCorrectOpt", + # "precipBiasCorrectOpt", + # "cfsv2EnsMember", + # "customSuppPcpFreq", + # "customFcstFreq", + # "rqiMethod", "nwmVersion", "nwmConfig", "forcing_output", @@ -1360,7 +1360,7 @@ ], "var_rename_map": {"config_path": "cfg_bmi"}, "extract_input_variable_attrs_map": { - "SuppPcp": "supp_precip_forcings", + # "SuppPcp": "supp_precip_forcings", "OutputFrequency": "output_freq", "SubOutputHour": "sub_output_hour", "SubOutFreq": "sub_output_freq", @@ -1376,19 +1376,19 @@ "SuppPcpMandatory": "supp_precip_mandatory", "RegridOptSuppPcp": "regrid_opt_supp_pcp", "SuppPcpTemporalInterpolation": "suppTemporalInterp", - "SuppPcpMaxHours": "supp_pcp_max_hours", "SuppPcpInputOffsets": "supp_input_offsets", - "SuppPcpParamDirs": "supp_precip_param_dir", + "SuppPcpParamDir": "supp_precip_param_dir", + "SuppPcpForcingTypes": "supp_precip_file_types", }, "extract_input_variable_attrs_map_precip_only": { "customSuppPcpFreq": "customSuppPcpFreq", }, "extract_input_variable_attrs_map_not_precip_only": { + # "InputForcings": "input_forcings", # np "ForecastInputHorizons": "fcst_input_horizons", # np "ForecastInputOffsets": "fcst_input_offsets", # np "IgnoredBorderWidths": "ignored_border_widths", # np "RegridOpt": "regrid_opt", # np - "RegridWeightsDir": "weightsDir", # np "ForcingTemporalInterpolation": "forceTemoralInterp", # np "TemperatureDownscaling": "t2dDownscaleOpt", # np "PressureDownscaling": "psfcDownscaleOpt", # np @@ -1402,7 +1402,6 @@ "SwBiasCorrection": "swBiasCorrectOpt", # np #yes "LwBiasCorrection": "lwBiasCorrectOpt", # np #yes "PrecipBiasCorrection": "precipBiasCorrectOpt", # np #yes - "InputForcings": "input_forcings", # np "InputForcingTypes": "input_force_types", # np "InputForcingDirectories": "input_force_dirs", # np "InputMandatory": "input_force_mandatory", # np @@ -1424,13 +1423,16 @@ "includeLQFrac": "include_lqfrac", "floatOutput": "useFloats", "Output": "forcing_output", + "SuppPcpMaxHours": "supp_pcp_max_hours", + "RegridWeightsDir": "weightsDir", # np }, "try_config_get_except_attr_map": { "RefcstBDateProc": "b_date_proc", "Geopackage": "geopackage", "GeogridIn": "geogrid", "SpatialMetaIn": "spatial_meta", - "SuppPcpForcingTypes": "supp_precip_file_types", + + }, "file_types": ["GRIB1", "GRIB2", "NETCDF", "NETCDF4", "NWM", "ZARR"], } From 88146fc09ea673710b6afb5179cfe670b7de89a8 Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Tue, 19 May 2026 16:21:46 -0500 Subject: [PATCH 037/132] handle circular imports --- .../NextGen_Forcings_Engine/os_utils.py | 2 +- .../NextGen_Forcings_Engine/retry_utils.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/os_utils.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/os_utils.py index 67827d49..7853da24 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/os_utils.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/os_utils.py @@ -5,7 +5,7 @@ import typing from typing import TYPE_CHECKING -from . import retry_utils +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine import retry_utils if TYPE_CHECKING: from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/retry_utils.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/retry_utils.py index f542c3ad..81594156 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/retry_utils.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/retry_utils.py @@ -4,14 +4,15 @@ import time import traceback import types -from typing import TYPE_CHECKING - -from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import MpiConfig +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( ConfigOptions, ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import ( + MpiConfig, + ) def retry_w_mpi_context( @@ -47,6 +48,13 @@ def wrapper( *args, **kwargs, ): + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( + ConfigOptions, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import ( + MpiConfig, + ) + if not isinstance(mpi_config, MpiConfig): raise TypeError( f"Expected type {MpiConfig} for mpi_config, got: {type(mpi_config)}" From 786cffdfc126496c40868feafc541fc43fb1e611 Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Tue, 19 May 2026 16:22:53 -0500 Subject: [PATCH 038/132] remove commented out attributes --- .../NextGen_Forcings_Engine/core/consts.py | 69 ------------------- 1 file changed, 69 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py index 5a4490f2..c1a95c50 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py @@ -1264,90 +1264,25 @@ "ConfigOptions": [ "bmi_time", "current_time", - # "input_forcings", - # "supp_precip_forcings", - # "input_force_dirs", - # "input_force_types", "supp_precip_dirs", - # "supp_precip_file_types", "supp_precip_param_dir", - # "input_force_mandatory", "supp_precip_mandatory", - # "supp_pcp_max_hours", - # "number_inputs", - # "number_supp_pcp", - # "output_freq", - # "sub_output_hour", - # "sub_output_freq", - # "scratch_dir", - # "num_output_steps", - # "num_supp_output_steps", - # "actual_output_steps", - # "realtime_flag", - # "refcst_flag", - # "ana_flag", "e_date_proc", "first_fcst_cycle", "current_fcst_cycle", "current_output_step", - # "cycle_length_minutes", "prev_output_date", "current_output_date", - # "look_back", "future_time", - # "fcst_freq", "nFcsts", - # "fcst_shift", - # "fcst_input_horizons", - # "fcst_input_offsets", "process_window", - # "spatial_meta", - # "grid_type", "grid_meta", "ExactExtract", - # "lat_var", - # "lon_var", - # "hgt_var", - # "cosalpha_var", - # "sinalpha_var", - # "slope_var", - # "slope_azimuth_var", - # "slope_var_elem", - # "slope_azimuth_var_elem", - # "nodecoords_var", - # "elemcoords_var", - # "elemconn_var", - # "numelemconn_var", - # "element_id_var", - # "hgt_elem_var", - # "ignored_border_widths", - # "regrid_opt", - # "weightsDir", - # "regrid_opt_supp_pcp", "errMsg", "statusMsg", "logFile", "logHandle", - # "dScaleParamDirs", "paramFlagArray", - # "forceTemoralInterp", - # "suppTemporalInterp", - # "t2dDownscaleOpt", - # "swDownscaleOpt", - # "psfcDownscaleOpt", - # "precipDownscaleOpt", - # "q2dDownscaleOpt", - # "t2BiasCorrectOpt", - # "psfcBiasCorrectOpt", - # "q2BiasCorrectOpt", - # "windBiasCorrect", - # "swBiasCorrectOpt", - # "lwBiasCorrectOpt", - # "precipBiasCorrectOpt", - # "cfsv2EnsMember", - # "customSuppPcpFreq", - # "customFcstFreq", - # "rqiMethod", "nwmVersion", "nwmConfig", "forcing_output", @@ -1360,7 +1295,6 @@ ], "var_rename_map": {"config_path": "cfg_bmi"}, "extract_input_variable_attrs_map": { - # "SuppPcp": "supp_precip_forcings", "OutputFrequency": "output_freq", "SubOutputHour": "sub_output_hour", "SubOutFreq": "sub_output_freq", @@ -1384,7 +1318,6 @@ "customSuppPcpFreq": "customSuppPcpFreq", }, "extract_input_variable_attrs_map_not_precip_only": { - # "InputForcings": "input_forcings", # np "ForecastInputHorizons": "fcst_input_horizons", # np "ForecastInputOffsets": "fcst_input_offsets", # np "IgnoredBorderWidths": "ignored_border_widths", # np @@ -1431,8 +1364,6 @@ "Geopackage": "geopackage", "GeogridIn": "geogrid", "SpatialMetaIn": "spatial_meta", - - }, "file_types": ["GRIB1", "GRIB2", "NETCDF", "NETCDF4", "NWM", "ZARR"], } From af922e091504210ba2c1f64992b5d49b053bf411 Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Tue, 19 May 2026 16:24:07 -0500 Subject: [PATCH 039/132] use dir instead of vars --- .../NextGen_Forcings_Engine/core/forcingInputMod.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py index 356c04cb..1cb97ef7 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py @@ -93,7 +93,8 @@ def _initialize_config_options(self) -> None: Check if the attibute allready exists before setting. """ - for key, val in list(vars(self.config_options).items()): + for key in dir(self.config_options): + val=getattr(self.config_options,key) if ( isinstance(val, list) and len(val) > 0 From 91550d0dd3b3d126b397aeac1ee1c3ced099183f Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Tue, 19 May 2026 16:24:13 -0500 Subject: [PATCH 040/132] debugging --- .../NextGen_Forcings_Engine/core/config.py | 103 ++++++++++-------- 1 file changed, 59 insertions(+), 44 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index f1b6c4d7..95dcc77e 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import configparser import json import logging @@ -5,6 +7,7 @@ import re import uuid from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Any # Use the Error, Warning, and Trapping System Package for logging import numpy as np @@ -59,12 +62,10 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No ) self.nwm_source = "s3://noaa-nwm-retrospective-3-0-pds" - self._scratch_dir_has_been_uniquefied = False self.supp_precip_forcings = self.extract_input_variable("SuppPcp") if not self.precip_only_flag: self.input_forcings = self.extract_input_variable("InputForcings") - # Create temporary array to hold flags if we need input parameter files. self.param_flag = np.zeros([len(self.input_forcings)], int) @@ -75,19 +76,30 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No setattr(self, attr, None) self.broadcast_new_64bit_uid() - for cfg_bmi_attr, config_options_attr in self.try_config_get_except_attr_map.items(): - setattr(self,config_options_attr,self.try_config_get(cfg_bmi_attr)) - + for ( + cfg_bmi_attr, + config_options_attr, + ) in self.try_config_get_except_attr_map.items(): + setattr(self, config_options_attr, self.try_config_get(cfg_bmi_attr)) + self.set_attrs(CONFIGOPTIONS["extract_input_variable_attrs_map"]) if self.precip_only_flag: self.set_attrs( CONFIGOPTIONS["extract_input_variable_attrs_map_precip_only"] ) + self.set_attrs( + CONFIGOPTIONS["extract_input_variable_attrs_map_not_precip_only"], + set_none=True, + ) else: self.set_attrs( CONFIGOPTIONS["extract_input_variable_attrs_map_not_precip_only"] ) + self.set_attrs( + CONFIGOPTIONS["extract_input_variable_attrs_map_precip_only"], + set_none=True, + ) if 27 in self.input_forcings: self.nwm_geogrid = self.extract_input_variable("NWMGeogridIn") @@ -99,14 +111,14 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No for cfg_bmi_attr, config_options_attr in CONFIGOPTIONS[ "extract_input_variable_set_default_attrs_map" ].items(): - if config_options_attr=="supp_pcp_max_hours": - default=None + if config_options_attr == "supp_pcp_max_hours": + default = None else: - default=0 + default = 0 setattr( self, config_options_attr, - self.extract_input_variable_set_default(cfg_bmi_attr,default), + self.extract_input_variable_set_default(cfg_bmi_attr, default), ) @property @@ -159,18 +171,24 @@ def precip_only_flag(self) -> bool: precip_only = True return precip_only - def set_attrs(self, attrs_dict: dict): + def set_attrs(self, attrs_dict: dict, set_none: bool = False): """Set the attributes of the class based on the configuration file. This is used to populate the attributes of the class after they have been read in and validated from the configuration file.""" for cfg_bmi_attr, config_options_attr in attrs_dict.items(): - setattr( - self, config_options_attr, self.extract_input_variable(cfg_bmi_attr) - ) - def set_attrs_use_default(self,attrs_dict:dict): + if set_none: + attr = None + else: + attr = self.extract_input_variable(cfg_bmi_attr) + setattr(self, config_options_attr, attr) + + def set_attrs_use_default(self, attrs_dict: dict): """Set the attributes of the class based on the configuration file. Set default value to default if not found in config file.""" for cfg_bmi_attr, config_options_attr in attrs_dict.items(): setattr( - self, config_options_attr, self.extract_input_variable_set_default(cfg_bmi_attr) + self, + config_options_attr, + self.extract_input_variable_set_default(cfg_bmi_attr), ) + def extract_input_variable(self, variable_name: str) -> str: """Extract the variable name from the configuration file for a given variable.""" try: @@ -199,7 +217,7 @@ def extract_input_variable_set_default(self, variable_name: str, default=0) -> s err_out_screen( f"Improper {variable_name} value: {self.cfg_bmi[variable_name]}", e ) - if default==0: + if default == 0: if variable not in [0, 1]: err_out_screen(f"Please choose a {variable_name} value of 0 or 1.") return variable @@ -219,7 +237,7 @@ def try_config_get(self, variable_name: str) -> str: ) def check_number_of_inputs( - self, value: list, variable_name: str, input_type: str,number_inputs:int + self, value: list, variable_name: str, input_type: str, number_inputs: int ) -> None: """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable.""" if len(value) != number_inputs: @@ -229,12 +247,14 @@ def check_number_of_inputs( def check_number_of_inputs_forcings(self, value: list, variable_name: str) -> None: """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable, specifically for input forcings variables which should match the number of input forcing options specified by the user in the configuration file.""" - return self.check_number_of_inputs(value, variable_name, " InputForcings",self.number_inputs) + return self.check_number_of_inputs( + value, variable_name, " InputForcings", self.number_inputs + ) def check_number_of_inputs_supp_pcp(self, value: list, variable_name: str) -> None: """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable, specifically for supplemental precip forcing variables which should match the number of supplemental precip forcing options specified by the user in the configuration file.""" return self.check_number_of_inputs( - value, variable_name, " SupplementalPrecipForcings",self.number_supp_pcp + value, variable_name, " SupplementalPrecipForcings", self.number_supp_pcp ) def check_input_values_in_range( @@ -262,6 +282,7 @@ def check_input_values_positive(self, value: list, variable_name: str) -> None: err_out_screen( f"Invalid {variable_name} value '{val}' specified in configuration file. Please specify values greater than zero." ) + def uniquefy_scratch_dir_as_child(self, uid: str) -> None: """Modify the existing scratch dir by adding the UID string available to all ranks from the MpiConfig class. @@ -321,7 +342,7 @@ def supp_precip_forcings(self): @supp_precip_forcings.setter def supp_precip_forcings(self, value: list) -> None: """Set the list of supplemental precip forcing options specified by the user in the configuration file. This is used to control which supplemental precip forcings are processed and how they are processed based on the other configuration options specified for each supplemental precip forcing.""" - if len(value)>0: + if len(value) > 0: self.check_input_values_in_range( [int(i) for i in value], "SuppPcp", @@ -572,7 +593,6 @@ def geogrid(self, value: str) -> None: ) self.try_make_dir(geogrid_parent, " esmf_mesh") - def try_make_dir(self, directory: str, optional_str: str = "") -> None: """Try to make a directory, and catch any errors.""" if not os.path.isdir(directory): @@ -653,7 +673,6 @@ def input_force_types(self, value: list) -> None: ) self._input_force_types = value - @property def file_types(self): """Get the list of input forcing file types specified by the user in the configuration file. This is used to control how input forcings are read in and processed based on the file type specified for each input forcing in the configuration file.""" @@ -662,7 +681,10 @@ def file_types(self): @property def input_force_dirs(self) -> list: """Get the list of input forcing directories specified by the user in the configuration file. This is used to control where input forcings are read in from for each input forcing specified by the user in the configuration file.""" - return self._input_force_dirs + if self._input_force_dirs: + return self._input_force_dirs + else: + None @input_force_dirs.setter def input_force_dirs(self, value: list) -> None: @@ -703,7 +725,6 @@ def input_force_mandatory(self, value: list) -> None: self.check_input_values_in_range(value, "InputMandatory", [0, 1]) self._input_force_mandatory = value - @property def customSuppPcpFreq(self) -> int: """Get the custom supplemental precip output frequency specified by the user in the configuration file. This is used to control the output frequency of supplemental precip forcings if the user has chosen to run the supplemental precip forcings module only.""" @@ -856,12 +877,6 @@ def grid_type(self, value: str) -> None: ) self._grid_type = value.lower() - def raise_grid_type_error(self, grid_type: str, variable_name: str) -> None: - """Raise an error if a variable is requested that is not valid for the given grid type.""" - err_out_screen( - f"{variable_name} is not a valid variable for grid type {grid_type}. Please check your configuration file." - ) - @property def lon_var(self) -> str: """Naming convention of the longitude variable within the "GeogridIn" file the user has specified. Variable naming convention ONLY for gridded domain configurations. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. In the case for "gridded" domain configuration options and a user specifying downscaling options while only specifying a height variable feature on the grid, this netcdf variable (LONVAR) is then EXPECTED to contain a netcdf metadata attribute called "dx" that specifies the grid spacing in the longtiudinal direction. Otherwise, it will throw an error and not be able to calculate the slope and tilt of each grid cell. @@ -870,8 +885,6 @@ def lon_var(self) -> str: """ if self.grid_type == "gridded": return self.extract_input_variable("LONVAR") - else: - self.raise_grid_type_error(self.grid_type, "LONVAR") @property def lat_var(self) -> str: @@ -881,8 +894,6 @@ def lat_var(self) -> str: """ if self.grid_type == "gridded": return self.extract_input_variable("LATVAR") - else: - self.raise_grid_type_error(self.grid_type, "LATVAR") @property def nodecoords_var(self) -> str: @@ -892,8 +903,6 @@ def nodecoords_var(self) -> str: """ if self.grid_type in ["unstructured", "hydrofabric"]: return self.extract_input_variable("NodeCoords") - else: - self.raise_grid_type_error(self.grid_type, "NodeCoords") @property def elemcoords_var(self) -> str: @@ -903,8 +912,6 @@ def elemcoords_var(self) -> str: """ if self.grid_type in ["unstructured", "hydrofabric"]: return self.extract_input_variable("ElemCoords") - else: - self.raise_grid_type_error(self.grid_type, "ElemCoords") @property def elemconn_var(self) -> str: @@ -914,8 +921,6 @@ def elemconn_var(self) -> str: """ if self.grid_type in ["unstructured", "hydrofabric"]: return self.extract_input_variable("ElemConn") - else: - self.raise_grid_type_error(self.grid_type, "ElemConn") @property def numelemconn_var(self) -> str: @@ -925,8 +930,6 @@ def numelemconn_var(self) -> str: """ if self.grid_type in ["unstructured", "hydrofabric"]: return self.extract_input_variable("NumElemConn") - else: - self.raise_grid_type_error(self.grid_type, "NumElemConn") @property def element_id_var(self) -> str: @@ -936,8 +939,6 @@ def element_id_var(self) -> str: """ if self.grid_type == "hydrofabric": return self.extract_input_variable("ElemID") - else: - self.raise_grid_type_error(self.grid_type, "ElemID") @property def ignored_border_widths(self) -> list: @@ -970,6 +971,8 @@ def regrid_opt(self, value: list) -> None: self.check_number_of_inputs_forcings(value, "RegridOpt") self.check_input_values_in_range(value, "RegridOpt", [1, 2, 3]) self._regrid_opt = value + else: + self._regrid_opt = None @property def weightsDir(self) -> str: @@ -1378,6 +1381,8 @@ def supp_precip_mandatory(self, value): if self.number_supp_pcp > 0: self.check_input_values_in_range(value, "SuppPcpMandatory", [0, 1]) self._supp_precip_mandatory = value + else: + self._supp_precip_mandatory = None @property def regrid_opt_supp_pcp(self): @@ -1393,6 +1398,8 @@ def regrid_opt_supp_pcp(self, value): if self.number_supp_pcp > 0: self.check_input_values_in_range(value, "RegridOptSuppPcp", [1, 2, 3]) self._regrid_opt_supp_pcp = value + else: + self._regrid_opt_supp_pcp = None @property def suppTemporalInterp(self): @@ -1410,6 +1417,8 @@ def suppTemporalInterp(self, value): value, "SuppPcpTemporalInterpolation", [0, 1, 2] ) self._suppTemporalInterp = value + else: + self._suppTemporalInterp = None @property def supp_pcp_max_hours(self): @@ -1425,6 +1434,8 @@ def supp_pcp_max_hours(self, value): elif isinstance(value, float) or isinstance(value, int): value = [value] * self.number_supp_pcp self._supp_pcp_max_hours = value + else: + self._supp_pcp_max_hours = None @property def supp_input_offsets(self): @@ -1439,6 +1450,9 @@ def supp_input_offsets(self, value): """Set the list of time offsets to apply to supplemental precipitation input forcing files specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are processed based on the time offset specified for each supplemental precipitation input forcing in the configuration file.""" if self.number_supp_pcp > 0: self.check_number_of_inputs_supp_pcp(value, "SuppPcpInputOffsets") + self._supp_input_offsets = value + else: + self._supp_input_offsets = None @property def supp_precip_dirs(self): @@ -1490,6 +1504,7 @@ def supp_precip_param_dir(self, value): @property def cfsv2EnsMember(self): """Set the CFSv2 ensemble member to process specified by the user in the configuration file. This is used to control which CFSv2 ensemble member is processed for CFSv2 input forcings based on the ensemble member specified in the configuration file.""" + value = None if not self.precip_only_flag: # Read in Ensemble information # Read in CFS ensemble member information IF we have chosen CFSv2 as an input From 6ac79f0145ccca384f5c7c8a79882b1d68f94a6e Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Wed, 20 May 2026 14:32:27 -0500 Subject: [PATCH 041/132] fix regrid_opt --- .../NextGen_Forcings_Engine/core/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 95dcc77e..037fff30 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -111,7 +111,7 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No for cfg_bmi_attr, config_options_attr in CONFIGOPTIONS[ "extract_input_variable_set_default_attrs_map" ].items(): - if config_options_attr == "supp_pcp_max_hours": + if config_options_attr in ["supp_pcp_max_hours","weightsDir"]: default = None else: default = 0 @@ -967,7 +967,7 @@ def regrid_opt(self): @regrid_opt.setter def regrid_opt(self, value: list) -> None: """Set the list of regridding options specified by the user in the configuration file. This is used to control how input forcings are regridded based on the regridding option specified for each input forcing in the configuration file.""" - if self.precip_only_flag: + if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "RegridOpt") self.check_input_values_in_range(value, "RegridOpt", [1, 2, 3]) self._regrid_opt = value From 5ba2579e543ed0ce281540eed818138d8779168e Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Wed, 27 May 2026 13:51:03 -0500 Subject: [PATCH 042/132] refactor initialize method --- .../NextGen_Forcings_Engine/bmi_model.py | 408 +++++++++--------- NextGen_Forcings_Engine_BMI/run_bmi_model.py | 14 +- 2 files changed, 218 insertions(+), 204 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 06fe6d13..47d79538 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -2,14 +2,15 @@ # This is needed for get_var_bytes import gc import hashlib +import logging import os # time debugging import time from collections import defaultdict -from pathlib import Path from datetime import datetime, timezone -import logging +from pathlib import Path + import netCDF4 as nc # import data_tools @@ -128,25 +129,15 @@ class NWMv3_Forcing_Engine_BMI_model_Base(Bmi): It includes methods for initializing the model, updating it, accessing model variables, and managing model configuration. This class is responsible for interacting with geospatial data and forcing inputs for the model simulation. - - Attributes - ---------- - _values : dict - Dictionary storing model values. - _start_time : float - The start time for the simulation. - _end_time : float - The end time for the simulation. - _model : object - The model object. - _comm : object - The MPI communicator. - var_array_lengths : int - Length of the variable arrays. - """ - def __init__(self): + def __init__( + self, + config_file: str, + b_date: str = None, + geogrid: str = None, + output_path: str = None, + ) -> None: """Create a model that is ready for initialization. Initializes the model with default values for time, variables, and grid types. @@ -177,7 +168,7 @@ def __init__(self): self.cfg_bmi = None self._job_meta = None self._mpi_meta = None - self.geo_meta = None + self._geo_meta = None self._grid_type = None self._grids = None self._grid_map = None @@ -188,122 +179,121 @@ def __init__(self): self._var_units_map = None self._input_forcing_mod = None self._supp_pcp_mod = None + self._output_obj = None self._model_parameters_list = [] - # Diagnostic timing setup - self._call_counts = defaultdict(int) self._call_times = defaultdict(float) self._total_start = None + self._att_map = BMI_MODEL["att_map"] + self._input_var_names = [] + self._model_parameters_list = [] + self._config_file = config_file + self._geogrid = geogrid + self.get_grid_edge_count - # ---------------------------------------------- - # Required, static attributes of the model - # ---------------------------------------------- - _att_map = BMI_MODEL["att_map"] - - # --------------------------------------------- - # Input variable names (CSDMS standard names) - # --------------------------------------------- - # Forcings engine requires no inputs currently - # and only provides model output - _input_var_names = [] - - _input_var_types = {} - - # ------------------------------------------------------ - # A list of static attributes/parameters. - # ------------------------------------------------------ - _model_parameters_list = [] - - # ------------------------------------------------------------ - # ------------------------------------------------------------ - # BMI: Model Control Functions - # ------------------------------------------------------------ - # ------------------------------------------------------------ - - # ------------------------------------------------------------------- - def initialize(self, config_file: str, output_path: str | None = None) -> None: - """Initialize the model using a configuration file. - - This function is part of the BMI (Basic Model Interface) specification and is automatically - invoked by the BMI system. When running standalone, call `initialize_with_params()` instead, - which sets additional parameters such as `b_date`, `geogrid`, and `output_path`. - - This function is responsible for: - - Setting up core model attributes, grids, and MPI communication. - - Reading the BMI configuration file and initializing basic model components. - - :param config_file: The path to the configuration file for model initialization. - :raises RuntimeError: If the configuration file is invalid or missing. - """ - - LOG.info("---------------------------") - LOG.info( - f"BMI Forcing Engine initializing with {config_file}{Pld(St.INITTING, modnm=MODNM)}" - ) + # Now that _job_meta is set, call initialize() to set up the core model + self.initialize(config_file, output_path=output_path) - # -------------- Read in the BMI configuration -------------------------# - if not isinstance(config_file, str) or len(config_file) == 0: + @property + def bmi_cfg_file(self): + """Validate and return the BMI configuration file path.""" + if not isinstance(self._config_file, str) or len(self._config_file) == 0: LOG.critical("No BMI initialize configuration provided, nothing to do...") raise RuntimeError( "No BMI initialize configuration provided, nothing to do..." ) - - bmi_cfg_file = Path(config_file).resolve() + bmi_cfg_file = Path(self._config_file).resolve() if not bmi_cfg_file.is_file(): LOG.critical(f"Config file {bmi_cfg_file} not found, nothing to do...") raise RuntimeError( f"Config file {bmi_cfg_file} not found, nothing to do..." ) - - LOG.info(f"Reading config file: {bmi_cfg_file}") - with bmi_cfg_file.open("r") as fp: + return bmi_cfg_file + + @property + def cfg_bmi(self): + """Read and parse the BMI configuration file.""" + if self._cfg_bmi is not None: + return self._cfg_bmi + LOG.info(f"Reading config file: {self.bmi_cfg_file}") + with self.bmi_cfg_file.open("r") as fp: cfg = yaml.safe_load(fp) - - self.cfg_bmi = parse_config(cfg) - - # If _job_meta was not set by initialize_with_params(), create a default one - if self._job_meta is None: - self._job_meta = ConfigOptions(self.cfg_bmi) - - # Parse the configuration options - # try: - # self._job_meta.validate_config(self.cfg_bmi) - # except KeyboardInterrupt as e: - # err_handler.err_out_screen("User keyboard interrupt", e) - # except ImportError as e: - # err_handler.err_out_screen("Missing Python packages", e) - # except InterruptedError as e: - # err_handler.err_out_screen("External kill signal detected", e) - # except Exception as e: - # err_handler.err_out_screen("Unhandled exception", e) - - # Set NWM version and config, if provided in the config - if self.cfg_bmi.get("NWM_VERSION") is not None: - self._job_meta.nwmVersion = self.cfg_bmi["NWM_VERSION"] - - # Place NWM configuration (if provided by the user). This will be placed into the final - # output files as a global attribute. - if self.cfg_bmi.get("NWM_CONFIG") is not None: - self._job_meta.nwmConfig = self.cfg_bmi["NWM_CONFIG"] - - # Initialize MPI communication - self._mpi_meta = MpiConfig(self._job_meta) - - self.geo_meta = HydrofabricGeoMeta(self._job_meta, self._mpi_meta) - + return parse_config(cfg) + + @cfg_bmi.setter + def cfg_bmi(self, value): + """Set the BMI configuration.""" + self._cfg_bmi = value + + @property + def _job_meta(self): + """Return the job metadata object.""" + return self.__job_meta + + @_job_meta.setter + def _job_meta(self, value): + """Set the job metadata object.""" + if value is None: + value = ConfigOptions( + self.cfg_bmi, b_date=self._b_date, geogrid_arg=self._geogrid + ) + try: + value.validate_config(self.cfg_bmi) + except KeyboardInterrupt as e: + err_handler.err_out_screen("User keyboard interrupt", e) + except ImportError as e: + err_handler.err_out_screen("Missing Python packages", e) + except InterruptedError as e: + err_handler.err_out_screen("External kill signal detected", e) + except Exception as e: + err_handler.err_out_screen("Unhandled exception", e) + value.nwmVersion = self.cfg_bmi.get("NWM_VERSION") + value.nwmConfig = self.cfg_bmi.get("NWM_CONFIG") + self.__job_meta = value + + @property + def _mpi_meta(self): + """Return the MPI metadata object.""" + if self.__mpi_meta is None: + self.__mpi_meta = MpiConfig(self._job_meta) + return self.__mpi_meta + + @_mpi_meta.setter + def _mpi_meta(self, value): + """Set the MPI metadata object.""" + self.__mpi_meta = value + + @property + def geo_meta(self): + """Return the geospatial metadata object.""" + if self._geo_meta is None: + self._geo_meta = HydrofabricGeoMeta(self._job_meta, self._mpi_meta) + return self._geo_meta + + @geo_meta.setter + def geo_meta(self, value): + """Set the geospatial metadata object.""" + self._geo_meta = value + + def init_mpi(self): + """Set up MPI communication for the model.""" try: comm = MPI.Comm.f2py(self._comm) if self._comm is not None else None self._mpi_meta.initialize_comm(comm=comm) except Exception as e: err_handler.err_out_screen(self._job_meta.errMsg, e) - ### Reassign the scratch dir to a new child dir of the current scratch dir, - ### applying uniqueness to the final path. This must be called by all ranks, once. + def init_scratch_dir(self): + """Set up the scratch directory for the model, ensuring it is unique for each job. + + Reassign the scratch dir to a new child dir of the current scratch dir, + applying uniqueness to the final path. This must be called by all ranks, once. + """ self._job_meta.uniquefy_scratch_dir_as_child(self._mpi_meta.uid64) - # LOG.debug(f"self._job_meta type: {type(self._job_meta)}") - # Call ESMF mesh creation process + def create_esmf_mesh(self): + """Create the ESMF mesh for the model.""" if self._mpi_meta.rank == 0: cat_ids = esmf_creation.create_mesh(self._job_meta) cat_count = np.array([ @@ -314,7 +304,13 @@ def initialize(self, config_file: str, output_path: str | None = None) -> None: cat_ids = np.empty(cat_count[0], dtype=np.int64) self._mpi_meta.comm.Bcast(cat_ids, root=0) - # Call forcing_extraction process + def fetch_raw_forcing_data(self): + """Fetch raw forcing data for the model. + + This function is responsible for retrieving the raw forcing data needed for the model simulation. + It is called during the initialization process and ensures that all necessary data is available + before the model runs. + """ if self._job_meta.nwmConfig not in ["AORC", "NWM"]: if self._mpi_meta.rank == 0: err_handler.log_msg( @@ -332,54 +328,79 @@ def initialize(self, config_file: str, output_path: str | None = None) -> None: ) self._mpi_meta.comm.Barrier() - # Assign grid type to BMI class for grid information - self._grid_type = self._job_meta.grid_type.lower() - self.set_var_names() + @property + def _grid_type(self): + """Return the grid type of the model.""" + return self._job_meta.grid_type.lower() - # ----- Create some lookup tabels from the long variable names --------# - self._var_name_map_long_first = { + @property + def _var_name_map_long_first(self): + """Return the variable name mapping from long names to short names.""" + return { long_name: self._var_name_units_map[long_name][0] for long_name in self._var_name_units_map.keys() } - self._var_name_map_short_first = { + + @property + def _var_name_map_short_first(self): + """Return the variable name mapping from short names to long names.""" + return { self._var_name_units_map[long_name][0]: long_name for long_name in self._var_name_units_map.keys() } - self._var_units_map = { + + @property + def _var_units_map(self): + """Return the variable units mapping.""" + return { long_name: self._var_name_units_map[long_name][1] for long_name in self._var_name_units_map.keys() } - # Check to make sure we have enough dimensionality to run regridding. We assume that hydrofabric discretizations are large - # enough that 1x1 (single catchment) will provide enough points. For gridded and unstructured domains, we need to make sure - # that the local grid size for each processor is at least 2x2 to run the regridding process. - # forcing_input dimensionality is checked in regrid.py. + @property + def dimensionality(self): + """Return the dimensionality of the model grid based on the grid type. - dimensionality = 1 if self._grid_type == "hydrofabric" else 2 + Check to make sure we have enough dimensionality to run regridding. We assume that hydrofabric discretizations are large + enough that 1x1 (single catchment) will provide enough points. For gridded and unstructured domains, we need to make sure + that the local grid size for each processor is at least 2x2 to run the regridding process. + forcing_input dimensionality is checked in regrid.py. + """ + if self._grid_type == "hydrofabric": + return 1 + else: + return 2 + def check_dimensionality(self): + """Check that the local grid size is sufficient for the specified number of cores.""" if ( - self.geo_meta.nx_local < dimensionality - or self.geo_meta.ny_local < dimensionality + self.geo_meta.nx_local < self.dimensionality + or self.geo_meta.ny_local < self.dimensionality ): self._job_meta.errMsg = ( f"You have specified too many cores for your WRF-Hydro grid. " - f"Local grid Must have x/y dimension size of {dimensionality}." + f"Local grid Must have x/y dimension size of {self.dimensionality}." ) err_handler.err_out_screen_para(self._job_meta.errMsg, self._mpi_meta) err_handler.check_program_status(self._job_meta, self._mpi_meta) - # Initialize our output object, which includes local slabs from the output grid. + def init_output_obj(self): + """Initialize our output object, which includes local slabs from the output grid.""" try: self._output_obj = ioMod.OutputObj(self._job_meta, self.geo_meta) except Exception as e: err_handler.err_out_screen_para(self._job_meta, self._mpi_meta) err_handler.check_program_status(self._job_meta, self._mpi_meta) - # Next, initialize our input forcing classes. These objects will contain - # information about our source products (I.E. data type, grid sizes, etc). - # Information will be mapped via the options specified by the user. - # In addition, input ESMF grid objects will be created to hold data for - # downscaling and regridding purposes. + def init_input_forcing_mod(self): + """Initialize the input forcing module. + + Next, initialize our input forcing classes. These objects will contain + information about our source products (I.E. data type, grid sizes, etc). + Information will be mapped via the options specified by the user. + In addition, input ESMF grid objects will be created to hold data for + downscaling and regridding purposes. + """ try: self._input_forcing_mod = forcingInputMod.init_dict( self._job_meta, self.geo_meta, self._mpi_meta @@ -388,84 +409,59 @@ def initialize(self, config_file: str, output_path: str | None = None) -> None: err_handler.err_out_screen_para(self._job_meta, self._mpi_meta) err_handler.check_program_status(self._job_meta, self._mpi_meta) - # If we have specified supplemental precipitation products, initialize - # the supp class. + def init_supp_pcp_mod(self): + """Initialize the supplemental precipitation module, if applicable.""" if self._job_meta.number_supp_pcp > 0: self._supp_pcp_mod = suppPrecipMod.init_dict(self._job_meta, self.geo_meta) else: self._supp_pcp_mod = None err_handler.check_program_status(self._job_meta, self._mpi_meta) - # ------------- Initialize the parameters, inputs and outputs ----------# + def initialize_parameters(self): + """Initialize the parameters, inputs and outputs.""" for parm in self._model_parameters_list: self._values[self._var_name_map_short_first[parm]] = self.cfg_bmi[parm] - self.get_size_of_arrays() - - # for model_input in self.get_input_var_names(): - # self._values[model_input] = np.zeros(self._varsize, dtype=float) - - # Set initial time, step, and true catchment IDs + def set_initial_time_and_step(self) -> None: + """Set the initial time and time step size for the model.""" self._values["current_model_time"] = self.cfg_bmi["initial_time"] self._values["time_step_size"] = self.cfg_bmi["time_step_seconds"] self._values["CAT-ID"] = cat_ids - # Initialize the Forcings Engine model - self._model = NWMv3ForcingEngineModel() + def initialize(self, config_file: str) -> None: + """Initialize the model using a configuration file. - self._configure_output_path(output_path) + This function is part of the BMI (Basic Model Interface) specification and is automatically + invoked by the BMI system. When running standalone, call `initialize_with_params()` instead, + which sets additional parameters such as `b_date`, `geogrid`, and `output_path`. - LOG.info(f"BMI Forcing Engine initialized{Pld(St.INITTED, modnm=MODNM)}") + This function is responsible for: + - Setting up core model attributes, grids, and MPI communication. + - Reading the BMI configuration file and initializing basic model components. - def initialize_with_params( - self, - config_file: str, - b_date: str = None, - geogrid: str = None, - output_path: str = None, - ) -> None: - """Initialize the NWMv3 Forcings Engine model with additional job metadata parameters. - - This function **must be called by the user** to fully initialize the NWMv3 Forcings Engine model, - including both core model setup and additional job metadata configuration (such as b_date, geogrid, and output path). - - It performs the following: - - Sets up job metadata (b_date, geogrid) by calling `config_options`. - - Calls the `initialize()` function to handle core model setup (reading the config file, - initializing basic model attributes like MPI, grids, etc.). - - Handles additional configuration options, such as determining the output path - for model results. - - **DO NOT call `initialize()` directly**. Always use this function, which ensures proper - initialization of all necessary parameters and job metadata. - - :param config_file: The configuration file path for the model initialization. - :param b_date: The start date for the simulation. Typically the forecast cycle start time. - :param geogrid: The path to the geospatial grid data, such as a geospatial file for the grid. - :param output_path: The output path for model results. If omitted, a default path will be generated. - :raises ValueError: If an invalid grid type is specified, an exception is raised. + :param config_file: The path to the configuration file for model initialization. + :raises RuntimeError: If the configuration file is invalid or missing. """ - # This is required prior to the first log message. - LOG.bind() - - bmi_cfg_file = Path(config_file).resolve() - if not bmi_cfg_file.is_file(): - LOG.critical(f"Config file {bmi_cfg_file} not found, nothing to do...") - raise RuntimeError( - f"Config file {bmi_cfg_file} not found, nothing to do..." - ) + self.init_mpi() + self.init_scratch_dir() + self.create_esmf_mesh() + self.fetch_raw_forcing_data() + self.set_var_names() + self.check_dimensionality() + self.init_output_obj() + self.init_input_forcing_mod() + self.init_supp_pcp_mod() + self.get_size_of_arrays() + self.set_initial_time_and_step() - LOG.info(f"Reading config file: {bmi_cfg_file}") - with bmi_cfg_file.open("r") as fp: - cfg = yaml.safe_load(fp) + # Initialize the Forcings Engine model + self._model = NWMv3ForcingEngineModel() - self.cfg_bmi = parse_config(cfg) - # Set the job metadata parameters (b_date, geogrid) using config_options - self.cfg_bmi = parse_config(cfg) - self._job_meta = ConfigOptions(self.cfg_bmi, b_date=b_date, geogrid=geogrid) + # Set catchment ids if using hydrofabric + if self._grid_type == "hydrofabric": + self._values["CAT-ID"] = self.geo_meta.element_ids_global - # Now that _job_meta is set, call initialize() to set up the core model - self.initialize(config_file, output_path=output_path) + self._configure_output_path(self.output_path) def _configure_output_path(self, output_path: str | None = None) -> None: """Set the output path and initializes the output NetCDF file if forcing output is enabled. @@ -1646,12 +1642,18 @@ class NWMv3_Forcing_Engine_BMI_model_Gridded(NWMv3_Forcing_Engine_BMI_model_Base geospatial data and forcing inputs for the model simulation. """ - def __init__(self): + def __init__( + self, + config_file: str, + b_date: str = None, + geogrid: str = None, + output_path: str = None, + ): """Create a model that is ready for initialization. Initializes the model with default values for time, variables, and grid types. """ - super().__init__() + super().__init__(config_file, b_date, geogrid, output_path) self.GeoMeta = GriddedGeoMeta def grid_ranks(self) -> list[int]: @@ -1712,12 +1714,18 @@ class NWMv3_Forcing_Engine_BMI_model_HydroFabric(NWMv3_Forcing_Engine_BMI_model_ geospatial data and forcing inputs for the model simulation. """ - def __init__(self): + def __init__( + self, + config_file: str, + b_date: str = None, + geogrid: str = None, + output_path: str = None, + ): """Create a model that is ready for initialization. Initializes the model with default values for time, variables, and grid types. """ - super().__init__() + super().__init__(config_file, b_date, geogrid, output_path) self.GeoMeta = HydrofabricGeoMeta def grid_ranks(self) -> list[int]: @@ -1774,12 +1782,18 @@ class NWMv3_Forcing_Engine_BMI_model_Unstructured(NWMv3_Forcing_Engine_BMI_model geospatial data and forcing inputs for the model simulation. """ - def __init__(self): + def __init__( + self, + config_file: str, + b_date: str = None, + geogrid: str = None, + output_path: str = None, + ): """Create a model that is ready for initialization. Initializes the model with default values for time, variables, and grid types. """ - super().__init__() + super().__init__(config_file, b_date, geogrid, output_path) self.GeoMeta = UnstructuredGeoMeta def grid_ranks(self) -> list[int]: diff --git a/NextGen_Forcings_Engine_BMI/run_bmi_model.py b/NextGen_Forcings_Engine_BMI/run_bmi_model.py index 70c56d52..85cb5de2 100755 --- a/NextGen_Forcings_Engine_BMI/run_bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/run_bmi_model.py @@ -343,17 +343,17 @@ def run_bmi( config = parse_config(yaml.safe_load(fp)) print("Creating an instance of the BMI model object") - model = BMIMODEL[config.get("GRID_TYPE")]() + model = BMIMODEL[config.get("GRID_TYPE")](cfg_path,b_date,geogrid,output_path=str(output_path) if output_path else None) # IMPORTANT: We are not calling initialize() directly here. # Instead, we call initialize_with_params(), which handles # the initialization process and internally calls initialize(). - model.initialize_with_params( - cfg_path, - b_date=b_date, - geogrid=geogrid, - output_path=str(output_path) if output_path else None, - ) + # model.initialize_with_params( + # cfg_path, + # b_date=b_date, + # geogrid=geogrid, + # output_path=str(output_path) if output_path else None, + # ) ngen_datetimes, start_time, end_time = get_date_times(start_time, end_time) num_iterations = len(ngen_datetimes) print_init(model, num_iterations) From 9f615dc9783a81209f475d7f4c719830dcddb032 Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Thu, 28 May 2026 08:05:34 -0500 Subject: [PATCH 043/132] remove excessive code comments --- .../NextGen_Forcings_Engine/bmi_model.py | 226 ++++++------------ 1 file changed, 77 insertions(+), 149 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 47d79538..5904bf67 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -35,6 +35,7 @@ ) from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.consts import BMI_MODEL from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.geoMod import ( + GeoMeta, GriddedGeoMeta, HydrofabricGeoMeta, UnstructuredGeoMeta, @@ -142,17 +143,11 @@ def __init__( Initializes the model with default values for time, variables, and grid types. """ - # This is required prior to the first log message. - if FORCING_USE_EWTS: - val = getenv_any("EWTS_USE_NGEN_BRIDGE", "").strip().lower() - if val in {"1", "true", "yes", "on"}: - configure_existing_logger(LOG) - else: - _configure_stdout_logging() - LOG.warning("ewts package installed but EWTS_USE_NGEN_BRIDGE not on. Falling back to default logging.") - else: - _configure_stdout_logging() + self.init_log(config_file) + self._config_file = config_file + self._geogrid = geogrid + self._b_date= b_date super(NWMv3_Forcing_Engine_BMI_model_Base, self).__init__() self._values = {} self._start_time = 0.0 @@ -169,14 +164,10 @@ def __init__( self._job_meta = None self._mpi_meta = None self._geo_meta = None - self._grid_type = None self._grids = None self._grid_map = None self._output_var_names = None self._var_name_units_map = None - self._var_name_map_long_first = None - self._var_name_map_short_first = None - self._var_units_map = None self._input_forcing_mod = None self._supp_pcp_mod = None self._output_obj = None @@ -188,15 +179,31 @@ def __init__( self._att_map = BMI_MODEL["att_map"] self._input_var_names = [] self._model_parameters_list = [] - self._config_file = config_file - self._geogrid = geogrid + self.get_grid_edge_count # Now that _job_meta is set, call initialize() to set up the core model - self.initialize(config_file, output_path=output_path) + self.initialize(config_file) + + self._model = NWMv3ForcingEngineModel() + + def init_log(self, config_file: str) -> None: + """Initialize the logging system for the model.""" + # This is required prior to the first log message. + if FORCING_USE_EWTS: + val = getenv_any("EWTS_USE_NGEN_BRIDGE", "").strip().lower() + if val in {"1", "true", "yes", "on"}: + configure_existing_logger(LOG) + else: + _configure_stdout_logging() + LOG.warning("ewts package installed but EWTS_USE_NGEN_BRIDGE not on. Falling back to default logging.") + else: + _configure_stdout_logging() + LOG.info("-" * 30) + LOG.info(f"BMI Forcing Engine initialized with {config_file}") @property - def bmi_cfg_file(self): + def bmi_cfg_file(self) -> Path: """Validate and return the BMI configuration file path.""" if not isinstance(self._config_file, str) or len(self._config_file) == 0: LOG.critical("No BMI initialize configuration provided, nothing to do...") @@ -212,7 +219,7 @@ def bmi_cfg_file(self): return bmi_cfg_file @property - def cfg_bmi(self): + def cfg_bmi(self) -> dict: """Read and parse the BMI configuration file.""" if self._cfg_bmi is not None: return self._cfg_bmi @@ -222,17 +229,17 @@ def cfg_bmi(self): return parse_config(cfg) @cfg_bmi.setter - def cfg_bmi(self, value): + def cfg_bmi(self, value: dict) -> None: """Set the BMI configuration.""" self._cfg_bmi = value @property - def _job_meta(self): + def _job_meta(self) -> ConfigOptions: """Return the job metadata object.""" return self.__job_meta @_job_meta.setter - def _job_meta(self, value): + def _job_meta(self, value: ConfigOptions) -> None: """Set the job metadata object.""" if value is None: value = ConfigOptions( @@ -253,30 +260,30 @@ def _job_meta(self, value): self.__job_meta = value @property - def _mpi_meta(self): + def _mpi_meta(self) -> MpiConfig: """Return the MPI metadata object.""" if self.__mpi_meta is None: self.__mpi_meta = MpiConfig(self._job_meta) return self.__mpi_meta @_mpi_meta.setter - def _mpi_meta(self, value): + def _mpi_meta(self, value: MpiConfig) -> None: """Set the MPI metadata object.""" self.__mpi_meta = value @property - def geo_meta(self): + def geo_meta(self) -> GeoMeta: """Return the geospatial metadata object.""" if self._geo_meta is None: self._geo_meta = HydrofabricGeoMeta(self._job_meta, self._mpi_meta) return self._geo_meta @geo_meta.setter - def geo_meta(self, value): + def geo_meta(self, value: GeoMeta) -> None: """Set the geospatial metadata object.""" self._geo_meta = value - def init_mpi(self): + def init_mpi(self) -> None: """Set up MPI communication for the model.""" try: comm = MPI.Comm.f2py(self._comm) if self._comm is not None else None @@ -284,7 +291,7 @@ def init_mpi(self): except Exception as e: err_handler.err_out_screen(self._job_meta.errMsg, e) - def init_scratch_dir(self): + def init_scratch_dir(self) -> None: """Set up the scratch directory for the model, ensuring it is unique for each job. Reassign the scratch dir to a new child dir of the current scratch dir, @@ -292,7 +299,7 @@ def init_scratch_dir(self): """ self._job_meta.uniquefy_scratch_dir_as_child(self._mpi_meta.uid64) - def create_esmf_mesh(self): + def create_esmf_mesh(self) -> None: """Create the ESMF mesh for the model.""" if self._mpi_meta.rank == 0: cat_ids = esmf_creation.create_mesh(self._job_meta) @@ -304,7 +311,7 @@ def create_esmf_mesh(self): cat_ids = np.empty(cat_count[0], dtype=np.int64) self._mpi_meta.comm.Bcast(cat_ids, root=0) - def fetch_raw_forcing_data(self): + def fetch_raw_forcing_data(self) -> None: """Fetch raw forcing data for the model. This function is responsible for retrieving the raw forcing data needed for the model simulation. @@ -329,12 +336,12 @@ def fetch_raw_forcing_data(self): self._mpi_meta.comm.Barrier() @property - def _grid_type(self): + def _grid_type(self) -> str: """Return the grid type of the model.""" return self._job_meta.grid_type.lower() @property - def _var_name_map_long_first(self): + def _var_name_map_long_first(self) -> dict: """Return the variable name mapping from long names to short names.""" return { long_name: self._var_name_units_map[long_name][0] @@ -342,7 +349,7 @@ def _var_name_map_long_first(self): } @property - def _var_name_map_short_first(self): + def _var_name_map_short_first(self) -> dict: """Return the variable name mapping from short names to long names.""" return { self._var_name_units_map[long_name][0]: long_name @@ -350,7 +357,7 @@ def _var_name_map_short_first(self): } @property - def _var_units_map(self): + def _var_units_map(self) -> dict: """Return the variable units mapping.""" return { long_name: self._var_name_units_map[long_name][1] @@ -358,7 +365,7 @@ def _var_units_map(self): } @property - def dimensionality(self): + def dimensionality(self) -> int: """Return the dimensionality of the model grid based on the grid type. Check to make sure we have enough dimensionality to run regridding. We assume that hydrofabric discretizations are large @@ -366,12 +373,9 @@ def dimensionality(self): that the local grid size for each processor is at least 2x2 to run the regridding process. forcing_input dimensionality is checked in regrid.py. """ - if self._grid_type == "hydrofabric": - return 1 - else: - return 2 + return {"hydrofabric": 1}.get(self._grid_type, 2) - def check_dimensionality(self): + def check_dimensionality(self) -> None: """Check that the local grid size is sufficient for the specified number of cores.""" if ( self.geo_meta.nx_local < self.dimensionality @@ -384,7 +388,7 @@ def check_dimensionality(self): err_handler.err_out_screen_para(self._job_meta.errMsg, self._mpi_meta) err_handler.check_program_status(self._job_meta, self._mpi_meta) - def init_output_obj(self): + def init_output_obj(self) -> None: """Initialize our output object, which includes local slabs from the output grid.""" try: self._output_obj = ioMod.OutputObj(self._job_meta, self.geo_meta) @@ -392,7 +396,7 @@ def init_output_obj(self): err_handler.err_out_screen_para(self._job_meta, self._mpi_meta) err_handler.check_program_status(self._job_meta, self._mpi_meta) - def init_input_forcing_mod(self): + def init_input_forcing_mod(self) -> None: """Initialize the input forcing module. Next, initialize our input forcing classes. These objects will contain @@ -409,7 +413,7 @@ def init_input_forcing_mod(self): err_handler.err_out_screen_para(self._job_meta, self._mpi_meta) err_handler.check_program_status(self._job_meta, self._mpi_meta) - def init_supp_pcp_mod(self): + def init_supp_pcp_mod(self) -> None: """Initialize the supplemental precipitation module, if applicable.""" if self._job_meta.number_supp_pcp > 0: self._supp_pcp_mod = suppPrecipMod.init_dict(self._job_meta, self.geo_meta) @@ -417,7 +421,7 @@ def init_supp_pcp_mod(self): self._supp_pcp_mod = None err_handler.check_program_status(self._job_meta, self._mpi_meta) - def initialize_parameters(self): + def initialize_parameters(self) -> None: """Initialize the parameters, inputs and outputs.""" for parm in self._model_parameters_list: self._values[self._var_name_map_short_first[parm]] = self.cfg_bmi[parm] @@ -428,6 +432,11 @@ def set_initial_time_and_step(self) -> None: self._values["time_step_size"] = self.cfg_bmi["time_step_seconds"] self._values["CAT-ID"] = cat_ids + def set_catchment_ids(self) -> None: + """Set catchment ids if using hydrofabric.""" + if self._grid_type == "hydrofabric": + self._values["CAT-ID"] = self.geo_meta.element_ids_global + def initialize(self, config_file: str) -> None: """Initialize the model using a configuration file. @@ -451,24 +460,17 @@ def initialize(self, config_file: str) -> None: self.init_output_obj() self.init_input_forcing_mod() self.init_supp_pcp_mod() + self.initialize_parameters() self.get_size_of_arrays() self.set_initial_time_and_step() + self.set_catchment_ids() - # Initialize the Forcings Engine model - self._model = NWMv3ForcingEngineModel() - - # Set catchment ids if using hydrofabric - if self._grid_type == "hydrofabric": - self._values["CAT-ID"] = self.geo_meta.element_ids_global - - self._configure_output_path(self.output_path) + self._configure_output_path() - def _configure_output_path(self, output_path: str | None = None) -> None: + def _configure_output_path(self) -> None: """Set the output path and initializes the output NetCDF file if forcing output is enabled. This is safe to call once after model initialization. - - :param output_path: Optional override path. """ gpkg_key = self._job_meta.geopackage time_key = str(time.time()).replace(".", "") @@ -484,14 +486,10 @@ def _configure_output_path(self, output_path: str | None = None) -> None: if ext is None: raise ValueError(f"Invalid grid_type: {self._job_meta.grid_type}") - if output_path: - self._output_obj.outPath = output_path + if self.output_path: + self._output_obj.outPath = self.output_path else: - filename = ( - f"NextGen_Forcings_Engine_{ext}_{gpkg_hash}_{time_hash}_output_" - + pd.Timestamp(self._job_meta.b_date_proc).strftime("%Y%m%d%H%M") - + ".nc" - ) + filename = f"NextGen_Forcings_Engine_{ext}_{gpkg_hash}_{time_hash}_output_{pd.Timestamp(self._job_meta.b_date_proc).strftime('%Y%m%d%H%M')}.nc" self._output_obj.outPath = os.path.join( self._job_meta.scratch_dir, filename ) @@ -501,8 +499,7 @@ def _configure_output_path(self, output_path: str | None = None) -> None: ) self._output_configured = True - # ------------------------------------------------------------ - def update(self): + def update(self) -> None: """Update the model by advancing one time step. This method increments the current model time by the time step size @@ -511,13 +508,11 @@ def update(self): :return: None """ - # Run the model to the next timestep self.update_until( self._values["current_model_time"] + self._values["time_step_size"] ) - # ------------------------------------------------------------ - def update_until(self, future_time: float): + def update_until(self, future_time: float) -> None: """Update the model to a specified future time. This method updates the model by running time steps until the @@ -529,9 +524,6 @@ def update_until(self, future_time: float): :return: None """ - # Method for running the model on the initial time if the model has not been run, - # and the future time is the same as the initial time. - if ( self._values["current_model_time"] == future_time @@ -565,8 +557,7 @@ def update_until(self, future_time: float): self._output_obj, ) - # ------------------------------------------------------------ - def finalize(self): + def finalize(self) -> None: """Finalize the model, performing necessary cleanup tasks. This method cleans up any temporary files created during the model run, @@ -582,10 +573,8 @@ def finalize(self): ) # Force destruction of ESMF objects - self.geo_meta = None - self._input_forcing_mod = None - self._supp_pcp_mod = None - self._model = None + for attr in ["geo_meta", "_input_forcing_mod", "_supp_pcp_mod", "_model"]: + setattr(self, attr, None) # Try moving this after all of the ESMF and model bits have # been disposed of - maybe they were keeping something open. @@ -597,13 +586,7 @@ def finalize(self): gc.collect() # make sure objects are deleted from memory LOG.info(Pld(St.COMPLETE, msg="Finishing BMI finalize()", modnm=MODNM)) - # ------------------------------------------------------------------- - # ------------------------------------------------------------------- - # BMI: Model Information Functions - # ------------------------------------------------------------------- - # ------------------------------------------------------------------- - - def get_attribute(self, att_name): + def get_attribute(self, att_name: str) -> Any: """Retrieve an attribute from the model's attribute map. This method searches the `_att_map` dictionary for the specified attribute name @@ -617,11 +600,7 @@ def get_attribute(self, att_name): except Exception as e: LOG.error(f"Could not find attribute: {att_name} - {e}") - # -------------------------------------------------------- - # Note: These are currently variables needed from other - # components vs. those read from files or GUI. - # -------------------------------------------------------- - def get_input_var_names(self): + def get_input_var_names(self) -> list[str]: """Get the list of input variable names. This method returns the list of input variable names defined in the model. @@ -630,7 +609,7 @@ def get_input_var_names(self): """ return self._input_var_names - def get_output_var_names(self): + def get_output_var_names(self) -> list[str]: """Get the list of output variable names. This method returns the list of output variable names defined in the model. @@ -639,8 +618,7 @@ def get_output_var_names(self): """ return self._output_var_names - # ------------------------------------------------------------ - def get_component_name(self): + def get_component_name(self) -> str: """Get the name of the component. This method retrieves the model name using the `get_attribute` method. @@ -649,8 +627,7 @@ def get_component_name(self): """ return self.get_attribute("model_name") - # ------------------------------------------------------------ - def get_input_item_count(self): + def get_input_item_count(self) -> int: """Get the count of input variables. This method returns the total number of input variables defined in the model. @@ -659,8 +636,7 @@ def get_input_item_count(self): """ return len(self._input_var_names) - # ------------------------------------------------------------ - def get_output_item_count(self): + def get_output_item_count(self) -> int: """Get the count of output variables. This method returns the total number of output variables defined in the model. @@ -669,7 +645,6 @@ def get_output_item_count(self): """ return len(self._output_var_names) - # ------------------------------------------------------------ def get_value(self, var_name: str, dest: NDArray[Any]) -> NDArray[Any]: """Copy the values of a variable into the provided destination array. @@ -722,7 +697,6 @@ def get_value(self, var_name: str, dest: NDArray[Any]) -> NDArray[Any]: return dest - # ------------------------------------------------------------------- def get_value_ptr(self, var_name: str) -> NDArray[Any]: """Get a reference to the values of a variable. @@ -819,12 +793,7 @@ def get_value_ptr(self, var_name: str) -> NDArray[Any]: # LOG.debug(f"[BMI get_value_ptr] Returning ravelled array for variable '{var_name}'") return arr.ravel() - # ------------------------------------------------------------------- - # ------------------------------------------------------------------- - # BMI: Variable Information Functions - # ------------------------------------------------------------------- - # ------------------------------------------------------------------- - def get_var_name(self, long_var_name): + def get_var_name(self, long_var_name: str) -> str: """Get the short name of the variable corresponding to the long variable name. :param long_var_name: The long variable name as defined in the model. @@ -832,8 +801,7 @@ def get_var_name(self, long_var_name): """ return self._var_name_map_long_first[long_var_name] - # ------------------------------------------------------------------- - def get_var_units(self, long_var_name): + def get_var_units(self, long_var_name: str) -> str: """Get the units of the variable corresponding to the long variable name. :param long_var_name: The long variable name as defined in the model. @@ -841,7 +809,6 @@ def get_var_units(self, long_var_name): """ return self._var_units_map[long_var_name] - # ------------------------------------------------------------------- def get_var_type(self, var_name: str) -> str: """Get the data type of a variable. @@ -851,8 +818,7 @@ def get_var_type(self, var_name: str) -> str: """ return str(self.get_value_ptr(var_name).dtype) - # ------------------------------------------------------------ - def get_var_grid(self, name): + def get_var_grid(self, name: str) -> int: """Get the grid associated with a variable. :param name: The name of the variable. @@ -873,8 +839,7 @@ def get_var_grid(self, name): return self._var_grid_id raise (UnknownBMIVariable(f"No known variable in BMI model: {name}")) - # ------------------------------------------------------------ - def get_var_itemsize(self, name): + def get_var_itemsize(self, name: str) -> int: """Get the item size (in bytes) of a variable. This function retrieves the memory size (in bytes) for each element of the variable @@ -885,8 +850,7 @@ def get_var_itemsize(self, name): """ return self.get_value_ptr(name).itemsize - # ------------------------------------------------------------ - def get_var_location(self, name): + def get_var_location(self, name: str) -> str: """Get the location of a variable in the grid. This function determines the location of a variable (whether it's at a "face" @@ -905,8 +869,7 @@ def get_var_location(self, name): else: raise ValueError(f"get_var_location: grid_id {self._var_grid_id} unknown") - # ------------------------------------------------------------------- - def get_var_rank(self, long_var_name): + def get_var_rank(self, long_var_name: str) -> np.int16: """Get the rank of a variable. This function retrieves the rank (number of dimensions) of a variable @@ -918,7 +881,6 @@ def get_var_rank(self, long_var_name): """ return np.int16(0) - # ------------------------------------------------------------------- def get_start_time(self) -> float: """Get the model's start time. @@ -929,8 +891,6 @@ def get_start_time(self) -> float: """ return self._start_time - # ------------------------------------------------------------------- - def get_end_time(self) -> float: """Get the model's end time. @@ -942,8 +902,6 @@ def get_end_time(self) -> float: """ return self._end_time - # ------------------------------------------------------------------- - def get_current_time(self) -> float: """Get the current time of the model. @@ -954,7 +912,6 @@ def get_current_time(self) -> float: """ return self._values["current_model_time"] - # ------------------------------------------------------------------- def get_time_step(self) -> float: """Get the model's time step size. @@ -965,7 +922,6 @@ def get_time_step(self) -> float: """ return self._values["time_step_size"] - # ------------------------------------------------------------------- def get_time_units(self) -> str: """Get the units of time for the model. @@ -976,8 +932,6 @@ def get_time_units(self) -> str: """ return self.get_attribute("time_units") - # ------------------------------------------------------------------- - def set_value(self, var_name: str, values: NDArray[Any]): """Set model values for the provided BMI variable. @@ -993,7 +947,6 @@ def set_value(self, var_name: str, values: NDArray[Any]): else: self._values[var_name][:] = values - # ------------------------------------------------------------ def set_value_at_indices( self, var_name: str, indices: NDArray[np.int_], src: NDArray[Any] ): @@ -1011,7 +964,6 @@ def set_value_at_indices( bmi_var_value_index = indices[i] self.get_value_ptr(var_name)[bmi_var_value_index] = src[i] - # ------------------------------------------------------------ def get_var_nbytes(self, var_name) -> int: """Get the number of bytes required for a variable. @@ -1023,7 +975,6 @@ def get_var_nbytes(self, var_name) -> int: """ return self.get_value_ptr(var_name).nbytes - # ------------------------------------------------------------ def get_value_at_indices( self, var_name: str, dest: NDArray[Any], indices: NDArray[np.int_] ) -> NDArray[Any]: @@ -1046,7 +997,6 @@ def get_value_at_indices( # JG Note: remaining grid funcs do not apply for type 'scalar' # Yet all functions in the BMI must be implemented # See https://bmi.readthedocs.io/en/latest/bmi.best_practices.html - # ------------------------------------------------------------ def get_grid_edge_count(self, grid_id: int) -> int: """Retrieve the number of edges for the specified grid. @@ -1113,7 +1063,6 @@ def get_grid_edge_count(self, grid_id: int) -> int: # If no valid grid is found, raise an exception or handle accordingly. raise ValueError("No valid grid found to calculate edge count.") - # ------------------------------------------------------------ def get_grid_edge_nodes( self, grid_id: int, edge_nodes: NDArray[np.int_] ) -> NDArray[np.int_]: @@ -1177,7 +1126,6 @@ def get_grid_edge_nodes( raise Exception("Unexpected error in retrieving edge nodes") - # ------------------------------------------------------------ def get_grid_face_count(self, grid_id: int) -> int: """Retrieve the number of faces for the specified grid. @@ -1204,7 +1152,6 @@ def get_grid_face_count(self, grid_id: int) -> int: # If the loop doesn't return, raise an exception indicating grid ID not found. raise ValueError("Grid ID not found in _grids.") - # ------------------------------------------------------------ def get_grid_face_edges( self, grid_id: int, face_edges: NDArray[np.int_] ) -> NDArray[np.int_]: @@ -1270,7 +1217,6 @@ def get_grid_face_edges( # If the loop doesn't return, raise an exception indicating an unexpected error raise Exception("Unexpected error in retrieving face edges.") - # ------------------------------------------------------------ def get_grid_face_nodes( self, grid_id: int, face_nodes: NDArray[np.int_] ) -> NDArray[np.int_]: @@ -1309,7 +1255,6 @@ def get_grid_face_nodes( # If the loop doesn't return, raise an exception indicating an unexpected error raise Exception("Unexpected error in retrieving face nodes.") - # ------------------------------------------------------------ def get_grid_node_count(self, grid_id: int) -> int: """Retrieve the number of nodes for the specified grid. @@ -1337,7 +1282,6 @@ def get_grid_node_count(self, grid_id: int) -> int: # If the loop doesn't return within the for loop, raise an exception raise ValueError("Grid ID not found in _grids.") - # ------------------------------------------------------------ def get_grid_nodes_per_face( self, grid_id: int, nodes_per_face: NDArray[np.int_] ) -> NDArray[np.int_]: @@ -1370,7 +1314,6 @@ def get_grid_nodes_per_face( # If the loop doesn't return, raise an exception indicating an unexpected error raise Exception("Unexpected error in retrieving nodes per face.") - # ------------------------------------------------------------ def get_grid_origin( self, grid_id: int, origin: NDArray[np.float64] ) -> NDArray[np.float64]: @@ -1389,7 +1332,6 @@ def get_grid_origin( return origin raise ValueError(f"get_grid_origin: grid_id {grid_id} unknown") - # ------------------------------------------------------------ def get_grid_rank(self, grid_id: int) -> int: """Retrieve the rank of the specified grid. @@ -1404,7 +1346,6 @@ def get_grid_rank(self, grid_id: int) -> int: return grid.rank raise ValueError(f"get_grid_rank: grid_id {grid_id} unknown") - # ------------------------------------------------------------ def get_grid_shape(self, grid_id: int, shape: NDArray[np.int_]) -> NDArray[np.int_]: """Retrieve the shape (dimensions) of the specified grid. @@ -1421,7 +1362,6 @@ def get_grid_shape(self, grid_id: int, shape: NDArray[np.int_]) -> NDArray[np.in return shape raise ValueError(f"get_grid_shape: grid_id {grid_id} unknown") - # ------------------------------------------------------------ def get_grid_size(self, grid_id: int) -> int: """Retrieve the size (total number of elements) of the specified grid. @@ -1436,7 +1376,6 @@ def get_grid_size(self, grid_id: int) -> int: return grid.size raise ValueError(f"get_grid_size: grid_id {grid_id} unknown") - # ------------------------------------------------------------ def get_grid_spacing( self, grid_id: int, spacing: NDArray[np.float64] ) -> NDArray[np.float64]: @@ -1455,8 +1394,6 @@ def get_grid_spacing( return spacing raise ValueError(f"get_grid_spacing: grid_id {grid_id} unknown") - # ------------------------------------------------------------ - def get_grid_type(self, grid_id: int) -> str: """Retrieve the type of the specified grid. @@ -1471,7 +1408,6 @@ def get_grid_type(self, grid_id: int) -> str: return grid.type raise ValueError(f"get_grid_type: grid_id {grid_id} unknown") - # ------------------------------------------------------------ def get_grid_x(self, grid_id: int, x: NDArray[np.float64]) -> NDArray[np.float64]: """Retrieve the x-coordinates (longitude or grid points) for the specified grid. @@ -1491,7 +1427,6 @@ def get_grid_x(self, grid_id: int, x: NDArray[np.float64]) -> NDArray[np.float64 return x raise ValueError(f"get_grid_x: grid_id {grid_id} unknown") - # ------------------------------------------------------------ def get_grid_y(self, grid_id: int, y: NDArray[np.float64]) -> NDArray[np.float64]: """Retrieve the y-coordinates (latitude or grid points) for the specified grid. @@ -1511,7 +1446,6 @@ def get_grid_y(self, grid_id: int, y: NDArray[np.float64]) -> NDArray[np.float64 return y raise ValueError(f"get_grid_y: grid_id {grid_id} unknown") - # ------------------------------------------------------------ def get_grid_z(self, grid_id: int, z: NDArray[np.float64]) -> NDArray[np.float64]: """Retrieve the z-coordinates (depth or grid points) for the specified grid. @@ -1531,12 +1465,6 @@ def get_grid_z(self, grid_id: int, z: NDArray[np.float64]) -> NDArray[np.float64 return z raise ValueError(f"get_grid_z: grid_id {grid_id} unknown") - # ------------------------------------------------------------ - # ------------------------------------------------------------ - # -- Random utility functions - # ------------------------------------------------------------ - # ------------------------------------------------------------ - def parse_config(cfg: dict) -> dict: """Parse the provided configuration dictionary (`cfg`) and modifies it based on certain rules. From 2b1cf0ca3eaf2de7e78e30d9091db2d8063de5cd Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Fri, 26 Jun 2026 14:45:34 -0500 Subject: [PATCH 044/132] move init none attrs to consts.py --- .../NextGen_Forcings_Engine/bmi_model.py | 19 +++---------------- .../NextGen_Forcings_Engine/core/consts.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 5904bf67..f007f576 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -147,35 +147,22 @@ def __init__( self._config_file = config_file self._geogrid = geogrid - self._b_date= b_date + self._b_date = b_date super(NWMv3_Forcing_Engine_BMI_model_Base, self).__init__() + for attr in BMI_MODEL[self.__class__.__base__.__name__]: + setattr(self, attr, None) self._values = {} self._start_time = 0.0 self._end_time = np.finfo(float).max - self._model = None - self._comm = None self.var_array_lengths = 1 # Track output configuration status self._output_configured = False # Initialize attributes in __init__ to avoid PyCharm errors - self.cfg_bmi = None - self._job_meta = None - self._mpi_meta = None - self._geo_meta = None - self._grids = None - self._grid_map = None - self._output_var_names = None - self._var_name_units_map = None - self._input_forcing_mod = None - self._supp_pcp_mod = None - self._output_obj = None self._model_parameters_list = [] - self._call_counts = defaultdict(int) self._call_times = defaultdict(float) - self._total_start = None self._att_map = BMI_MODEL["att_map"] self._input_var_names = [] self._model_parameters_list = [] diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py index c1a95c50..4420e8cb 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py @@ -10,6 +10,22 @@ ) BMI_MODEL = { + "NWMv3_Forcing_Engine_BMI_model_Base": [ + "_model", + "_comm", + "cfg_bmi", + "_job_meta", + "_mpi_meta", + "_geo_meta", + "_grids", + "_grid_map", + "_output_var_names", + "_var_name_units_map", + "_input_forcing_mod", + "_supp_pcp_mod", + "_output_obj", + "_total_start", + ], "att_map": { "model_name": "NWMv3.0 Forcings Engine BMI Python", "version": "1.0", From 57ce76cfa128088e1a5c623a16c5175119e92c22 Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Wed, 1 Jul 2026 14:53:46 -0500 Subject: [PATCH 045/132] merge bmi_model refactor --- .../NextGen_Forcings_Engine/bmi_model.py | 36 +++++++++---------- .../NextGen_Forcings_Engine/core/config.py | 2 ++ .../NextGen_Forcings_Engine/core/regrid.py | 6 ++-- NextGen_Forcings_Engine_BMI/run_bmi_model.py | 16 +++------ 4 files changed, 27 insertions(+), 33 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index f007f576..da812bab 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -9,6 +9,7 @@ import time from collections import defaultdict from datetime import datetime, timezone +from functools import cached_property from pathlib import Path import netCDF4 as nc @@ -134,7 +135,6 @@ class NWMv3_Forcing_Engine_BMI_model_Base(Bmi): def __init__( self, - config_file: str, b_date: str = None, geogrid: str = None, output_path: str = None, @@ -143,14 +143,10 @@ def __init__( Initializes the model with default values for time, variables, and grid types. """ - self.init_log(config_file) - - self._config_file = config_file + self.output_path = output_path self._geogrid = geogrid self._b_date = b_date - super(NWMv3_Forcing_Engine_BMI_model_Base, self).__init__() - for attr in BMI_MODEL[self.__class__.__base__.__name__]: - setattr(self, attr, None) + self._values = {} self._start_time = 0.0 self._end_time = np.finfo(float).max @@ -167,14 +163,9 @@ def __init__( self._input_var_names = [] self._model_parameters_list = [] - self.get_grid_edge_count - - # Now that _job_meta is set, call initialize() to set up the core model - self.initialize(config_file) - - self._model = NWMv3ForcingEngineModel() + super(NWMv3_Forcing_Engine_BMI_model_Base, self).__init__() - def init_log(self, config_file: str) -> None: + def init_log(self) -> None: """Initialize the logging system for the model.""" # This is required prior to the first log message. if FORCING_USE_EWTS: @@ -187,9 +178,11 @@ def init_log(self, config_file: str) -> None: else: _configure_stdout_logging() LOG.info("-" * 30) - LOG.info(f"BMI Forcing Engine initialized with {config_file}") + LOG.info( + f"BMI Forcing Engine initialized with {self._config_file}{Pld(St.INITTING, modnm=MODNM)}" + ) - @property + @cached_property def bmi_cfg_file(self) -> Path: """Validate and return the BMI configuration file path.""" if not isinstance(self._config_file, str) or len(self._config_file) == 0: @@ -213,7 +206,8 @@ def cfg_bmi(self) -> dict: LOG.info(f"Reading config file: {self.bmi_cfg_file}") with self.bmi_cfg_file.open("r") as fp: cfg = yaml.safe_load(fp) - return parse_config(cfg) + self._cfg_bmi = parse_config(cfg) + return self._cfg_bmi @cfg_bmi.setter def cfg_bmi(self, value: dict) -> None: @@ -438,6 +432,11 @@ def initialize(self, config_file: str) -> None: :param config_file: The path to the configuration file for model initialization. :raises RuntimeError: If the configuration file is invalid or missing. """ + self._config_file = config_file + for attr in BMI_MODEL[self.__class__.__base__.__name__]: + setattr(self, attr, None) + self._model = NWMv3ForcingEngineModel() + self.init_log() self.init_mpi() self.init_scratch_dir() self.create_esmf_mesh() @@ -1631,7 +1630,6 @@ class NWMv3_Forcing_Engine_BMI_model_HydroFabric(NWMv3_Forcing_Engine_BMI_model_ def __init__( self, - config_file: str, b_date: str = None, geogrid: str = None, output_path: str = None, @@ -1640,7 +1638,7 @@ def __init__( Initializes the model with default values for time, variables, and grid types. """ - super().__init__(config_file, b_date, geogrid, output_path) + super().__init__(b_date, geogrid, output_path) self.GeoMeta = HydrofabricGeoMeta def grid_ranks(self) -> list[int]: diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 037fff30..3bc060a3 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -1128,6 +1128,8 @@ def perform_downscaling(self) -> bool: or 2 in self.t2dDownscaleOpt ): return True + else: + return False @property def t2BiasCorrectOpt(self) -> list: diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py index fda67297..92ff26d8 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py @@ -3985,7 +3985,7 @@ def regrid_nwm(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config): ) input_forcings.height = None - if mpi_config.rank == 0: + if mpi_config.rank == 0 and config_options.perform_downscaling: pt.log_debug( f"Unable to locate HGT_surface in: {input_forcings.file_in2}. Downscaling will not be available." ) @@ -4282,7 +4282,7 @@ def regrid_nwm_aws(input_forcings, config_options, wrf_hydro_geo_meta, mpi_confi ) input_forcings.height = None - if mpi_config.rank == 0: + if mpi_config.rank == 0 and config_options.perform_downscaling: pt.log_info( f"Unable to locate HGT_surface in: {input_forcings.file_in2}. Downscaling will not be available." ) @@ -4849,7 +4849,7 @@ def regrid_custom_hourly_netcdf( else: input_forcings.height = None - if mpi_config.rank == 0: + if mpi_config.rank == 0 and config_options.perform_downscaling: pt.log_info( f"Unable to locate HGT_surface in: {input_forcings.file_in2}. Downscaling will not be available." ) diff --git a/NextGen_Forcings_Engine_BMI/run_bmi_model.py b/NextGen_Forcings_Engine_BMI/run_bmi_model.py index 85cb5de2..553b6cf6 100755 --- a/NextGen_Forcings_Engine_BMI/run_bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/run_bmi_model.py @@ -343,17 +343,11 @@ def run_bmi( config = parse_config(yaml.safe_load(fp)) print("Creating an instance of the BMI model object") - model = BMIMODEL[config.get("GRID_TYPE")](cfg_path,b_date,geogrid,output_path=str(output_path) if output_path else None) - - # IMPORTANT: We are not calling initialize() directly here. - # Instead, we call initialize_with_params(), which handles - # the initialization process and internally calls initialize(). - # model.initialize_with_params( - # cfg_path, - # b_date=b_date, - # geogrid=geogrid, - # output_path=str(output_path) if output_path else None, - # ) + model = BMIMODEL[config.get("GRID_TYPE")]( + b_date, geogrid, output_path=str(output_path) if output_path else None + ) + model.initialize(cfg_path) + ngen_datetimes, start_time, end_time = get_date_times(start_time, end_time) num_iterations = len(ngen_datetimes) print_init(model, num_iterations) From 3f66d53668618e6c20a0a2fa028e9b50a7857517 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 24 Apr 2026 15:30:47 -0400 Subject: [PATCH 046/132] Simplify var passing within NWMv3ForcingEngineModel.run --- .../NextGen_Forcings_Engine/model.py | 176 ++++++------------ tests/test_utils.py | 40 +--- 2 files changed, 66 insertions(+), 150 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index c9cc75f4..4fde3687 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -15,6 +15,7 @@ disaggregateMod, downscale, err_handler, + forcingInputMod, layeringMod, ) from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( @@ -129,42 +130,12 @@ def run( :raises RuntimeError: If the model fails to initialize or if required arguments are missing. """ - LOG.debug( - f"{Pld(St.INPROG, msg=f'Starting timestep with future_time={future_time}', modnm=MODNM)}", - ) - ( - future_time, - config_options, - ) = self.determine_forecast( - future_time, - config_options, - ) - ( - config_options, - input_forcing_mod, - mpi_config, - ) = self.adjust_precip( - config_options, - input_forcing_mod, - mpi_config, - ) - ( - config_options, - mpi_config, - ) = self.log_forecast( - config_options, - mpi_config, - ) - ( - future_time, - config_options, - wrf_hydro_geo_meta, - input_forcing_mod, - supp_pcp_mod, - mpi_config, - output_obj, - input_forcings, - ) = self.loop_through_forcing_products( + + self.determine_forecast(future_time, config_options) + self.adjust_precip(config_options, input_forcing_mod, mpi_config) + self.log_forecast(config_options, mpi_config) + # TODO look into input_forcings usage in `process_suplemental_precip` and in `loop_through_forcing_products` at `disaggregate_fun`. + input_forcings = self.loop_through_forcing_products( future_time, config_options, wrf_hydro_geo_meta, @@ -173,13 +144,7 @@ def run( mpi_config, output_obj, ) - ( - config_options, - wrf_hydro_geo_meta, - supp_pcp_mod, - mpi_config, - output_obj, - ) = self.process_suplemental_precip( + self.process_suplemental_precip( config_options, wrf_hydro_geo_meta, supp_pcp_mod, @@ -187,23 +152,13 @@ def run( output_obj, input_forcings, ) - ( - config_options, - wrf_hydro_geo_meta, - mpi_config, - output_obj, - ) = self.write_output( + self.write_output( config_options, wrf_hydro_geo_meta, mpi_config, output_obj, ) - ( - model, - config_options, - wrf_hydro_geo_meta, - output_obj, - ) = self.update_dict( + self.update_dict( model, config_options, wrf_hydro_geo_meta, @@ -218,8 +173,13 @@ def determine_forecast( self, future_time: float, config_options: ConfigOptions, - ): - """Determine the forecast for the given future time and configuration.""" + ) -> None: + """Determine the forecast for the given future time and configuration. + + Warnings + -------- + Modifies mutable arguments in-place. + """ # Assign the future time to the configuration config_options.bmi_time = future_time self.disaggregate_fun = disaggregateMod.disaggregate_factory(config_options) @@ -275,38 +235,38 @@ def determine_forecast( if config_options.first_fcst_cycle is None: config_options.first_fcst_cycle = config_options.current_fcst_cycle - return ( - future_time, - config_options, - ) - @time_function def adjust_precip( self, config_options: ConfigOptions, input_forcing_mod: dict, mpi_config: MpiConfig, - ): - """Adjust precipitation for the given forecast cycle.""" + ) -> None: + """Adjust precipitation for the given forecast cycle. + + Warnings + -------- + Modifies mutable arguments in-place. + """ if not config_options.precip_only_flag: # reset skips if present for force_key in config_options.input_forcings: input_forcing_mod[force_key].skip = False err_handler.check_program_status(config_options, mpi_config) - return ( - config_options, - input_forcing_mod, - mpi_config, - ) @time_function def log_forecast( self, config_options: ConfigOptions, mpi_config: MpiConfig, - ): - """Log information about the current forecast cycle.""" + ) -> None: + """Log information about the current forecast cycle. + + Warnings + -------- + Modifies mutable arguments in-place. + """ # Log information about this forecast cycle if mpi_config.rank == 0: config_options.statusMsg = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" @@ -324,11 +284,6 @@ def log_forecast( err_handler.log_msg(config_options, mpi_config, True) # mpi_config.comm.barrier() - return ( - config_options, - mpi_config, - ) - @time_function def loop_through_forcing_products( self, @@ -339,8 +294,13 @@ def loop_through_forcing_products( supp_pcp_mod: dict, mpi_config: MpiConfig, output_obj: OutputObj, - ): - """Loop through each forcing product and process it for the current forecast cycle.""" + ) -> forcingInputMod.InputForcingsHydrofabric: + """Loop through each forcing product and process it for the current forecast cycle. + + Warnings + -------- + Modifies mutable arguments in-place. + """ # Loop through each output timestep. Perform the following functions: # 1.) Calculate all necessary input files per user options. # 2.) Read in input forcings from GRIB/NetCDF files. @@ -642,16 +602,7 @@ def loop_through_forcing_products( # err_handler.check_program_status(config_options, mpi_config) ############################################################################################## - return ( - future_time, - config_options, - wrf_hydro_geo_meta, - input_forcing_mod, - supp_pcp_mod, - mpi_config, - output_obj, - input_forcings, - ) + return input_forcings @time_function def process_suplemental_precip( @@ -662,8 +613,13 @@ def process_suplemental_precip( mpi_config: MpiConfig, output_obj: OutputObj, input_forcings: dict, - ): - """Process supplemental precipitation for the current forecast cycle.""" + ) -> None: + """Process supplemental precipitation for the current forecast cycle. + + Warnings + -------- + Modifies mutable arguments in-place. + """ if config_options.customSuppPcpFreq is not None: # Process supplemental precipitation if we specified in the configuration file. if config_options.number_supp_pcp > 0: @@ -717,14 +673,6 @@ def process_suplemental_precip( ) err_handler.check_program_status(config_options, mpi_config) - return ( - config_options, - wrf_hydro_geo_meta, - supp_pcp_mod, - mpi_config, - output_obj, - ) - @time_function def write_output( self, @@ -732,8 +680,13 @@ def write_output( wrf_hydro_geo_meta: GeoMeta, mpi_config: MpiConfig, output_obj: OutputObj, - ): - """Write the output for the current forecast cycle.""" + ) -> None: + """Write the output for the current forecast cycle. + + Warnings + -------- + Modifies mutable arguments in-place. + """ # If user requests output for given domain, then call # the I/O module to update opened netcdf file with forcing fields if ( @@ -743,12 +696,6 @@ def write_output( output_obj.gather_global_outputs( config_options, wrf_hydro_geo_meta, mpi_config ) - return ( - config_options, - wrf_hydro_geo_meta, - mpi_config, - output_obj, - ) """##################Step 6: flatten and update dict##########################################################################""" @@ -759,8 +706,13 @@ def update_dict( config_options: ConfigOptions, wrf_hydro_geo_meta: GeoMeta, output_obj: OutputObj, - ): - """Flatten the Forcings Engine output object and update the BMI dictionary.""" + ) -> None: + """Flatten the Forcings Engine output object and update the BMI dictionary. + + Warnings + -------- + Modifies mutable arguments in-place. + """ # Now loop through Forcings Engine output object # and flatten the 2D forcing array and append to # the BMI object to advertise to BMIinterface @@ -813,10 +765,4 @@ def update_dict( model[variable + "_ELEMENT"] = output_obj.output_global[ count, : ].flatten() - - return ( - model, - config_options, - wrf_hydro_geo_meta, - output_obj, - ) + model["CAT-ID"] = wrf_hydro_geo_meta.element_ids_global diff --git a/tests/test_utils.py b/tests/test_utils.py index e21a9d5c..049578e1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -673,43 +673,13 @@ def pre_regrid(self) -> None: ) model = self.bmi_model._model - ### NOTE with the exception of setting the skip flag, the below - ### block is copied verbatim from NWMv3ForcingEngineModel.run() - ( - future_time, - config_options, - ) = model.determine_forecast( - future_time, - config_options, - ) - ( - config_options, - input_forcing_mod, - mpi_config, - ) = model.adjust_precip( - config_options, - input_forcing_mod, - mpi_config, - ) - ( - config_options, - mpi_config, - ) = model.log_forecast( - config_options, - mpi_config, - ) + ### NOTE this should mimic NWMv3ForcingEngineModel.run() with the exception of setting the skip flag + model.determine_forecast(future_time, config_options) + model.adjust_precip(config_options, input_forcing_mod, mpi_config) + model.log_forecast(config_options, mpi_config) ### NOTE setting the flag causes the regrid step to be skipped self.set_input_forcings_skip_flags() - ( - future_time, - config_options, - geo_meta, - input_forcing_mod, - supp_pcp_mod, - mpi_config, - output_obj, - input_forcings, - ) = model.loop_through_forcing_products( + model.loop_through_forcing_products( future_time, config_options, geo_meta, From 65d9e1f15f1f8954b325451cf968dc6e7a8bfb19 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 10:10:43 -0400 Subject: [PATCH 047/132] Raise error on unexpected grid_type --- .../NextGen_Forcings_Engine/model.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 4fde3687..ed8fec99 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -320,6 +320,10 @@ def loop_through_forcing_products( elif config_options.grid_type == "hydrofabric": # Reset out final grids to missing values. output_obj.output_local[:, :] = config_options.globalNdv + else: + raise ValueError( + f"Unexpected grid_type: {repr(config_options.grid_type)}" + ) # Increment or initialize output step count if config_options.current_output_step is None: @@ -496,6 +500,10 @@ def loop_through_forcing_products( input_forcings.regridded_forcings1[:, :] = ( input_forcings.regridded_forcings2[:, :] ) + else: + raise ValueError( + f"Unexpected grid_type: {repr(config_options.grid_type)}" + ) # Re-calculate the neighbor files. input_forcings.calc_neighbor_files( config_options, output_obj.outDate, mpi_config @@ -766,3 +774,5 @@ def update_dict( count, : ].flatten() model["CAT-ID"] = wrf_hydro_geo_meta.element_ids_global + else: + raise ValueError(f"Unexpected grid_type: {repr(config_options.grid_type)}") From e7eb5b1ed6d183a585a2d82646e10dffe415c48f Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 12:10:35 -0400 Subject: [PATCH 048/132] Use `time.perf_counter` instead of `time.time` --- .../NextGen_Forcings_Engine/model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index ed8fec99..229fe863 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -2,7 +2,7 @@ import logging import os from contextlib import contextmanager -from time import time +from time import time, perf_counter import numpy as np import pandas as pd @@ -48,9 +48,9 @@ def timing_block(step_str: str): step_str: Description of the step being timed. """ - start = time() + start = perf_counter() yield - end = time() + end = perf_counter() LOG.debug(f" Execution time for {step_str}: {round(end - start, 2)} seconds") From 7e64e3f4fe97e5ccef03ddb1c5fca4a4d355c948 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 13:37:59 -0400 Subject: [PATCH 049/132] Start encapsulating BMI model into model.py NWMv3ForcingEngineModel starting with _values dict --- .../NextGen_Forcings_Engine/bmi_model.py | 2 -- .../NextGen_Forcings_Engine/model.py | 36 +++++++++++-------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index da812bab..c7e7aaee 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -516,7 +516,6 @@ def update_until(self, future_time: float) -> None: == self.cfg_bmi["initial_time"] ): self._model.run( - self._values, future_time, self._job_meta, self.geo_meta, @@ -533,7 +532,6 @@ def update_until(self, future_time: float) -> None: self._values["current_model_time"] += self._values["time_step_size"] # Run the model for the new current time and update the state. self._model.run( - self._values, self._values["current_model_time"], self._job_meta, self.geo_meta, diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 229fe863..575221f1 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -1,8 +1,11 @@ +from __future__ import annotations + import datetime import logging import os from contextlib import contextmanager -from time import time, perf_counter +from time import perf_counter, time +from typing import TYPE_CHECKING import numpy as np import pandas as pd @@ -35,6 +38,12 @@ NWMV3PuertoRicoProcessor, ) +if TYPE_CHECKING: + # To allow type hint without circular import error + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.bmi_model import ( + NWMv3_Forcing_Engine_BMI_model_Base, + ) + LOG = logging.getLogger("FORCING") MODNM = ModuleKey.FORCING.value @@ -68,9 +77,10 @@ def wrapper(*args, **kwargs): class NWMv3ForcingEngineModel: """NextGen Forcings Engine BMI model class for NWMv3 forcings.""" - def __init__(self): + def __init__(self, bmi_model: NWMv3_Forcing_Engine_BMI_model_Base): """Initialize the NWMv3 Forcing Engine Model.""" self.source_data_processor = None + self._bmi = bmi_model # TODO: refactor the bmi_model.py file and this to have this type maintain its own state. # def __init__(self): @@ -83,7 +93,6 @@ def __init__(self): def run( self, - model: dict, future_time: float, config_options: ConfigOptions, wrf_hydro_geo_meta: GeoMeta, @@ -94,10 +103,10 @@ def run( ) -> None: """Execute the full forcings engine BMI pipeline for a given future timestep. - This method updates the `model` state dictionary with atmospheric forcings computed from + This method updates the `self._bmi._values` state dictionary with atmospheric forcings computed from available input datasets. It handles initialization, AWS Zarr loading, regridding, temporal interpolation, bias correction, downscaling, supplemental precipitation processing, and output - population into the model structure. + population into the self._bmi._values structure. The following steps are performed: @@ -116,10 +125,9 @@ def run( b. Disaggregate and interpolate. c. Layer into the final output. 5. Write output to NetCDF forcing files if requested. - 6. Update the model state dictionary with flattened arrays. + 6. Update the self._bmi._values state dictionary with flattened arrays. 7. Advance the BMI time index. - :param model: The model state dictionary that will be updated with new forcing data. :param future_time: The number of seconds into the future to advance the model. :param config_options: Configuration object containing all model options, flags, and paths. :param wrf_hydro_geo_meta: Geospatial metadata needed for regridding and interpolation. @@ -159,7 +167,6 @@ def run( output_obj, ) self.update_dict( - model, config_options, wrf_hydro_geo_meta, output_obj, @@ -710,7 +717,6 @@ def write_output( @time_function def update_dict( self, - model: dict, config_options: ConfigOptions, wrf_hydro_geo_meta: GeoMeta, output_obj: OutputObj, @@ -759,20 +765,22 @@ def update_dict( ] if config_options.grid_type == "gridded": for count, variable in enumerate(variables): - model[variable + "_ELEMENT"] = output_obj.output_local[ + self._bmi._values[variable + "_ELEMENT"] = output_obj.output_local[ count, :, : ].flatten() elif config_options.grid_type == "unstructured": for count, variable in enumerate(variables): - model[variable + "_ELEMENT"] = output_obj.output_local_elem[ + self._bmi._values[variable + "_ELEMENT"] = output_obj.output_local_elem[ + count, : + ].flatten() + self._bmi._values[variable + "_NODE"] = output_obj.output_local[ count, : ].flatten() - model[variable + "_NODE"] = output_obj.output_local[count, :].flatten() elif config_options.grid_type == "hydrofabric": for count, variable in enumerate(variables): - model[variable + "_ELEMENT"] = output_obj.output_global[ + self._bmi._values[variable + "_ELEMENT"] = output_obj.output_global[ count, : ].flatten() - model["CAT-ID"] = wrf_hydro_geo_meta.element_ids_global + self._bmi._values["CAT-ID"] = wrf_hydro_geo_meta.element_ids_global else: raise ValueError(f"Unexpected grid_type: {repr(config_options.grid_type)}") From e7d92b9a12cdd0823698a53de666da05f7a2f44d Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 14:34:31 -0400 Subject: [PATCH 050/132] Encapsulate ConfigOptions instance --- .../NextGen_Forcings_Engine/bmi_model.py | 2 - .../NextGen_Forcings_Engine/model.py | 397 ++++++++++-------- tests/test_utils.py | 8 +- 3 files changed, 218 insertions(+), 189 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index c7e7aaee..606a06af 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -517,7 +517,6 @@ def update_until(self, future_time: float) -> None: ): self._model.run( future_time, - self._job_meta, self.geo_meta, self._input_forcing_mod, self._supp_pcp_mod, @@ -533,7 +532,6 @@ def update_until(self, future_time: float) -> None: # Run the model for the new current time and update the state. self._model.run( self._values["current_model_time"], - self._job_meta, self.geo_meta, self._input_forcing_mod, self._supp_pcp_mod, diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 575221f1..6a045262 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -94,7 +94,6 @@ def __init__(self, bmi_model: NWMv3_Forcing_Engine_BMI_model_Base): def run( self, future_time: float, - config_options: ConfigOptions, wrf_hydro_geo_meta: GeoMeta, input_forcing_mod: dict, supp_pcp_mod: dict, @@ -108,6 +107,8 @@ def run( interpolation, bias correction, downscaling, supplemental precipitation processing, and output population into the self._bmi._values structure. + `self._bmi._job_meta`, an instance of ConfigOptions is also updated in-place, for example for time handling. + The following steps are performed: 1. Determine the current forecast and output times based on the future timestamp @@ -129,7 +130,6 @@ def run( 7. Advance the BMI time index. :param future_time: The number of seconds into the future to advance the model. - :param config_options: Configuration object containing all model options, flags, and paths. :param wrf_hydro_geo_meta: Geospatial metadata needed for regridding and interpolation. :param input_forcing_mod: Dictionary of initialized input forcing modules indexed by forcing key. :param supp_pcp_mod: Dictionary of supplemental precipitation modules indexed by key. @@ -139,13 +139,12 @@ def run( :raises RuntimeError: If the model fails to initialize or if required arguments are missing. """ - self.determine_forecast(future_time, config_options) - self.adjust_precip(config_options, input_forcing_mod, mpi_config) - self.log_forecast(config_options, mpi_config) + self.determine_forecast(future_time) + self.adjust_precip(input_forcing_mod, mpi_config) + self.log_forecast(mpi_config) # TODO look into input_forcings usage in `process_suplemental_precip` and in `loop_through_forcing_products` at `disaggregate_fun`. input_forcings = self.loop_through_forcing_products( future_time, - config_options, wrf_hydro_geo_meta, input_forcing_mod, supp_pcp_mod, @@ -153,7 +152,6 @@ def run( output_obj, ) self.process_suplemental_precip( - config_options, wrf_hydro_geo_meta, supp_pcp_mod, mpi_config, @@ -161,25 +159,22 @@ def run( input_forcings, ) self.write_output( - config_options, wrf_hydro_geo_meta, mpi_config, output_obj, ) self.update_dict( - config_options, wrf_hydro_geo_meta, output_obj, ) ## Update BMI model time index to next iteration - config_options.bmi_time_index += 1 + self._bmi._job_meta.bmi_time_index += 1 @time_function def determine_forecast( self, future_time: float, - config_options: ConfigOptions, ) -> None: """Determine the forecast for the given future time and configuration. @@ -188,64 +183,67 @@ def determine_forecast( Modifies mutable arguments in-place. """ # Assign the future time to the configuration - config_options.bmi_time = future_time - self.disaggregate_fun = disaggregateMod.disaggregate_factory(config_options) + self._bmi._job_meta.bmi_time = future_time + self.disaggregate_fun = disaggregateMod.disaggregate_factory( + self._bmi._job_meta + ) # Calculate current time stamp based on operational configuration - if config_options.ana_flag: + if self._bmi._job_meta.ana_flag: # If we're in an AnA configuration, then must offset the BMI future # timestamp to account for the "lookback" period being properly iterated # over between 3-28 hour look back time period and operation configuration - - # if config_options.input_forcings[0] in [20, 22]: - # config_options.current_fcst_cycle = ( - # config_options.b_date_proc - # + pd.TimedeltaIndex( - # np.array([future_time - 7200.0], dtype=float), "s" - # )[0] - # ) - # config_options.current_time = ( - # config_options.b_date_proc - # + pd.TimedeltaIndex( - # np.array([future_time - 7200.0], dtype=float), "s" - # )[0] - # ) - # config_options.future_time = future_time - # else: - - # Puerto Rico / Hawaii AnA: 1-hour lookback (based on 6-hourly forecast cycles) - config_options.current_fcst_cycle = ( - config_options.b_date_proc - + pd.TimedeltaIndex(np.array([future_time - 3600.0], dtype=float), "s")[ - 0 - ] - ) - config_options.current_time = ( - config_options.b_date_proc - + pd.TimedeltaIndex(np.array([future_time - 3600.0], dtype=float), "s")[ - 0 - ] - ) + if self._bmi._job_meta.input_forcings[0] in [20, 22]: + self._bmi._job_meta.current_fcst_cycle = ( + self._bmi._job_meta.b_date_proc + + pd.TimedeltaIndex( + np.array([future_time - 7200.0], dtype=float), "s" + )[0] + ) + self._bmi._job_meta.current_time = ( + self._bmi._job_meta.b_date_proc + + pd.TimedeltaIndex( + np.array([future_time - 7200.0], dtype=float), "s" + )[0] + ) + self._bmi._job_meta.future_time = future_time + else: + # Puerto Rico / Hawaii AnA: 1-hour lookback (based on 6-hourly forecast cycles) + self._bmi._job_meta.current_fcst_cycle = ( + self._bmi._job_meta.b_date_proc + + pd.TimedeltaIndex( + np.array([future_time - 3600.0], dtype=float), "s" + )[0] + ) + self._bmi._job_meta.current_time = ( + self._bmi._job_meta.b_date_proc + + pd.TimedeltaIndex( + np.array([future_time - 3600.0], dtype=float), "s" + )[0] + ) else: # Forecast-only mode — use BMI timestamp as-is - config_options.current_fcst_cycle = config_options.b_date_proc - config_options.current_time = pd.Timestamp( - config_options.b_date_proc + self._bmi._job_meta.current_fcst_cycle = self._bmi._job_meta.b_date_proc + self._bmi._job_meta.current_time = pd.Timestamp( + self._bmi._job_meta.b_date_proc ) + pd.to_timedelta(future_time, unit="s") LOG.debug( "NextGen Forcings Engine processing meteorological forcings for BMI timestamp" ) - LOG.debug(f"Model.py current time: {config_options.current_time}") - LOG.debug(f"Model.py current fcst cycle: {config_options.current_fcst_cycle}") + LOG.debug(f"Model.py current time: {self._bmi._job_meta.current_time}") + LOG.debug( + f"Model.py current fcst cycle: {self._bmi._job_meta.current_fcst_cycle}" + ) - if config_options.first_fcst_cycle is None: - config_options.first_fcst_cycle = config_options.current_fcst_cycle + if self._bmi._job_meta.first_fcst_cycle is None: + self._bmi._job_meta.first_fcst_cycle = ( + self._bmi._job_meta.current_fcst_cycle + ) @time_function def adjust_precip( self, - config_options: ConfigOptions, input_forcing_mod: dict, mpi_config: MpiConfig, ) -> None: @@ -255,17 +253,16 @@ def adjust_precip( -------- Modifies mutable arguments in-place. """ - if not config_options.precip_only_flag: + if not self._bmi._job_meta.precip_only_flag: # reset skips if present - for force_key in config_options.input_forcings: + for force_key in self._bmi._job_meta.input_forcings: input_forcing_mod[force_key].skip = False - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status(self._bmi._job_meta, mpi_config) @time_function def log_forecast( self, - config_options: ConfigOptions, mpi_config: MpiConfig, ) -> None: """Log information about the current forecast cycle. @@ -276,26 +273,25 @@ def log_forecast( """ # Log information about this forecast cycle if mpi_config.rank == 0: - config_options.statusMsg = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" - err_handler.log_msg(config_options, mpi_config, True) - config_options.statusMsg = ( + self._bmi._job_meta.statusMsg = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + err_handler.log_msg(self._bmi._job_meta, mpi_config, True) + self._bmi._job_meta.statusMsg = ( "Processing Forecast Cycle: " - + config_options.current_fcst_cycle.strftime("%Y-%m-%d %H:%M") + + self._bmi._job_meta.current_fcst_cycle.strftime("%Y-%m-%d %H:%M") ) - err_handler.log_msg(config_options, mpi_config, True) - config_options.statusMsg = ( + err_handler.log_msg(self._bmi._job_meta, mpi_config, True) + self._bmi._job_meta.statusMsg = ( "Forecast Cycle Length is: " - + str(config_options.cycle_length_minutes) + + str(self._bmi._job_meta.cycle_length_minutes) + " minutes" ) - err_handler.log_msg(config_options, mpi_config, True) + err_handler.log_msg(self._bmi._job_meta, mpi_config, True) # mpi_config.comm.barrier() @time_function def loop_through_forcing_products( self, future_time: float, - config_options: ConfigOptions, wrf_hydro_geo_meta: GeoMeta, input_forcing_mod: dict, supp_pcp_mod: dict, @@ -314,93 +310,99 @@ def loop_through_forcing_products( # 3.) Regrid the forcings, and temporally interpolate. # 4.) Downscale. # 5.) Layer, and output as necessary. - ana_factor = 1 if config_options.ana_flag is False else 0 + ana_factor = 1 if self._bmi._job_meta.ana_flag is False else 0 show_message = True - if not config_options.precip_only_flag: - if config_options.grid_type == "gridded": + if not self._bmi._job_meta.precip_only_flag: + if self._bmi._job_meta.grid_type == "gridded": # Reset out final grids to missing values. - output_obj.output_local[:, :, :] = config_options.globalNdv - elif config_options.grid_type == "unstructured": + output_obj.output_local[:, :, :] = self._bmi._job_meta.globalNdv + elif self._bmi._job_meta.grid_type == "unstructured": # Reset out final grids to missing values. - output_obj.output_local[:, :] = config_options.globalNdv - output_obj.output_local_elem[:, :] = config_options.globalNdv - elif config_options.grid_type == "hydrofabric": + output_obj.output_local[:, :] = self._bmi._job_meta.globalNdv + output_obj.output_local_elem[:, :] = self._bmi._job_meta.globalNdv + elif self._bmi._job_meta.grid_type == "hydrofabric": # Reset out final grids to missing values. - output_obj.output_local[:, :] = config_options.globalNdv + output_obj.output_local[:, :] = self._bmi._job_meta.globalNdv else: raise ValueError( - f"Unexpected grid_type: {repr(config_options.grid_type)}" + f"Unexpected grid_type: {repr(self._bmi._job_meta.grid_type)}" ) # Increment or initialize output step count - if config_options.current_output_step is None: - config_options.current_output_step = 1 + if self._bmi._job_meta.current_output_step is None: + self._bmi._job_meta.current_output_step = 1 else: - config_options.current_output_step += 1 + self._bmi._job_meta.current_output_step += 1 # Optional sub-output timestamp - if config_options.sub_output_hour is not None: + if self._bmi._job_meta.sub_output_hour is not None: # TODO This is not used - subOutDate = config_options.first_fcst_cycle + datetime.timedelta( - hours=config_options.sub_output_hour + subOutDate = self._bmi._job_meta.first_fcst_cycle + datetime.timedelta( + hours=self._bmi._job_meta.sub_output_hour ) # Compute the output timestamp for this step - if config_options.ana_flag: + if self._bmi._job_meta.ana_flag: output_obj.outDate = ( - config_options.current_fcst_cycle - + datetime.timedelta(seconds=config_options.output_freq * 60) + self._bmi._job_meta.current_fcst_cycle + + datetime.timedelta(seconds=self._bmi._job_meta.output_freq * 60) ) else: output_obj.outDate = ( - config_options.current_fcst_cycle + self._bmi._job_meta.current_fcst_cycle + datetime.timedelta(seconds=future_time) ) - config_options.current_output_date = output_obj.outDate + self._bmi._job_meta.current_output_date = output_obj.outDate # Adjust file_date for AnA if needed file_date = ( output_obj.outDate - - datetime.timedelta(seconds=config_options.output_freq * 60) - if config_options.ana_flag + - datetime.timedelta(seconds=self._bmi._job_meta.output_freq * 60) + if self._bmi._job_meta.ana_flag else output_obj.outDate ) # Compute previous output date (used for downscaling logic) - if config_options.current_output_step == ana_factor: - config_options.prev_output_date = config_options.current_output_date + if self._bmi._job_meta.current_output_step == ana_factor: + self._bmi._job_meta.prev_output_date = ( + self._bmi._job_meta.current_output_date + ) else: - config_options.prev_output_date = ( - config_options.current_output_date + self._bmi._job_meta.prev_output_date = ( + self._bmi._job_meta.current_output_date - datetime.timedelta(seconds=future_time) ) # Print message on log file indicating the timestamp # we are currently processing for forcings if mpi_config.rank == 0 and show_message: - config_options.statusMsg = "=========================================" - err_handler.log_msg(config_options, mpi_config, True) - config_options.statusMsg = f"Processing for output timestep: {file_date.strftime('%Y-%m-%d %H:%M')}" - err_handler.log_msg(config_options, mpi_config, True) - - config_options.currentForceNum = 0 - config_options.currentCustomForceNum = 0 - LOG.debug(f"config_options.input_forcings: {config_options.input_forcings}") + self._bmi._job_meta.statusMsg = ( + "=========================================" + ) + err_handler.log_msg(self._bmi._job_meta, mpi_config, True) + self._bmi._job_meta.statusMsg = f"Processing for output timestep: {file_date.strftime('%Y-%m-%d %H:%M')}" + err_handler.log_msg(self._bmi._job_meta, mpi_config, True) + + self._bmi._job_meta.currentForceNum = 0 + self._bmi._job_meta.currentCustomForceNum = 0 + LOG.debug( + f"config_options.input_forcings: {self._bmi._job_meta.input_forcings}" + ) # Loop over each of the input forcings specified. LOG.debug( - f"Model.py forcing loop: {len(config_options.input_forcings)} forcings configured: {config_options.input_forcings}" + f"Model.py forcing loop: {len(self._bmi._job_meta.input_forcings)} forcings configured: {self._bmi._job_meta.input_forcings}" ) - for force_key in config_options.input_forcings: + for force_key in self._bmi._job_meta.input_forcings: LOG.debug(f"force_key: {force_key}") - LOG.debug(f"config_options.aws: {config_options.aws}") + LOG.debug(f"config_options.aws: {self._bmi._job_meta.aws}") # Pass these methods for AORC data is ERA5-Interim blend is requested # so we can finish filling in the missing gaps if ( force_key == 23 - and 12 in config_options.input_forcings - and 21 in config_options.input_forcings + and 12 in self._bmi._job_meta.input_forcings + and 21 in self._bmi._job_meta.input_forcings ): input_forcings = input_forcing_mod[force_key] @@ -410,35 +412,39 @@ def loop_through_forcing_products( else: input_forcings = input_forcing_mod[force_key] input_forcings.calc_neighbor_files( - config_options, output_obj.outDate, mpi_config + self._bmi._job_meta, output_obj.outDate, mpi_config ) if force_key in [12, 21, 27]: - if config_options.aws is None: + if self._bmi._job_meta.aws is None: # Calculate the previous and next input cycle files from the inputs. input_forcings.calc_neighbor_files( - config_options, output_obj.outDate, mpi_config + self._bmi._job_meta, output_obj.outDate, mpi_config + ) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) else: # Flag to indicate the AWS .zarr AORC method if force_key == 12: if self.source_data_processor is None: self.source_data_processor = AORCConusProcessor( - config_options, mpi_config, wrf_hydro_geo_meta + self._bmi._job_meta, mpi_config, wrf_hydro_geo_meta ) elif force_key == 21: if self.source_data_processor is None: self.source_data_processor = AORCAlaskaProcessor( - config_options, mpi_config, wrf_hydro_geo_meta + self._bmi._job_meta, mpi_config, wrf_hydro_geo_meta ) # Flag to indicate the AWS .zarr NWMv3 Forcing file method elif force_key == 27: if self.source_data_processor is None: - if config_options.nwm_domain == "CONUS": + if self._bmi._job_meta.nwm_domain == "CONUS": self.source_data_processor = NWMV3ConusProcessor( - config_options, mpi_config, wrf_hydro_geo_meta + self._bmi._job_meta, + mpi_config, + wrf_hydro_geo_meta, ) elif config_options.nwm_domain == "Hawaii": self.source_data_processor = NWMV3HawaiiProcessor( @@ -455,16 +461,18 @@ def loop_through_forcing_products( ) elif config_options.nwm_domain == "Alaska": self.source_data_processor = NWMV3AlaskaProcessor( - config_options, mpi_config, wrf_hydro_geo_meta + self._bmi._job_meta, + mpi_config, + wrf_hydro_geo_meta, ) else: raise ValueError( - f"Unsupported domain type ({config_options.nwm_domain} for forcing type: {force_key} )" + f"Unsupported domain type ({self._bmi._job_meta.nwm_domain} for forcing type: {force_key} )" ) - config_options.aws_obj = ( + self._bmi._job_meta.aws_obj = ( self.source_data_processor.process_historical_data( - config_options.current_time + self._bmi._job_meta.current_time ) ) @@ -474,15 +482,15 @@ def loop_through_forcing_products( break # Regrid forcings. input_forcings.regrid_inputs( - config_options, wrf_hydro_geo_meta, mpi_config + self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status(self._bmi._job_meta, mpi_config) # Run check on regridded fields for reasonable values that are not missing values. err_handler.check_forcing_bounds( - config_options, input_forcings, mpi_config + self._bmi._job_meta, input_forcings, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status(self._bmi._job_meta, mpi_config) # If we are restarting a forecast cycle, re-calculate the neighboring files, and regrid the # next set of forcings as the previous step just regridded the previous forcing. @@ -492,83 +500,89 @@ def loop_through_forcing_products( and input_forcings.regridded_forcings2 is not None ): # Set the forcings back to reflect we just regridded the previous set of inputs, not the next. - if config_options.grid_type == "gridded": + if self._bmi._job_meta.grid_type == "gridded": input_forcings.regridded_forcings1[:, :, :] = ( input_forcings.regridded_forcings2[:, :, :] ) - elif config_options.grid_type == "unstructured": + elif self._bmi._job_meta.grid_type == "unstructured": input_forcings.regridded_forcings1[:, :] = ( input_forcings.regridded_forcings2[:, :] ) input_forcings.regridded_forcings1_elem[:, :] = ( input_forcings.regridded_forcings2_elem[:, :] ) - elif config_options.grid_type == "hydrofabric": + elif self._bmi._job_meta.grid_type == "hydrofabric": input_forcings.regridded_forcings1[:, :] = ( input_forcings.regridded_forcings2[:, :] ) else: raise ValueError( - f"Unexpected grid_type: {repr(config_options.grid_type)}" + f"Unexpected grid_type: {repr(self._bmi._job_meta.grid_type)}" ) # Re-calculate the neighbor files. input_forcings.calc_neighbor_files( - config_options, output_obj.outDate, mpi_config + self._bmi._job_meta, output_obj.outDate, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status(self._bmi._job_meta, mpi_config) # Regrid the forcings for the end of the window. input_forcings.regrid_inputs( - config_options, wrf_hydro_geo_meta, mpi_config + self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status(self._bmi._job_meta, mpi_config) input_forcings.rstFlag = 0 # Run temporal interpolation on the grids. - input_forcings.temporal_interpolate_inputs(config_options, mpi_config) - err_handler.check_program_status(config_options, mpi_config) + input_forcings.temporal_interpolate_inputs( + self._bmi._job_meta, mpi_config + ) + err_handler.check_program_status(self._bmi._job_meta, mpi_config) # Run bias correction. bias_correction.run_bias_correction( - input_forcings, config_options, wrf_hydro_geo_meta, mpi_config + input_forcings, self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status(self._bmi._job_meta, mpi_config) # Run downscaling on grids for this output timestep. downscale.run_downscaling( - input_forcings, config_options, wrf_hydro_geo_meta, mpi_config + input_forcings, self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status(self._bmi._job_meta, mpi_config) # Layer in forcings from this product. layeringMod.layer_final_forcings( - output_obj, input_forcings, config_options, mpi_config + output_obj, input_forcings, self._bmi._job_meta, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status(self._bmi._job_meta, mpi_config) - config_options.currentForceNum += 1 + self._bmi._job_meta.currentForceNum += 1 if force_key == 10: - config_options.currentCustomForceNum += 1 + self._bmi._job_meta.currentCustomForceNum += 1 LOG.debug(f"End of loop for force_key {force_key}") # Process supplemental precipitation if we specified in the configuration file. - if config_options.number_supp_pcp > 0: - for supp_pcp_key in config_options.supp_precip_forcings: + if self._bmi._job_meta.number_supp_pcp > 0: + for supp_pcp_key in self._bmi._job_meta.supp_precip_forcings: if supp_pcp_key != 13: # Like with input forcings, calculate the neighboring files to use. supp_pcp_mod[supp_pcp_key].calc_neighbor_files( - config_options, output_obj.outDate, mpi_config + self._bmi._job_meta, output_obj.outDate, mpi_config + ) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) # Regrid the supplemental precipitation. supp_pcp_mod[supp_pcp_key].regrid_inputs( - config_options, wrf_hydro_geo_meta, mpi_config + self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config + ) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) if ( supp_pcp_mod[supp_pcp_key].regridded_precip1 is not None @@ -576,45 +590,53 @@ def loop_through_forcing_products( ): # Run check on regridded fields for reasonable values that are not missing values. err_handler.check_supp_pcp_bounds( - config_options, + self._bmi._job_meta, supp_pcp_mod[supp_pcp_key], mpi_config, wrf_hydro_geo_meta, ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config + ) # TODO input_forcings has not yet been initialized, so this is a bug waiting to happen self.disaggregate_fun( input_forcings, supp_pcp_mod[supp_pcp_key], - config_options, + self._bmi._job_meta, mpi_config, ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config + ) # Run temporal interpolation on the grids. supp_pcp_mod[supp_pcp_key].temporal_interpolate_inputs( - config_options, mpi_config + self._bmi._job_meta, mpi_config + ) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) # Layer in the supplemental precipitation into the current output object. layeringMod.layer_supplemental_forcing( output_obj, supp_pcp_mod[supp_pcp_key], - config_options, + self._bmi._job_meta, mpi_config, ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config + ) # Call the output routines # adjust date for AnA if necessary - if config_options.ana_flag: + if self._bmi._job_meta.ana_flag: output_obj.outDate = file_date ################ Commenting this out to bypass NWM forcing file output functionality ######### - # output_obj.output_final_ldasin(config_options, wrf_hydro_geo_meta, mpi_config) - # err_handler.check_program_status(config_options, mpi_config) + # output_obj.output_final_ldasin(self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config) + # err_handler.check_program_status(self._bmi._job_meta, mpi_config) ############################################################################################## return input_forcings @@ -622,7 +644,6 @@ def loop_through_forcing_products( @time_function def process_suplemental_precip( self, - config_options: ConfigOptions, wrf_hydro_geo_meta: GeoMeta, supp_pcp_mod: dict, mpi_config: MpiConfig, @@ -635,22 +656,26 @@ def process_suplemental_precip( -------- Modifies mutable arguments in-place. """ - if config_options.customSuppPcpFreq is not None: + if self._bmi._job_meta.customSuppPcpFreq is not None: # Process supplemental precipitation if we specified in the configuration file. - if config_options.number_supp_pcp > 0: - for supp_pcp_key in config_options.supp_precip_forcings: + if self._bmi._job_meta.number_supp_pcp > 0: + for supp_pcp_key in self._bmi._job_meta.supp_precip_forcings: if supp_pcp_key == 14: # Like with input forcings, calculate the neighboring files to use. supp_pcp_mod[supp_pcp_key].calc_neighbor_files( - config_options, output_obj.outDate, mpi_config + self._bmi._job_meta, output_obj.outDate, mpi_config + ) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) # Regrid the supplemental precipitation. supp_pcp_mod[supp_pcp_key].regrid_inputs( - config_options, wrf_hydro_geo_meta, mpi_config + self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config + ) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) if ( supp_pcp_mod[supp_pcp_key].regridded_precip1 is not None @@ -658,40 +683,47 @@ def process_suplemental_precip( ): # Run check on regridded fields for reasonable values that are not missing values. err_handler.check_supp_pcp_bounds( - config_options, + self._bmi._job_meta, supp_pcp_mod[supp_pcp_key], mpi_config, wrf_hydro_geo_meta, ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config + ) self.disaggregate_fun( input_forcings, supp_pcp_mod[supp_pcp_key], - config_options, + self._bmi._job_meta, mpi_config, ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config + ) # Run temporal interpolation on the grids. supp_pcp_mod[supp_pcp_key].temporal_interpolate_inputs( - config_options, mpi_config + self._bmi._job_meta, mpi_config + ) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config ) - err_handler.check_program_status(config_options, mpi_config) # Layer in the supplemental precipitation into the current output object. layeringMod.layer_supplemental_forcing( output_obj, supp_pcp_mod[supp_pcp_key], - config_options, + self._bmi._job_meta, mpi_config, ) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status( + self._bmi._job_meta, mpi_config + ) @time_function def write_output( self, - config_options: ConfigOptions, wrf_hydro_geo_meta: GeoMeta, mpi_config: MpiConfig, output_obj: OutputObj, @@ -705,11 +737,11 @@ def write_output( # If user requests output for given domain, then call # the I/O module to update opened netcdf file with forcing fields if ( - config_options.forcing_output == 1 - or config_options.grid_type == "hydrofabric" + self._bmi._job_meta.forcing_output == 1 + or self._bmi._job_meta.grid_type == "hydrofabric" ): output_obj.gather_global_outputs( - config_options, wrf_hydro_geo_meta, mpi_config + self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config ) """##################Step 6: flatten and update dict##########################################################################""" @@ -717,7 +749,6 @@ def write_output( @time_function def update_dict( self, - config_options: ConfigOptions, wrf_hydro_geo_meta: GeoMeta, output_obj: OutputObj, ) -> None: @@ -740,7 +771,7 @@ def update_dict( # 7.) Surface incoming shortwave radiation flux (W/m^2) # 8.) Liquid Precipitation Fraction (%), Only available in certain operational configurations - if config_options.include_lqfrac == 1: + if self._bmi._job_meta.include_lqfrac == 1: variables = [ "U2D", "V2D", @@ -763,12 +794,12 @@ def update_dict( "PSFC", "SWDOWN", ] - if config_options.grid_type == "gridded": + if self._bmi._job_meta.grid_type == "gridded": for count, variable in enumerate(variables): self._bmi._values[variable + "_ELEMENT"] = output_obj.output_local[ count, :, : ].flatten() - elif config_options.grid_type == "unstructured": + elif self._bmi._job_meta.grid_type == "unstructured": for count, variable in enumerate(variables): self._bmi._values[variable + "_ELEMENT"] = output_obj.output_local_elem[ count, : @@ -776,11 +807,13 @@ def update_dict( self._bmi._values[variable + "_NODE"] = output_obj.output_local[ count, : ].flatten() - elif config_options.grid_type == "hydrofabric": + elif self._bmi._job_meta.grid_type == "hydrofabric": for count, variable in enumerate(variables): self._bmi._values[variable + "_ELEMENT"] = output_obj.output_global[ count, : ].flatten() self._bmi._values["CAT-ID"] = wrf_hydro_geo_meta.element_ids_global else: - raise ValueError(f"Unexpected grid_type: {repr(config_options.grid_type)}") + raise ValueError( + f"Unexpected grid_type: {repr(self._bmi._job_meta.grid_type)}" + ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 049578e1..b86ad8a1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -660,7 +660,6 @@ def pre_regrid(self) -> None: f"In pre_regrid, expected state to be either None or 'post_ran' but got {repr(self._state)}. The test is set up incorrectly." ) - config_options = self.config_options mpi_config = self.mpi_config geo_meta = self.geo_meta supp_pcp_mod = self.bmi_model._supp_pcp_mod @@ -674,14 +673,13 @@ def pre_regrid(self) -> None: model = self.bmi_model._model ### NOTE this should mimic NWMv3ForcingEngineModel.run() with the exception of setting the skip flag - model.determine_forecast(future_time, config_options) - model.adjust_precip(config_options, input_forcing_mod, mpi_config) - model.log_forecast(config_options, mpi_config) + model.determine_forecast(future_time) + model.adjust_precip(input_forcing_mod, mpi_config) + model.log_forecast(mpi_config) ### NOTE setting the flag causes the regrid step to be skipped self.set_input_forcings_skip_flags() model.loop_through_forcing_products( future_time, - config_options, geo_meta, input_forcing_mod, supp_pcp_mod, From cbe219742030758fa60ff43b60db70172e7004da Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 14:53:12 -0400 Subject: [PATCH 051/132] Encapsulate GeoMeta instance --- .../NextGen_Forcings_Engine/bmi_model.py | 2 - .../NextGen_Forcings_Engine/model.py | 51 +++++++++---------- tests/test_utils.py | 2 - 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 606a06af..2c890f97 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -517,7 +517,6 @@ def update_until(self, future_time: float) -> None: ): self._model.run( future_time, - self.geo_meta, self._input_forcing_mod, self._supp_pcp_mod, self._mpi_meta, @@ -532,7 +531,6 @@ def update_until(self, future_time: float) -> None: # Run the model for the new current time and update the state. self._model.run( self._values["current_model_time"], - self.geo_meta, self._input_forcing_mod, self._supp_pcp_mod, self._mpi_meta, diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 6a045262..633886fe 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -94,7 +94,6 @@ def __init__(self, bmi_model: NWMv3_Forcing_Engine_BMI_model_Base): def run( self, future_time: float, - wrf_hydro_geo_meta: GeoMeta, input_forcing_mod: dict, supp_pcp_mod: dict, mpi_config: MpiConfig, @@ -130,7 +129,6 @@ def run( 7. Advance the BMI time index. :param future_time: The number of seconds into the future to advance the model. - :param wrf_hydro_geo_meta: Geospatial metadata needed for regridding and interpolation. :param input_forcing_mod: Dictionary of initialized input forcing modules indexed by forcing key. :param supp_pcp_mod: Dictionary of supplemental precipitation modules indexed by key. :param mpi_config: Object containing MPI communication settings such as rank and communicator. @@ -145,26 +143,22 @@ def run( # TODO look into input_forcings usage in `process_suplemental_precip` and in `loop_through_forcing_products` at `disaggregate_fun`. input_forcings = self.loop_through_forcing_products( future_time, - wrf_hydro_geo_meta, input_forcing_mod, supp_pcp_mod, mpi_config, output_obj, ) self.process_suplemental_precip( - wrf_hydro_geo_meta, supp_pcp_mod, mpi_config, output_obj, input_forcings, ) self.write_output( - wrf_hydro_geo_meta, mpi_config, output_obj, ) self.update_dict( - wrf_hydro_geo_meta, output_obj, ) @@ -292,7 +286,6 @@ def log_forecast( def loop_through_forcing_products( self, future_time: float, - wrf_hydro_geo_meta: GeoMeta, input_forcing_mod: dict, supp_pcp_mod: dict, mpi_config: MpiConfig, @@ -429,12 +422,16 @@ def loop_through_forcing_products( if force_key == 12: if self.source_data_processor is None: self.source_data_processor = AORCConusProcessor( - self._bmi._job_meta, mpi_config, wrf_hydro_geo_meta + self._bmi._job_meta, + mpi_config, + self._bmi.geo_meta, ) elif force_key == 21: if self.source_data_processor is None: self.source_data_processor = AORCAlaskaProcessor( - self._bmi._job_meta, mpi_config, wrf_hydro_geo_meta + self._bmi._job_meta, + mpi_config, + self._bmi.geo_meta, ) # Flag to indicate the AWS .zarr NWMv3 Forcing file method @@ -444,13 +441,12 @@ def loop_through_forcing_products( self.source_data_processor = NWMV3ConusProcessor( self._bmi._job_meta, mpi_config, - wrf_hydro_geo_meta, + self._bmi.geo_meta, ) elif config_options.nwm_domain == "Hawaii": self.source_data_processor = NWMV3HawaiiProcessor( config_options, mpi_config, wrf_hydro_geo_meta ) - elif config_options.nwm_domain == "PR": self.source_data_processor = ( NWMV3PuertoRicoProcessor( @@ -463,7 +459,7 @@ def loop_through_forcing_products( self.source_data_processor = NWMV3AlaskaProcessor( self._bmi._job_meta, mpi_config, - wrf_hydro_geo_meta, + self._bmi.geo_meta, ) else: raise ValueError( @@ -482,7 +478,7 @@ def loop_through_forcing_products( break # Regrid forcings. input_forcings.regrid_inputs( - self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config + self._bmi._job_meta, self._bmi.geo_meta, mpi_config ) err_handler.check_program_status(self._bmi._job_meta, mpi_config) @@ -527,7 +523,7 @@ def loop_through_forcing_products( # Regrid the forcings for the end of the window. input_forcings.regrid_inputs( - self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config + self._bmi._job_meta, self._bmi.geo_meta, mpi_config ) err_handler.check_program_status(self._bmi._job_meta, mpi_config) @@ -541,13 +537,19 @@ def loop_through_forcing_products( # Run bias correction. bias_correction.run_bias_correction( - input_forcings, self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config + input_forcings, + self._bmi._job_meta, + self._bmi.geo_meta, + mpi_config, ) err_handler.check_program_status(self._bmi._job_meta, mpi_config) # Run downscaling on grids for this output timestep. downscale.run_downscaling( - input_forcings, self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config + input_forcings, + self._bmi._job_meta, + self._bmi.geo_meta, + mpi_config, ) err_handler.check_program_status(self._bmi._job_meta, mpi_config) @@ -578,7 +580,7 @@ def loop_through_forcing_products( # Regrid the supplemental precipitation. supp_pcp_mod[supp_pcp_key].regrid_inputs( - self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config + self._bmi._job_meta, self._bmi.geo_meta, mpi_config ) err_handler.check_program_status( self._bmi._job_meta, mpi_config @@ -593,7 +595,7 @@ def loop_through_forcing_products( self._bmi._job_meta, supp_pcp_mod[supp_pcp_key], mpi_config, - wrf_hydro_geo_meta, + self._bmi.geo_meta, ) err_handler.check_program_status( self._bmi._job_meta, mpi_config @@ -635,7 +637,7 @@ def loop_through_forcing_products( output_obj.outDate = file_date ################ Commenting this out to bypass NWM forcing file output functionality ######### - # output_obj.output_final_ldasin(self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config) + # output_obj.output_final_ldasin(self._bmi._job_meta, self._bmi.geo_meta, mpi_config) # err_handler.check_program_status(self._bmi._job_meta, mpi_config) ############################################################################################## @@ -644,7 +646,6 @@ def loop_through_forcing_products( @time_function def process_suplemental_precip( self, - wrf_hydro_geo_meta: GeoMeta, supp_pcp_mod: dict, mpi_config: MpiConfig, output_obj: OutputObj, @@ -671,7 +672,7 @@ def process_suplemental_precip( # Regrid the supplemental precipitation. supp_pcp_mod[supp_pcp_key].regrid_inputs( - self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config + self._bmi._job_meta, self._bmi.geo_meta, mpi_config ) err_handler.check_program_status( self._bmi._job_meta, mpi_config @@ -686,7 +687,7 @@ def process_suplemental_precip( self._bmi._job_meta, supp_pcp_mod[supp_pcp_key], mpi_config, - wrf_hydro_geo_meta, + self._bmi.geo_meta, ) err_handler.check_program_status( self._bmi._job_meta, mpi_config @@ -724,7 +725,6 @@ def process_suplemental_precip( @time_function def write_output( self, - wrf_hydro_geo_meta: GeoMeta, mpi_config: MpiConfig, output_obj: OutputObj, ) -> None: @@ -741,7 +741,7 @@ def write_output( or self._bmi._job_meta.grid_type == "hydrofabric" ): output_obj.gather_global_outputs( - self._bmi._job_meta, wrf_hydro_geo_meta, mpi_config + self._bmi._job_meta, self._bmi.geo_meta, mpi_config ) """##################Step 6: flatten and update dict##########################################################################""" @@ -749,7 +749,6 @@ def write_output( @time_function def update_dict( self, - wrf_hydro_geo_meta: GeoMeta, output_obj: OutputObj, ) -> None: """Flatten the Forcings Engine output object and update the BMI dictionary. @@ -812,7 +811,7 @@ def update_dict( self._bmi._values[variable + "_ELEMENT"] = output_obj.output_global[ count, : ].flatten() - self._bmi._values["CAT-ID"] = wrf_hydro_geo_meta.element_ids_global + self._bmi._values["CAT-ID"] = self._bmi.geo_meta.element_ids_global else: raise ValueError( f"Unexpected grid_type: {repr(self._bmi._job_meta.grid_type)}" diff --git a/tests/test_utils.py b/tests/test_utils.py index b86ad8a1..81a92fd7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -661,7 +661,6 @@ def pre_regrid(self) -> None: ) mpi_config = self.mpi_config - geo_meta = self.geo_meta supp_pcp_mod = self.bmi_model._supp_pcp_mod output_obj = self.bmi_model._output_obj input_forcing_mod = self.bmi_model._input_forcing_mod @@ -680,7 +679,6 @@ def pre_regrid(self) -> None: self.set_input_forcings_skip_flags() model.loop_through_forcing_products( future_time, - geo_meta, input_forcing_mod, supp_pcp_mod, mpi_config, From 98ce2afbf82e283045d9a71e5ad224ced2d796f0 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 15:36:11 -0400 Subject: [PATCH 052/132] Encapsulate input forcing mod dict --- .../NextGen_Forcings_Engine/bmi_model.py | 2 -- .../NextGen_Forcings_Engine/model.py | 13 ++++--------- tests/test_utils.py | 6 ++---- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 2c890f97..9365226a 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -517,7 +517,6 @@ def update_until(self, future_time: float) -> None: ): self._model.run( future_time, - self._input_forcing_mod, self._supp_pcp_mod, self._mpi_meta, self._output_obj, @@ -531,7 +530,6 @@ def update_until(self, future_time: float) -> None: # Run the model for the new current time and update the state. self._model.run( self._values["current_model_time"], - self._input_forcing_mod, self._supp_pcp_mod, self._mpi_meta, self._output_obj, diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 633886fe..2a0ea617 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -94,7 +94,6 @@ def __init__(self, bmi_model: NWMv3_Forcing_Engine_BMI_model_Base): def run( self, future_time: float, - input_forcing_mod: dict, supp_pcp_mod: dict, mpi_config: MpiConfig, output_obj: OutputObj, @@ -129,7 +128,6 @@ def run( 7. Advance the BMI time index. :param future_time: The number of seconds into the future to advance the model. - :param input_forcing_mod: Dictionary of initialized input forcing modules indexed by forcing key. :param supp_pcp_mod: Dictionary of supplemental precipitation modules indexed by key. :param mpi_config: Object containing MPI communication settings such as rank and communicator. :param output_obj: Output object that stores the generated atmospheric forcing arrays. @@ -138,12 +136,11 @@ def run( """ self.determine_forecast(future_time) - self.adjust_precip(input_forcing_mod, mpi_config) + self.adjust_precip(mpi_config) self.log_forecast(mpi_config) # TODO look into input_forcings usage in `process_suplemental_precip` and in `loop_through_forcing_products` at `disaggregate_fun`. input_forcings = self.loop_through_forcing_products( future_time, - input_forcing_mod, supp_pcp_mod, mpi_config, output_obj, @@ -238,7 +235,6 @@ def determine_forecast( @time_function def adjust_precip( self, - input_forcing_mod: dict, mpi_config: MpiConfig, ) -> None: """Adjust precipitation for the given forecast cycle. @@ -250,7 +246,7 @@ def adjust_precip( if not self._bmi._job_meta.precip_only_flag: # reset skips if present for force_key in self._bmi._job_meta.input_forcings: - input_forcing_mod[force_key].skip = False + self._bmi._input_forcing_mod[force_key].skip = False err_handler.check_program_status(self._bmi._job_meta, mpi_config) @@ -286,7 +282,6 @@ def log_forecast( def loop_through_forcing_products( self, future_time: float, - input_forcing_mod: dict, supp_pcp_mod: dict, mpi_config: MpiConfig, output_obj: OutputObj, @@ -397,13 +392,13 @@ def loop_through_forcing_products( and 12 in self._bmi._job_meta.input_forcings and 21 in self._bmi._job_meta.input_forcings ): - input_forcings = input_forcing_mod[force_key] + input_forcings = self._bmi._input_forcing_mod[force_key] # These are not used # AORC_mask = input_forcings.regridded_mask_AORC # AORC_elem_mask = input_forcings.regridded_mask_elem_AORC else: - input_forcings = input_forcing_mod[force_key] + input_forcings = self._bmi._input_forcing_mod[force_key] input_forcings.calc_neighbor_files( self._bmi._job_meta, output_obj.outDate, mpi_config ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 81a92fd7..8186c0dc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -663,7 +663,6 @@ def pre_regrid(self) -> None: mpi_config = self.mpi_config supp_pcp_mod = self.bmi_model._supp_pcp_mod output_obj = self.bmi_model._output_obj - input_forcing_mod = self.bmi_model._input_forcing_mod future_time = ( self.bmi_model._values["current_model_time"] @@ -673,13 +672,12 @@ def pre_regrid(self) -> None: ### NOTE this should mimic NWMv3ForcingEngineModel.run() with the exception of setting the skip flag model.determine_forecast(future_time) - model.adjust_precip(input_forcing_mod, mpi_config) + model.adjust_precip(mpi_config) model.log_forecast(mpi_config) ### NOTE setting the flag causes the regrid step to be skipped self.set_input_forcings_skip_flags() model.loop_through_forcing_products( future_time, - input_forcing_mod, supp_pcp_mod, mpi_config, output_obj, @@ -691,7 +689,7 @@ def pre_regrid(self) -> None: def set_input_forcings_skip_flags(self) -> None: """Set the `skip` flag on the InputForcings object so that forcing regrid will not occur during loop_through_forcing_products().""" logging.debug( - "Setting input_forcing.skip = True for each value in dict self.input_forcing_mod" + "Setting input_forcing.skip = True for each value in dict self.bmi_model._input_forcing_mod" ) for force_key, input_forcing in self.bmi_model._input_forcing_mod.items(): input_forcing.skip = True From 433d180cf5b815b3086eee29925a173296fe3a95 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 15:43:48 -0400 Subject: [PATCH 053/132] Encapsulate supp pcp mod --- .../NextGen_Forcings_Engine/bmi_model.py | 2 - .../NextGen_Forcings_Engine/model.py | 46 ++++++++++--------- tests/test_utils.py | 2 - 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 9365226a..7204df83 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -517,7 +517,6 @@ def update_until(self, future_time: float) -> None: ): self._model.run( future_time, - self._supp_pcp_mod, self._mpi_meta, self._output_obj, ) @@ -530,7 +529,6 @@ def update_until(self, future_time: float) -> None: # Run the model for the new current time and update the state. self._model.run( self._values["current_model_time"], - self._supp_pcp_mod, self._mpi_meta, self._output_obj, ) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 2a0ea617..fd1bb0e2 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -94,7 +94,6 @@ def __init__(self, bmi_model: NWMv3_Forcing_Engine_BMI_model_Base): def run( self, future_time: float, - supp_pcp_mod: dict, mpi_config: MpiConfig, output_obj: OutputObj, ) -> None: @@ -128,7 +127,6 @@ def run( 7. Advance the BMI time index. :param future_time: The number of seconds into the future to advance the model. - :param supp_pcp_mod: Dictionary of supplemental precipitation modules indexed by key. :param mpi_config: Object containing MPI communication settings such as rank and communicator. :param output_obj: Output object that stores the generated atmospheric forcing arrays. @@ -141,12 +139,10 @@ def run( # TODO look into input_forcings usage in `process_suplemental_precip` and in `loop_through_forcing_products` at `disaggregate_fun`. input_forcings = self.loop_through_forcing_products( future_time, - supp_pcp_mod, mpi_config, output_obj, ) self.process_suplemental_precip( - supp_pcp_mod, mpi_config, output_obj, input_forcings, @@ -282,7 +278,6 @@ def log_forecast( def loop_through_forcing_products( self, future_time: float, - supp_pcp_mod: dict, mpi_config: MpiConfig, output_obj: OutputObj, ) -> forcingInputMod.InputForcingsHydrofabric: @@ -566,7 +561,7 @@ def loop_through_forcing_products( for supp_pcp_key in self._bmi._job_meta.supp_precip_forcings: if supp_pcp_key != 13: # Like with input forcings, calculate the neighboring files to use. - supp_pcp_mod[supp_pcp_key].calc_neighbor_files( + self._bmi._supp_pcp_mod[supp_pcp_key].calc_neighbor_files( self._bmi._job_meta, output_obj.outDate, mpi_config ) err_handler.check_program_status( @@ -574,7 +569,7 @@ def loop_through_forcing_products( ) # Regrid the supplemental precipitation. - supp_pcp_mod[supp_pcp_key].regrid_inputs( + self._bmi._supp_pcp_mod[supp_pcp_key].regrid_inputs( self._bmi._job_meta, self._bmi.geo_meta, mpi_config ) err_handler.check_program_status( @@ -582,13 +577,15 @@ def loop_through_forcing_products( ) if ( - supp_pcp_mod[supp_pcp_key].regridded_precip1 is not None - and supp_pcp_mod[supp_pcp_key].regridded_precip2 is not None + self._bmi._supp_pcp_mod[supp_pcp_key].regridded_precip1 + is not None + and self._bmi._supp_pcp_mod[supp_pcp_key].regridded_precip2 + is not None ): # Run check on regridded fields for reasonable values that are not missing values. err_handler.check_supp_pcp_bounds( self._bmi._job_meta, - supp_pcp_mod[supp_pcp_key], + self._bmi._supp_pcp_mod[supp_pcp_key], mpi_config, self._bmi.geo_meta, ) @@ -599,7 +596,7 @@ def loop_through_forcing_products( # TODO input_forcings has not yet been initialized, so this is a bug waiting to happen self.disaggregate_fun( input_forcings, - supp_pcp_mod[supp_pcp_key], + self._bmi._supp_pcp_mod[supp_pcp_key], self._bmi._job_meta, mpi_config, ) @@ -608,7 +605,9 @@ def loop_through_forcing_products( ) # Run temporal interpolation on the grids. - supp_pcp_mod[supp_pcp_key].temporal_interpolate_inputs( + self._bmi._supp_pcp_mod[ + supp_pcp_key + ].temporal_interpolate_inputs( self._bmi._job_meta, mpi_config ) err_handler.check_program_status( @@ -618,7 +617,7 @@ def loop_through_forcing_products( # Layer in the supplemental precipitation into the current output object. layeringMod.layer_supplemental_forcing( output_obj, - supp_pcp_mod[supp_pcp_key], + self._bmi._supp_pcp_mod[supp_pcp_key], self._bmi._job_meta, mpi_config, ) @@ -641,7 +640,6 @@ def loop_through_forcing_products( @time_function def process_suplemental_precip( self, - supp_pcp_mod: dict, mpi_config: MpiConfig, output_obj: OutputObj, input_forcings: dict, @@ -658,7 +656,7 @@ def process_suplemental_precip( for supp_pcp_key in self._bmi._job_meta.supp_precip_forcings: if supp_pcp_key == 14: # Like with input forcings, calculate the neighboring files to use. - supp_pcp_mod[supp_pcp_key].calc_neighbor_files( + self._bmi._supp_pcp_mod[supp_pcp_key].calc_neighbor_files( self._bmi._job_meta, output_obj.outDate, mpi_config ) err_handler.check_program_status( @@ -666,7 +664,7 @@ def process_suplemental_precip( ) # Regrid the supplemental precipitation. - supp_pcp_mod[supp_pcp_key].regrid_inputs( + self._bmi._supp_pcp_mod[supp_pcp_key].regrid_inputs( self._bmi._job_meta, self._bmi.geo_meta, mpi_config ) err_handler.check_program_status( @@ -674,13 +672,15 @@ def process_suplemental_precip( ) if ( - supp_pcp_mod[supp_pcp_key].regridded_precip1 is not None - and supp_pcp_mod[supp_pcp_key].regridded_precip2 is not None + self._bmi._supp_pcp_mod[supp_pcp_key].regridded_precip1 + is not None + and self._bmi._supp_pcp_mod[supp_pcp_key].regridded_precip2 + is not None ): # Run check on regridded fields for reasonable values that are not missing values. err_handler.check_supp_pcp_bounds( self._bmi._job_meta, - supp_pcp_mod[supp_pcp_key], + self._bmi._supp_pcp_mod[supp_pcp_key], mpi_config, self._bmi.geo_meta, ) @@ -690,7 +690,7 @@ def process_suplemental_precip( self.disaggregate_fun( input_forcings, - supp_pcp_mod[supp_pcp_key], + self._bmi._supp_pcp_mod[supp_pcp_key], self._bmi._job_meta, mpi_config, ) @@ -699,7 +699,9 @@ def process_suplemental_precip( ) # Run temporal interpolation on the grids. - supp_pcp_mod[supp_pcp_key].temporal_interpolate_inputs( + self._bmi._supp_pcp_mod[ + supp_pcp_key + ].temporal_interpolate_inputs( self._bmi._job_meta, mpi_config ) err_handler.check_program_status( @@ -709,7 +711,7 @@ def process_suplemental_precip( # Layer in the supplemental precipitation into the current output object. layeringMod.layer_supplemental_forcing( output_obj, - supp_pcp_mod[supp_pcp_key], + self._bmi._supp_pcp_mod[supp_pcp_key], self._bmi._job_meta, mpi_config, ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 8186c0dc..dc53e411 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -661,7 +661,6 @@ def pre_regrid(self) -> None: ) mpi_config = self.mpi_config - supp_pcp_mod = self.bmi_model._supp_pcp_mod output_obj = self.bmi_model._output_obj future_time = ( @@ -678,7 +677,6 @@ def pre_regrid(self) -> None: self.set_input_forcings_skip_flags() model.loop_through_forcing_products( future_time, - supp_pcp_mod, mpi_config, output_obj, ) From 105547c9a7bc9f38d809d1bbd8e53a86023bebaf Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 15:50:57 -0400 Subject: [PATCH 054/132] Encapsulate MpiConfig --- .../NextGen_Forcings_Engine/bmi_model.py | 2 - .../NextGen_Forcings_Engine/model.py | 160 +++++++++--------- tests/test_utils.py | 11 +- 3 files changed, 83 insertions(+), 90 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 7204df83..a4a53cab 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -517,7 +517,6 @@ def update_until(self, future_time: float) -> None: ): self._model.run( future_time, - self._mpi_meta, self._output_obj, ) else: @@ -529,7 +528,6 @@ def update_until(self, future_time: float) -> None: # Run the model for the new current time and update the state. self._model.run( self._values["current_model_time"], - self._mpi_meta, self._output_obj, ) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index fd1bb0e2..1577688a 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -94,7 +94,6 @@ def __init__(self, bmi_model: NWMv3_Forcing_Engine_BMI_model_Base): def run( self, future_time: float, - mpi_config: MpiConfig, output_obj: OutputObj, ) -> None: """Execute the full forcings engine BMI pipeline for a given future timestep. @@ -127,28 +126,24 @@ def run( 7. Advance the BMI time index. :param future_time: The number of seconds into the future to advance the model. - :param mpi_config: Object containing MPI communication settings such as rank and communicator. :param output_obj: Output object that stores the generated atmospheric forcing arrays. :raises RuntimeError: If the model fails to initialize or if required arguments are missing. """ self.determine_forecast(future_time) - self.adjust_precip(mpi_config) - self.log_forecast(mpi_config) + self.adjust_precip() + self.log_forecast() # TODO look into input_forcings usage in `process_suplemental_precip` and in `loop_through_forcing_products` at `disaggregate_fun`. input_forcings = self.loop_through_forcing_products( future_time, - mpi_config, output_obj, ) self.process_suplemental_precip( - mpi_config, output_obj, input_forcings, ) self.write_output( - mpi_config, output_obj, ) self.update_dict( @@ -229,10 +224,7 @@ def determine_forecast( ) @time_function - def adjust_precip( - self, - mpi_config: MpiConfig, - ) -> None: + def adjust_precip(self) -> None: """Adjust precipitation for the given forecast cycle. Warnings @@ -244,13 +236,10 @@ def adjust_precip( for force_key in self._bmi._job_meta.input_forcings: self._bmi._input_forcing_mod[force_key].skip = False - err_handler.check_program_status(self._bmi._job_meta, mpi_config) + err_handler.check_program_status(self._bmi._job_meta, self._bmi._mpi_meta) @time_function - def log_forecast( - self, - mpi_config: MpiConfig, - ) -> None: + def log_forecast(self) -> None: """Log information about the current forecast cycle. Warnings @@ -258,28 +247,25 @@ def log_forecast( Modifies mutable arguments in-place. """ # Log information about this forecast cycle - if mpi_config.rank == 0: + if self._bmi._mpi_meta.rank == 0: self._bmi._job_meta.statusMsg = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" - err_handler.log_msg(self._bmi._job_meta, mpi_config, True) + err_handler.log_msg(self._bmi._job_meta, self._bmi._mpi_meta, True) self._bmi._job_meta.statusMsg = ( "Processing Forecast Cycle: " + self._bmi._job_meta.current_fcst_cycle.strftime("%Y-%m-%d %H:%M") ) - err_handler.log_msg(self._bmi._job_meta, mpi_config, True) + err_handler.log_msg(self._bmi._job_meta, self._bmi._mpi_meta, True) self._bmi._job_meta.statusMsg = ( "Forecast Cycle Length is: " + str(self._bmi._job_meta.cycle_length_minutes) + " minutes" ) - err_handler.log_msg(self._bmi._job_meta, mpi_config, True) - # mpi_config.comm.barrier() + err_handler.log_msg(self._bmi._job_meta, self._bmi._mpi_meta, True) + # self._bmi._mpi_meta.comm.barrier() @time_function def loop_through_forcing_products( - self, - future_time: float, - mpi_config: MpiConfig, - output_obj: OutputObj, + self, future_time: float, output_obj: OutputObj ) -> forcingInputMod.InputForcingsHydrofabric: """Loop through each forcing product and process it for the current forecast cycle. @@ -359,13 +345,13 @@ def loop_through_forcing_products( # Print message on log file indicating the timestamp # we are currently processing for forcings - if mpi_config.rank == 0 and show_message: + if self._bmi._mpi_meta.rank == 0 and show_message: self._bmi._job_meta.statusMsg = ( "=========================================" ) - err_handler.log_msg(self._bmi._job_meta, mpi_config, True) + err_handler.log_msg(self._bmi._job_meta, self._bmi._mpi_meta, True) self._bmi._job_meta.statusMsg = f"Processing for output timestep: {file_date.strftime('%Y-%m-%d %H:%M')}" - err_handler.log_msg(self._bmi._job_meta, mpi_config, True) + err_handler.log_msg(self._bmi._job_meta, self._bmi._mpi_meta, True) self._bmi._job_meta.currentForceNum = 0 self._bmi._job_meta.currentCustomForceNum = 0 @@ -395,17 +381,17 @@ def loop_through_forcing_products( else: input_forcings = self._bmi._input_forcing_mod[force_key] input_forcings.calc_neighbor_files( - self._bmi._job_meta, output_obj.outDate, mpi_config + self._bmi._job_meta, output_obj.outDate, self._bmi._mpi_meta ) if force_key in [12, 21, 27]: if self._bmi._job_meta.aws is None: # Calculate the previous and next input cycle files from the inputs. input_forcings.calc_neighbor_files( - self._bmi._job_meta, output_obj.outDate, mpi_config + self._bmi._job_meta, output_obj.outDate, self._bmi._mpi_meta ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) else: # Flag to indicate the AWS .zarr AORC method @@ -413,14 +399,14 @@ def loop_through_forcing_products( if self.source_data_processor is None: self.source_data_processor = AORCConusProcessor( self._bmi._job_meta, - mpi_config, + self._bmi._mpi_meta, self._bmi.geo_meta, ) elif force_key == 21: if self.source_data_processor is None: self.source_data_processor = AORCAlaskaProcessor( self._bmi._job_meta, - mpi_config, + self._bmi._mpi_meta, self._bmi.geo_meta, ) @@ -430,7 +416,7 @@ def loop_through_forcing_products( if self._bmi._job_meta.nwm_domain == "CONUS": self.source_data_processor = NWMV3ConusProcessor( self._bmi._job_meta, - mpi_config, + self._bmi._mpi_meta, self._bmi.geo_meta, ) elif config_options.nwm_domain == "Hawaii": @@ -448,7 +434,7 @@ def loop_through_forcing_products( elif config_options.nwm_domain == "Alaska": self.source_data_processor = NWMV3AlaskaProcessor( self._bmi._job_meta, - mpi_config, + self._bmi._mpi_meta, self._bmi.geo_meta, ) else: @@ -468,15 +454,19 @@ def loop_through_forcing_products( break # Regrid forcings. input_forcings.regrid_inputs( - self._bmi._job_meta, self._bmi.geo_meta, mpi_config + self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta + ) + err_handler.check_program_status( + self._bmi._job_meta, self._bmi._mpi_meta ) - err_handler.check_program_status(self._bmi._job_meta, mpi_config) # Run check on regridded fields for reasonable values that are not missing values. err_handler.check_forcing_bounds( - self._bmi._job_meta, input_forcings, mpi_config + self._bmi._job_meta, input_forcings, self._bmi._mpi_meta + ) + err_handler.check_program_status( + self._bmi._job_meta, self._bmi._mpi_meta ) - err_handler.check_program_status(self._bmi._job_meta, mpi_config) # If we are restarting a forecast cycle, re-calculate the neighboring files, and regrid the # next set of forcings as the previous step just regridded the previous forcing. @@ -507,47 +497,59 @@ def loop_through_forcing_products( ) # Re-calculate the neighbor files. input_forcings.calc_neighbor_files( - self._bmi._job_meta, output_obj.outDate, mpi_config + self._bmi._job_meta, output_obj.outDate, self._bmi._mpi_meta + ) + err_handler.check_program_status( + self._bmi._job_meta, self._bmi._mpi_meta ) - err_handler.check_program_status(self._bmi._job_meta, mpi_config) # Regrid the forcings for the end of the window. input_forcings.regrid_inputs( - self._bmi._job_meta, self._bmi.geo_meta, mpi_config + self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta + ) + err_handler.check_program_status( + self._bmi._job_meta, self._bmi._mpi_meta ) - err_handler.check_program_status(self._bmi._job_meta, mpi_config) input_forcings.rstFlag = 0 # Run temporal interpolation on the grids. input_forcings.temporal_interpolate_inputs( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta + ) + err_handler.check_program_status( + self._bmi._job_meta, self._bmi._mpi_meta ) - err_handler.check_program_status(self._bmi._job_meta, mpi_config) # Run bias correction. bias_correction.run_bias_correction( input_forcings, self._bmi._job_meta, self._bmi.geo_meta, - mpi_config, + self._bmi._mpi_meta, + ) + err_handler.check_program_status( + self._bmi._job_meta, self._bmi._mpi_meta ) - err_handler.check_program_status(self._bmi._job_meta, mpi_config) # Run downscaling on grids for this output timestep. downscale.run_downscaling( input_forcings, self._bmi._job_meta, self._bmi.geo_meta, - mpi_config, + self._bmi._mpi_meta, + ) + err_handler.check_program_status( + self._bmi._job_meta, self._bmi._mpi_meta ) - err_handler.check_program_status(self._bmi._job_meta, mpi_config) # Layer in forcings from this product. layeringMod.layer_final_forcings( - output_obj, input_forcings, self._bmi._job_meta, mpi_config + output_obj, input_forcings, self._bmi._job_meta, self._bmi._mpi_meta + ) + err_handler.check_program_status( + self._bmi._job_meta, self._bmi._mpi_meta ) - err_handler.check_program_status(self._bmi._job_meta, mpi_config) self._bmi._job_meta.currentForceNum += 1 @@ -562,18 +564,18 @@ def loop_through_forcing_products( if supp_pcp_key != 13: # Like with input forcings, calculate the neighboring files to use. self._bmi._supp_pcp_mod[supp_pcp_key].calc_neighbor_files( - self._bmi._job_meta, output_obj.outDate, mpi_config + self._bmi._job_meta, output_obj.outDate, self._bmi._mpi_meta ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) # Regrid the supplemental precipitation. self._bmi._supp_pcp_mod[supp_pcp_key].regrid_inputs( - self._bmi._job_meta, self._bmi.geo_meta, mpi_config + self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) if ( @@ -586,11 +588,11 @@ def loop_through_forcing_products( err_handler.check_supp_pcp_bounds( self._bmi._job_meta, self._bmi._supp_pcp_mod[supp_pcp_key], - mpi_config, + self._bmi._mpi_meta, self._bmi.geo_meta, ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) # TODO input_forcings has not yet been initialized, so this is a bug waiting to happen @@ -598,20 +600,20 @@ def loop_through_forcing_products( input_forcings, self._bmi._supp_pcp_mod[supp_pcp_key], self._bmi._job_meta, - mpi_config, + self._bmi._mpi_meta, ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) # Run temporal interpolation on the grids. self._bmi._supp_pcp_mod[ supp_pcp_key ].temporal_interpolate_inputs( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) # Layer in the supplemental precipitation into the current output object. @@ -619,10 +621,10 @@ def loop_through_forcing_products( output_obj, self._bmi._supp_pcp_mod[supp_pcp_key], self._bmi._job_meta, - mpi_config, + self._bmi._mpi_meta, ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) # Call the output routines @@ -631,8 +633,8 @@ def loop_through_forcing_products( output_obj.outDate = file_date ################ Commenting this out to bypass NWM forcing file output functionality ######### - # output_obj.output_final_ldasin(self._bmi._job_meta, self._bmi.geo_meta, mpi_config) - # err_handler.check_program_status(self._bmi._job_meta, mpi_config) + # output_obj.output_final_ldasin(self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta) + # err_handler.check_program_status(self._bmi._job_meta, self._bmi._mpi_meta) ############################################################################################## return input_forcings @@ -640,7 +642,6 @@ def loop_through_forcing_products( @time_function def process_suplemental_precip( self, - mpi_config: MpiConfig, output_obj: OutputObj, input_forcings: dict, ) -> None: @@ -657,18 +658,18 @@ def process_suplemental_precip( if supp_pcp_key == 14: # Like with input forcings, calculate the neighboring files to use. self._bmi._supp_pcp_mod[supp_pcp_key].calc_neighbor_files( - self._bmi._job_meta, output_obj.outDate, mpi_config + self._bmi._job_meta, output_obj.outDate, self._bmi._mpi_meta ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) # Regrid the supplemental precipitation. self._bmi._supp_pcp_mod[supp_pcp_key].regrid_inputs( - self._bmi._job_meta, self._bmi.geo_meta, mpi_config + self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) if ( @@ -681,31 +682,31 @@ def process_suplemental_precip( err_handler.check_supp_pcp_bounds( self._bmi._job_meta, self._bmi._supp_pcp_mod[supp_pcp_key], - mpi_config, + self._bmi._mpi_meta, self._bmi.geo_meta, ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) self.disaggregate_fun( input_forcings, self._bmi._supp_pcp_mod[supp_pcp_key], self._bmi._job_meta, - mpi_config, + self._bmi._mpi_meta, ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) # Run temporal interpolation on the grids. self._bmi._supp_pcp_mod[ supp_pcp_key ].temporal_interpolate_inputs( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) # Layer in the supplemental precipitation into the current output object. @@ -713,16 +714,15 @@ def process_suplemental_precip( output_obj, self._bmi._supp_pcp_mod[supp_pcp_key], self._bmi._job_meta, - mpi_config, + self._bmi._mpi_meta, ) err_handler.check_program_status( - self._bmi._job_meta, mpi_config + self._bmi._job_meta, self._bmi._mpi_meta ) @time_function def write_output( self, - mpi_config: MpiConfig, output_obj: OutputObj, ) -> None: """Write the output for the current forecast cycle. @@ -738,7 +738,7 @@ def write_output( or self._bmi._job_meta.grid_type == "hydrofabric" ): output_obj.gather_global_outputs( - self._bmi._job_meta, self._bmi.geo_meta, mpi_config + self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta ) """##################Step 6: flatten and update dict##########################################################################""" diff --git a/tests/test_utils.py b/tests/test_utils.py index dc53e411..efba3ca9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -660,7 +660,6 @@ def pre_regrid(self) -> None: f"In pre_regrid, expected state to be either None or 'post_ran' but got {repr(self._state)}. The test is set up incorrectly." ) - mpi_config = self.mpi_config output_obj = self.bmi_model._output_obj future_time = ( @@ -671,15 +670,11 @@ def pre_regrid(self) -> None: ### NOTE this should mimic NWMv3ForcingEngineModel.run() with the exception of setting the skip flag model.determine_forecast(future_time) - model.adjust_precip(mpi_config) - model.log_forecast(mpi_config) + model.adjust_precip() + model.log_forecast() ### NOTE setting the flag causes the regrid step to be skipped self.set_input_forcings_skip_flags() - model.loop_through_forcing_products( - future_time, - mpi_config, - output_obj, - ) + model.loop_through_forcing_products(future_time, output_obj) # Update test fixture status self._state = "pre_ran" From 5fbf642f4d386c392401fb6de4652e0ab92d722e Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 16:01:33 -0400 Subject: [PATCH 055/132] Encapsulate OutputObj instance --- .../NextGen_Forcings_Engine/bmi_model.py | 10 +- .../NextGen_Forcings_Engine/model.py | 125 ++++++++---------- tests/test_utils.py | 4 +- 3 files changed, 61 insertions(+), 78 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index a4a53cab..a780e744 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -515,10 +515,7 @@ def update_until(self, future_time: float) -> None: == future_time == self.cfg_bmi["initial_time"] ): - self._model.run( - future_time, - self._output_obj, - ) + self._model.run(future_time) else: # Start a while loop to iterate the model time step by step until the # current model time reaches or exceeds the future_time. @@ -526,10 +523,7 @@ def update_until(self, future_time: float) -> None: # Advance the model time by the defined time step size. self._values["current_model_time"] += self._values["time_step_size"] # Run the model for the new current time and update the state. - self._model.run( - self._values["current_model_time"], - self._output_obj, - ) + self._model.run(self._values["current_model_time"]) def finalize(self) -> None: """Finalize the model, performing necessary cleanup tasks. diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 1577688a..2f101c34 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -91,11 +91,7 @@ def __init__(self, bmi_model: NWMv3_Forcing_Engine_BMI_model_Base): # def aws_obj(files): # return xr.open_mfdataset(files, engine="zarr", parallel=True, consolidated=True) - def run( - self, - future_time: float, - output_obj: OutputObj, - ) -> None: + def run(self, future_time: float) -> None: """Execute the full forcings engine BMI pipeline for a given future timestep. This method updates the `self._bmi._values` state dictionary with atmospheric forcings computed from @@ -126,7 +122,6 @@ def run( 7. Advance the BMI time index. :param future_time: The number of seconds into the future to advance the model. - :param output_obj: Output object that stores the generated atmospheric forcing arrays. :raises RuntimeError: If the model fails to initialize or if required arguments are missing. """ @@ -137,27 +132,16 @@ def run( # TODO look into input_forcings usage in `process_suplemental_precip` and in `loop_through_forcing_products` at `disaggregate_fun`. input_forcings = self.loop_through_forcing_products( future_time, - output_obj, - ) - self.process_suplemental_precip( - output_obj, - input_forcings, - ) - self.write_output( - output_obj, - ) - self.update_dict( - output_obj, ) + self.process_suplemental_precip(input_forcings) + self.write_output() + self.update_dict() ## Update BMI model time index to next iteration self._bmi._job_meta.bmi_time_index += 1 @time_function - def determine_forecast( - self, - future_time: float, - ) -> None: + def determine_forecast(self, future_time: float) -> None: """Determine the forecast for the given future time and configuration. Warnings @@ -265,7 +249,7 @@ def log_forecast(self) -> None: @time_function def loop_through_forcing_products( - self, future_time: float, output_obj: OutputObj + self, future_time: float ) -> forcingInputMod.InputForcingsHydrofabric: """Loop through each forcing product and process it for the current forecast cycle. @@ -284,14 +268,18 @@ def loop_through_forcing_products( if not self._bmi._job_meta.precip_only_flag: if self._bmi._job_meta.grid_type == "gridded": # Reset out final grids to missing values. - output_obj.output_local[:, :, :] = self._bmi._job_meta.globalNdv + self._bmi._output_obj.output_local[:, :, :] = ( + self._bmi._job_meta.globalNdv + ) elif self._bmi._job_meta.grid_type == "unstructured": # Reset out final grids to missing values. - output_obj.output_local[:, :] = self._bmi._job_meta.globalNdv - output_obj.output_local_elem[:, :] = self._bmi._job_meta.globalNdv + self._bmi._output_obj.output_local[:, :] = self._bmi._job_meta.globalNdv + self._bmi._output_obj.output_local_elem[:, :] = ( + self._bmi._job_meta.globalNdv + ) elif self._bmi._job_meta.grid_type == "hydrofabric": # Reset out final grids to missing values. - output_obj.output_local[:, :] = self._bmi._job_meta.globalNdv + self._bmi._output_obj.output_local[:, :] = self._bmi._job_meta.globalNdv else: raise ValueError( f"Unexpected grid_type: {repr(self._bmi._job_meta.grid_type)}" @@ -312,24 +300,24 @@ def loop_through_forcing_products( # Compute the output timestamp for this step if self._bmi._job_meta.ana_flag: - output_obj.outDate = ( + self._bmi._output_obj.outDate = ( self._bmi._job_meta.current_fcst_cycle + datetime.timedelta(seconds=self._bmi._job_meta.output_freq * 60) ) else: - output_obj.outDate = ( + self._bmi._output_obj.outDate = ( self._bmi._job_meta.current_fcst_cycle + datetime.timedelta(seconds=future_time) ) - self._bmi._job_meta.current_output_date = output_obj.outDate + self._bmi._job_meta.current_output_date = self._bmi._output_obj.outDate # Adjust file_date for AnA if needed file_date = ( - output_obj.outDate + self._bmi._output_obj.outDate - datetime.timedelta(seconds=self._bmi._job_meta.output_freq * 60) if self._bmi._job_meta.ana_flag - else output_obj.outDate + else self._bmi._output_obj.outDate ) # Compute previous output date (used for downscaling logic) @@ -381,14 +369,18 @@ def loop_through_forcing_products( else: input_forcings = self._bmi._input_forcing_mod[force_key] input_forcings.calc_neighbor_files( - self._bmi._job_meta, output_obj.outDate, self._bmi._mpi_meta + self._bmi._job_meta, + self._bmi._output_obj.outDate, + self._bmi._mpi_meta, ) if force_key in [12, 21, 27]: if self._bmi._job_meta.aws is None: # Calculate the previous and next input cycle files from the inputs. input_forcings.calc_neighbor_files( - self._bmi._job_meta, output_obj.outDate, self._bmi._mpi_meta + self._bmi._job_meta, + self._bmi._output_obj.outDate, + self._bmi._mpi_meta, ) err_handler.check_program_status( self._bmi._job_meta, self._bmi._mpi_meta @@ -497,7 +489,9 @@ def loop_through_forcing_products( ) # Re-calculate the neighbor files. input_forcings.calc_neighbor_files( - self._bmi._job_meta, output_obj.outDate, self._bmi._mpi_meta + self._bmi._job_meta, + self._bmi._output_obj.outDate, + self._bmi._mpi_meta, ) err_handler.check_program_status( self._bmi._job_meta, self._bmi._mpi_meta @@ -545,7 +539,10 @@ def loop_through_forcing_products( # Layer in forcings from this product. layeringMod.layer_final_forcings( - output_obj, input_forcings, self._bmi._job_meta, self._bmi._mpi_meta + self._bmi._output_obj, + input_forcings, + self._bmi._job_meta, + self._bmi._mpi_meta, ) err_handler.check_program_status( self._bmi._job_meta, self._bmi._mpi_meta @@ -564,7 +561,9 @@ def loop_through_forcing_products( if supp_pcp_key != 13: # Like with input forcings, calculate the neighboring files to use. self._bmi._supp_pcp_mod[supp_pcp_key].calc_neighbor_files( - self._bmi._job_meta, output_obj.outDate, self._bmi._mpi_meta + self._bmi._job_meta, + self._bmi._output_obj.outDate, + self._bmi._mpi_meta, ) err_handler.check_program_status( self._bmi._job_meta, self._bmi._mpi_meta @@ -618,7 +617,7 @@ def loop_through_forcing_products( # Layer in the supplemental precipitation into the current output object. layeringMod.layer_supplemental_forcing( - output_obj, + self._bmi._output_obj, self._bmi._supp_pcp_mod[supp_pcp_key], self._bmi._job_meta, self._bmi._mpi_meta, @@ -630,21 +629,17 @@ def loop_through_forcing_products( # Call the output routines # adjust date for AnA if necessary if self._bmi._job_meta.ana_flag: - output_obj.outDate = file_date + self._bmi._output_obj.outDate = file_date ################ Commenting this out to bypass NWM forcing file output functionality ######### - # output_obj.output_final_ldasin(self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta) + # self._bmi._output_obj.output_final_ldasin(self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta) # err_handler.check_program_status(self._bmi._job_meta, self._bmi._mpi_meta) ############################################################################################## return input_forcings @time_function - def process_suplemental_precip( - self, - output_obj: OutputObj, - input_forcings: dict, - ) -> None: + def process_suplemental_precip(self, input_forcings: dict) -> None: """Process supplemental precipitation for the current forecast cycle. Warnings @@ -658,7 +653,9 @@ def process_suplemental_precip( if supp_pcp_key == 14: # Like with input forcings, calculate the neighboring files to use. self._bmi._supp_pcp_mod[supp_pcp_key].calc_neighbor_files( - self._bmi._job_meta, output_obj.outDate, self._bmi._mpi_meta + self._bmi._job_meta, + self._bmi._output_obj.outDate, + self._bmi._mpi_meta, ) err_handler.check_program_status( self._bmi._job_meta, self._bmi._mpi_meta @@ -711,7 +708,7 @@ def process_suplemental_precip( # Layer in the supplemental precipitation into the current output object. layeringMod.layer_supplemental_forcing( - output_obj, + self._bmi._output_obj, self._bmi._supp_pcp_mod[supp_pcp_key], self._bmi._job_meta, self._bmi._mpi_meta, @@ -721,10 +718,7 @@ def process_suplemental_precip( ) @time_function - def write_output( - self, - output_obj: OutputObj, - ) -> None: + def write_output(self) -> None: """Write the output for the current forecast cycle. Warnings @@ -737,17 +731,14 @@ def write_output( self._bmi._job_meta.forcing_output == 1 or self._bmi._job_meta.grid_type == "hydrofabric" ): - output_obj.gather_global_outputs( + self._bmi._output_obj.gather_global_outputs( self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta ) """##################Step 6: flatten and update dict##########################################################################""" @time_function - def update_dict( - self, - output_obj: OutputObj, - ) -> None: + def update_dict(self) -> None: """Flatten the Forcings Engine output object and update the BMI dictionary. Warnings @@ -792,22 +783,22 @@ def update_dict( ] if self._bmi._job_meta.grid_type == "gridded": for count, variable in enumerate(variables): - self._bmi._values[variable + "_ELEMENT"] = output_obj.output_local[ - count, :, : - ].flatten() + self._bmi._values[variable + "_ELEMENT"] = ( + self._bmi._output_obj.output_local[count, :, :].flatten() + ) elif self._bmi._job_meta.grid_type == "unstructured": for count, variable in enumerate(variables): - self._bmi._values[variable + "_ELEMENT"] = output_obj.output_local_elem[ - count, : - ].flatten() - self._bmi._values[variable + "_NODE"] = output_obj.output_local[ - count, : - ].flatten() + self._bmi._values[variable + "_ELEMENT"] = ( + self._bmi._output_obj.output_local_elem[count, :].flatten() + ) + self._bmi._values[variable + "_NODE"] = ( + self._bmi._output_obj.output_local[count, :].flatten() + ) elif self._bmi._job_meta.grid_type == "hydrofabric": for count, variable in enumerate(variables): - self._bmi._values[variable + "_ELEMENT"] = output_obj.output_global[ - count, : - ].flatten() + self._bmi._values[variable + "_ELEMENT"] = ( + self._bmi._output_obj.output_global[count, :].flatten() + ) self._bmi._values["CAT-ID"] = self._bmi.geo_meta.element_ids_global else: raise ValueError( diff --git a/tests/test_utils.py b/tests/test_utils.py index efba3ca9..56accdd5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -660,8 +660,6 @@ def pre_regrid(self) -> None: f"In pre_regrid, expected state to be either None or 'post_ran' but got {repr(self._state)}. The test is set up incorrectly." ) - output_obj = self.bmi_model._output_obj - future_time = ( self.bmi_model._values["current_model_time"] + self.bmi_model._values["time_step_size"] @@ -674,7 +672,7 @@ def pre_regrid(self) -> None: model.log_forecast() ### NOTE setting the flag causes the regrid step to be skipped self.set_input_forcings_skip_flags() - model.loop_through_forcing_products(future_time, output_obj) + model.loop_through_forcing_products(future_time) # Update test fixture status self._state = "pre_ran" From f37216fab1ae666b37ae151acddf6c9c2adfedd0 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 16:03:13 -0400 Subject: [PATCH 056/132] Remove unused imports --- .../NextGen_Forcings_Engine/model.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 2f101c34..ad417d17 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -2,15 +2,12 @@ import datetime import logging -import os from contextlib import contextmanager -from time import perf_counter, time +from time import perf_counter from typing import TYPE_CHECKING import numpy as np import pandas as pd -from ewts import Payload as Pld -from ewts import Status as St from ewts.modules import ModuleKey from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core import ( @@ -21,14 +18,6 @@ forcingInputMod, layeringMod, ) -from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( - ConfigOptions, -) -from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.geoMod import ( - GeoMeta, -) -from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.ioMod import OutputObj -from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import MpiConfig from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.historical_forcing import ( AORCAlaskaProcessor, AORCConusProcessor, From 0ac49fe54c453d0b5d3d7346da16876e241dae5b Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 16:09:41 -0400 Subject: [PATCH 057/132] Remove hard-coded msg control --- NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index ad417d17..436a1510 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -253,7 +253,6 @@ def loop_through_forcing_products( # 4.) Downscale. # 5.) Layer, and output as necessary. ana_factor = 1 if self._bmi._job_meta.ana_flag is False else 0 - show_message = True if not self._bmi._job_meta.precip_only_flag: if self._bmi._job_meta.grid_type == "gridded": # Reset out final grids to missing values. @@ -322,7 +321,7 @@ def loop_through_forcing_products( # Print message on log file indicating the timestamp # we are currently processing for forcings - if self._bmi._mpi_meta.rank == 0 and show_message: + if self._bmi._mpi_meta.rank == 0: self._bmi._job_meta.statusMsg = ( "=========================================" ) From d4e283f5ee4a0cb385d8f5dd7cd5d99998b073d7 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 16:57:44 -0400 Subject: [PATCH 058/132] DRYify some AnA deltas --- .../NextGen_Forcings_Engine/model.py | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 436a1510..9fd5d3a0 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -148,33 +148,28 @@ def determine_forecast(self, future_time: float) -> None: # If we're in an AnA configuration, then must offset the BMI future # timestamp to account for the "lookback" period being properly iterated # over between 3-28 hour look back time period and operation configuration + # TODO confirm these codes, and should they consider all input_forcings not just [0]? if self._bmi._job_meta.input_forcings[0] in [20, 22]: + delta = pd.TimedeltaIndex( + np.array([future_time - 7200.0], dtype=float), "s" + )[0] self._bmi._job_meta.current_fcst_cycle = ( - self._bmi._job_meta.b_date_proc - + pd.TimedeltaIndex( - np.array([future_time - 7200.0], dtype=float), "s" - )[0] + self._bmi._job_meta.b_date_proc + delta ) self._bmi._job_meta.current_time = ( - self._bmi._job_meta.b_date_proc - + pd.TimedeltaIndex( - np.array([future_time - 7200.0], dtype=float), "s" - )[0] + self._bmi._job_meta.b_date_proc + delta ) self._bmi._job_meta.future_time = future_time else: # Puerto Rico / Hawaii AnA: 1-hour lookback (based on 6-hourly forecast cycles) + delta = pd.TimedeltaIndex( + np.array([future_time - 3600.0], dtype=float), "s" + )[0] self._bmi._job_meta.current_fcst_cycle = ( - self._bmi._job_meta.b_date_proc - + pd.TimedeltaIndex( - np.array([future_time - 3600.0], dtype=float), "s" - )[0] + self._bmi._job_meta.b_date_proc + delta ) self._bmi._job_meta.current_time = ( - self._bmi._job_meta.b_date_proc - + pd.TimedeltaIndex( - np.array([future_time - 3600.0], dtype=float), "s" - )[0] + self._bmi._job_meta.b_date_proc + delta ) else: # Forecast-only mode — use BMI timestamp as-is From 15764175a7ff4f3ed9421b2f5369d9180fe8c6f7 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 17:25:16 -0400 Subject: [PATCH 059/132] DRYify calls to check_program_status --- .../NextGen_Forcings_Engine/model.py | 92 ++++++------------- 1 file changed, 27 insertions(+), 65 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 9fd5d3a0..1b328fd2 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -80,6 +80,10 @@ def __init__(self, bmi_model: NWMv3_Forcing_Engine_BMI_model_Base): # def aws_obj(files): # return xr.open_mfdataset(files, engine="zarr", parallel=True, consolidated=True) + def check_program_status(self) -> None: + """Call err_handler.check_program_status""" + err_handler.check_program_status(self._bmi._job_meta, self._bmi._mpi_meta) + def run(self, future_time: float) -> None: """Execute the full forcings engine BMI pipeline for a given future timestep. @@ -204,7 +208,7 @@ def adjust_precip(self) -> None: for force_key in self._bmi._job_meta.input_forcings: self._bmi._input_forcing_mod[force_key].skip = False - err_handler.check_program_status(self._bmi._job_meta, self._bmi._mpi_meta) + self.check_program_status() @time_function def log_forecast(self) -> None: @@ -365,9 +369,7 @@ def loop_through_forcing_products( self._bmi._output_obj.outDate, self._bmi._mpi_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() else: # Flag to indicate the AWS .zarr AORC method if force_key == 12: @@ -431,17 +433,13 @@ def loop_through_forcing_products( input_forcings.regrid_inputs( self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # Run check on regridded fields for reasonable values that are not missing values. err_handler.check_forcing_bounds( self._bmi._job_meta, input_forcings, self._bmi._mpi_meta ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # If we are restarting a forecast cycle, re-calculate the neighboring files, and regrid the # next set of forcings as the previous step just regridded the previous forcing. @@ -476,17 +474,13 @@ def loop_through_forcing_products( self._bmi._output_obj.outDate, self._bmi._mpi_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # Regrid the forcings for the end of the window. input_forcings.regrid_inputs( self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() input_forcings.rstFlag = 0 @@ -494,9 +488,7 @@ def loop_through_forcing_products( input_forcings.temporal_interpolate_inputs( self._bmi._job_meta, self._bmi._mpi_meta ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # Run bias correction. bias_correction.run_bias_correction( @@ -505,9 +497,7 @@ def loop_through_forcing_products( self._bmi.geo_meta, self._bmi._mpi_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # Run downscaling on grids for this output timestep. downscale.run_downscaling( @@ -516,9 +506,7 @@ def loop_through_forcing_products( self._bmi.geo_meta, self._bmi._mpi_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # Layer in forcings from this product. layeringMod.layer_final_forcings( @@ -527,9 +515,7 @@ def loop_through_forcing_products( self._bmi._job_meta, self._bmi._mpi_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() self._bmi._job_meta.currentForceNum += 1 @@ -548,17 +534,13 @@ def loop_through_forcing_products( self._bmi._output_obj.outDate, self._bmi._mpi_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # Regrid the supplemental precipitation. self._bmi._supp_pcp_mod[supp_pcp_key].regrid_inputs( self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() if ( self._bmi._supp_pcp_mod[supp_pcp_key].regridded_precip1 @@ -573,9 +555,7 @@ def loop_through_forcing_products( self._bmi._mpi_meta, self._bmi.geo_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # TODO input_forcings has not yet been initialized, so this is a bug waiting to happen self.disaggregate_fun( @@ -584,9 +564,7 @@ def loop_through_forcing_products( self._bmi._job_meta, self._bmi._mpi_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # Run temporal interpolation on the grids. self._bmi._supp_pcp_mod[ @@ -594,9 +572,7 @@ def loop_through_forcing_products( ].temporal_interpolate_inputs( self._bmi._job_meta, self._bmi._mpi_meta ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # Layer in the supplemental precipitation into the current output object. layeringMod.layer_supplemental_forcing( @@ -605,9 +581,7 @@ def loop_through_forcing_products( self._bmi._job_meta, self._bmi._mpi_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # Call the output routines # adjust date for AnA if necessary @@ -616,7 +590,7 @@ def loop_through_forcing_products( ################ Commenting this out to bypass NWM forcing file output functionality ######### # self._bmi._output_obj.output_final_ldasin(self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta) - # err_handler.check_program_status(self._bmi._job_meta, self._bmi._mpi_meta) + # self.check_program_status() ############################################################################################## return input_forcings @@ -640,17 +614,13 @@ def process_suplemental_precip(self, input_forcings: dict) -> None: self._bmi._output_obj.outDate, self._bmi._mpi_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # Regrid the supplemental precipitation. self._bmi._supp_pcp_mod[supp_pcp_key].regrid_inputs( self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() if ( self._bmi._supp_pcp_mod[supp_pcp_key].regridded_precip1 @@ -665,9 +635,7 @@ def process_suplemental_precip(self, input_forcings: dict) -> None: self._bmi._mpi_meta, self._bmi.geo_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() self.disaggregate_fun( input_forcings, @@ -675,9 +643,7 @@ def process_suplemental_precip(self, input_forcings: dict) -> None: self._bmi._job_meta, self._bmi._mpi_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # Run temporal interpolation on the grids. self._bmi._supp_pcp_mod[ @@ -685,9 +651,7 @@ def process_suplemental_precip(self, input_forcings: dict) -> None: ].temporal_interpolate_inputs( self._bmi._job_meta, self._bmi._mpi_meta ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() # Layer in the supplemental precipitation into the current output object. layeringMod.layer_supplemental_forcing( @@ -696,9 +660,7 @@ def process_suplemental_precip(self, input_forcings: dict) -> None: self._bmi._job_meta, self._bmi._mpi_meta, ) - err_handler.check_program_status( - self._bmi._job_meta, self._bmi._mpi_meta - ) + self.check_program_status() @time_function def write_output(self) -> None: From b1ddfcd0ddcd8c805d3f294cf7c3ae59f8650306 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 27 Apr 2026 17:32:54 -0400 Subject: [PATCH 060/132] Clean up comments and docstrings --- .../NextGen_Forcings_Engine/model.py | 63 +++++++++---------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 1b328fd2..28c68c61 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -1,3 +1,5 @@ +"""NWMv3ForcingEngineModel, to be constructed and managed by inheritors of NWMv3_Forcing_Engine_BMI_model_Base from bmi_model.py""" + from __future__ import annotations import datetime @@ -28,7 +30,6 @@ ) if TYPE_CHECKING: - # To allow type hint without circular import error from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.bmi_model import ( NWMv3_Forcing_Engine_BMI_model_Base, ) @@ -64,22 +65,15 @@ def wrapper(*args, **kwargs): class NWMv3ForcingEngineModel: - """NextGen Forcings Engine BMI model class for NWMv3 forcings.""" + """NextGen Forcings Engine BMI model class for NWMv3 forcings. + To be constructed and managed by inheritors of NWMv3_Forcing_Engine_BMI_model_Base from bmi_model.py. + """ def __init__(self, bmi_model: NWMv3_Forcing_Engine_BMI_model_Base): """Initialize the NWMv3 Forcing Engine Model.""" self.source_data_processor = None self._bmi = bmi_model - # TODO: refactor the bmi_model.py file and this to have this type maintain its own state. - # def __init__(self): - # super(ngen_model, self).__init__() - # #self._model = model - - # @dask.delayed - # def aws_obj(files): - # return xr.open_mfdataset(files, engine="zarr", parallel=True, consolidated=True) - def check_program_status(self) -> None: """Call err_handler.check_program_status""" err_handler.check_program_status(self._bmi._job_meta, self._bmi._mpi_meta) @@ -218,7 +212,6 @@ def log_forecast(self) -> None: -------- Modifies mutable arguments in-place. """ - # Log information about this forecast cycle if self._bmi._mpi_meta.rank == 0: self._bmi._job_meta.statusMsg = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" err_handler.log_msg(self._bmi._job_meta, self._bmi._mpi_meta, True) @@ -241,16 +234,17 @@ def loop_through_forcing_products( ) -> forcingInputMod.InputForcingsHydrofabric: """Loop through each forcing product and process it for the current forecast cycle. + Loop through each output timestep. Perform the following functions: + 1.) Calculate all necessary input files per user options. + 2.) Read in input forcings from GRIB/NetCDF files. + 3.) Regrid the forcings, and temporally interpolate. + 4.) Downscale. + 5.) Layer, and output as necessary. + Warnings -------- Modifies mutable arguments in-place. """ - # Loop through each output timestep. Perform the following functions: - # 1.) Calculate all necessary input files per user options. - # 2.) Read in input forcings from GRIB/NetCDF files. - # 3.) Regrid the forcings, and temporally interpolate. - # 4.) Downscale. - # 5.) Layer, and output as necessary. ana_factor = 1 if self._bmi._job_meta.ana_flag is False else 0 if not self._bmi._job_meta.precip_only_flag: if self._bmi._job_meta.grid_type == "gridded": @@ -371,6 +365,7 @@ def loop_through_forcing_products( ) self.check_program_status() else: + # TODO assert one force_key? # Flag to indicate the AWS .zarr AORC method if force_key == 12: if self.source_data_processor is None: @@ -519,6 +514,7 @@ def loop_through_forcing_products( self._bmi._job_meta.currentForceNum += 1 + # TODO what is this? if force_key == 10: self._bmi._job_meta.currentCustomForceNum += 1 @@ -665,13 +661,13 @@ def process_suplemental_precip(self, input_forcings: dict) -> None: @time_function def write_output(self) -> None: """Write the output for the current forecast cycle. + If user requests output for given domain, then call + the I/O module to update opened netcdf file with forcing fields. Warnings -------- Modifies mutable arguments in-place. """ - # If user requests output for given domain, then call - # the I/O module to update opened netcdf file with forcing fields if ( self._bmi._job_meta.forcing_output == 1 or self._bmi._job_meta.grid_type == "hydrofabric" @@ -680,28 +676,27 @@ def write_output(self) -> None: self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta ) - """##################Step 6: flatten and update dict##########################################################################""" - @time_function def update_dict(self) -> None: """Flatten the Forcings Engine output object and update the BMI dictionary. + Loop through Forcings Engine output object + and flatten the 2D forcing array and append to + the BMI object to advertise to BMIinterface. + 0.) U-Wind (m/s) + 1.) V-Wind (m/s) + 2.) Surface incoming longwave radiation flux (W/m^2) + 3.) Precipitation rate (mm/s) + 4.) 2-meter temperature (K) + 5.) 2-meter specific humidity (kg/kg) + 6.) Surface pressure (Pa) + 7.) Surface incoming shortwave radiation flux (W/m^2) + 8.) Liquid Precipitation Fraction (%), Only available in certain operational configurations + Warnings -------- Modifies mutable arguments in-place. """ - # Now loop through Forcings Engine output object - # and flatten the 2D forcing array and append to - # the BMI object to advertise to BMIinterface - # 0.) U-Wind (m/s) - # 1.) V-Wind (m/s) - # 2.) Surface incoming longwave radiation flux (W/m^2) - # 3.) Precipitation rate (mm/s) - # 4.) 2-meter temperature (K) - # 5.) 2-meter specific humidity (kg/kg) - # 6.) Surface pressure (Pa) - # 7.) Surface incoming shortwave radiation flux (W/m^2) - # 8.) Liquid Precipitation Fraction (%), Only available in certain operational configurations if self._bmi._job_meta.include_lqfrac == 1: variables = [ From 2e372103ce4f23ea33eccd0784b66adfe666042b Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 29 Apr 2026 07:30:16 -0400 Subject: [PATCH 061/132] Add forcing key count assertion --- NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 28c68c61..4438e287 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -365,7 +365,10 @@ def loop_through_forcing_products( ) self.check_program_status() else: - # TODO assert one force_key? + if len(self._bmi._job_meta.input_forcings) != 1: + raise ValueError( + f"Expected to have 1 forcing key, but have {len(self._bmi._job_meta.input_forcings)}: {list(self._bmi._job_meta.input_forcings)}" + ) # Flag to indicate the AWS .zarr AORC method if force_key == 12: if self.source_data_processor is None: From 034a3df2e654f7956fdd7687f963a25c5d9c1f1f Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 29 Apr 2026 07:32:27 -0400 Subject: [PATCH 062/132] Update docstrings --- .../NextGen_Forcings_Engine/model.py | 26 ++----------------- 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 4438e287..8a788e8c 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -191,12 +191,7 @@ def determine_forecast(self, future_time: float) -> None: @time_function def adjust_precip(self) -> None: - """Adjust precipitation for the given forecast cycle. - - Warnings - -------- - Modifies mutable arguments in-place. - """ + """Adjust precipitation for the given forecast cycle.""" if not self._bmi._job_meta.precip_only_flag: # reset skips if present for force_key in self._bmi._job_meta.input_forcings: @@ -206,12 +201,7 @@ def adjust_precip(self) -> None: @time_function def log_forecast(self) -> None: - """Log information about the current forecast cycle. - - Warnings - -------- - Modifies mutable arguments in-place. - """ + """Log information about the current forecast cycle.""" if self._bmi._mpi_meta.rank == 0: self._bmi._job_meta.statusMsg = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" err_handler.log_msg(self._bmi._job_meta, self._bmi._mpi_meta, True) @@ -240,10 +230,6 @@ def loop_through_forcing_products( 3.) Regrid the forcings, and temporally interpolate. 4.) Downscale. 5.) Layer, and output as necessary. - - Warnings - -------- - Modifies mutable arguments in-place. """ ana_factor = 1 if self._bmi._job_meta.ana_flag is False else 0 if not self._bmi._job_meta.precip_only_flag: @@ -666,10 +652,6 @@ def write_output(self) -> None: """Write the output for the current forecast cycle. If user requests output for given domain, then call the I/O module to update opened netcdf file with forcing fields. - - Warnings - -------- - Modifies mutable arguments in-place. """ if ( self._bmi._job_meta.forcing_output == 1 @@ -695,10 +677,6 @@ def update_dict(self) -> None: 6.) Surface pressure (Pa) 7.) Surface incoming shortwave radiation flux (W/m^2) 8.) Liquid Precipitation Fraction (%), Only available in certain operational configurations - - Warnings - -------- - Modifies mutable arguments in-place. """ if self._bmi._job_meta.include_lqfrac == 1: From 84a21c648442a4eb4208398c8d24a38f372f3ac8 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 29 Apr 2026 07:48:47 -0400 Subject: [PATCH 063/132] Move block using `rstFlag` to new private method --- .../NextGen_Forcings_Engine/model.py | 89 ++++++++++--------- 1 file changed, 49 insertions(+), 40 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 8a788e8c..9f90d7de 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -427,46 +427,7 @@ def loop_through_forcing_products( # If we are restarting a forecast cycle, re-calculate the neighboring files, and regrid the # next set of forcings as the previous step just regridded the previous forcing. - if input_forcings.rstFlag == 1: - if ( - input_forcings.regridded_forcings1 is not None - and input_forcings.regridded_forcings2 is not None - ): - # Set the forcings back to reflect we just regridded the previous set of inputs, not the next. - if self._bmi._job_meta.grid_type == "gridded": - input_forcings.regridded_forcings1[:, :, :] = ( - input_forcings.regridded_forcings2[:, :, :] - ) - elif self._bmi._job_meta.grid_type == "unstructured": - input_forcings.regridded_forcings1[:, :] = ( - input_forcings.regridded_forcings2[:, :] - ) - input_forcings.regridded_forcings1_elem[:, :] = ( - input_forcings.regridded_forcings2_elem[:, :] - ) - elif self._bmi._job_meta.grid_type == "hydrofabric": - input_forcings.regridded_forcings1[:, :] = ( - input_forcings.regridded_forcings2[:, :] - ) - else: - raise ValueError( - f"Unexpected grid_type: {repr(self._bmi._job_meta.grid_type)}" - ) - # Re-calculate the neighbor files. - input_forcings.calc_neighbor_files( - self._bmi._job_meta, - self._bmi._output_obj.outDate, - self._bmi._mpi_meta, - ) - self.check_program_status() - - # Regrid the forcings for the end of the window. - input_forcings.regrid_inputs( - self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta - ) - self.check_program_status() - - input_forcings.rstFlag = 0 + self.__use_rstFlag(input_forcings) # Run temporal interpolation on the grids. input_forcings.temporal_interpolate_inputs( @@ -580,6 +541,54 @@ def loop_through_forcing_products( return input_forcings + def __use_rstFlag(self, input_forcings): + """ + If we are restarting a forecast cycle, re-calculate the neighboring files, and regrid the + next set of forcings as the previous step just regridded the previous forcing. + + This code block was cut and pasted from method `loop_through_forcing_products` during refactor. + """ + if input_forcings.rstFlag == 1: + if ( + input_forcings.regridded_forcings1 is not None + and input_forcings.regridded_forcings2 is not None + ): + # Set the forcings back to reflect we just regridded the previous set of inputs, not the next. + if self._bmi._job_meta.grid_type == "gridded": + input_forcings.regridded_forcings1[:, :, :] = ( + input_forcings.regridded_forcings2[:, :, :] + ) + elif self._bmi._job_meta.grid_type == "unstructured": + input_forcings.regridded_forcings1[:, :] = ( + input_forcings.regridded_forcings2[:, :] + ) + input_forcings.regridded_forcings1_elem[:, :] = ( + input_forcings.regridded_forcings2_elem[:, :] + ) + elif self._bmi._job_meta.grid_type == "hydrofabric": + input_forcings.regridded_forcings1[:, :] = ( + input_forcings.regridded_forcings2[:, :] + ) + else: + raise ValueError( + f"Unexpected grid_type: {repr(self._bmi._job_meta.grid_type)}" + ) + # Re-calculate the neighbor files. + input_forcings.calc_neighbor_files( + self._bmi._job_meta, + self._bmi._output_obj.outDate, + self._bmi._mpi_meta, + ) + self.check_program_status() + + # Regrid the forcings for the end of the window. + input_forcings.regrid_inputs( + self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta + ) + self.check_program_status() + + input_forcings.rstFlag = 0 + @time_function def process_suplemental_precip(self, input_forcings: dict) -> None: """Process supplemental precipitation for the current forecast cycle. From 5ab9812c7a45ecc0f94978c500dfc0396d17b40f Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 29 Apr 2026 08:25:03 -0400 Subject: [PATCH 064/132] Move block for supplemental precip handling to new private method --- .../NextGen_Forcings_Engine/model.py | 166 +++++++----------- 1 file changed, 59 insertions(+), 107 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 9f90d7de..9bfa401f 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -474,60 +474,9 @@ def loop_through_forcing_products( if self._bmi._job_meta.number_supp_pcp > 0: for supp_pcp_key in self._bmi._job_meta.supp_precip_forcings: if supp_pcp_key != 13: - # Like with input forcings, calculate the neighboring files to use. - self._bmi._supp_pcp_mod[supp_pcp_key].calc_neighbor_files( - self._bmi._job_meta, - self._bmi._output_obj.outDate, - self._bmi._mpi_meta, - ) - self.check_program_status() - - # Regrid the supplemental precipitation. - self._bmi._supp_pcp_mod[supp_pcp_key].regrid_inputs( - self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta - ) - self.check_program_status() - - if ( - self._bmi._supp_pcp_mod[supp_pcp_key].regridded_precip1 - is not None - and self._bmi._supp_pcp_mod[supp_pcp_key].regridded_precip2 - is not None - ): - # Run check on regridded fields for reasonable values that are not missing values. - err_handler.check_supp_pcp_bounds( - self._bmi._job_meta, - self._bmi._supp_pcp_mod[supp_pcp_key], - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) - self.check_program_status() - - # TODO input_forcings has not yet been initialized, so this is a bug waiting to happen - self.disaggregate_fun( - input_forcings, - self._bmi._supp_pcp_mod[supp_pcp_key], - self._bmi._job_meta, - self._bmi._mpi_meta, - ) - self.check_program_status() - - # Run temporal interpolation on the grids. - self._bmi._supp_pcp_mod[ - supp_pcp_key - ].temporal_interpolate_inputs( - self._bmi._job_meta, self._bmi._mpi_meta - ) - self.check_program_status() - - # Layer in the supplemental precipitation into the current output object. - layeringMod.layer_supplemental_forcing( - self._bmi._output_obj, - self._bmi._supp_pcp_mod[supp_pcp_key], - self._bmi._job_meta, - self._bmi._mpi_meta, - ) - self.check_program_status() + # Below comment copied from earlier code, the comment had been just above the call to `disaggregate_fun`. + # TODO input_forcings has not yet been initialized, so this is a bug waiting to happen + self.__process_supp_precip_key(input_forcings, supp_pcp_key) # Call the output routines # adjust date for AnA if necessary @@ -541,6 +490,61 @@ def loop_through_forcing_products( return input_forcings + def __process_supp_precip_key(self, input_forcings: dict, supp_pcp_key: int): + """Process supplemental precipitation for one supplemental precipitation key. + + This code block was cut and pasted from methods `loop_through_forcing_products` and `process_suplemental_precip` during refactor. + """ + # Like with input forcings, calculate the neighboring files to use. + self._bmi._supp_pcp_mod[supp_pcp_key].calc_neighbor_files( + self._bmi._job_meta, + self._bmi._output_obj.outDate, + self._bmi._mpi_meta, + ) + self.check_program_status() + + # Regrid the supplemental precipitation. + self._bmi._supp_pcp_mod[supp_pcp_key].regrid_inputs( + self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta + ) + self.check_program_status() + + if ( + self._bmi._supp_pcp_mod[supp_pcp_key].regridded_precip1 is not None + and self._bmi._supp_pcp_mod[supp_pcp_key].regridded_precip2 is not None + ): + # Run check on regridded fields for reasonable values that are not missing values. + err_handler.check_supp_pcp_bounds( + self._bmi._job_meta, + self._bmi._supp_pcp_mod[supp_pcp_key], + self._bmi._mpi_meta, + self._bmi.geo_meta, + ) + self.check_program_status() + + self.disaggregate_fun( + input_forcings, + self._bmi._supp_pcp_mod[supp_pcp_key], + self._bmi._job_meta, + self._bmi._mpi_meta, + ) + self.check_program_status() + + # Run temporal interpolation on the grids. + self._bmi._supp_pcp_mod[supp_pcp_key].temporal_interpolate_inputs( + self._bmi._job_meta, self._bmi._mpi_meta + ) + self.check_program_status() + + # Layer in the supplemental precipitation into the current output object. + layeringMod.layer_supplemental_forcing( + self._bmi._output_obj, + self._bmi._supp_pcp_mod[supp_pcp_key], + self._bmi._job_meta, + self._bmi._mpi_meta, + ) + self.check_program_status() + def __use_rstFlag(self, input_forcings): """ If we are restarting a forecast cycle, re-calculate the neighboring files, and regrid the @@ -602,59 +606,7 @@ def process_suplemental_precip(self, input_forcings: dict) -> None: if self._bmi._job_meta.number_supp_pcp > 0: for supp_pcp_key in self._bmi._job_meta.supp_precip_forcings: if supp_pcp_key == 14: - # Like with input forcings, calculate the neighboring files to use. - self._bmi._supp_pcp_mod[supp_pcp_key].calc_neighbor_files( - self._bmi._job_meta, - self._bmi._output_obj.outDate, - self._bmi._mpi_meta, - ) - self.check_program_status() - - # Regrid the supplemental precipitation. - self._bmi._supp_pcp_mod[supp_pcp_key].regrid_inputs( - self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta - ) - self.check_program_status() - - if ( - self._bmi._supp_pcp_mod[supp_pcp_key].regridded_precip1 - is not None - and self._bmi._supp_pcp_mod[supp_pcp_key].regridded_precip2 - is not None - ): - # Run check on regridded fields for reasonable values that are not missing values. - err_handler.check_supp_pcp_bounds( - self._bmi._job_meta, - self._bmi._supp_pcp_mod[supp_pcp_key], - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) - self.check_program_status() - - self.disaggregate_fun( - input_forcings, - self._bmi._supp_pcp_mod[supp_pcp_key], - self._bmi._job_meta, - self._bmi._mpi_meta, - ) - self.check_program_status() - - # Run temporal interpolation on the grids. - self._bmi._supp_pcp_mod[ - supp_pcp_key - ].temporal_interpolate_inputs( - self._bmi._job_meta, self._bmi._mpi_meta - ) - self.check_program_status() - - # Layer in the supplemental precipitation into the current output object. - layeringMod.layer_supplemental_forcing( - self._bmi._output_obj, - self._bmi._supp_pcp_mod[supp_pcp_key], - self._bmi._job_meta, - self._bmi._mpi_meta, - ) - self.check_program_status() + self.__process_supp_precip_key(input_forcings, supp_pcp_key) @time_function def write_output(self) -> None: From 7f8eead3111c18e1b62b2ca1eff228e3a20d862a Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 29 Apr 2026 08:28:11 -0400 Subject: [PATCH 065/132] docstrings and type hints --- .../NextGen_Forcings_Engine/model.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 9bfa401f..3cb1d814 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -490,10 +490,16 @@ def loop_through_forcing_products( return input_forcings - def __process_supp_precip_key(self, input_forcings: dict, supp_pcp_key: int): + def __process_supp_precip_key( + self, input_forcings: dict, supp_pcp_key: int + ) -> None: """Process supplemental precipitation for one supplemental precipitation key. This code block was cut and pasted from methods `loop_through_forcing_products` and `process_suplemental_precip` during refactor. + + Warnings + -------- + Modifies mutable arguments in-place. """ # Like with input forcings, calculate the neighboring files to use. self._bmi._supp_pcp_mod[supp_pcp_key].calc_neighbor_files( @@ -545,12 +551,16 @@ def __process_supp_precip_key(self, input_forcings: dict, supp_pcp_key: int): ) self.check_program_status() - def __use_rstFlag(self, input_forcings): + def __use_rstFlag(self, input_forcings: dict) -> None: """ If we are restarting a forecast cycle, re-calculate the neighboring files, and regrid the next set of forcings as the previous step just regridded the previous forcing. This code block was cut and pasted from method `loop_through_forcing_products` during refactor. + + Warnings + -------- + Modifies mutable arguments in-place. """ if input_forcings.rstFlag == 1: if ( From 5fcef77e8d4fb44fe440fa46e7f76415447e463f Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 29 Apr 2026 08:42:10 -0400 Subject: [PATCH 066/132] Move block for AORC and NWM handling to new private method --- .../NextGen_Forcings_Engine/core/parallel.py | 7 +- .../NextGen_Forcings_Engine/model.py | 148 ++++++++++-------- 2 files changed, 85 insertions(+), 70 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index dee8126c..dd87f7e3 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -5,11 +5,13 @@ import signal import sys from functools import partial -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypeVar import mpi4py import numpy as np +from . import err_handler, mpi_utils + mpi4py.rc.threads = False from mpi4py import MPI # noqa: E402 @@ -18,7 +20,6 @@ from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( ConfigOptions, ) -from . import err_handler, mpi_utils # If MPI was initialized outside of python, # disable initialization/finalization behavior @@ -26,7 +27,7 @@ mpi4py.rc.initialize = False mpi4py.rc.finalize = False -if typing.TYPE_CHECKING: +if TYPE_CHECKING: from .config import ConfigOptions from .geoMod import GriddedGeoMeta diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 3cb1d814..829f7968 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -341,73 +341,8 @@ def loop_through_forcing_products( self._bmi._mpi_meta, ) - if force_key in [12, 21, 27]: - if self._bmi._job_meta.aws is None: - # Calculate the previous and next input cycle files from the inputs. - input_forcings.calc_neighbor_files( - self._bmi._job_meta, - self._bmi._output_obj.outDate, - self._bmi._mpi_meta, - ) - self.check_program_status() - else: - if len(self._bmi._job_meta.input_forcings) != 1: - raise ValueError( - f"Expected to have 1 forcing key, but have {len(self._bmi._job_meta.input_forcings)}: {list(self._bmi._job_meta.input_forcings)}" - ) - # Flag to indicate the AWS .zarr AORC method - if force_key == 12: - if self.source_data_processor is None: - self.source_data_processor = AORCConusProcessor( - self._bmi._job_meta, - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) - elif force_key == 21: - if self.source_data_processor is None: - self.source_data_processor = AORCAlaskaProcessor( - self._bmi._job_meta, - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) - - # Flag to indicate the AWS .zarr NWMv3 Forcing file method - elif force_key == 27: - if self.source_data_processor is None: - if self._bmi._job_meta.nwm_domain == "CONUS": - self.source_data_processor = NWMV3ConusProcessor( - self._bmi._job_meta, - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) - elif config_options.nwm_domain == "Hawaii": - self.source_data_processor = NWMV3HawaiiProcessor( - config_options, mpi_config, wrf_hydro_geo_meta - ) - elif config_options.nwm_domain == "PR": - self.source_data_processor = ( - NWMV3PuertoRicoProcessor( - config_options, - mpi_config, - wrf_hydro_geo_meta, - ) - ) - elif config_options.nwm_domain == "Alaska": - self.source_data_processor = NWMV3AlaskaProcessor( - self._bmi._job_meta, - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) - else: - raise ValueError( - f"Unsupported domain type ({self._bmi._job_meta.nwm_domain} for forcing type: {force_key} )" - ) - - self._bmi._job_meta.aws_obj = ( - self.source_data_processor.process_historical_data( - self._bmi._job_meta.current_time - ) - ) + # Handle AORC and NWM force keys + self.__handle_aorc_and_nwm_force_keys(input_forcings, force_key) # If skipping this forcing, continue early if input_forcings.skip is True: @@ -490,6 +425,85 @@ def loop_through_forcing_products( return input_forcings + def __handle_aorc_and_nwm_force_keys(self, input_forcings, force_key: int) -> None: + """During `loop_through_forcing_products`, handle the case of the force key being AORC or NWM. + + This code block was cut and pasted from methods `loop_through_forcing_products` during refactor. + + Warnings + -------- + Modifies mutable arguments in-place. + """ + if force_key in [12, 21, 27]: + if self._bmi._job_meta.aws is None: + # Calculate the previous and next input cycle files from the inputs. + input_forcings.calc_neighbor_files( + self._bmi._job_meta, + self._bmi._output_obj.outDate, + self._bmi._mpi_meta, + ) + self.check_program_status() + else: + if len(self._bmi._job_meta.input_forcings) != 1: + raise ValueError( + f"Expected to have 1 forcing key, but have {len(self._bmi._job_meta.input_forcings)}: {list(self._bmi._job_meta.input_forcings)}" + ) + # Flag to indicate the AWS .zarr AORC method + if force_key == 12: + if self.source_data_processor is None: + self.source_data_processor = AORCConusProcessor( + self._bmi._job_meta, + self._bmi._mpi_meta, + self._bmi.geo_meta, + ) + elif force_key == 21: + if self.source_data_processor is None: + self.source_data_processor = AORCAlaskaProcessor( + self._bmi._job_meta, + self._bmi._mpi_meta, + self._bmi.geo_meta, + ) + + # Flag to indicate the AWS .zarr NWMv3 Forcing file method + elif force_key == 27: + if self.source_data_processor is None: + if self._bmi._job_meta.nwm_domain == "CONUS": + self.source_data_processor = NWMV3ConusProcessor( + self._bmi._job_meta, + self._bmi._mpi_meta, + self._bmi.geo_meta, + ) + elif self._bmi._job_meta.nwm_domain == "Hawaii": + self.source_data_processor = NWMV3HawaiiProcessor( + self._bmi._job_meta, + self._bmi._mpi_meta, + self._bmi.geo_meta, + ) + elif self._bmi._job_meta.nwm_domain == "PR": + self.source_data_processor = ( + NWMV3PuertoRicoProcessor( + self._bmi._job_meta, + self._bmi._mpi_meta, + self._bmi.geo_meta, + ) + ) + elif self._bmi._job_meta.nwm_domain == "Alaska": + self.source_data_processor = NWMV3AlaskaProcessor( + self._bmi._job_meta, + self._bmi._mpi_meta, + self._bmi.geo_meta, + ) + else: + raise ValueError( + f"Unsupported domain type ({self._bmi._job_meta.nwm_domain} for forcing type: {force_key} )" + ) + + self._bmi._job_meta.aws_obj = ( + self.source_data_processor.process_historical_data( + self._bmi._job_meta.current_time + ) + ) + def __process_supp_precip_key( self, input_forcings: dict, supp_pcp_key: int ) -> None: From 2cb35bb5af17d7a248ff2ec83dff1f4adfb01f6f Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 29 Apr 2026 08:46:40 -0400 Subject: [PATCH 067/132] Comments and type hints --- .../NextGen_Forcings_Engine/model.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 829f7968..52e70650 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -345,9 +345,11 @@ def loop_through_forcing_products( self.__handle_aorc_and_nwm_force_keys(input_forcings, force_key) # If skipping this forcing, continue early + # NOTE this is used by the esmf regrid pytests, to halt the loop before "manually" calling a particular regrid function. if input_forcings.skip is True: LOG.debug(f"Breaking loop for force_key {force_key}") break + # Regrid forcings. input_forcings.regrid_inputs( self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta @@ -505,7 +507,7 @@ def __handle_aorc_and_nwm_force_keys(self, input_forcings, force_key: int) -> No ) def __process_supp_precip_key( - self, input_forcings: dict, supp_pcp_key: int + self, input_forcings: forcingInputMod.InputForcings, supp_pcp_key: int ) -> None: """Process supplemental precipitation for one supplemental precipitation key. @@ -565,7 +567,7 @@ def __process_supp_precip_key( ) self.check_program_status() - def __use_rstFlag(self, input_forcings: dict) -> None: + def __use_rstFlag(self, input_forcings: forcingInputMod.InputForcings) -> None: """ If we are restarting a forecast cycle, re-calculate the neighboring files, and regrid the next set of forcings as the previous step just regridded the previous forcing. @@ -618,7 +620,9 @@ def __use_rstFlag(self, input_forcings: dict) -> None: input_forcings.rstFlag = 0 @time_function - def process_suplemental_precip(self, input_forcings: dict) -> None: + def process_suplemental_precip( + self, input_forcings: forcingInputMod.InputForcings + ) -> None: """Process supplemental precipitation for the current forecast cycle. Warnings From 04a4db54d7b39ccfc02115cd1637a1d21e670f4e Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 29 Apr 2026 08:59:01 -0400 Subject: [PATCH 068/132] Move model.py BMI variables to consts.py --- .../NextGen_Forcings_Engine/core/consts.py | 17 ++++++++++- .../NextGen_Forcings_Engine/model.py | 28 ++++--------------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py index 4420e8cb..104a012d 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py @@ -1245,7 +1245,22 @@ 0: timeInterpMod.no_interpolation_supp_pcp, 1: timeInterpMod.nearest_neighbor_supp_pcp, 2: timeInterpMod.weighted_average_supp_pcp, - }, + } +} + +MODEL = { + # Used by method `model.NWMv3ForcingEngineModel.update_dict` + "update_dict_base_vars": [ + "U2D", + "V2D", + "LWDOWN", + "RAINRATE", + "T2D", + "Q2D", + "PSFC", + "SWDOWN", + ], + "update_dict_var_include_lqfraq": "LQFRAC", } TEST_UTILS = { diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 52e70650..4df95802 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -20,6 +20,9 @@ forcingInputMod, layeringMod, ) +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.consts import ( + MODEL as model_consts, +) from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.historical_forcing import ( AORCAlaskaProcessor, AORCConusProcessor, @@ -668,29 +671,10 @@ def update_dict(self) -> None: 8.) Liquid Precipitation Fraction (%), Only available in certain operational configurations """ + variables = model_consts["update_dict_base_vars"] if self._bmi._job_meta.include_lqfrac == 1: - variables = [ - "U2D", - "V2D", - "LWDOWN", - "RAINRATE", - "T2D", - "Q2D", - "PSFC", - "SWDOWN", - "LQFRAC", - ] - else: - variables = [ - "U2D", - "V2D", - "LWDOWN", - "RAINRATE", - "T2D", - "Q2D", - "PSFC", - "SWDOWN", - ] + variables.append(model_consts["update_dict_var_include_lqfraq"]) + if self._bmi._job_meta.grid_type == "gridded": for count, variable in enumerate(variables): self._bmi._values[variable + "_ELEMENT"] = ( From 361087ad3563a6376ebeb8811dd030196b475d18 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 29 Apr 2026 09:17:45 -0400 Subject: [PATCH 069/132] Fix usage of new consts.MODEL list --- NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 4df95802..5fb826a8 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import datetime import logging from contextlib import contextmanager @@ -671,7 +672,7 @@ def update_dict(self) -> None: 8.) Liquid Precipitation Fraction (%), Only available in certain operational configurations """ - variables = model_consts["update_dict_base_vars"] + variables = copy.deepcopy(model_consts["update_dict_base_vars"]) if self._bmi._job_meta.include_lqfrac == 1: variables.append(model_consts["update_dict_var_include_lqfraq"]) From e2ce719b08e499c24c716bb278ec8274d977da3a Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 29 Apr 2026 13:24:28 -0400 Subject: [PATCH 070/132] Comments --- NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 5fb826a8..c2a8046a 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -152,6 +152,7 @@ def determine_forecast(self, future_time: float) -> None: # over between 3-28 hour look back time period and operation configuration # TODO confirm these codes, and should they consider all input_forcings not just [0]? if self._bmi._job_meta.input_forcings[0] in [20, 22]: + # NOTE This appears to be intending to operate on Alaska-only AnA. delta = pd.TimedeltaIndex( np.array([future_time - 7200.0], dtype=float), "s" )[0] @@ -163,6 +164,7 @@ def determine_forecast(self, future_time: float) -> None: ) self._bmi._job_meta.future_time = future_time else: + # NOTE below comment was original, but this appears to be operating on all non-Alaska AnA, not just Puerto Rico / Hawaii AnA. # Puerto Rico / Hawaii AnA: 1-hour lookback (based on 6-hourly forecast cycles) delta = pd.TimedeltaIndex( np.array([future_time - 3600.0], dtype=float), "s" From 06f2d87dd02418519810d1e21006486696dd78a9 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Thu, 30 Apr 2026 16:19:38 -0400 Subject: [PATCH 071/132] Add NotImplementedError for SubOutputHour --- NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index c2a8046a..00895b6d 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -266,6 +266,9 @@ def loop_through_forcing_products( # Optional sub-output timestamp if self._bmi._job_meta.sub_output_hour is not None: + raise NotImplementedError( + f"sub_output_hour (config SubOutputHour) is {repr(self._bmi._job_meta.sub_output_hour)} (not None) but is not used." + ) # TODO This is not used subOutDate = self._bmi._job_meta.first_fcst_cycle + datetime.timedelta( hours=self._bmi._job_meta.sub_output_hour From a3cfee139b40e6dadb859e70a0bf6d6005909b5d Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 1 May 2026 14:04:18 -0400 Subject: [PATCH 072/132] Type hints and docstrings --- .../NextGen_Forcings_Engine/model.py | 75 +++++++++++++++---- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 00895b6d..00cb9417 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -45,11 +45,12 @@ @contextmanager def timing_block(step_str: str): - """Context manager for timing code execution. - - Args: - step_str: Description of the step being timed. + """Context manager for timing code execution. Used by decorator `time_function`. + Parameters + ---------- + step_str : str + Description of the step being timed. """ start = perf_counter() yield @@ -58,7 +59,7 @@ def timing_block(step_str: str): def time_function(func): - """Measure the execution time of a function.""" + """Decorator for measuring the execution time of a function.""" def wrapper(*args, **kwargs): with timing_block(f"Executing {func.__name__}"): @@ -70,11 +71,17 @@ def wrapper(*args, **kwargs): class NWMv3ForcingEngineModel: """NextGen Forcings Engine BMI model class for NWMv3 forcings. + To be constructed and managed by inheritors of NWMv3_Forcing_Engine_BMI_model_Base from bmi_model.py. """ def __init__(self, bmi_model: NWMv3_Forcing_Engine_BMI_model_Base): - """Initialize the NWMv3 Forcing Engine Model.""" + """Initialize the NWMv3 Forcing Engine Model. + + Parameters + ---------- + bmi_model : NWMv3_Forcing_Engine_BMI_model_Base + """ self.source_data_processor = None self._bmi = bmi_model @@ -112,9 +119,17 @@ def run(self, future_time: float) -> None: 6. Update the self._bmi._values state dictionary with flattened arrays. 7. Advance the BMI time index. - :param future_time: The number of seconds into the future to advance the model. - - :raises RuntimeError: If the model fails to initialize or if required arguments are missing. + Parameters + ---------- + future_time : float + Timestamp, represented as *seconds relative to overall start time*, to advance to before returning. + Since this is relative to overall start time, it is unaware of the actual UTC datetimestamp of the start. + For example, since 1-hour timesteps are typical, the first value of this would typically be 3600, the second value 7200, etc. + + Raises + ------ + RuntimeError + If the model fails to initialize or if required arguments are missing. """ self.determine_forecast(future_time) @@ -137,7 +152,7 @@ def determine_forecast(self, future_time: float) -> None: Warnings -------- - Modifies mutable arguments in-place. + Modifies mutable arguments in-place. """ # Assign the future time to the configuration self._bmi._job_meta.bmi_time = future_time @@ -236,6 +251,15 @@ def loop_through_forcing_products( 3.) Regrid the forcings, and temporally interpolate. 4.) Downscale. 5.) Layer, and output as necessary. + + Parameters + ---------- + future_time : float + See description in `self.run`. + + Returns + ---------- + input_forcings: forcingInputMod.InputForcings """ ana_factor = 1 if self._bmi._job_meta.ana_flag is False else 0 if not self._bmi._job_meta.precip_only_flag: @@ -436,14 +460,21 @@ def loop_through_forcing_products( return input_forcings - def __handle_aorc_and_nwm_force_keys(self, input_forcings, force_key: int) -> None: + def __handle_aorc_and_nwm_force_keys( + self, input_forcings: forcingInputMod.InputForcings, force_key: int + ) -> None: """During `loop_through_forcing_products`, handle the case of the force key being AORC or NWM. This code block was cut and pasted from methods `loop_through_forcing_products` during refactor. + Parameters + ---------- + input_forcings : forcingInputMod.InputForcings + force_key : int + Warnings -------- - Modifies mutable arguments in-place. + Modifies mutable arguments in-place. """ if force_key in [12, 21, 27]: if self._bmi._job_meta.aws is None: @@ -522,9 +553,14 @@ def __process_supp_precip_key( This code block was cut and pasted from methods `loop_through_forcing_products` and `process_suplemental_precip` during refactor. + Parameters + ---------- + input_forcings : forcingInputMod.InputForcings + supp_pcp_key : int + Warnings -------- - Modifies mutable arguments in-place. + Modifies mutable arguments in-place. """ # Like with input forcings, calculate the neighboring files to use. self._bmi._supp_pcp_mod[supp_pcp_key].calc_neighbor_files( @@ -583,9 +619,13 @@ def __use_rstFlag(self, input_forcings: forcingInputMod.InputForcings) -> None: This code block was cut and pasted from method `loop_through_forcing_products` during refactor. + Parameters + ---------- + input_forcings : forcingInputMod.InputForcings + Warnings -------- - Modifies mutable arguments in-place. + Modifies mutable arguments in-place. """ if input_forcings.rstFlag == 1: if ( @@ -634,9 +674,13 @@ def process_suplemental_precip( ) -> None: """Process supplemental precipitation for the current forecast cycle. + Parameters + ---------- + input_forcings : forcingInputMod.InputForcings + Warnings -------- - Modifies mutable arguments in-place. + Modifies mutable arguments in-place. """ if self._bmi._job_meta.customSuppPcpFreq is not None: # Process supplemental precipitation if we specified in the configuration file. @@ -648,6 +692,7 @@ def process_suplemental_precip( @time_function def write_output(self) -> None: """Write the output for the current forecast cycle. + If user requests output for given domain, then call the I/O module to update opened netcdf file with forcing fields. """ From 2d5be5e4e6e3fcfd872c4e0e15cb486b0c8acbbc Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 1 May 2026 19:42:16 -0400 Subject: [PATCH 073/132] Rename methods --- .../NextGen_Forcings_Engine/core/consts.py | 2 +- .../NextGen_Forcings_Engine/model.py | 23 ++++++++----------- tests/test_utils.py | 9 ++++---- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py index 104a012d..077048b3 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py @@ -1249,7 +1249,7 @@ } MODEL = { - # Used by method `model.NWMv3ForcingEngineModel.update_dict` + # Used by method `model.NWMv3ForcingEngineModel.update_bmi_output_dict` "update_dict_base_vars": [ "U2D", "V2D", diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 00cb9417..ba2a8df9 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -132,22 +132,19 @@ def run(self, future_time: float) -> None: If the model fails to initialize or if required arguments are missing. """ - self.determine_forecast(future_time) - self.adjust_precip() - self.log_forecast() - # TODO look into input_forcings usage in `process_suplemental_precip` and in `loop_through_forcing_products` at `disaggregate_fun`. - input_forcings = self.loop_through_forcing_products( - future_time, - ) + self.set_cycle_timing_attrs(future_time) + self.set_skip_flags() + self.log_cycle() + input_forcings = self.loop_through_forcing_products(future_time) self.process_suplemental_precip(input_forcings) self.write_output() - self.update_dict() + self.update_bmi_output_dict() ## Update BMI model time index to next iteration self._bmi._job_meta.bmi_time_index += 1 @time_function - def determine_forecast(self, future_time: float) -> None: + def set_cycle_timing_attrs(self, future_time: float) -> None: """Determine the forecast for the given future time and configuration. Warnings @@ -211,7 +208,7 @@ def determine_forecast(self, future_time: float) -> None: ) @time_function - def adjust_precip(self) -> None: + def set_skip_flags(self) -> None: """Adjust precipitation for the given forecast cycle.""" if not self._bmi._job_meta.precip_only_flag: # reset skips if present @@ -221,7 +218,7 @@ def adjust_precip(self) -> None: self.check_program_status() @time_function - def log_forecast(self) -> None: + def log_cycle(self) -> None: """Log information about the current forecast cycle.""" if self._bmi._mpi_meta.rank == 0: self._bmi._job_meta.statusMsg = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" @@ -259,7 +256,7 @@ def loop_through_forcing_products( Returns ---------- - input_forcings: forcingInputMod.InputForcings + input_forcings: forcingInputMod.InputForcings | None """ ana_factor = 1 if self._bmi._job_meta.ana_flag is False else 0 if not self._bmi._job_meta.precip_only_flag: @@ -705,7 +702,7 @@ def write_output(self) -> None: ) @time_function - def update_dict(self) -> None: + def update_bmi_output_dict(self) -> None: """Flatten the Forcings Engine output object and update the BMI dictionary. Loop through Forcings Engine output object diff --git a/tests/test_utils.py b/tests/test_utils.py index 56accdd5..2b0d4a68 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -666,10 +666,11 @@ def pre_regrid(self) -> None: ) model = self.bmi_model._model - ### NOTE this should mimic NWMv3ForcingEngineModel.run() with the exception of setting the skip flag - model.determine_forecast(future_time) - model.adjust_precip() - model.log_forecast() + # NOTE this should mimic NWMv3ForcingEngineModel.run() + # with the exception of externally setting the skip flags within this class. + model.set_cycle_timing_attrs(future_time) + model.set_skip_flags() + model.log_cycle() ### NOTE setting the flag causes the regrid step to be skipped self.set_input_forcings_skip_flags() model.loop_through_forcing_products(future_time) From ec356bf13293314a6cb73443fc319f4ad3304d59 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 1 May 2026 19:43:47 -0400 Subject: [PATCH 074/132] Fix input_forcings reference (return None conditionally) --- NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index ba2a8df9..bf1797b2 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -239,7 +239,7 @@ def log_cycle(self) -> None: @time_function def loop_through_forcing_products( self, future_time: float - ) -> forcingInputMod.InputForcingsHydrofabric: + ) -> forcingInputMod.InputForcingsHydrofabric | None: """Loop through each forcing product and process it for the current forecast cycle. Loop through each output timestep. Perform the following functions: @@ -454,6 +454,8 @@ def loop_through_forcing_products( # self._bmi._output_obj.output_final_ldasin(self._bmi._job_meta, self._bmi.geo_meta, self._bmi._mpi_meta) # self.check_program_status() ############################################################################################## + else: + input_forcings = None return input_forcings From 9d287ddb9c255212e464529f7074f0bfa1637c9a Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 1 May 2026 20:02:45 -0400 Subject: [PATCH 075/132] Use partials for log calls and use MPI-aware log methods --- .../NextGen_Forcings_Engine/model.py | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index bf1797b2..b565fb82 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -6,6 +6,7 @@ import datetime import logging from contextlib import contextmanager +from functools import partial from time import perf_counter from typing import TYPE_CHECKING @@ -55,7 +56,7 @@ def timing_block(step_str: str): start = perf_counter() yield end = perf_counter() - LOG.debug(f" Execution time for {step_str}: {round(end - start, 2)} seconds") + LOG.debug(msg=f" Execution time for {step_str}: {round(end - start, 2)} seconds") def time_function(func): @@ -84,6 +85,13 @@ def __init__(self, bmi_model: NWMv3_Forcing_Engine_BMI_model_Base): """ self.source_data_processor = None self._bmi = bmi_model + # Partials + self.log_info = partial( + err_handler.log_msg, self._bmi._job_meta, self._bmi._mpi_meta, False + ) + self.log_debug = partial( + err_handler.log_msg, self._bmi._job_meta, self._bmi._mpi_meta, True + ) def check_program_status(self) -> None: """Call err_handler.check_program_status""" @@ -194,12 +202,12 @@ def set_cycle_timing_attrs(self, future_time: float) -> None: self._bmi._job_meta.b_date_proc ) + pd.to_timedelta(future_time, unit="s") - LOG.debug( - "NextGen Forcings Engine processing meteorological forcings for BMI timestamp" + self.log_debug( + msg="NextGen Forcings Engine processing meteorological forcings for BMI timestamp" ) - LOG.debug(f"Model.py current time: {self._bmi._job_meta.current_time}") - LOG.debug( - f"Model.py current fcst cycle: {self._bmi._job_meta.current_fcst_cycle}" + self.log_debug(msg=f"Model.py current time: {self._bmi._job_meta.current_time}") + self.log_debug( + msg=f"Model.py current fcst cycle: {self._bmi._job_meta.current_fcst_cycle}" ) if self._bmi._job_meta.first_fcst_cycle is None: @@ -221,19 +229,13 @@ def set_skip_flags(self) -> None: def log_cycle(self) -> None: """Log information about the current forecast cycle.""" if self._bmi._mpi_meta.rank == 0: - self._bmi._job_meta.statusMsg = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" - err_handler.log_msg(self._bmi._job_meta, self._bmi._mpi_meta, True) - self._bmi._job_meta.statusMsg = ( - "Processing Forecast Cycle: " - + self._bmi._job_meta.current_fcst_cycle.strftime("%Y-%m-%d %H:%M") + self.log_debug(msg="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") + self.log_debug( + msg=f"Processing Forecast Cycle: {self._bmi._job_meta.current_fcst_cycle.strftime('%Y-%m-%d %H:%M')}" ) - err_handler.log_msg(self._bmi._job_meta, self._bmi._mpi_meta, True) - self._bmi._job_meta.statusMsg = ( - "Forecast Cycle Length is: " - + str(self._bmi._job_meta.cycle_length_minutes) - + " minutes" + self.log_debug( + msg=f"Forecast Cycle Length is: {self._bmi._job_meta.cycle_length_minutes!s} minutes" ) - err_handler.log_msg(self._bmi._job_meta, self._bmi._mpi_meta, True) # self._bmi._mpi_meta.comm.barrier() @time_function @@ -331,26 +333,24 @@ def loop_through_forcing_products( # Print message on log file indicating the timestamp # we are currently processing for forcings if self._bmi._mpi_meta.rank == 0: - self._bmi._job_meta.statusMsg = ( - "=========================================" + self.log_debug(msg="=========================================") + self.log_debug( + msg=f"Processing for output timestep: {file_date.strftime('%Y-%m-%d %H:%M')}" ) - err_handler.log_msg(self._bmi._job_meta, self._bmi._mpi_meta, True) - self._bmi._job_meta.statusMsg = f"Processing for output timestep: {file_date.strftime('%Y-%m-%d %H:%M')}" - err_handler.log_msg(self._bmi._job_meta, self._bmi._mpi_meta, True) self._bmi._job_meta.currentForceNum = 0 self._bmi._job_meta.currentCustomForceNum = 0 - LOG.debug( - f"config_options.input_forcings: {self._bmi._job_meta.input_forcings}" + self.log_debug( + msg=f"config_options.input_forcings: {self._bmi._job_meta.input_forcings}" ) # Loop over each of the input forcings specified. - LOG.debug( - f"Model.py forcing loop: {len(self._bmi._job_meta.input_forcings)} forcings configured: {self._bmi._job_meta.input_forcings}" + self.log_debug( + msg=f"Model.py forcing loop: {len(self._bmi._job_meta.input_forcings)} forcings configured: {self._bmi._job_meta.input_forcings}" ) for force_key in self._bmi._job_meta.input_forcings: - LOG.debug(f"force_key: {force_key}") - LOG.debug(f"config_options.aws: {self._bmi._job_meta.aws}") + self.log_debug(msg=f"force_key: {force_key}") + self.log_debug(msg=f"config_options.aws: {self._bmi._job_meta.aws}") # Pass these methods for AORC data is ERA5-Interim blend is requested # so we can finish filling in the missing gaps if ( @@ -377,7 +377,7 @@ def loop_through_forcing_products( # If skipping this forcing, continue early # NOTE this is used by the esmf regrid pytests, to halt the loop before "manually" calling a particular regrid function. if input_forcings.skip is True: - LOG.debug(f"Breaking loop for force_key {force_key}") + self.log_debug(msg=f"Breaking loop for force_key {force_key}") break # Regrid forcings. @@ -435,7 +435,7 @@ def loop_through_forcing_products( if force_key == 10: self._bmi._job_meta.currentCustomForceNum += 1 - LOG.debug(f"End of loop for force_key {force_key}") + self.log_debug(msg=f"End of loop for force_key {force_key}") # Process supplemental precipitation if we specified in the configuration file. if self._bmi._job_meta.number_supp_pcp > 0: From 9304958bbca27e59cbab92ea6c0abf4226740602 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 1 May 2026 20:03:09 -0400 Subject: [PATCH 076/132] Run flynt -tc -ll 9999 --- .../NextGen_Forcings_Engine/model.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index b565fb82..01555f1d 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -727,20 +727,20 @@ def update_bmi_output_dict(self) -> None: if self._bmi._job_meta.grid_type == "gridded": for count, variable in enumerate(variables): - self._bmi._values[variable + "_ELEMENT"] = ( + self._bmi._values[f"{variable}_ELEMENT"] = ( self._bmi._output_obj.output_local[count, :, :].flatten() ) elif self._bmi._job_meta.grid_type == "unstructured": for count, variable in enumerate(variables): - self._bmi._values[variable + "_ELEMENT"] = ( + self._bmi._values[f"{variable}_ELEMENT"] = ( self._bmi._output_obj.output_local_elem[count, :].flatten() ) - self._bmi._values[variable + "_NODE"] = ( + self._bmi._values[f"{variable}_NODE"] = ( self._bmi._output_obj.output_local[count, :].flatten() ) elif self._bmi._job_meta.grid_type == "hydrofabric": for count, variable in enumerate(variables): - self._bmi._values[variable + "_ELEMENT"] = ( + self._bmi._values[f"{variable}_ELEMENT"] = ( self._bmi._output_obj.output_global[count, :].flatten() ) self._bmi._values["CAT-ID"] = self._bmi.geo_meta.element_ids_global From b3a472a41c855c95c97519914c80936c7a57fec2 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 1 May 2026 20:19:10 -0400 Subject: [PATCH 077/132] Format docstrings to reST --- .../NextGen_Forcings_Engine/model.py | 180 ++++++++---------- 1 file changed, 79 insertions(+), 101 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 01555f1d..5fc169b8 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -46,12 +46,9 @@ @contextmanager def timing_block(step_str: str): - """Context manager for timing code execution. Used by decorator `time_function`. + """Context manager for timing code execution. Used by the decorator ``time_function``. - Parameters - ---------- - step_str : str - Description of the step being timed. + :param str step_str: Description of the step being timed. """ start = perf_counter() yield @@ -77,11 +74,9 @@ class NWMv3ForcingEngineModel: """ def __init__(self, bmi_model: NWMv3_Forcing_Engine_BMI_model_Base): - """Initialize the NWMv3 Forcing Engine Model. + """Initialize the NWMv3 Forcing Engine model. - Parameters - ---------- - bmi_model : NWMv3_Forcing_Engine_BMI_model_Base + :param bmi_model NWMv3_Forcing_Engine_BMI_model_Base: BMI model instance to initialize. """ self.source_data_processor = None self._bmi = bmi_model @@ -100,44 +95,47 @@ def check_program_status(self) -> None: def run(self, future_time: float) -> None: """Execute the full forcings engine BMI pipeline for a given future timestep. - This method updates the `self._bmi._values` state dictionary with atmospheric forcings computed from - available input datasets. It handles initialization, AWS Zarr loading, regridding, temporal - interpolation, bias correction, downscaling, supplemental precipitation processing, and output - population into the self._bmi._values structure. + This method updates the ``self._bmi._values`` state dictionary with atmospheric + forcings computed from available input datasets. It handles initialization, + AWS Zarr loading, regridding, temporal interpolation, bias correction, + downscaling, supplemental precipitation processing, and output population into + the ``self._bmi._values`` structure. - `self._bmi._job_meta`, an instance of ConfigOptions is also updated in-place, for example for time handling. + ``self._bmi._job_meta``, an instance of ``ConfigOptions``, is also updated + in-place, for example for forecast time handling. The following steps are performed: 1. Determine the current forecast and output times based on the future timestamp - and analysis mode (AnA or forecast). + and analysis mode (AnA or forecast). 2. Initialize or reset output grids and step counters. 3. Loop over each input forcing product: - a. Calculate neighboring input files. - b. Load AWS-hosted Zarr datasets if needed. - c. Regrid input forcings to the model grid. - d. Perform temporal interpolation. - e. Apply bias correction and downscaling. - f. Layer final forcings into the output object. + + a. Calculate neighboring input files. + b. Load AWS-hosted Zarr datasets if needed. + c. Regrid input forcings to the model grid. + d. Perform temporal interpolation. + e. Apply bias correction and downscaling. + f. Layer final forcings into the output object. + 4. Optionally process supplemental precipitation forcings: - a. Regrid and validate. - b. Disaggregate and interpolate. - c. Layer into the final output. + + a. Regrid and validate. + b. Disaggregate and interpolate. + c. Layer into the final output. + 5. Write output to NetCDF forcing files if requested. - 6. Update the self._bmi._values state dictionary with flattened arrays. + 6. Update the ``self._bmi._values`` state dictionary with flattened arrays. 7. Advance the BMI time index. - Parameters - ---------- - future_time : float - Timestamp, represented as *seconds relative to overall start time*, to advance to before returning. - Since this is relative to overall start time, it is unaware of the actual UTC datetimestamp of the start. - For example, since 1-hour timesteps are typical, the first value of this would typically be 3600, the second value 7200, etc. - - Raises - ------ - RuntimeError - If the model fails to initialize or if required arguments are missing. + :param float future_time: Timestamp, represented as *seconds relative to overall + start time*, to advance to before returning. Since this value is relative + to the overall start time, it is unaware of the actual UTC datetimestamp of + the start. For example, since 1-hour timesteps are typical, the first value + would typically be 3600, the second value 7200, etc. + + :raises RuntimeError: If the model fails to initialize or if required arguments + are missing. """ self.set_cycle_timing_attrs(future_time) @@ -155,9 +153,7 @@ def run(self, future_time: float) -> None: def set_cycle_timing_attrs(self, future_time: float) -> None: """Determine the forecast for the given future time and configuration. - Warnings - -------- - Modifies mutable arguments in-place. + :warning: Modifies mutable arguments in-place """ # Assign the future time to the configuration self._bmi._job_meta.bmi_time = future_time @@ -244,21 +240,17 @@ def loop_through_forcing_products( ) -> forcingInputMod.InputForcingsHydrofabric | None: """Loop through each forcing product and process it for the current forecast cycle. - Loop through each output timestep. Perform the following functions: - 1.) Calculate all necessary input files per user options. - 2.) Read in input forcings from GRIB/NetCDF files. - 3.) Regrid the forcings, and temporally interpolate. - 4.) Downscale. - 5.) Layer, and output as necessary. - - Parameters - ---------- - future_time : float - See description in `self.run`. - - Returns - ---------- - input_forcings: forcingInputMod.InputForcings | None + Loop through each output timestep and perform the following steps: + + 1. Calculate all necessary input files per user options. + 2. Read input forcings from GRIB/NetCDF files. + 3. Regrid the forcings and perform temporal interpolation. + 4. Downscale. + 5. Layer and write output as necessary. + + :param float future_time: See description in ``self.run``. + :returns: Processed input forcings for the current timestep. + :rtype: forcingInputMod.InputForcings | None """ ana_factor = 1 if self._bmi._job_meta.ana_flag is False else 0 if not self._bmi._job_meta.precip_only_flag: @@ -462,18 +454,14 @@ def loop_through_forcing_products( def __handle_aorc_and_nwm_force_keys( self, input_forcings: forcingInputMod.InputForcings, force_key: int ) -> None: - """During `loop_through_forcing_products`, handle the case of the force key being AORC or NWM. + """During ``loop_through_forcing_products``, handle the case where the force key is AORC or NWM. - This code block was cut and pasted from methods `loop_through_forcing_products` during refactor. + This code block was cut and pasted from the method ``loop_through_forcing_products`` during refactor. - Parameters - ---------- - input_forcings : forcingInputMod.InputForcings - force_key : int + :param input_forcings forcingInputMod.InputForcings: Input forcings object to be modified. + :param int force_key: Identifier for the forcing type. - Warnings - -------- - Modifies mutable arguments in-place. + :warning: Modifies mutable arguments in-place. """ if force_key in [12, 21, 27]: if self._bmi._job_meta.aws is None: @@ -548,18 +536,15 @@ def __handle_aorc_and_nwm_force_keys( def __process_supp_precip_key( self, input_forcings: forcingInputMod.InputForcings, supp_pcp_key: int ) -> None: - """Process supplemental precipitation for one supplemental precipitation key. + """Process supplemental precipitation for a single supplemental precipitation key. - This code block was cut and pasted from methods `loop_through_forcing_products` and `process_suplemental_precip` during refactor. + This code block was cut and pasted from the methods + ``loop_through_forcing_products`` and ``process_suplemental_precip`` during refactor. - Parameters - ---------- - input_forcings : forcingInputMod.InputForcings - supp_pcp_key : int + :param input_forcings forcingInputMod.InputForcings: Input forcings object to be modified. + :param int supp_pcp_key: Identifier for the supplemental precipitation forcing. - Warnings - -------- - Modifies mutable arguments in-place. + :warning: Modifies mutable arguments in-place. """ # Like with input forcings, calculate the neighboring files to use. self._bmi._supp_pcp_mod[supp_pcp_key].calc_neighbor_files( @@ -612,19 +597,15 @@ def __process_supp_precip_key( self.check_program_status() def __use_rstFlag(self, input_forcings: forcingInputMod.InputForcings) -> None: - """ - If we are restarting a forecast cycle, re-calculate the neighboring files, and regrid the - next set of forcings as the previous step just regridded the previous forcing. + """If restarting a forecast cycle, re-calculate neighboring files and regrid the + next set of forcings, as the previous step regridded the prior forcing. - This code block was cut and pasted from method `loop_through_forcing_products` during refactor. + This code block was cut and pasted from the method + ``loop_through_forcing_products`` during refactor. - Parameters - ---------- - input_forcings : forcingInputMod.InputForcings + :param input_forcings forcingInputMod.InputForcings: Input forcings object to be modified. - Warnings - -------- - Modifies mutable arguments in-place. + :warning: Modifies mutable arguments in-place. """ if input_forcings.rstFlag == 1: if ( @@ -673,13 +654,9 @@ def process_suplemental_precip( ) -> None: """Process supplemental precipitation for the current forecast cycle. - Parameters - ---------- - input_forcings : forcingInputMod.InputForcings + :param input_forcings forcingInputMod.InputForcings: Input forcings object to be modified. - Warnings - -------- - Modifies mutable arguments in-place. + :warning: Modifies mutable arguments in-place. """ if self._bmi._job_meta.customSuppPcpFreq is not None: # Process supplemental precipitation if we specified in the configuration file. @@ -707,20 +684,21 @@ def write_output(self) -> None: def update_bmi_output_dict(self) -> None: """Flatten the Forcings Engine output object and update the BMI dictionary. - Loop through Forcings Engine output object - and flatten the 2D forcing array and append to - the BMI object to advertise to BMIinterface. - 0.) U-Wind (m/s) - 1.) V-Wind (m/s) - 2.) Surface incoming longwave radiation flux (W/m^2) - 3.) Precipitation rate (mm/s) - 4.) 2-meter temperature (K) - 5.) 2-meter specific humidity (kg/kg) - 6.) Surface pressure (Pa) - 7.) Surface incoming shortwave radiation flux (W/m^2) - 8.) Liquid Precipitation Fraction (%), Only available in certain operational configurations - """ + Loop through the Forcings Engine output object, flatten the 2D forcing arrays, + and append them to the BMI object for advertisement through the BMI interface. + The flattened variables are ordered as follows: + + 0. U-wind (m/s) + 1. V-wind (m/s) + 2. Surface incoming longwave radiation flux (W/m²) + 3. Precipitation rate (mm/s) + 4. 2-meter air temperature (K) + 5. 2-meter specific humidity (kg/kg) + 6. Surface pressure (Pa) + 7. Surface incoming shortwave radiation flux (W/m²) + 8. Liquid precipitation fraction (%), available only in certain operational configurations + """ variables = copy.deepcopy(model_consts["update_dict_base_vars"]) if self._bmi._job_meta.include_lqfrac == 1: variables.append(model_consts["update_dict_var_include_lqfraq"]) From b9c6850c7e01654eba0b029218de4f601135c65b Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 1 May 2026 20:35:50 -0400 Subject: [PATCH 078/132] DRYify --- .../NextGen_Forcings_Engine/model.py | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 5fc169b8..ac691d09 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -423,7 +423,7 @@ def loop_through_forcing_products( self._bmi._job_meta.currentForceNum += 1 - # TODO what is this? + # NOTE currentCustomForceNum does not appear to be used. if force_key == 10: self._bmi._job_meta.currentCustomForceNum += 1 @@ -463,6 +463,8 @@ def __handle_aorc_and_nwm_force_keys( :warning: Modifies mutable arguments in-place. """ + proc_args = (self._bmi._job_meta, self._bmi._mpi_meta, self._bmi.geo_meta) + if force_key in [12, 21, 27]: if self._bmi._job_meta.aws is None: # Calculate the previous and next input cycle files from the inputs. @@ -480,18 +482,10 @@ def __handle_aorc_and_nwm_force_keys( # Flag to indicate the AWS .zarr AORC method if force_key == 12: if self.source_data_processor is None: - self.source_data_processor = AORCConusProcessor( - self._bmi._job_meta, - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) + proc_cls = AORCConusProcessor elif force_key == 21: if self.source_data_processor is None: - self.source_data_processor = AORCAlaskaProcessor( - self._bmi._job_meta, - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) + proc_cls = AORCAlaskaProcessor # Flag to indicate the AWS .zarr NWMv3 Forcing file method elif force_key == 27: @@ -517,16 +511,13 @@ def __handle_aorc_and_nwm_force_keys( ) ) elif self._bmi._job_meta.nwm_domain == "Alaska": - self.source_data_processor = NWMV3AlaskaProcessor( - self._bmi._job_meta, - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) + proc_cls = NWMV3AlaskaProcessor else: raise ValueError( f"Unsupported domain type ({self._bmi._job_meta.nwm_domain} for forcing type: {force_key} )" ) + self.source_data_processor = proc_cls(*proc_args) self._bmi._job_meta.aws_obj = ( self.source_data_processor.process_historical_data( self._bmi._job_meta.current_time From 6cf6a1f944791cc53a7a9e674950af024eec6776 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 1 May 2026 20:36:50 -0400 Subject: [PATCH 079/132] Add assertion --- NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index ac691d09..843791d0 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -516,6 +516,8 @@ def __handle_aorc_and_nwm_force_keys( raise ValueError( f"Unsupported domain type ({self._bmi._job_meta.nwm_domain} for forcing type: {force_key} )" ) + else: + raise ValueError(f"Unexpected force_key: {force_key}") self.source_data_processor = proc_cls(*proc_args) self._bmi._job_meta.aws_obj = ( From b2cefbf74fd091daa093ed1ce79d4e9115f6d85b Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 1 May 2026 21:04:34 -0400 Subject: [PATCH 080/132] Fix source_data_processor sets --- .../NextGen_Forcings_Engine/model.py | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 843791d0..567c5b87 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -479,17 +479,14 @@ def __handle_aorc_and_nwm_force_keys( raise ValueError( f"Expected to have 1 forcing key, but have {len(self._bmi._job_meta.input_forcings)}: {list(self._bmi._job_meta.input_forcings)}" ) - # Flag to indicate the AWS .zarr AORC method - if force_key == 12: - if self.source_data_processor is None: + if self.source_data_processor is None: + # Flag to indicate the AWS .zarr AORC method + if force_key == 12: proc_cls = AORCConusProcessor - elif force_key == 21: - if self.source_data_processor is None: + elif force_key == 21: proc_cls = AORCAlaskaProcessor - - # Flag to indicate the AWS .zarr NWMv3 Forcing file method - elif force_key == 27: - if self.source_data_processor is None: + # Flag to indicate the AWS .zarr NWMv3 Forcing file method + elif force_key == 27: if self._bmi._job_meta.nwm_domain == "CONUS": self.source_data_processor = NWMV3ConusProcessor( self._bmi._job_meta, @@ -516,10 +513,10 @@ def __handle_aorc_and_nwm_force_keys( raise ValueError( f"Unsupported domain type ({self._bmi._job_meta.nwm_domain} for forcing type: {force_key} )" ) - else: - raise ValueError(f"Unexpected force_key: {force_key}") + else: + raise ValueError(f"Unexpected force_key: {force_key}") + self.source_data_processor = proc_cls(*proc_args) - self.source_data_processor = proc_cls(*proc_args) self._bmi._job_meta.aws_obj = ( self.source_data_processor.process_historical_data( self._bmi._job_meta.current_time From b27beb3470692ccb87e7d2a7b02e5870aed7355d Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 1 May 2026 21:07:30 -0400 Subject: [PATCH 081/132] Docstrings --- .../NextGen_Forcings_Engine/model.py | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 567c5b87..b72e3e2f 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -110,20 +110,16 @@ def run(self, future_time: float) -> None: and analysis mode (AnA or forecast). 2. Initialize or reset output grids and step counters. 3. Loop over each input forcing product: - - a. Calculate neighboring input files. - b. Load AWS-hosted Zarr datasets if needed. - c. Regrid input forcings to the model grid. - d. Perform temporal interpolation. - e. Apply bias correction and downscaling. - f. Layer final forcings into the output object. - + a. Calculate neighboring input files. + b. Load AWS-hosted Zarr datasets if needed. + c. Regrid input forcings to the model grid. + d. Perform temporal interpolation. + e. Apply bias correction and downscaling. + f. Layer final forcings into the output object. 4. Optionally process supplemental precipitation forcings: - - a. Regrid and validate. - b. Disaggregate and interpolate. - c. Layer into the final output. - + a. Regrid and validate. + b. Disaggregate and interpolate. + c. Layer into the final output. 5. Write output to NetCDF forcing files if requested. 6. Update the ``self._bmi._values`` state dictionary with flattened arrays. 7. Advance the BMI time index. @@ -433,7 +429,7 @@ def loop_through_forcing_products( if self._bmi._job_meta.number_supp_pcp > 0: for supp_pcp_key in self._bmi._job_meta.supp_precip_forcings: if supp_pcp_key != 13: - # Below comment copied from earlier code, the comment had been just above the call to `disaggregate_fun`. + # Below comment copied from earlier code, the comment had been just above the call to ``disaggregate_fun``. # TODO input_forcings has not yet been initialized, so this is a bug waiting to happen self.__process_supp_precip_key(input_forcings, supp_pcp_key) From 281c88ddf12931e64a0052c5c4ccb9943446ce7c Mon Sep 17 00:00:00 2001 From: "Matthew.Deshotel" Date: Mon, 13 Jul 2026 10:05:59 -0500 Subject: [PATCH 082/132] comment out not implemented error. --- .../NextGen_Forcings_Engine/model.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index b72e3e2f..9e5a9a36 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -276,14 +276,18 @@ def loop_through_forcing_products( self._bmi._job_meta.current_output_step += 1 # Optional sub-output timestamp - if self._bmi._job_meta.sub_output_hour is not None: - raise NotImplementedError( - f"sub_output_hour (config SubOutputHour) is {repr(self._bmi._job_meta.sub_output_hour)} (not None) but is not used." - ) - # TODO This is not used - subOutDate = self._bmi._job_meta.first_fcst_cycle + datetime.timedelta( - hours=self._bmi._job_meta.sub_output_hour - ) + # if self._bmi._job_meta.sub_output_hour is not None: + # raise NotImplementedError( + # f"sub_output_hour (config SubOutputHour) is {repr(self._bmi._job_meta.sub_output_hour)} (not None) but is not used." + # ) + # # TODO This is not used. The raise not implemented error causes a fail on medium range blen due to the sub_output_hour being + # specified in the config file. Testing was performed and not specifying sub_output_hour produces the same results for medium range blend + # as of 7/13/2026. Not sure what this was intended to do but it is not used/effective at this time. Commenting it out to ensure medium range blend completes + # and it is retained in case the intent is realized and it should be resurrected. + + # subOutDate = self._bmi._job_meta.first_fcst_cycle + datetime.timedelta( + # hours=self._bmi._job_meta.sub_output_hour + # ) # Compute the output timestamp for this step if self._bmi._job_meta.ana_flag: From 4424fb9d5a1069a698de1189edb19524470d70c2 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Thu, 13 Aug 2026 14:03:08 -0400 Subject: [PATCH 083/132] Remove duplicate method --- .../NextGen_Forcings_Engine/core/config.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 3bc060a3..8f03e093 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -157,11 +157,6 @@ def supp_precip_count(self) -> int: # return len(SUPPPRECIPMOD["suppPrecipMod"]["PRODUCT_NAMES"]) return 15 - @property - def number_supp_pcp(self) -> int: - """Calculate the number of supplemental precip forcings specified by the user in the configuration file.""" - return len(self.supp_precip_forcings) - @property def precip_only_flag(self) -> bool: """Flag to indicate whether the user has chosen to run the supplemental precip forcings module only, which will trigger some different processing pathways and error checking for certain configuration options.""" From 2368e02c4522c6a280fae166cd3ed519b26a3dae Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 17 Aug 2026 08:47:32 -0400 Subject: [PATCH 084/132] Following refactor+rebase: update CAT-ID assignments and init calls, fix namespace error, remove call to `validate_config` (validation occurs automatically in new design) --- .../NextGen_Forcings_Engine/bmi_model.py | 21 ++++++++----------- .../NextGen_Forcings_Engine/model.py | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index a780e744..c63d0afa 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -223,11 +223,10 @@ def _job_meta(self) -> ConfigOptions: def _job_meta(self, value: ConfigOptions) -> None: """Set the job metadata object.""" if value is None: - value = ConfigOptions( - self.cfg_bmi, b_date=self._b_date, geogrid_arg=self._geogrid - ) try: - value.validate_config(self.cfg_bmi) + value = ConfigOptions( + self.cfg_bmi, b_date=self._b_date, geogrid=self._geogrid + ) except KeyboardInterrupt as e: err_handler.err_out_screen("User keyboard interrupt", e) except ImportError as e: @@ -281,7 +280,7 @@ def init_scratch_dir(self) -> None: self._job_meta.uniquefy_scratch_dir_as_child(self._mpi_meta.uid64) def create_esmf_mesh(self) -> None: - """Create the ESMF mesh for the model.""" + """Create the ESMF mesh for the model and set ``self._cat_ids`` (later used as BMI variable "CAT-ID").""" if self._mpi_meta.rank == 0: cat_ids = esmf_creation.create_mesh(self._job_meta) cat_count = np.array([ @@ -291,6 +290,7 @@ def create_esmf_mesh(self) -> None: if self._mpi_meta.rank != 0: cat_ids = np.empty(cat_count[0], dtype=np.int64) self._mpi_meta.comm.Bcast(cat_ids, root=0) + self._cat_ids = cat_ids def fetch_raw_forcing_data(self) -> None: """Fetch raw forcing data for the model. @@ -411,12 +411,11 @@ def set_initial_time_and_step(self) -> None: """Set the initial time and time step size for the model.""" self._values["current_model_time"] = self.cfg_bmi["initial_time"] self._values["time_step_size"] = self.cfg_bmi["time_step_seconds"] - self._values["CAT-ID"] = cat_ids def set_catchment_ids(self) -> None: """Set catchment ids if using hydrofabric.""" if self._grid_type == "hydrofabric": - self._values["CAT-ID"] = self.geo_meta.element_ids_global + self._values["CAT-ID"] = self._cat_ids def initialize(self, config_file: str) -> None: """Initialize the model using a configuration file. @@ -435,7 +434,7 @@ def initialize(self, config_file: str) -> None: self._config_file = config_file for attr in BMI_MODEL[self.__class__.__base__.__name__]: setattr(self, attr, None) - self._model = NWMv3ForcingEngineModel() + self._model = NWMv3ForcingEngineModel(self) self.init_log() self.init_mpi() self.init_scratch_dir() @@ -1540,7 +1539,6 @@ class NWMv3_Forcing_Engine_BMI_model_Gridded(NWMv3_Forcing_Engine_BMI_model_Base def __init__( self, - config_file: str, b_date: str = None, geogrid: str = None, output_path: str = None, @@ -1549,7 +1547,7 @@ def __init__( Initializes the model with default values for time, variables, and grid types. """ - super().__init__(config_file, b_date, geogrid, output_path) + super().__init__(b_date, geogrid, output_path) self.GeoMeta = GriddedGeoMeta def grid_ranks(self) -> list[int]: @@ -1679,7 +1677,6 @@ class NWMv3_Forcing_Engine_BMI_model_Unstructured(NWMv3_Forcing_Engine_BMI_model def __init__( self, - config_file: str, b_date: str = None, geogrid: str = None, output_path: str = None, @@ -1688,7 +1685,7 @@ def __init__( Initializes the model with default values for time, variables, and grid types. """ - super().__init__(config_file, b_date, geogrid, output_path) + super().__init__(b_date, geogrid, output_path) self.GeoMeta = UnstructuredGeoMeta def grid_ranks(self) -> list[int]: diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 9e5a9a36..41fad2b9 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -711,7 +711,7 @@ def update_bmi_output_dict(self) -> None: self._bmi._values[f"{variable}_ELEMENT"] = ( self._bmi._output_obj.output_global[count, :].flatten() ) - self._bmi._values["CAT-ID"] = self._bmi.geo_meta.element_ids_global + self._bmi._values["CAT-ID"] = self._bmi._cat_ids else: raise ValueError( f"Unexpected grid_type: {repr(self._bmi._job_meta.grid_type)}" From e79516c70563a131c481a402d68e0d2910e5629e Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 17 Aug 2026 18:13:32 -0400 Subject: [PATCH 085/132] Ruff reorder imports --- tests/test_utils.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 2b0d4a68..e50aac9e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -12,17 +12,6 @@ import test_config_classes # noqa: F401 # Used by test implementations, more convenient to have it in here rather than using more importlib import test_consts # noqa: F401 # Used by test implementations, more convenient to have it in here rather than using more importlib import xarray as xr -from test_config_classes import ( - TestConfig_AnA, - TestConfig_Base, - TestConfig_BmiModel, - TestConfig_ConfigOptions, - TestConfig_GeoMod, - TestConfig_InputForcing, - TestConfig_Regrid, - TestConfig_SuppPrecip, -) - from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.bmi_model import ( BMIMODEL, NWMv3_Forcing_Engine_BMI_model_Base, @@ -44,6 +33,16 @@ assert_equal_with_tol, serialize_to_json, ) +from test_config_classes import ( + TestConfig_AnA, + TestConfig_Base, + TestConfig_BmiModel, + TestConfig_ConfigOptions, + TestConfig_GeoMod, + TestConfig_InputForcing, + TestConfig_Regrid, + TestConfig_SuppPrecip, +) OS_VAR__CREATE_TEST_EXPECT_DATA = "FORCING_PYTEST_WRITE_TEST_EXPECTED_DATA" From 272bb158f1a52bab500b685e67c80bf048014afb Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 17 Aug 2026 18:14:36 -0400 Subject: [PATCH 086/132] Tests: update for new BMI forcing initialization method --- tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index e50aac9e..23233fe6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -284,7 +284,7 @@ class BMIForcingFixture: def __init__(self, cfg: TestConfig_Base) -> None: """Initialize BMIForcingFixture.""" self.bmi_model: NWMv3_Forcing_Engine_BMI_model_Base = BMIMODEL[cfg.grid_type]() - self.bmi_model.initialize_with_params(config_file=cfg.config_file) + self.bmi_model.initialize(cfg.config_file) self.bmi_model_values = self.bmi_model._values self.mpi_config: MpiConfig = self.bmi_model._mpi_meta From afc9964bc9dd9a5c6d2a2ec5cc2fa8dd398a7dcd Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 17 Aug 2026 18:15:25 -0400 Subject: [PATCH 087/132] Tests: improve test data serialization/deserialization logic --- tests/test_utils.py | 76 +++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 23233fe6..43012925 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -386,40 +386,35 @@ def deserial_expected(self, suffix: str, current_output_step: str = "") -> dict: """Get the expected metadata results as a deserialized dictionary.""" file_path = self.expected_results_file_path(suffix, current_output_step) - if os.environ.get(OS_VAR__CREATE_TEST_EXPECT_DATA, "").lower() == "true": - # Dump current results to disk, to save it as "expected" results for later test runs. - # Should only be used when committing new test results to the repository. - logging.warning(f"Writing test data: {file_path}") - deserial_expected = self.deserial_actual( - suffix, current_output_step, write_to_file=False - ) - with open(file_path, "w") as f: - f.write(serialize_to_json(deserial_expected, sort_keys=True)) - # Remove keys that should be excluded from comparison (match read-exclusion) - deserial_expected = remove_key(deserial_expected, self.keys_to_exclude) + try: + with open(file_path) as f: + deserial_expected = json.load(f) if self.map_old_to_new_var_names: deserial_expected = self.map_old_to_new_variable_names( deserial_expected ) self._trim_arrays_to_input_map_output(deserial_expected) - return deserial_expected - else: - try: - with open(file_path) as f: - deserial_expected = json.load(f) - if self.map_old_to_new_var_names: - deserial_expected = self.map_old_to_new_variable_names( - deserial_expected - ) - self._trim_arrays_to_input_map_output(deserial_expected) - # Remove keys that should be excluded from comparison (match write-exclusion) - deserial_expected = remove_key(deserial_expected, self.keys_to_exclude) - # order and reverse so private attributes are last - return OrderedDict(reversed(list(deserial_expected.items()))) - except FileNotFoundError as e: - raise FileNotFoundError( - f"Could not find {file_path}. Try running the test using OS var {OS_VAR__CREATE_TEST_EXPECT_DATA}=true first to set up the test results expected data." - ) from e + # Remove keys that should be excluded from comparison + deserial_expected = remove_key(deserial_expected, self.keys_to_exclude) + # order and reverse so private attributes are last + return OrderedDict(reversed(list(deserial_expected.items()))) + except FileNotFoundError as e: + raise FileNotFoundError( + f"Could not find {file_path}. Try running the test using OS var {OS_VAR__CREATE_TEST_EXPECT_DATA}=true first to set up the test results expected data." + ) from e + + def _write_expected_file(self, actual_data: dict, suffix: str, current_output_step: str = "") -> None: + """Write actual data to expected results file for test data generation. + + This is a separate explicit step in the workflow to avoid confusion between + expected and actual data. Should only be called when FORCING_PYTEST_WRITE_TEST_EXPECTED_DATA=true. + """ + file_path = self.expected_results_file_path(suffix, current_output_step) + # Remove excluded keys before writing + data_to_write = remove_key(dict(actual_data), self.keys_to_exclude) + logging.warning(f"Writing test data: {file_path}") + with open(file_path, "w") as f: + f.write(serialize_to_json(data_to_write, sort_keys=True)) def map_old_to_new_variable_names(self, data: dict) -> dict: """Map old variable names to new variable names in the expected results data.""" @@ -440,7 +435,11 @@ def after_intitialization_check(self) -> None: orig = self.keys_to_exclude self.keys_to_exclude = orig + self.keys_to_exclude_at_init try: - self.compare(self.deserial_actual("init"), self.deserial_expected("init")) + actual = self.deserial_actual("init") + if os.environ.get(OS_VAR__CREATE_TEST_EXPECT_DATA, "").lower() == "true": + self._write_expected_file(actual, "init") + expected = self.deserial_expected("init") + self.compare(actual, expected) finally: self.keys_to_exclude = orig @@ -474,17 +473,20 @@ def after_bmi_model_update(self, current_output_step: int) -> None: """ logging.info("Starting after_bmi_model_update()...") - self.compare( - self.deserial_actual("after_update", f"_step_{current_output_step}"), - self.deserial_expected("after_update", f"_step_{current_output_step}"), - ) + actual = self.deserial_actual("after_update", f"_step_{current_output_step}") + if os.environ.get(OS_VAR__CREATE_TEST_EXPECT_DATA, "").lower() == "true": + self._write_expected_file(actual, "after_update", f"_step_{current_output_step}") + expected = self.deserial_expected("after_update", f"_step_{current_output_step}") + self.compare(actual, expected) def after_finalize(self) -> None: """Run checks after bmi_model.finalize() has been called.""" logging.info("Starting after_finalize()...") - self.compare( - self.deserial_actual("finalize"), self.deserial_expected("finalize") - ) + actual = self.deserial_actual("finalize") + if os.environ.get(OS_VAR__CREATE_TEST_EXPECT_DATA, "").lower() == "true": + self._write_expected_file(actual, "finalize") + expected = self.deserial_expected("finalize") + self.compare(actual, expected) def actual_results_file_path( self, suffix: str, current_output_step: str = "" From 9ae1dfe359eb990920d3a35e50168fd0231cb195 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 17 Aug 2026 18:16:11 -0400 Subject: [PATCH 088/132] Tests: bmi_model: exclude more keys --- tests/bmi_model/test_bmi_model.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/bmi_model/test_bmi_model.py b/tests/bmi_model/test_bmi_model.py index 274d1795..80461061 100644 --- a/tests/bmi_model/test_bmi_model.py +++ b/tests/bmi_model/test_bmi_model.py @@ -21,7 +21,12 @@ config_file=consts.RETRO_FORCING_CONFIG_FILE__AORC_CONUS, keys_to_check=(), keys_to_exclude=tuple( - set(consts.KEYS_TO_EXCLUDE) | {"d_program_init", "geogrid", "scratch_dir", "Element_Elevation", "Element_Slope", "Element_Slope_Azmuith"} + set(consts.KEYS_TO_EXCLUDE) | { + "d_program_init", "geogrid", "scratch_dir", "Element_Elevation", "Element_Slope", "Element_Slope_Azmuith", + "geo_meta.config_options.cfg_bmi", + "geo_meta.mpi_config.config_options", + "mpi_config.config_options", + } ), grid_type=consts.GRID_TYPE, test_file_name_prefix="bmi_model", From be3ac8e29ea559021910d34ac92ed66b54bb15a7 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 17 Aug 2026 21:19:32 -0400 Subject: [PATCH 089/132] Fix various config handling issues following refactor + rebase. Notably if rqiMethod is 0 (unused) then rqiThresh property now resolves to None regardless of the value of the RqiThresh key in the config file. --- .../NextGen_Forcings_Engine/core/config.py | 174 ++++++++++++------ 1 file changed, 120 insertions(+), 54 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 8f03e093..868d7d1c 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -46,6 +46,14 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No else: self.user_provided_geogrid_flag = False + # If b_date not provided, try to read from config file + if b_date is None: + b_date = cfg_bmi.get("RefcstBDateProc", None) + if b_date is None: + err_out_screen( + "RefcstBDateProc is either missing or None in configuration file." + ) + self.b_date_proc = b_date self.cfg_bmi = cfg_bmi self.geogrid = geogrid @@ -63,19 +71,44 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No self.nwm_source = "s3://noaa-nwm-retrospective-3-0-pds" self._scratch_dir_has_been_uniquefied = False - self.supp_precip_forcings = self.extract_input_variable("SuppPcp") - if not self.precip_only_flag: - self.input_forcings = self.extract_input_variable("InputForcings") - - # Create temporary array to hold flags if we need input parameter files. - self.param_flag = np.zeros([len(self.input_forcings)], int) - - # set list of attibutes from consts.py to None. - # These are indexed from the consts dictionary using the class name + + # These must exist (as None) before the properties are accessed + self._supp_precip_forcings = None + self._b_date_proc = None + self._input_forcings = None + self._nwm_geogrid = None + self._output_freq = None + self._sub_output_hour = None + self._sub_output_freq = None + self._scratch_dir = None + self._useCompression = None + self._ana_flag = None + self._look_back = None + self._fcst_freq = None + self._spatial_meta = None + self._geopackage = None + self._geogrid = None + self._grid_type = None + + # set list of attributes from consts.py to None early on in the init process. + # These are indexed from the consts dictionary. + # This must happen before accessing properties like precip_only_flag for attr in CONFIGOPTIONS[self.__class__.__name__]: setattr(self, attr, None) self.broadcast_new_64bit_uid() + # Extract optional forcing inputs (SuppPcp, InputForcings) from config; default to empty lists if not provided since code assumes these are always iterable + supp_pcp = self.extract_input_variable("SuppPcp") + self.supp_precip_forcings = supp_pcp if supp_pcp is not None else [] + if not self.precip_only_flag: + input_forc = self.extract_input_variable("InputForcings") + self.input_forcings = input_forc if input_forc is not None else [] + # Create temporary array to hold flags if we need input parameter files. + self.param_flag = np.zeros([len(self.input_forcings)], int) + else: + self.input_forcings = [] + self.param_flag = np.array([], int) + for ( cfg_bmi_attr, config_options_attr, @@ -107,6 +140,11 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No self.set_attrs(CONFIGOPTIONS["downscaling_attrs_map"]) if self.grid_type == "unstructured": self.set_attrs(CONFIGOPTIONS["downscaling_unstructred_attrs_map"]) + else: + # Initialize downscaling attributes to None even if downscaling is not performed + self.set_attrs(CONFIGOPTIONS["downscaling_attrs_map"], set_none=True) + if self.grid_type == "unstructured": + self.set_attrs(CONFIGOPTIONS["downscaling_unstructred_attrs_map"], set_none=True) for cfg_bmi_attr, config_options_attr in CONFIGOPTIONS[ "extract_input_variable_set_default_attrs_map" @@ -121,10 +159,28 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No self.extract_input_variable_set_default(cfg_bmi_attr, default), ) + # Call post_init to perform any calculations that depend on multiple attributes + self.post_init() + + def post_init(self) -> None: + """Attr setting/calculating operations that depend on others already being set. + + This method is called at the end of __init__ to perform any side-effect calculations + that require multiple attributes to be initialized. This ensures independence of initialization order + and matches the original pre-refactored code behavior, where calculations + were performed after all attrs were read. + """ + # Calculate the beginning/ending processing dates if we are running realtime or if look_back != -9999 + if self.look_back is not None and self.look_back != -9999: + calculate_lookback_window(self) + elif self.realtime_flag: + calculate_lookback_window(self) + @property def try_config_get_except_attr_map(self) -> dict: """Get the mapping of configuration variable names to class attribute names for variables that are extracted directly from the configuration file without any additional processing. This is used to control how variables are extracted from the configuration file and assigned to class attributes in a consistent way based on the mapping specified in the consts.py file.""" - dict_map = CONFIGOPTIONS["try_config_get_except_attr_map"] + # Don't mutate the module-level CONFIGOPTIONS object. Operate on a copy instead (and return that modified copy). + dict_map = CONFIGOPTIONS["try_config_get_except_attr_map"].copy() if self._b_date_proc is not None and "RefcstBDateProc" in dict_map: dict_map.pop("RefcstBDateProc") if self.geogrid is not None and "GeogridIn" in dict_map: @@ -161,7 +217,7 @@ def supp_precip_count(self) -> int: def precip_only_flag(self) -> bool: """Flag to indicate whether the user has chosen to run the supplemental precip forcings module only, which will trigger some different processing pathways and error checking for certain configuration options.""" precip_only = False - if self.number_supp_pcp == 1: + if self.supp_precip_forcings is not None and len(self.supp_precip_forcings) > 0: if int(self.supp_precip_forcings[0]) == 14: precip_only = True return precip_only @@ -337,7 +393,7 @@ def supp_precip_forcings(self): @supp_precip_forcings.setter def supp_precip_forcings(self, value: list) -> None: """Set the list of supplemental precip forcing options specified by the user in the configuration file. This is used to control which supplemental precip forcings are processed and how they are processed based on the other configuration options specified for each supplemental precip forcing.""" - if len(value) > 0: + if value is not None and len(value) > 0: self.check_input_values_in_range( [int(i) for i in value], "SuppPcp", @@ -456,8 +512,7 @@ def look_back(self, value: int) -> None: """Set the look back window in hours specified by the user in the configuration file. This is used to calculate the processing window for reforecast simulations, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" if value <= 0 and value != -9999: err_out_screen("Please specify a positive LookBack or -9999 for realtime.") - if value != -9999: - calculate_lookback_window(self) + # NOTE: Side effect (calculate_lookback_window) removed - now called in post_init() self._look_back = value @property @@ -510,8 +565,12 @@ def b_date_proc(self) -> str: @b_date_proc.setter def b_date_proc(self, value: str | datetime) -> None: """Set the beginning date of processing for reforecast simulations. This is used to calculate the processing window for reforecast simulations.""" + if value is None: + self._b_date_proc = None + return if isinstance(value, datetime): self._b_date_proc = value + return if value != -9999: if isinstance(value, str) and len(value) != 12: err_out_screen( @@ -537,9 +596,7 @@ def realtime_flag(self) -> bool: value = True else: value = False - # Calculate the beginning/ending processing dates if we are running realtime - if value: - calculate_lookback_window(self) + # NOTE: Side effect (calculate_lookback_window) removed from property getter - now called in post_init() return value @property @@ -573,11 +630,13 @@ def geogrid(self) -> str: @geogrid.setter def geogrid(self, value: str) -> None: """Set the pathway to the geogrid file to be used for processing. This is used to specify the grid information for regridding input forcings, and is only necessary if the user is running a simulation that requires regridding of input forcings.""" + # If user provided geogrid, use it as-is if self.user_provided_geogrid_flag: self._geogrid = value - if value is None: + # If value is None, just set to None + elif value is None: self._geogrid = value - # err_out_screen("Unable to locate GeogridIn in the configuration file.") + # Otherwise, process the path value from config with uid prefix else: geogrid_parent = os.path.dirname(value) geogrid_filename = os.path.basename(value) @@ -607,7 +666,7 @@ def input_forcings(self) -> list: @input_forcings.setter def input_forcings(self, value: list) -> None: """Set the list of input forcing options specified by the user in the configuration file. This is used to control which input forcings are processed and how they are processed based on the other configuration options specified for each input forcing.""" - if not self.precip_only_flag: + if value is not None and not self.precip_only_flag: self.check_input_values_in_range( value, "InputForcings", list(range(1, self.force_count + 1)) ) @@ -616,12 +675,15 @@ def input_forcings(self, value: list) -> None: @property def number_inputs(self) -> int: """Calculate the number of input forcing options specified by the user in the configuration file. This is used for error checking to ensure users specify valid input forcing options in the configuration file, and to control the flow of the program based on how many input forcings are being processed.""" + if self.input_forcings is None: + return 0 if not self.precip_only_flag: if len(self.input_forcings) == 0: err_out_screen( "Please choose at least one InputForcings dataset to process" ) return len(self.input_forcings) + return 0 @property def number_custom_inputs(self) -> int: @@ -635,6 +697,11 @@ def number_custom_inputs(self) -> int: else: return 0 + @number_custom_inputs.setter + def number_custom_inputs(self, value: int) -> None: + """This is a read-only computed property based on input_forcings.""" + raise AttributeError(f"number_custom_inputs is read-only (tried to set to: {value})") + @property def nwm_geogrid(self) -> str: """Only for the NWM v3 retorspective forcing module option (27) that requires the geo_em_NWM_DOMAIN.nc file as input for the NextGen Forcings Engine to properly setup up the ESMF grid object for the NWM forcing files since that information is not readily available in the NWM v3 retrospective forcing files.""" @@ -643,7 +710,7 @@ def nwm_geogrid(self) -> str: @nwm_geogrid.setter def nwm_geogrid(self, value: str) -> None: """Set the pathway to the NWM geogrid file specified by the user in the configuration file. This is used to specify the grid information for regridding NWM input forcings, and is only necessary if the user has chosen to regrid NWM input forcings in the configuration file.""" - if not self.precip_only_flag and 27 in self.input_forcings: + if not self.precip_only_flag and self.input_forcings is not None and 27 in self.input_forcings: self._nwm_geogrid = value else: self._nwm_geogrid = None @@ -746,9 +813,7 @@ def fcst_shift(self) -> int: def fcst_shift(self, value: int) -> None: if True: # was: self.realtime_flag: self.check_input_values_non_negative([value], "ForecastShift") - # Calculate the beginning/ending processing dates if we are running realtime - if self.realtime_flag: - calculate_lookback_window(self) + # NOTE: Side effect (calculate_lookback_window) removed - now called in post_init() self._fcst_shift = value # NOTE this commented out code copied from pre-refactored code on 5/6/2026 @@ -1284,6 +1349,8 @@ def runCfsNldasBiasCorrect(self) -> bool: @property def number_supp_pcp(self) -> int: """Get the number of supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how many supplemental precipitation input forcings are processed based on the number of supplemental precipitation input forcings specified in the configuration file.""" + if self.supp_precip_forcings is None: + return 0 return len(self.supp_precip_forcings) @property @@ -1312,7 +1379,7 @@ def supplemental_precip_file_type_options(self) -> list: return ["GRIB1", "GRIB2", "NETCDF"] @property - def rqiMethod(self) -> list: + def rqiMethod(self) -> int | list[int] | None: """Optional RQI method for radar-based data. 0 - Do not use any RQI filtering. Use all radar-based estimates. 1 - Use hourly MRMS Radar Quality Index grids, 2 - Use NWM monthly climatology grids (NWM only!!!!). Example- RqiMethod: 2 @@ -1322,46 +1389,45 @@ def rqiMethod(self) -> list: for suppOpt in self.supp_precip_forcings: # Read in RQI threshold to apply to radar products. if suppOpt in (1, 2, 7, 10, 11, 12): - value = self.extract_input_variable("RqiMethod") - - # Check that if we have more than one RqiMethod, it's the correct number - if type(value) is list: - self.check_number_of_inputs_supp_pcp(value, "RqiMethod") - elif type(value) is int: - # Support 'classic' mode of single method - value = [value] * self.number_supp_pcp - - # Make sure the RqiMethod(s) makes sense. - for method in value: - self.check_input_values_in_range(method, "RqiMethod", [0, 1, 2]) + # Returns None if key missing (not configured for this product) + value = self.cfg_bmi.get("RqiMethod") + if value is not None: + # Validate + if type(value) is list: + self.check_number_of_inputs_supp_pcp(value, "RqiMethod") + elif type(value) in (int, type(None)): + # Support configuration file representing this with a single scalar value, apply to all + value = [value] * self.number_supp_pcp + + self.check_input_values_in_range(value, "RqiMethod", [0, 1, 2]) return value @property - def rqiThresh(self): + def rqiThresh(self) -> float | list[float] | None: """Optional RQI threshold to be used to mask out. Currently used for MRMS products. Please choose a value from 0.0-1.0. Associated radar quality index files will be expected from MRMS data. Example- RqiThreshold: 0.9 """ - value = 1.0 + value = None if self.number_supp_pcp > 0: for supp_opt in self.supp_precip_forcings: # Read in RQI threshold to apply to radar products. if supp_opt in (1, 2, 7, 10, 11, 12): - value = self.extract_input_variable("RqiThresh") - - # Check that if we have more than one RqiThresh, it's the correct number - if type(value) is list: - self.check_number_of_inputs_supp_pcp(value, "RqiThresh") - elif type(value) is (int, float): - # Support 'classic' mode of single threshold - value = [value] * self.number_supp_pcp - - # Make sure the RqiThresh(es) makes sense. - for threshold in value: - if threshold < 0.0 or threshold > 1.0: - err_out_screen( - "Please specify RqiThresholds between 0.0 and 1.0." - ) + # Returns None if key missing (not configured for this product) + value = self.cfg_bmi.get("RqiThresh") + if value is not None: + # Validate + if type(value) is list: + self.check_number_of_inputs_supp_pcp(value, "RqiThresh") + elif type(value) in (int, float, type(None)): + # Support configuration file representing this with a single scalar value, apply to all + value = [value] * self.number_supp_pcp + + for threshold in value: + if threshold < 0.0 or threshold > 1.0: + err_out_screen( + "Please specify RqiThresholds between 0.0 and 1.0." + ) return value @property From 49b45791b68e289d4536c5faa140f9a57d7acef4 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 17 Aug 2026 21:21:43 -0400 Subject: [PATCH 090/132] Test data: update expected json files for bmi_model and config_options tests relative to refactored class structures and new key exclusions in test utils --- ...i_model_after_update_n1_rank0__step_1.json | 405 ++-------- ...i_model_after_update_n1_rank0__step_2.json | 405 ++-------- ...i_model_after_update_n1_rank0__step_3.json | 405 ++-------- ...i_model_after_update_n2_rank0__step_1.json | 409 ++--------- ...i_model_after_update_n2_rank0__step_2.json | 409 ++--------- ...i_model_after_update_n2_rank0__step_3.json | 409 ++--------- ...i_model_after_update_n2_rank1__step_1.json | 407 ++--------- ...i_model_after_update_n2_rank1__step_2.json | 407 ++--------- ...i_model_after_update_n2_rank1__step_3.json | 407 ++--------- ...expected_bmi_model_finalize_n1_rank0_.json | 690 +++++++++++++++++- ...expected_bmi_model_finalize_n2_rank0_.json | 669 ++++++++++++++++- ...expected_bmi_model_finalize_n2_rank1_.json | 662 ++++++++++++++++- ...est_expected_bmi_model_init_n1_rank0_.json | 213 ++---- ...est_expected_bmi_model_init_n2_rank0_.json | 217 ++---- ...est_expected_bmi_model_init_n2_rank1_.json | 215 ++---- ...options_after_update_n1_rank0__step_1.json | 162 +++- ...options_after_update_n1_rank0__step_2.json | 162 +++- ...options_after_update_n1_rank0__step_3.json | 162 +++- ...options_after_update_n2_rank0__step_1.json | 162 +++- ...options_after_update_n2_rank0__step_2.json | 162 +++- ...options_after_update_n2_rank0__step_3.json | 162 +++- ...options_after_update_n2_rank1__step_1.json | 162 +++- ...options_after_update_n2_rank1__step_2.json | 162 +++- ...options_after_update_n2_rank1__step_3.json | 162 +++- ...ted_config_options_finalize_n1_rank0_.json | 160 +++- ...ted_config_options_finalize_n2_rank0_.json | 160 +++- ...ted_config_options_finalize_n2_rank1_.json | 160 +++- ...xpected_config_options_init_n1_rank0_.json | 160 +++- ...xpected_config_options_init_n2_rank0_.json | 160 +++- ...xpected_config_options_init_n2_rank1_.json | 160 +++- 30 files changed, 4961 insertions(+), 3786 deletions(-) diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json index 40136e80..283f9511 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,6 +118,7 @@ "initial_time": 0, "time_step_seconds": 3600 }, + "dimensionality": 1, "geo_meta": { "approx_centroid_global_xy": [ -72.05503025291092, @@ -323,10 +325,29 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_time": 3600, "bmi_time_index": 1, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -354,17 +375,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -406,6 +435,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -434,24 +467,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -464,9 +502,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 @@ -723,355 +766,6 @@ "mesh_inds_elem": null, "mpi_config": { "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "config_options": { - "ExactExtract": null, - "actual_output_steps": 71, - "ana_flag": 0, - "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", - "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", - "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", - "aorc_conus_year_url": "{source}/{year}.zarr", - "aws": true, - "aws_obj": { - "attrs": {}, - "coords": { - "spatial_ref": { - "attrs": { - "crs_wkt": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]", - "geographic_crs_name": "WGS 84", - "grid_mapping_name": "latitude_longitude", - "horizontal_datum_name": "World Geodetic System 1984", - "inverse_flattening": 298.257223563, - "longitude_of_prime_meridian": 0.0, - "prime_meridian_name": "Greenwich", - "reference_ellipsoid_name": "WGS 84", - "semi_major_axis": 6378137.0, - "semi_minor_axis": 6356752.314245179, - "spatial_ref": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]" - }, - "data": 0, - "dims": [] - }, - "time": { - "attrs": { - "long_name": "verification time generated by wgrib2 function verftime()", - "reference_date": "2013.01.01 00:00:00 UTC", - "reference_time": 1356998400.0, - "reference_time_description": "kind of product unclear, reference date is variable, min found reference date is given", - "reference_time_type": 0, - "time_step": 0.0, - "time_step_setting": "auto" - }, - "data": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "dims": [] - }, - "x": { - "attrs": { - "long_name": "longitude", - "units": "degrees_east" - }, - "data": "hash_-5399622063019475031_len_28", - "dims": [ - "x" - ] - }, - "y": { - "attrs": { - "long_name": "latitude", - "units": "degrees_north" - }, - "data": "hash_-1457289664996315734_len_37", - "dims": [ - "y" - ] - } - }, - "data_vars": { - "APCP_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Total Precipitation", - "short_name": "APCP_surface", - "units": "kg/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DLWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Long-Wave Rad. Flux", - "short_name": "DLWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DSWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Short-Wave Rad. Flux", - "short_name": "DSWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "PRES_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Pressure", - "short_name": "PRES_surface", - "units": "Pa" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "SPFH_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Specific Humidity", - "short_name": "SPFH_2maboveground", - "units": "kg/kg" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "TMP_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Temperature", - "short_name": "TMP_2maboveground", - "units": "K" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "UGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "U-Component of Wind", - "short_name": "UGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "VGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "V-Component of Wind", - "short_name": "VGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - } - }, - "dims": { - "x": 28, - "y": 37 - } - }, - "aws_time": null, - "b_date_proc": "2013-07-01T00:00:00", - "bmi_time": 3600, - "bmi_time_index": 1, - "cfsv2EnsMember": null, - "config_path": null, - "cosalpha_var": null, - "currentCustomForceNum": 0, - "currentForceNum": 1, - "current_fcst_cycle": "2013-07-01T00:00:00", - "current_output_date": "2013-07-01T01:00:00", - "current_output_step": 1, - "current_time": "2013-07-01T01:00:00", - "customFcstFreq": [], - "customSuppPcpFreq": null, - "cycle_length_minutes": 4260, - "dScaleParamDirs": [ - "/ngen-app/data" - ], - "e_date_proc": null, - "elemconn_var": "elementConn", - "elemcoords_var": "centerCoords", - "element_id_var": "element_id", - "errFlag": 0, - "errMsg": null, - "fcst_freq": 60, - "fcst_input_horizons": [ - 4260 - ], - "fcst_input_offsets": [ - 0 - ], - "fcst_shift": 0, - "first_fcst_cycle": "2013-07-01T00:00:00", - "forceTemoralInterp": [ - 0 - ], - "forcing_output": 0, - "future_time": null, - "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", - "globalNdv": -9999.0, - "grid_meta": null, - "grid_type": "hydrofabric", - "hgt_elem_var": null, - "hgt_var": null, - "ignored_border_widths": [ - 0 - ], - "include_lqfrac": 1, - "input_force_dirs": [ - "s3://null" - ], - "input_force_mandatory": [ - 0 - ], - "input_force_types": [ - "GRIB2" - ], - "input_forcings": [ - 12 - ], - "lat_var": null, - "logFile": null, - "logHandle": null, - "lon_var": null, - "look_back": -9999, - "lwBiasCorrectOpt": [ - 0 - ], - "nFcsts": 1, - "nodecoords_var": "nodeCoords", - "num_output_steps": 71, - "num_supp_output_steps": null, - "number_custom_inputs": 0, - "number_inputs": 1, - "number_supp_pcp": 0, - "numelemconn_var": "numElementConn", - "nwmConfig": "AORC", - "nwmVersion": 4.0, - "nwm_domain": null, - "nwm_geogrid": null, - "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", - "nwm_url": null, - "output_freq": 60, - "paramFlagArray": null, - "precipBiasCorrectOpt": [ - 0 - ], - "precipDownscaleOpt": [ - 0 - ], - "precip_only_flag": false, - "prev_output_date": "2013-07-01T00:00:00", - "process_window": null, - "psfcBiasCorrectOpt": [ - 0 - ], - "psfcDownscaleOpt": [ - 0 - ], - "q2BiasCorrectOpt": [ - 0 - ], - "q2dDownscaleOpt": [ - 0 - ], - "realtime_flag": false, - "refcst_flag": true, - "regrid_opt": [ - 1 - ], - "regrid_opt_supp_pcp": null, - "rqiMethod": null, - "rqiThresh": 1.0, - "runCfsNldasBiasCorrect": false, - "sinalpha_var": null, - "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, - "slope_var": null, - "slope_var_elem": null, - "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", - "sub_output_freq": null, - "sub_output_hour": null, - "suppTemporalInterp": null, - "supp_pcp_max_hours": null, - "supp_precip_dirs": null, - "supp_precip_file_types": [], - "supp_precip_forcings": [], - "supp_precip_mandatory": null, - "supp_precip_param_dir": null, - "swBiasCorrectOpt": [ - 0 - ], - "swDownscaleOpt": [ - 0 - ], - "t2BiasCorrectOpt": [ - 0 - ], - "t2dDownscaleOpt": [ - 0 - ], - "useCompression": 0, - "useFloats": 0, - "use_data_at_current_time": true, - "weightsDir": null, - "windBiasCorrect": [ - 0 - ] - }, "rank": 0, "size": 1 }, @@ -1152,5 +846,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json index b495ac2e..26e1450f 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,6 +118,7 @@ "initial_time": 0, "time_step_seconds": 3600 }, + "dimensionality": 1, "geo_meta": { "approx_centroid_global_xy": [ -72.05503025291092, @@ -323,10 +325,29 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_time": 7200, "bmi_time_index": 2, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -354,17 +375,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -406,6 +435,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -434,24 +467,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -464,9 +502,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 @@ -723,355 +766,6 @@ "mesh_inds_elem": null, "mpi_config": { "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "config_options": { - "ExactExtract": null, - "actual_output_steps": 71, - "ana_flag": 0, - "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", - "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", - "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", - "aorc_conus_year_url": "{source}/{year}.zarr", - "aws": true, - "aws_obj": { - "attrs": {}, - "coords": { - "spatial_ref": { - "attrs": { - "crs_wkt": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]", - "geographic_crs_name": "WGS 84", - "grid_mapping_name": "latitude_longitude", - "horizontal_datum_name": "World Geodetic System 1984", - "inverse_flattening": 298.257223563, - "longitude_of_prime_meridian": 0.0, - "prime_meridian_name": "Greenwich", - "reference_ellipsoid_name": "WGS 84", - "semi_major_axis": 6378137.0, - "semi_minor_axis": 6356752.314245179, - "spatial_ref": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]" - }, - "data": 0, - "dims": [] - }, - "time": { - "attrs": { - "long_name": "verification time generated by wgrib2 function verftime()", - "reference_date": "2013.01.01 00:00:00 UTC", - "reference_time": 1356998400.0, - "reference_time_description": "kind of product unclear, reference date is variable, min found reference date is given", - "reference_time_type": 0, - "time_step": 0.0, - "time_step_setting": "auto" - }, - "data": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "dims": [] - }, - "x": { - "attrs": { - "long_name": "longitude", - "units": "degrees_east" - }, - "data": "hash_-5399622063019475031_len_28", - "dims": [ - "x" - ] - }, - "y": { - "attrs": { - "long_name": "latitude", - "units": "degrees_north" - }, - "data": "hash_-1457289664996315734_len_37", - "dims": [ - "y" - ] - } - }, - "data_vars": { - "APCP_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Total Precipitation", - "short_name": "APCP_surface", - "units": "kg/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DLWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Long-Wave Rad. Flux", - "short_name": "DLWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DSWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Short-Wave Rad. Flux", - "short_name": "DSWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "PRES_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Pressure", - "short_name": "PRES_surface", - "units": "Pa" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "SPFH_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Specific Humidity", - "short_name": "SPFH_2maboveground", - "units": "kg/kg" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "TMP_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Temperature", - "short_name": "TMP_2maboveground", - "units": "K" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "UGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "U-Component of Wind", - "short_name": "UGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "VGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "V-Component of Wind", - "short_name": "VGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - } - }, - "dims": { - "x": 28, - "y": 37 - } - }, - "aws_time": null, - "b_date_proc": "2013-07-01T00:00:00", - "bmi_time": 7200, - "bmi_time_index": 2, - "cfsv2EnsMember": null, - "config_path": null, - "cosalpha_var": null, - "currentCustomForceNum": 0, - "currentForceNum": 1, - "current_fcst_cycle": "2013-07-01T00:00:00", - "current_output_date": "2013-07-01T02:00:00", - "current_output_step": 2, - "current_time": "2013-07-01T02:00:00", - "customFcstFreq": [], - "customSuppPcpFreq": null, - "cycle_length_minutes": 4260, - "dScaleParamDirs": [ - "/ngen-app/data" - ], - "e_date_proc": null, - "elemconn_var": "elementConn", - "elemcoords_var": "centerCoords", - "element_id_var": "element_id", - "errFlag": 0, - "errMsg": null, - "fcst_freq": 60, - "fcst_input_horizons": [ - 4260 - ], - "fcst_input_offsets": [ - 0 - ], - "fcst_shift": 0, - "first_fcst_cycle": "2013-07-01T00:00:00", - "forceTemoralInterp": [ - 0 - ], - "forcing_output": 0, - "future_time": null, - "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", - "globalNdv": -9999.0, - "grid_meta": null, - "grid_type": "hydrofabric", - "hgt_elem_var": null, - "hgt_var": null, - "ignored_border_widths": [ - 0 - ], - "include_lqfrac": 1, - "input_force_dirs": [ - "s3://null" - ], - "input_force_mandatory": [ - 0 - ], - "input_force_types": [ - "GRIB2" - ], - "input_forcings": [ - 12 - ], - "lat_var": null, - "logFile": null, - "logHandle": null, - "lon_var": null, - "look_back": -9999, - "lwBiasCorrectOpt": [ - 0 - ], - "nFcsts": 1, - "nodecoords_var": "nodeCoords", - "num_output_steps": 71, - "num_supp_output_steps": null, - "number_custom_inputs": 0, - "number_inputs": 1, - "number_supp_pcp": 0, - "numelemconn_var": "numElementConn", - "nwmConfig": "AORC", - "nwmVersion": 4.0, - "nwm_domain": null, - "nwm_geogrid": null, - "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", - "nwm_url": null, - "output_freq": 60, - "paramFlagArray": null, - "precipBiasCorrectOpt": [ - 0 - ], - "precipDownscaleOpt": [ - 0 - ], - "precip_only_flag": false, - "prev_output_date": "2013-07-01T00:00:00", - "process_window": null, - "psfcBiasCorrectOpt": [ - 0 - ], - "psfcDownscaleOpt": [ - 0 - ], - "q2BiasCorrectOpt": [ - 0 - ], - "q2dDownscaleOpt": [ - 0 - ], - "realtime_flag": false, - "refcst_flag": true, - "regrid_opt": [ - 1 - ], - "regrid_opt_supp_pcp": null, - "rqiMethod": null, - "rqiThresh": 1.0, - "runCfsNldasBiasCorrect": false, - "sinalpha_var": null, - "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, - "slope_var": null, - "slope_var_elem": null, - "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", - "sub_output_freq": null, - "sub_output_hour": null, - "suppTemporalInterp": null, - "supp_pcp_max_hours": null, - "supp_precip_dirs": null, - "supp_precip_file_types": [], - "supp_precip_forcings": [], - "supp_precip_mandatory": null, - "supp_precip_param_dir": null, - "swBiasCorrectOpt": [ - 0 - ], - "swDownscaleOpt": [ - 0 - ], - "t2BiasCorrectOpt": [ - 0 - ], - "t2dDownscaleOpt": [ - 0 - ], - "useCompression": 0, - "useFloats": 0, - "use_data_at_current_time": true, - "weightsDir": null, - "windBiasCorrect": [ - 0 - ] - }, "rank": 0, "size": 1 }, @@ -1152,5 +846,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json index f7a6b582..78e899a3 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,6 +118,7 @@ "initial_time": 0, "time_step_seconds": 3600 }, + "dimensionality": 1, "geo_meta": { "approx_centroid_global_xy": [ -72.05503025291092, @@ -323,10 +325,29 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_time": 10800, "bmi_time_index": 3, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -354,17 +375,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -406,6 +435,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -434,24 +467,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -464,9 +502,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 @@ -723,355 +766,6 @@ "mesh_inds_elem": null, "mpi_config": { "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "config_options": { - "ExactExtract": null, - "actual_output_steps": 71, - "ana_flag": 0, - "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", - "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", - "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", - "aorc_conus_year_url": "{source}/{year}.zarr", - "aws": true, - "aws_obj": { - "attrs": {}, - "coords": { - "spatial_ref": { - "attrs": { - "crs_wkt": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]", - "geographic_crs_name": "WGS 84", - "grid_mapping_name": "latitude_longitude", - "horizontal_datum_name": "World Geodetic System 1984", - "inverse_flattening": 298.257223563, - "longitude_of_prime_meridian": 0.0, - "prime_meridian_name": "Greenwich", - "reference_ellipsoid_name": "WGS 84", - "semi_major_axis": 6378137.0, - "semi_minor_axis": 6356752.314245179, - "spatial_ref": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]" - }, - "data": 0, - "dims": [] - }, - "time": { - "attrs": { - "long_name": "verification time generated by wgrib2 function verftime()", - "reference_date": "2013.01.01 00:00:00 UTC", - "reference_time": 1356998400.0, - "reference_time_description": "kind of product unclear, reference date is variable, min found reference date is given", - "reference_time_type": 0, - "time_step": 0.0, - "time_step_setting": "auto" - }, - "data": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "dims": [] - }, - "x": { - "attrs": { - "long_name": "longitude", - "units": "degrees_east" - }, - "data": "hash_-5399622063019475031_len_28", - "dims": [ - "x" - ] - }, - "y": { - "attrs": { - "long_name": "latitude", - "units": "degrees_north" - }, - "data": "hash_-1457289664996315734_len_37", - "dims": [ - "y" - ] - } - }, - "data_vars": { - "APCP_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Total Precipitation", - "short_name": "APCP_surface", - "units": "kg/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DLWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Long-Wave Rad. Flux", - "short_name": "DLWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DSWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Short-Wave Rad. Flux", - "short_name": "DSWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "PRES_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Pressure", - "short_name": "PRES_surface", - "units": "Pa" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "SPFH_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Specific Humidity", - "short_name": "SPFH_2maboveground", - "units": "kg/kg" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "TMP_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Temperature", - "short_name": "TMP_2maboveground", - "units": "K" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "UGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "U-Component of Wind", - "short_name": "UGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "VGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "V-Component of Wind", - "short_name": "VGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - } - }, - "dims": { - "x": 28, - "y": 37 - } - }, - "aws_time": null, - "b_date_proc": "2013-07-01T00:00:00", - "bmi_time": 10800, - "bmi_time_index": 3, - "cfsv2EnsMember": null, - "config_path": null, - "cosalpha_var": null, - "currentCustomForceNum": 0, - "currentForceNum": 1, - "current_fcst_cycle": "2013-07-01T00:00:00", - "current_output_date": "2013-07-01T03:00:00", - "current_output_step": 3, - "current_time": "2013-07-01T03:00:00", - "customFcstFreq": [], - "customSuppPcpFreq": null, - "cycle_length_minutes": 4260, - "dScaleParamDirs": [ - "/ngen-app/data" - ], - "e_date_proc": null, - "elemconn_var": "elementConn", - "elemcoords_var": "centerCoords", - "element_id_var": "element_id", - "errFlag": 0, - "errMsg": null, - "fcst_freq": 60, - "fcst_input_horizons": [ - 4260 - ], - "fcst_input_offsets": [ - 0 - ], - "fcst_shift": 0, - "first_fcst_cycle": "2013-07-01T00:00:00", - "forceTemoralInterp": [ - 0 - ], - "forcing_output": 0, - "future_time": null, - "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", - "globalNdv": -9999.0, - "grid_meta": null, - "grid_type": "hydrofabric", - "hgt_elem_var": null, - "hgt_var": null, - "ignored_border_widths": [ - 0 - ], - "include_lqfrac": 1, - "input_force_dirs": [ - "s3://null" - ], - "input_force_mandatory": [ - 0 - ], - "input_force_types": [ - "GRIB2" - ], - "input_forcings": [ - 12 - ], - "lat_var": null, - "logFile": null, - "logHandle": null, - "lon_var": null, - "look_back": -9999, - "lwBiasCorrectOpt": [ - 0 - ], - "nFcsts": 1, - "nodecoords_var": "nodeCoords", - "num_output_steps": 71, - "num_supp_output_steps": null, - "number_custom_inputs": 0, - "number_inputs": 1, - "number_supp_pcp": 0, - "numelemconn_var": "numElementConn", - "nwmConfig": "AORC", - "nwmVersion": 4.0, - "nwm_domain": null, - "nwm_geogrid": null, - "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", - "nwm_url": null, - "output_freq": 60, - "paramFlagArray": null, - "precipBiasCorrectOpt": [ - 0 - ], - "precipDownscaleOpt": [ - 0 - ], - "precip_only_flag": false, - "prev_output_date": "2013-07-01T00:00:00", - "process_window": null, - "psfcBiasCorrectOpt": [ - 0 - ], - "psfcDownscaleOpt": [ - 0 - ], - "q2BiasCorrectOpt": [ - 0 - ], - "q2dDownscaleOpt": [ - 0 - ], - "realtime_flag": false, - "refcst_flag": true, - "regrid_opt": [ - 1 - ], - "regrid_opt_supp_pcp": null, - "rqiMethod": null, - "rqiThresh": 1.0, - "runCfsNldasBiasCorrect": false, - "sinalpha_var": null, - "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, - "slope_var": null, - "slope_var_elem": null, - "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", - "sub_output_freq": null, - "sub_output_hour": null, - "suppTemporalInterp": null, - "supp_pcp_max_hours": null, - "supp_precip_dirs": null, - "supp_precip_file_types": [], - "supp_precip_forcings": [], - "supp_precip_mandatory": null, - "supp_precip_param_dir": null, - "swBiasCorrectOpt": [ - 0 - ], - "swDownscaleOpt": [ - 0 - ], - "t2BiasCorrectOpt": [ - 0 - ], - "t2dDownscaleOpt": [ - 0 - ], - "useCompression": 0, - "useFloats": 0, - "use_data_at_current_time": true, - "weightsDir": null, - "windBiasCorrect": [ - 0 - ] - }, "rank": 0, "size": 1 }, @@ -1152,5 +846,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json index b07e8be3..93fc260a 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,6 +118,7 @@ "initial_time": 0, "time_step_seconds": 3600 }, + "dimensionality": 1, "geo_meta": { "approx_centroid_global_xy": [ -72.05503025291092, @@ -323,10 +325,29 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_time": 3600, "bmi_time_index": 1, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -354,17 +375,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -406,6 +435,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -434,24 +467,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -464,9 +502,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 @@ -531,7 +574,7 @@ "_coords": [ [ "hash_351698735832278484_len_364", - "hash_1673925891551318769_len_364" + "hash_-6066707896656524608_len_364" ], [ [ @@ -680,7 +723,7 @@ "height_elem": null, "heights_global": null, "inds": null, - "lat_bounds": "hash_2117205719057770460_len_582", + "lat_bounds": "hash_902249663378761532_len_582", "latitude_grid": [ 41.67904491423198, 41.73961258049449, @@ -705,355 +748,6 @@ "mesh_inds_elem": null, "mpi_config": { "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "config_options": { - "ExactExtract": null, - "actual_output_steps": 71, - "ana_flag": 0, - "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", - "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", - "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", - "aorc_conus_year_url": "{source}/{year}.zarr", - "aws": true, - "aws_obj": { - "attrs": {}, - "coords": { - "spatial_ref": { - "attrs": { - "crs_wkt": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]", - "geographic_crs_name": "WGS 84", - "grid_mapping_name": "latitude_longitude", - "horizontal_datum_name": "World Geodetic System 1984", - "inverse_flattening": 298.257223563, - "longitude_of_prime_meridian": 0.0, - "prime_meridian_name": "Greenwich", - "reference_ellipsoid_name": "WGS 84", - "semi_major_axis": 6378137.0, - "semi_minor_axis": 6356752.314245179, - "spatial_ref": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]" - }, - "data": 0, - "dims": [] - }, - "time": { - "attrs": { - "long_name": "verification time generated by wgrib2 function verftime()", - "reference_date": "2013.01.01 00:00:00 UTC", - "reference_time": 1356998400.0, - "reference_time_description": "kind of product unclear, reference date is variable, min found reference date is given", - "reference_time_type": 0, - "time_step": 0.0, - "time_step_setting": "auto" - }, - "data": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "dims": [] - }, - "x": { - "attrs": { - "long_name": "longitude", - "units": "degrees_east" - }, - "data": "hash_-5399622063019475031_len_28", - "dims": [ - "x" - ] - }, - "y": { - "attrs": { - "long_name": "latitude", - "units": "degrees_north" - }, - "data": "hash_-1457289664996315734_len_37", - "dims": [ - "y" - ] - } - }, - "data_vars": { - "APCP_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Total Precipitation", - "short_name": "APCP_surface", - "units": "kg/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DLWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Long-Wave Rad. Flux", - "short_name": "DLWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DSWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Short-Wave Rad. Flux", - "short_name": "DSWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "PRES_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Pressure", - "short_name": "PRES_surface", - "units": "Pa" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "SPFH_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Specific Humidity", - "short_name": "SPFH_2maboveground", - "units": "kg/kg" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "TMP_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Temperature", - "short_name": "TMP_2maboveground", - "units": "K" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "UGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "U-Component of Wind", - "short_name": "UGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "VGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "V-Component of Wind", - "short_name": "VGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - } - }, - "dims": { - "x": 28, - "y": 37 - } - }, - "aws_time": null, - "b_date_proc": "2013-07-01T00:00:00", - "bmi_time": 3600, - "bmi_time_index": 1, - "cfsv2EnsMember": null, - "config_path": null, - "cosalpha_var": null, - "currentCustomForceNum": 0, - "currentForceNum": 1, - "current_fcst_cycle": "2013-07-01T00:00:00", - "current_output_date": "2013-07-01T01:00:00", - "current_output_step": 1, - "current_time": "2013-07-01T01:00:00", - "customFcstFreq": [], - "customSuppPcpFreq": null, - "cycle_length_minutes": 4260, - "dScaleParamDirs": [ - "/ngen-app/data" - ], - "e_date_proc": null, - "elemconn_var": "elementConn", - "elemcoords_var": "centerCoords", - "element_id_var": "element_id", - "errFlag": 0, - "errMsg": null, - "fcst_freq": 60, - "fcst_input_horizons": [ - 4260 - ], - "fcst_input_offsets": [ - 0 - ], - "fcst_shift": 0, - "first_fcst_cycle": "2013-07-01T00:00:00", - "forceTemoralInterp": [ - 0 - ], - "forcing_output": 0, - "future_time": null, - "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", - "globalNdv": -9999.0, - "grid_meta": null, - "grid_type": "hydrofabric", - "hgt_elem_var": null, - "hgt_var": null, - "ignored_border_widths": [ - 0 - ], - "include_lqfrac": 1, - "input_force_dirs": [ - "s3://null" - ], - "input_force_mandatory": [ - 0 - ], - "input_force_types": [ - "GRIB2" - ], - "input_forcings": [ - 12 - ], - "lat_var": null, - "logFile": null, - "logHandle": null, - "lon_var": null, - "look_back": -9999, - "lwBiasCorrectOpt": [ - 0 - ], - "nFcsts": 1, - "nodecoords_var": "nodeCoords", - "num_output_steps": 71, - "num_supp_output_steps": null, - "number_custom_inputs": 0, - "number_inputs": 1, - "number_supp_pcp": 0, - "numelemconn_var": "numElementConn", - "nwmConfig": "AORC", - "nwmVersion": 4.0, - "nwm_domain": null, - "nwm_geogrid": null, - "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", - "nwm_url": null, - "output_freq": 60, - "paramFlagArray": null, - "precipBiasCorrectOpt": [ - 0 - ], - "precipDownscaleOpt": [ - 0 - ], - "precip_only_flag": false, - "prev_output_date": "2013-07-01T00:00:00", - "process_window": null, - "psfcBiasCorrectOpt": [ - 0 - ], - "psfcDownscaleOpt": [ - 0 - ], - "q2BiasCorrectOpt": [ - 0 - ], - "q2dDownscaleOpt": [ - 0 - ], - "realtime_flag": false, - "refcst_flag": true, - "regrid_opt": [ - 1 - ], - "regrid_opt_supp_pcp": null, - "rqiMethod": null, - "rqiThresh": 1.0, - "runCfsNldasBiasCorrect": false, - "sinalpha_var": null, - "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, - "slope_var": null, - "slope_var_elem": null, - "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", - "sub_output_freq": null, - "sub_output_hour": null, - "suppTemporalInterp": null, - "supp_pcp_max_hours": null, - "supp_precip_dirs": null, - "supp_precip_file_types": [], - "supp_precip_forcings": [], - "supp_precip_mandatory": null, - "supp_precip_param_dir": null, - "swBiasCorrectOpt": [ - 0 - ], - "swDownscaleOpt": [ - 0 - ], - "t2BiasCorrectOpt": [ - 0 - ], - "t2dDownscaleOpt": [ - 0 - ], - "useCompression": 0, - "useFloats": 0, - "use_data_at_current_time": true, - "weightsDir": null, - "windBiasCorrect": [ - 0 - ] - }, "rank": 0, "size": 2 }, @@ -1125,5 +819,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json index 26be099a..50cf0d86 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,6 +118,7 @@ "initial_time": 0, "time_step_seconds": 3600 }, + "dimensionality": 1, "geo_meta": { "approx_centroid_global_xy": [ -72.05503025291092, @@ -323,10 +325,29 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_time": 7200, "bmi_time_index": 2, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -354,17 +375,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -406,6 +435,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -434,24 +467,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -464,9 +502,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 @@ -531,7 +574,7 @@ "_coords": [ [ "hash_351698735832278484_len_364", - "hash_1673925891551318769_len_364" + "hash_-6066707896656524608_len_364" ], [ [ @@ -680,7 +723,7 @@ "height_elem": null, "heights_global": null, "inds": null, - "lat_bounds": "hash_2117205719057770460_len_582", + "lat_bounds": "hash_902249663378761532_len_582", "latitude_grid": [ 41.67904491423198, 41.73961258049449, @@ -705,355 +748,6 @@ "mesh_inds_elem": null, "mpi_config": { "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "config_options": { - "ExactExtract": null, - "actual_output_steps": 71, - "ana_flag": 0, - "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", - "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", - "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", - "aorc_conus_year_url": "{source}/{year}.zarr", - "aws": true, - "aws_obj": { - "attrs": {}, - "coords": { - "spatial_ref": { - "attrs": { - "crs_wkt": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]", - "geographic_crs_name": "WGS 84", - "grid_mapping_name": "latitude_longitude", - "horizontal_datum_name": "World Geodetic System 1984", - "inverse_flattening": 298.257223563, - "longitude_of_prime_meridian": 0.0, - "prime_meridian_name": "Greenwich", - "reference_ellipsoid_name": "WGS 84", - "semi_major_axis": 6378137.0, - "semi_minor_axis": 6356752.314245179, - "spatial_ref": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]" - }, - "data": 0, - "dims": [] - }, - "time": { - "attrs": { - "long_name": "verification time generated by wgrib2 function verftime()", - "reference_date": "2013.01.01 00:00:00 UTC", - "reference_time": 1356998400.0, - "reference_time_description": "kind of product unclear, reference date is variable, min found reference date is given", - "reference_time_type": 0, - "time_step": 0.0, - "time_step_setting": "auto" - }, - "data": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "dims": [] - }, - "x": { - "attrs": { - "long_name": "longitude", - "units": "degrees_east" - }, - "data": "hash_-5399622063019475031_len_28", - "dims": [ - "x" - ] - }, - "y": { - "attrs": { - "long_name": "latitude", - "units": "degrees_north" - }, - "data": "hash_-1457289664996315734_len_37", - "dims": [ - "y" - ] - } - }, - "data_vars": { - "APCP_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Total Precipitation", - "short_name": "APCP_surface", - "units": "kg/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DLWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Long-Wave Rad. Flux", - "short_name": "DLWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DSWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Short-Wave Rad. Flux", - "short_name": "DSWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "PRES_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Pressure", - "short_name": "PRES_surface", - "units": "Pa" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "SPFH_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Specific Humidity", - "short_name": "SPFH_2maboveground", - "units": "kg/kg" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "TMP_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Temperature", - "short_name": "TMP_2maboveground", - "units": "K" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "UGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "U-Component of Wind", - "short_name": "UGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "VGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "V-Component of Wind", - "short_name": "VGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - } - }, - "dims": { - "x": 28, - "y": 37 - } - }, - "aws_time": null, - "b_date_proc": "2013-07-01T00:00:00", - "bmi_time": 7200, - "bmi_time_index": 2, - "cfsv2EnsMember": null, - "config_path": null, - "cosalpha_var": null, - "currentCustomForceNum": 0, - "currentForceNum": 1, - "current_fcst_cycle": "2013-07-01T00:00:00", - "current_output_date": "2013-07-01T02:00:00", - "current_output_step": 2, - "current_time": "2013-07-01T02:00:00", - "customFcstFreq": [], - "customSuppPcpFreq": null, - "cycle_length_minutes": 4260, - "dScaleParamDirs": [ - "/ngen-app/data" - ], - "e_date_proc": null, - "elemconn_var": "elementConn", - "elemcoords_var": "centerCoords", - "element_id_var": "element_id", - "errFlag": 0, - "errMsg": null, - "fcst_freq": 60, - "fcst_input_horizons": [ - 4260 - ], - "fcst_input_offsets": [ - 0 - ], - "fcst_shift": 0, - "first_fcst_cycle": "2013-07-01T00:00:00", - "forceTemoralInterp": [ - 0 - ], - "forcing_output": 0, - "future_time": null, - "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", - "globalNdv": -9999.0, - "grid_meta": null, - "grid_type": "hydrofabric", - "hgt_elem_var": null, - "hgt_var": null, - "ignored_border_widths": [ - 0 - ], - "include_lqfrac": 1, - "input_force_dirs": [ - "s3://null" - ], - "input_force_mandatory": [ - 0 - ], - "input_force_types": [ - "GRIB2" - ], - "input_forcings": [ - 12 - ], - "lat_var": null, - "logFile": null, - "logHandle": null, - "lon_var": null, - "look_back": -9999, - "lwBiasCorrectOpt": [ - 0 - ], - "nFcsts": 1, - "nodecoords_var": "nodeCoords", - "num_output_steps": 71, - "num_supp_output_steps": null, - "number_custom_inputs": 0, - "number_inputs": 1, - "number_supp_pcp": 0, - "numelemconn_var": "numElementConn", - "nwmConfig": "AORC", - "nwmVersion": 4.0, - "nwm_domain": null, - "nwm_geogrid": null, - "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", - "nwm_url": null, - "output_freq": 60, - "paramFlagArray": null, - "precipBiasCorrectOpt": [ - 0 - ], - "precipDownscaleOpt": [ - 0 - ], - "precip_only_flag": false, - "prev_output_date": "2013-07-01T00:00:00", - "process_window": null, - "psfcBiasCorrectOpt": [ - 0 - ], - "psfcDownscaleOpt": [ - 0 - ], - "q2BiasCorrectOpt": [ - 0 - ], - "q2dDownscaleOpt": [ - 0 - ], - "realtime_flag": false, - "refcst_flag": true, - "regrid_opt": [ - 1 - ], - "regrid_opt_supp_pcp": null, - "rqiMethod": null, - "rqiThresh": 1.0, - "runCfsNldasBiasCorrect": false, - "sinalpha_var": null, - "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, - "slope_var": null, - "slope_var_elem": null, - "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", - "sub_output_freq": null, - "sub_output_hour": null, - "suppTemporalInterp": null, - "supp_pcp_max_hours": null, - "supp_precip_dirs": null, - "supp_precip_file_types": [], - "supp_precip_forcings": [], - "supp_precip_mandatory": null, - "supp_precip_param_dir": null, - "swBiasCorrectOpt": [ - 0 - ], - "swDownscaleOpt": [ - 0 - ], - "t2BiasCorrectOpt": [ - 0 - ], - "t2dDownscaleOpt": [ - 0 - ], - "useCompression": 0, - "useFloats": 0, - "use_data_at_current_time": true, - "weightsDir": null, - "windBiasCorrect": [ - 0 - ] - }, "rank": 0, "size": 2 }, @@ -1125,5 +819,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json index 211cc8ad..fe18aacb 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,6 +118,7 @@ "initial_time": 0, "time_step_seconds": 3600 }, + "dimensionality": 1, "geo_meta": { "approx_centroid_global_xy": [ -72.05503025291092, @@ -323,10 +325,29 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_time": 10800, "bmi_time_index": 3, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -354,17 +375,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -406,6 +435,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -434,24 +467,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -464,9 +502,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 @@ -531,7 +574,7 @@ "_coords": [ [ "hash_351698735832278484_len_364", - "hash_1673925891551318769_len_364" + "hash_-6066707896656524608_len_364" ], [ [ @@ -680,7 +723,7 @@ "height_elem": null, "heights_global": null, "inds": null, - "lat_bounds": "hash_2117205719057770460_len_582", + "lat_bounds": "hash_902249663378761532_len_582", "latitude_grid": [ 41.67904491423198, 41.73961258049449, @@ -705,355 +748,6 @@ "mesh_inds_elem": null, "mpi_config": { "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "config_options": { - "ExactExtract": null, - "actual_output_steps": 71, - "ana_flag": 0, - "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", - "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", - "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", - "aorc_conus_year_url": "{source}/{year}.zarr", - "aws": true, - "aws_obj": { - "attrs": {}, - "coords": { - "spatial_ref": { - "attrs": { - "crs_wkt": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]", - "geographic_crs_name": "WGS 84", - "grid_mapping_name": "latitude_longitude", - "horizontal_datum_name": "World Geodetic System 1984", - "inverse_flattening": 298.257223563, - "longitude_of_prime_meridian": 0.0, - "prime_meridian_name": "Greenwich", - "reference_ellipsoid_name": "WGS 84", - "semi_major_axis": 6378137.0, - "semi_minor_axis": 6356752.314245179, - "spatial_ref": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]" - }, - "data": 0, - "dims": [] - }, - "time": { - "attrs": { - "long_name": "verification time generated by wgrib2 function verftime()", - "reference_date": "2013.01.01 00:00:00 UTC", - "reference_time": 1356998400.0, - "reference_time_description": "kind of product unclear, reference date is variable, min found reference date is given", - "reference_time_type": 0, - "time_step": 0.0, - "time_step_setting": "auto" - }, - "data": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "dims": [] - }, - "x": { - "attrs": { - "long_name": "longitude", - "units": "degrees_east" - }, - "data": "hash_-5399622063019475031_len_28", - "dims": [ - "x" - ] - }, - "y": { - "attrs": { - "long_name": "latitude", - "units": "degrees_north" - }, - "data": "hash_-1457289664996315734_len_37", - "dims": [ - "y" - ] - } - }, - "data_vars": { - "APCP_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Total Precipitation", - "short_name": "APCP_surface", - "units": "kg/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DLWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Long-Wave Rad. Flux", - "short_name": "DLWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DSWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Short-Wave Rad. Flux", - "short_name": "DSWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "PRES_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Pressure", - "short_name": "PRES_surface", - "units": "Pa" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "SPFH_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Specific Humidity", - "short_name": "SPFH_2maboveground", - "units": "kg/kg" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "TMP_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Temperature", - "short_name": "TMP_2maboveground", - "units": "K" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "UGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "U-Component of Wind", - "short_name": "UGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "VGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "V-Component of Wind", - "short_name": "VGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - } - }, - "dims": { - "x": 28, - "y": 37 - } - }, - "aws_time": null, - "b_date_proc": "2013-07-01T00:00:00", - "bmi_time": 10800, - "bmi_time_index": 3, - "cfsv2EnsMember": null, - "config_path": null, - "cosalpha_var": null, - "currentCustomForceNum": 0, - "currentForceNum": 1, - "current_fcst_cycle": "2013-07-01T00:00:00", - "current_output_date": "2013-07-01T03:00:00", - "current_output_step": 3, - "current_time": "2013-07-01T03:00:00", - "customFcstFreq": [], - "customSuppPcpFreq": null, - "cycle_length_minutes": 4260, - "dScaleParamDirs": [ - "/ngen-app/data" - ], - "e_date_proc": null, - "elemconn_var": "elementConn", - "elemcoords_var": "centerCoords", - "element_id_var": "element_id", - "errFlag": 0, - "errMsg": null, - "fcst_freq": 60, - "fcst_input_horizons": [ - 4260 - ], - "fcst_input_offsets": [ - 0 - ], - "fcst_shift": 0, - "first_fcst_cycle": "2013-07-01T00:00:00", - "forceTemoralInterp": [ - 0 - ], - "forcing_output": 0, - "future_time": null, - "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", - "globalNdv": -9999.0, - "grid_meta": null, - "grid_type": "hydrofabric", - "hgt_elem_var": null, - "hgt_var": null, - "ignored_border_widths": [ - 0 - ], - "include_lqfrac": 1, - "input_force_dirs": [ - "s3://null" - ], - "input_force_mandatory": [ - 0 - ], - "input_force_types": [ - "GRIB2" - ], - "input_forcings": [ - 12 - ], - "lat_var": null, - "logFile": null, - "logHandle": null, - "lon_var": null, - "look_back": -9999, - "lwBiasCorrectOpt": [ - 0 - ], - "nFcsts": 1, - "nodecoords_var": "nodeCoords", - "num_output_steps": 71, - "num_supp_output_steps": null, - "number_custom_inputs": 0, - "number_inputs": 1, - "number_supp_pcp": 0, - "numelemconn_var": "numElementConn", - "nwmConfig": "AORC", - "nwmVersion": 4.0, - "nwm_domain": null, - "nwm_geogrid": null, - "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", - "nwm_url": null, - "output_freq": 60, - "paramFlagArray": null, - "precipBiasCorrectOpt": [ - 0 - ], - "precipDownscaleOpt": [ - 0 - ], - "precip_only_flag": false, - "prev_output_date": "2013-07-01T00:00:00", - "process_window": null, - "psfcBiasCorrectOpt": [ - 0 - ], - "psfcDownscaleOpt": [ - 0 - ], - "q2BiasCorrectOpt": [ - 0 - ], - "q2dDownscaleOpt": [ - 0 - ], - "realtime_flag": false, - "refcst_flag": true, - "regrid_opt": [ - 1 - ], - "regrid_opt_supp_pcp": null, - "rqiMethod": null, - "rqiThresh": 1.0, - "runCfsNldasBiasCorrect": false, - "sinalpha_var": null, - "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, - "slope_var": null, - "slope_var_elem": null, - "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", - "sub_output_freq": null, - "sub_output_hour": null, - "suppTemporalInterp": null, - "supp_pcp_max_hours": null, - "supp_precip_dirs": null, - "supp_precip_file_types": [], - "supp_precip_forcings": [], - "supp_precip_mandatory": null, - "supp_precip_param_dir": null, - "swBiasCorrectOpt": [ - 0 - ], - "swDownscaleOpt": [ - 0 - ], - "t2BiasCorrectOpt": [ - 0 - ], - "t2dDownscaleOpt": [ - 0 - ], - "useCompression": 0, - "useFloats": 0, - "use_data_at_current_time": true, - "weightsDir": null, - "windBiasCorrect": [ - 0 - ] - }, "rank": 0, "size": 2 }, @@ -1125,5 +819,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json index 0669c8f9..5f28473b 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,6 +118,7 @@ "initial_time": 0, "time_step_seconds": 3600 }, + "dimensionality": 1, "geo_meta": { "approx_centroid_global_xy": [ -72.05503025291092, @@ -323,10 +325,29 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_time": 3600, "bmi_time_index": 1, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -354,17 +375,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -406,6 +435,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -434,24 +467,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -464,9 +502,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 @@ -530,7 +573,7 @@ "_coords": [ [ "hash_7545881362083397004_len_218", - "hash_6855503821947747755_len_218" + "hash_-1022784215004640784_len_218" ], [ [ @@ -699,355 +742,6 @@ "mesh_inds_elem": null, "mpi_config": { "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "config_options": { - "ExactExtract": null, - "actual_output_steps": 71, - "ana_flag": 0, - "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", - "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", - "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", - "aorc_conus_year_url": "{source}/{year}.zarr", - "aws": true, - "aws_obj": { - "attrs": {}, - "coords": { - "spatial_ref": { - "attrs": { - "crs_wkt": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]", - "geographic_crs_name": "WGS 84", - "grid_mapping_name": "latitude_longitude", - "horizontal_datum_name": "World Geodetic System 1984", - "inverse_flattening": 298.257223563, - "longitude_of_prime_meridian": 0.0, - "prime_meridian_name": "Greenwich", - "reference_ellipsoid_name": "WGS 84", - "semi_major_axis": 6378137.0, - "semi_minor_axis": 6356752.314245179, - "spatial_ref": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]" - }, - "data": 0, - "dims": [] - }, - "time": { - "attrs": { - "long_name": "verification time generated by wgrib2 function verftime()", - "reference_date": "2013.01.01 00:00:00 UTC", - "reference_time": 1356998400.0, - "reference_time_description": "kind of product unclear, reference date is variable, min found reference date is given", - "reference_time_type": 0, - "time_step": 0.0, - "time_step_setting": "auto" - }, - "data": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "dims": [] - }, - "x": { - "attrs": { - "long_name": "longitude", - "units": "degrees_east" - }, - "data": "hash_-5399622063019475031_len_28", - "dims": [ - "x" - ] - }, - "y": { - "attrs": { - "long_name": "latitude", - "units": "degrees_north" - }, - "data": "hash_-1457289664996315734_len_37", - "dims": [ - "y" - ] - } - }, - "data_vars": { - "APCP_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Total Precipitation", - "short_name": "APCP_surface", - "units": "kg/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DLWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Long-Wave Rad. Flux", - "short_name": "DLWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DSWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Short-Wave Rad. Flux", - "short_name": "DSWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "PRES_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Pressure", - "short_name": "PRES_surface", - "units": "Pa" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "SPFH_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Specific Humidity", - "short_name": "SPFH_2maboveground", - "units": "kg/kg" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "TMP_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Temperature", - "short_name": "TMP_2maboveground", - "units": "K" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "UGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "U-Component of Wind", - "short_name": "UGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "VGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "V-Component of Wind", - "short_name": "VGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - } - }, - "dims": { - "x": 28, - "y": 37 - } - }, - "aws_time": null, - "b_date_proc": "2013-07-01T00:00:00", - "bmi_time": 3600, - "bmi_time_index": 1, - "cfsv2EnsMember": null, - "config_path": null, - "cosalpha_var": null, - "currentCustomForceNum": 0, - "currentForceNum": 1, - "current_fcst_cycle": "2013-07-01T00:00:00", - "current_output_date": "2013-07-01T01:00:00", - "current_output_step": 1, - "current_time": "2013-07-01T01:00:00", - "customFcstFreq": [], - "customSuppPcpFreq": null, - "cycle_length_minutes": 4260, - "dScaleParamDirs": [ - "/ngen-app/data" - ], - "e_date_proc": null, - "elemconn_var": "elementConn", - "elemcoords_var": "centerCoords", - "element_id_var": "element_id", - "errFlag": 0, - "errMsg": null, - "fcst_freq": 60, - "fcst_input_horizons": [ - 4260 - ], - "fcst_input_offsets": [ - 0 - ], - "fcst_shift": 0, - "first_fcst_cycle": "2013-07-01T00:00:00", - "forceTemoralInterp": [ - 0 - ], - "forcing_output": 0, - "future_time": null, - "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", - "globalNdv": -9999.0, - "grid_meta": null, - "grid_type": "hydrofabric", - "hgt_elem_var": null, - "hgt_var": null, - "ignored_border_widths": [ - 0 - ], - "include_lqfrac": 1, - "input_force_dirs": [ - "s3://null" - ], - "input_force_mandatory": [ - 0 - ], - "input_force_types": [ - "GRIB2" - ], - "input_forcings": [ - 12 - ], - "lat_var": null, - "logFile": null, - "logHandle": null, - "lon_var": null, - "look_back": -9999, - "lwBiasCorrectOpt": [ - 0 - ], - "nFcsts": 1, - "nodecoords_var": "nodeCoords", - "num_output_steps": 71, - "num_supp_output_steps": null, - "number_custom_inputs": 0, - "number_inputs": 1, - "number_supp_pcp": 0, - "numelemconn_var": "numElementConn", - "nwmConfig": "AORC", - "nwmVersion": 4.0, - "nwm_domain": null, - "nwm_geogrid": null, - "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", - "nwm_url": null, - "output_freq": 60, - "paramFlagArray": null, - "precipBiasCorrectOpt": [ - 0 - ], - "precipDownscaleOpt": [ - 0 - ], - "precip_only_flag": false, - "prev_output_date": "2013-07-01T00:00:00", - "process_window": null, - "psfcBiasCorrectOpt": [ - 0 - ], - "psfcDownscaleOpt": [ - 0 - ], - "q2BiasCorrectOpt": [ - 0 - ], - "q2dDownscaleOpt": [ - 0 - ], - "realtime_flag": false, - "refcst_flag": true, - "regrid_opt": [ - 1 - ], - "regrid_opt_supp_pcp": null, - "rqiMethod": null, - "rqiThresh": 1.0, - "runCfsNldasBiasCorrect": false, - "sinalpha_var": null, - "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, - "slope_var": null, - "slope_var_elem": null, - "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", - "sub_output_freq": null, - "sub_output_hour": null, - "suppTemporalInterp": null, - "supp_pcp_max_hours": null, - "supp_precip_dirs": null, - "supp_precip_file_types": [], - "supp_precip_forcings": [], - "supp_precip_mandatory": null, - "supp_precip_param_dir": null, - "swBiasCorrectOpt": [ - 0 - ], - "swDownscaleOpt": [ - 0 - ], - "t2BiasCorrectOpt": [ - 0 - ], - "t2dDownscaleOpt": [ - 0 - ], - "useCompression": 0, - "useFloats": 0, - "use_data_at_current_time": true, - "weightsDir": null, - "windBiasCorrect": [ - 0 - ] - }, "rank": 1, "size": 2 }, @@ -1116,5 +810,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json index 49febe29..65e8fc17 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,6 +118,7 @@ "initial_time": 0, "time_step_seconds": 3600 }, + "dimensionality": 1, "geo_meta": { "approx_centroid_global_xy": [ -72.05503025291092, @@ -323,10 +325,29 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_time": 7200, "bmi_time_index": 2, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -354,17 +375,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -406,6 +435,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -434,24 +467,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -464,9 +502,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 @@ -530,7 +573,7 @@ "_coords": [ [ "hash_7545881362083397004_len_218", - "hash_6855503821947747755_len_218" + "hash_-1022784215004640784_len_218" ], [ [ @@ -699,355 +742,6 @@ "mesh_inds_elem": null, "mpi_config": { "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "config_options": { - "ExactExtract": null, - "actual_output_steps": 71, - "ana_flag": 0, - "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", - "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", - "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", - "aorc_conus_year_url": "{source}/{year}.zarr", - "aws": true, - "aws_obj": { - "attrs": {}, - "coords": { - "spatial_ref": { - "attrs": { - "crs_wkt": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]", - "geographic_crs_name": "WGS 84", - "grid_mapping_name": "latitude_longitude", - "horizontal_datum_name": "World Geodetic System 1984", - "inverse_flattening": 298.257223563, - "longitude_of_prime_meridian": 0.0, - "prime_meridian_name": "Greenwich", - "reference_ellipsoid_name": "WGS 84", - "semi_major_axis": 6378137.0, - "semi_minor_axis": 6356752.314245179, - "spatial_ref": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]" - }, - "data": 0, - "dims": [] - }, - "time": { - "attrs": { - "long_name": "verification time generated by wgrib2 function verftime()", - "reference_date": "2013.01.01 00:00:00 UTC", - "reference_time": 1356998400.0, - "reference_time_description": "kind of product unclear, reference date is variable, min found reference date is given", - "reference_time_type": 0, - "time_step": 0.0, - "time_step_setting": "auto" - }, - "data": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "dims": [] - }, - "x": { - "attrs": { - "long_name": "longitude", - "units": "degrees_east" - }, - "data": "hash_-5399622063019475031_len_28", - "dims": [ - "x" - ] - }, - "y": { - "attrs": { - "long_name": "latitude", - "units": "degrees_north" - }, - "data": "hash_-1457289664996315734_len_37", - "dims": [ - "y" - ] - } - }, - "data_vars": { - "APCP_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Total Precipitation", - "short_name": "APCP_surface", - "units": "kg/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DLWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Long-Wave Rad. Flux", - "short_name": "DLWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DSWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Short-Wave Rad. Flux", - "short_name": "DSWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "PRES_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Pressure", - "short_name": "PRES_surface", - "units": "Pa" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "SPFH_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Specific Humidity", - "short_name": "SPFH_2maboveground", - "units": "kg/kg" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "TMP_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Temperature", - "short_name": "TMP_2maboveground", - "units": "K" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "UGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "U-Component of Wind", - "short_name": "UGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "VGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "V-Component of Wind", - "short_name": "VGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - } - }, - "dims": { - "x": 28, - "y": 37 - } - }, - "aws_time": null, - "b_date_proc": "2013-07-01T00:00:00", - "bmi_time": 7200, - "bmi_time_index": 2, - "cfsv2EnsMember": null, - "config_path": null, - "cosalpha_var": null, - "currentCustomForceNum": 0, - "currentForceNum": 1, - "current_fcst_cycle": "2013-07-01T00:00:00", - "current_output_date": "2013-07-01T02:00:00", - "current_output_step": 2, - "current_time": "2013-07-01T02:00:00", - "customFcstFreq": [], - "customSuppPcpFreq": null, - "cycle_length_minutes": 4260, - "dScaleParamDirs": [ - "/ngen-app/data" - ], - "e_date_proc": null, - "elemconn_var": "elementConn", - "elemcoords_var": "centerCoords", - "element_id_var": "element_id", - "errFlag": 0, - "errMsg": null, - "fcst_freq": 60, - "fcst_input_horizons": [ - 4260 - ], - "fcst_input_offsets": [ - 0 - ], - "fcst_shift": 0, - "first_fcst_cycle": "2013-07-01T00:00:00", - "forceTemoralInterp": [ - 0 - ], - "forcing_output": 0, - "future_time": null, - "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", - "globalNdv": -9999.0, - "grid_meta": null, - "grid_type": "hydrofabric", - "hgt_elem_var": null, - "hgt_var": null, - "ignored_border_widths": [ - 0 - ], - "include_lqfrac": 1, - "input_force_dirs": [ - "s3://null" - ], - "input_force_mandatory": [ - 0 - ], - "input_force_types": [ - "GRIB2" - ], - "input_forcings": [ - 12 - ], - "lat_var": null, - "logFile": null, - "logHandle": null, - "lon_var": null, - "look_back": -9999, - "lwBiasCorrectOpt": [ - 0 - ], - "nFcsts": 1, - "nodecoords_var": "nodeCoords", - "num_output_steps": 71, - "num_supp_output_steps": null, - "number_custom_inputs": 0, - "number_inputs": 1, - "number_supp_pcp": 0, - "numelemconn_var": "numElementConn", - "nwmConfig": "AORC", - "nwmVersion": 4.0, - "nwm_domain": null, - "nwm_geogrid": null, - "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", - "nwm_url": null, - "output_freq": 60, - "paramFlagArray": null, - "precipBiasCorrectOpt": [ - 0 - ], - "precipDownscaleOpt": [ - 0 - ], - "precip_only_flag": false, - "prev_output_date": "2013-07-01T00:00:00", - "process_window": null, - "psfcBiasCorrectOpt": [ - 0 - ], - "psfcDownscaleOpt": [ - 0 - ], - "q2BiasCorrectOpt": [ - 0 - ], - "q2dDownscaleOpt": [ - 0 - ], - "realtime_flag": false, - "refcst_flag": true, - "regrid_opt": [ - 1 - ], - "regrid_opt_supp_pcp": null, - "rqiMethod": null, - "rqiThresh": 1.0, - "runCfsNldasBiasCorrect": false, - "sinalpha_var": null, - "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, - "slope_var": null, - "slope_var_elem": null, - "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", - "sub_output_freq": null, - "sub_output_hour": null, - "suppTemporalInterp": null, - "supp_pcp_max_hours": null, - "supp_precip_dirs": null, - "supp_precip_file_types": [], - "supp_precip_forcings": [], - "supp_precip_mandatory": null, - "supp_precip_param_dir": null, - "swBiasCorrectOpt": [ - 0 - ], - "swDownscaleOpt": [ - 0 - ], - "t2BiasCorrectOpt": [ - 0 - ], - "t2dDownscaleOpt": [ - 0 - ], - "useCompression": 0, - "useFloats": 0, - "use_data_at_current_time": true, - "weightsDir": null, - "windBiasCorrect": [ - 0 - ] - }, "rank": 1, "size": 2 }, @@ -1116,5 +810,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json index 9fbaea9a..ca8f708a 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,6 +118,7 @@ "initial_time": 0, "time_step_seconds": 3600 }, + "dimensionality": 1, "geo_meta": { "approx_centroid_global_xy": [ -72.05503025291092, @@ -323,10 +325,29 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_time": 10800, "bmi_time_index": 3, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -354,17 +375,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -406,6 +435,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -434,24 +467,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -464,9 +502,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 @@ -530,7 +573,7 @@ "_coords": [ [ "hash_7545881362083397004_len_218", - "hash_6855503821947747755_len_218" + "hash_-1022784215004640784_len_218" ], [ [ @@ -699,355 +742,6 @@ "mesh_inds_elem": null, "mpi_config": { "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "config_options": { - "ExactExtract": null, - "actual_output_steps": 71, - "ana_flag": 0, - "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", - "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", - "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", - "aorc_conus_year_url": "{source}/{year}.zarr", - "aws": true, - "aws_obj": { - "attrs": {}, - "coords": { - "spatial_ref": { - "attrs": { - "crs_wkt": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]", - "geographic_crs_name": "WGS 84", - "grid_mapping_name": "latitude_longitude", - "horizontal_datum_name": "World Geodetic System 1984", - "inverse_flattening": 298.257223563, - "longitude_of_prime_meridian": 0.0, - "prime_meridian_name": "Greenwich", - "reference_ellipsoid_name": "WGS 84", - "semi_major_axis": 6378137.0, - "semi_minor_axis": 6356752.314245179, - "spatial_ref": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]" - }, - "data": 0, - "dims": [] - }, - "time": { - "attrs": { - "long_name": "verification time generated by wgrib2 function verftime()", - "reference_date": "2013.01.01 00:00:00 UTC", - "reference_time": 1356998400.0, - "reference_time_description": "kind of product unclear, reference date is variable, min found reference date is given", - "reference_time_type": 0, - "time_step": 0.0, - "time_step_setting": "auto" - }, - "data": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "dims": [] - }, - "x": { - "attrs": { - "long_name": "longitude", - "units": "degrees_east" - }, - "data": "hash_-5399622063019475031_len_28", - "dims": [ - "x" - ] - }, - "y": { - "attrs": { - "long_name": "latitude", - "units": "degrees_north" - }, - "data": "hash_-1457289664996315734_len_37", - "dims": [ - "y" - ] - } - }, - "data_vars": { - "APCP_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Total Precipitation", - "short_name": "APCP_surface", - "units": "kg/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DLWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Long-Wave Rad. Flux", - "short_name": "DLWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "DSWRF_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Downward Short-Wave Rad. Flux", - "short_name": "DSWRF_surface", - "units": "W/m^2" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "PRES_surface": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "surface", - "long_name": "Pressure", - "short_name": "PRES_surface", - "units": "Pa" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "SPFH_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Specific Humidity", - "short_name": "SPFH_2maboveground", - "units": "kg/kg" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "TMP_2maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "2 m above ground", - "long_name": "Temperature", - "short_name": "TMP_2maboveground", - "units": "K" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "UGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "U-Component of Wind", - "short_name": "UGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - }, - "VGRD_10maboveground": { - "attrs": { - "AORC_Contact": "aorc.info@noaa.gov", - "aorc_version": "v1.1", - "crs": "EPSG:4326", - "level": "10 m above ground", - "long_name": "V-Component of Wind", - "short_name": "VGRD_10maboveground", - "units": "m/s" - }, - "data": null, - "dims": [ - "y", - "x" - ] - } - }, - "dims": { - "x": 28, - "y": 37 - } - }, - "aws_time": null, - "b_date_proc": "2013-07-01T00:00:00", - "bmi_time": 10800, - "bmi_time_index": 3, - "cfsv2EnsMember": null, - "config_path": null, - "cosalpha_var": null, - "currentCustomForceNum": 0, - "currentForceNum": 1, - "current_fcst_cycle": "2013-07-01T00:00:00", - "current_output_date": "2013-07-01T03:00:00", - "current_output_step": 3, - "current_time": "2013-07-01T03:00:00", - "customFcstFreq": [], - "customSuppPcpFreq": null, - "cycle_length_minutes": 4260, - "dScaleParamDirs": [ - "/ngen-app/data" - ], - "e_date_proc": null, - "elemconn_var": "elementConn", - "elemcoords_var": "centerCoords", - "element_id_var": "element_id", - "errFlag": 0, - "errMsg": null, - "fcst_freq": 60, - "fcst_input_horizons": [ - 4260 - ], - "fcst_input_offsets": [ - 0 - ], - "fcst_shift": 0, - "first_fcst_cycle": "2013-07-01T00:00:00", - "forceTemoralInterp": [ - 0 - ], - "forcing_output": 0, - "future_time": null, - "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", - "globalNdv": -9999.0, - "grid_meta": null, - "grid_type": "hydrofabric", - "hgt_elem_var": null, - "hgt_var": null, - "ignored_border_widths": [ - 0 - ], - "include_lqfrac": 1, - "input_force_dirs": [ - "s3://null" - ], - "input_force_mandatory": [ - 0 - ], - "input_force_types": [ - "GRIB2" - ], - "input_forcings": [ - 12 - ], - "lat_var": null, - "logFile": null, - "logHandle": null, - "lon_var": null, - "look_back": -9999, - "lwBiasCorrectOpt": [ - 0 - ], - "nFcsts": 1, - "nodecoords_var": "nodeCoords", - "num_output_steps": 71, - "num_supp_output_steps": null, - "number_custom_inputs": 0, - "number_inputs": 1, - "number_supp_pcp": 0, - "numelemconn_var": "numElementConn", - "nwmConfig": "AORC", - "nwmVersion": 4.0, - "nwm_domain": null, - "nwm_geogrid": null, - "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", - "nwm_url": null, - "output_freq": 60, - "paramFlagArray": null, - "precipBiasCorrectOpt": [ - 0 - ], - "precipDownscaleOpt": [ - 0 - ], - "precip_only_flag": false, - "prev_output_date": "2013-07-01T00:00:00", - "process_window": null, - "psfcBiasCorrectOpt": [ - 0 - ], - "psfcDownscaleOpt": [ - 0 - ], - "q2BiasCorrectOpt": [ - 0 - ], - "q2dDownscaleOpt": [ - 0 - ], - "realtime_flag": false, - "refcst_flag": true, - "regrid_opt": [ - 1 - ], - "regrid_opt_supp_pcp": null, - "rqiMethod": null, - "rqiThresh": 1.0, - "runCfsNldasBiasCorrect": false, - "sinalpha_var": null, - "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, - "slope_var": null, - "slope_var_elem": null, - "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", - "sub_output_freq": null, - "sub_output_hour": null, - "suppTemporalInterp": null, - "supp_pcp_max_hours": null, - "supp_precip_dirs": null, - "supp_precip_file_types": [], - "supp_precip_forcings": [], - "supp_precip_mandatory": null, - "supp_precip_param_dir": null, - "swBiasCorrectOpt": [ - 0 - ], - "swDownscaleOpt": [ - 0 - ], - "t2BiasCorrectOpt": [ - 0 - ], - "t2dDownscaleOpt": [ - 0 - ], - "useCompression": 0, - "useFloats": 0, - "use_data_at_current_time": true, - "weightsDir": null, - "windBiasCorrect": [ - 0 - ] - }, "rank": 1, "size": 2 }, @@ -1116,5 +810,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n1_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n1_rank0_.json index f82eb7a5..5086d888 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n1_rank0_.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,7 +118,693 @@ "initial_time": 0, "time_step_seconds": 3600 }, - "geo_meta": null, + "dimensionality": 1, + "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], + "centerCoords": null, + "config_options": { + "ExactExtract": null, + "actual_output_steps": 71, + "ana_flag": 0, + "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", + "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", + "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", + "aorc_conus_year_url": "{source}/{year}.zarr", + "aws": true, + "aws_obj": { + "attrs": {}, + "coords": { + "spatial_ref": { + "attrs": { + "crs_wkt": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]", + "geographic_crs_name": "WGS 84", + "grid_mapping_name": "latitude_longitude", + "horizontal_datum_name": "World Geodetic System 1984", + "inverse_flattening": 298.257223563, + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "reference_ellipsoid_name": "WGS 84", + "semi_major_axis": 6378137.0, + "semi_minor_axis": 6356752.314245179, + "spatial_ref": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]" + }, + "data": 0, + "dims": [] + }, + "time": { + "attrs": { + "long_name": "verification time generated by wgrib2 function verftime()", + "reference_date": "2013.01.01 00:00:00 UTC", + "reference_time": 1356998400.0, + "reference_time_description": "kind of product unclear, reference date is variable, min found reference date is given", + "reference_time_type": 0, + "time_step": 0.0, + "time_step_setting": "auto" + }, + "data": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", + "dims": [] + }, + "x": { + "attrs": { + "long_name": "longitude", + "units": "degrees_east" + }, + "data": "hash_-5399622063019475031_len_28", + "dims": [ + "x" + ] + }, + "y": { + "attrs": { + "long_name": "latitude", + "units": "degrees_north" + }, + "data": "hash_-1457289664996315734_len_37", + "dims": [ + "y" + ] + } + }, + "data_vars": { + "APCP_surface": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "surface", + "long_name": "Total Precipitation", + "short_name": "APCP_surface", + "units": "kg/m^2" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "DLWRF_surface": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "surface", + "long_name": "Downward Long-Wave Rad. Flux", + "short_name": "DLWRF_surface", + "units": "W/m^2" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "DSWRF_surface": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "surface", + "long_name": "Downward Short-Wave Rad. Flux", + "short_name": "DSWRF_surface", + "units": "W/m^2" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "PRES_surface": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "surface", + "long_name": "Pressure", + "short_name": "PRES_surface", + "units": "Pa" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "SPFH_2maboveground": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "2 m above ground", + "long_name": "Specific Humidity", + "short_name": "SPFH_2maboveground", + "units": "kg/kg" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "TMP_2maboveground": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "2 m above ground", + "long_name": "Temperature", + "short_name": "TMP_2maboveground", + "units": "K" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "UGRD_10maboveground": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "10 m above ground", + "long_name": "U-Component of Wind", + "short_name": "UGRD_10maboveground", + "units": "m/s" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "VGRD_10maboveground": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "10 m above ground", + "long_name": "V-Component of Wind", + "short_name": "VGRD_10maboveground", + "units": "m/s" + }, + "data": null, + "dims": [ + "y", + "x" + ] + } + }, + "dims": { + "x": 28, + "y": 37 + } + }, + "aws_time": null, + "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, + "bmi_time": 10800, + "bmi_time_index": 3, + "cfsv2EnsMember": null, + "cosalpha_var": null, + "currentCustomForceNum": 0, + "currentForceNum": 1, + "current_fcst_cycle": "2013-07-01T00:00:00", + "current_output_date": "2013-07-01T03:00:00", + "current_output_step": 3, + "current_time": "2013-07-01T03:00:00", + "customFcstFreq": [], + "customSuppPcpFreq": null, + "cycle_length_minutes": 4260, + "dScaleParamDirs": [ + "/ngen-app/data" + ], + "e_date_proc": null, + "elemconn_var": "elementConn", + "elemcoords_var": "centerCoords", + "element_id_var": "element_id", + "errFlag": 0, + "errMsg": null, + "fcst_freq": 60, + "fcst_input_horizons": [ + 4260 + ], + "fcst_input_offsets": [ + 0 + ], + "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], + "first_fcst_cycle": "2013-07-01T00:00:00", + "forceTemoralInterp": [ + 0 + ], + "force_count": 27, + "forcing_output": 0, + "future_time": null, + "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "globalNdv": -9999.0, + "grid_meta": null, + "grid_type": "hydrofabric", + "hgt_var": null, + "ignored_border_widths": [ + 0 + ], + "include_lqfrac": 1, + "input_force_dirs": [ + "s3://null" + ], + "input_force_mandatory": [ + 0 + ], + "input_force_types": [ + "GRIB2" + ], + "input_forcings": [ + 12 + ], + "lat_var": null, + "logFile": null, + "logHandle": null, + "lon_var": null, + "look_back": -9999, + "lwBiasCorrectOpt": [ + 0 + ], + "nFcsts": 1, + "nodecoords_var": "nodeCoords", + "num_output_steps": 71, + "num_supp_output_steps": null, + "number_custom_inputs": 0, + "number_inputs": 1, + "number_supp_pcp": 0, + "numelemconn_var": "numElementConn", + "nwmConfig": "AORC", + "nwmVersion": 4.0, + "nwm_domain": null, + "nwm_geogrid": null, + "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", + "nwm_url": null, + "output_freq": 60, + "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, + "precipBiasCorrectOpt": [ + 0 + ], + "precipDownscaleOpt": [ + 0 + ], + "precip_only_flag": false, + "prev_output_date": "2013-07-01T00:00:00", + "process_window": null, + "psfcBiasCorrectOpt": [ + 0 + ], + "psfcDownscaleOpt": [ + 0 + ], + "q2BiasCorrectOpt": [ + 0 + ], + "q2dDownscaleOpt": [ + 0 + ], + "realtime_flag": false, + "refcst_flag": true, + "regrid_opt": [ + 1 + ], + "regrid_opt_supp_pcp": null, + "rqiMethod": null, + "rqiThresh": null, + "runCfsNldasBiasCorrect": false, + "sinalpha_var": null, + "slope_azimuth_var": null, + "slope_var": null, + "spatial_meta": null, + "statusMsg": "Starting BMI finalize()", + "sub_output_freq": null, + "sub_output_hour": null, + "suppTemporalInterp": null, + "supp_input_offsets": null, + "supp_pcp_max_hours": null, + "supp_precip_count": 15, + "supp_precip_dirs": null, + "supp_precip_file_types": [], + "supp_precip_forcings": [], + "supp_precip_mandatory": null, + "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], + "swBiasCorrectOpt": [ + 0 + ], + "swDownscaleOpt": [ + 0 + ], + "t2BiasCorrectOpt": [ + 0 + ], + "t2dDownscaleOpt": [ + 0 + ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, + "useCompression": 0, + "useFloats": 0, + "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, + "weightsDir": null, + "windBiasCorrect": [ + 0 + ] + }, + "cosa_grid": null, + "crs_atts": null, + "dx_meters": null, + "dy_meters": null, + "element_ids": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ], + "element_ids_global": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ], + "elementcoords_global": [ + [ + -72.05141087733058, + 41.67904491423198 + ], + [ + -72.05118973471748, + 41.739612580494466 + ], + [ + -72.0334056658147, + 41.72174218772185 + ], + [ + -72.06530692999942, + 41.77071194478257 + ], + [ + -72.08488656934233, + 41.78930917764987 + ], + [ + -72.04984213921676, + 41.79151575447281 + ], + [ + -72.04916985395522, + 41.82451694207896 + ] + ], + "esmf_ds": null, + "esmf_grid": { + "_area": [ + null, + null + ], + "_coord_sys": null, + "_coords": [ + [ + "hash_2904886930620536845_len_582", + "hash_902249663378761532_len_582" + ], + [ + [ + -72.05141087733058, + -72.05118973471748, + -72.0334056658147, + -72.06530692999942, + -72.08488656934233, + -72.04984213921676, + -72.04916985395522 + ], + [ + 41.67904491423198, + 41.739612580494466, + 41.72174218772185, + 41.77071194478257, + 41.78930917764987, + 41.79151575447281, + 41.82451694207896 + ] + ] + ], + "_finalized": false, + "_mask": [ + null, + null + ], + "_meta": {}, + "_parametric_dim": 2, + "_rank": 1, + "_size": [ + 582, + 7 + ], + "_size_owned": [ + 582, + 7 + ], + "_spatial_dim": null, + "_struct": {} + }, + "esmf_lat": null, + "esmf_lon": null, + "geogrid_ds": { + "attrs": { + "gridType": "unstructured", + "version": "0.9" + }, + "coords": {}, + "data_vars": { + "centerCoords": { + "attrs": { + "units": "degrees" + }, + "data": [ + [ + -72.05141087733058, + 41.67904491423198 + ], + [ + -72.05118973471748, + 41.739612580494466 + ], + [ + -72.0334056658147, + 41.72174218772185 + ], + [ + -72.06530692999942, + 41.77071194478257 + ], + [ + -72.08488656934233, + 41.78930917764987 + ], + [ + -72.04984213921676, + 41.79151575447281 + ], + [ + -72.04916985395522, + 41.82451694207896 + ] + ], + "dims": [ + "elementCount", + "coordDim" + ] + }, + "elementConn": { + "attrs": { + "long_name": "Node Indices that define the element connectivity" + }, + "data": "hash_5853656558824341831_len_781", + "dims": [ + "connectionCount" + ] + }, + "element_id": { + "attrs": { + "long_name": "False 32-bit catchment IDs use for ESMF mesh generation" + }, + "data": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ], + "dims": [ + "elementCount" + ] + }, + "nodeCoords": { + "attrs": { + "units": "degrees" + }, + "data": null, + "dims": [ + "nodeCount", + "coordDim" + ] + }, + "numElementConn": { + "attrs": { + "long_name": "Number of nodes per element" + }, + "data": [ + 160, + 119, + 95, + 68, + 69, + 173, + 97 + ], + "dims": [ + "elementCount" + ] + } + }, + "dims": { + "connectionCount": 781, + "coordDim": 2, + "elementCount": 7, + "nodeCount": 582 + } + }, + "height": null, + "height_elem": null, + "heights_global": null, + "inds": null, + "lat_bounds": "hash_902249663378761532_len_582", + "latitude_grid": [ + 41.67904491423198, + 41.739612580494466, + 41.72174218772185, + 41.77071194478257, + 41.78930917764987, + 41.79151575447281, + 41.82451694207896 + ], + "latitude_grid_elem": null, + "lon_bounds": "hash_2904886930620536845_len_582", + "longitude_grid": [ + -72.05141087733058, + -72.05118973471748, + -72.0334056658147, + -72.06530692999942, + -72.08488656934233, + -72.04984213921676, + -72.04916985395522 + ], + "longitude_grid_elem": null, + "mesh_inds": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ], + "mesh_inds_elem": null, + "mpi_config": { + "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", + "rank": 0, + "size": 1 + }, + "nodeCoords": null, + "nx_global": 7, + "nx_global_elem": null, + "nx_local": 7, + "nx_local_elem": null, + "ny_global": 7, + "ny_global_elem": null, + "ny_local": 7, + "ny_local_elem": null, + "pet_element_inds": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ], + "sina_grid": null, + "slope": null, + "slope_elem": null, + "slopes_global": null, + "slp_azi": null, + "slp_azi_elem": null, + "slp_azi_global": null, + "spatial_global_atts": null, + "spatial_metadata_exists": false, + "x_coord_atts": null, + "x_coords": null, + "x_lower_bound": null, + "x_upper_bound": null, + "y_coord_atts": null, + "y_coords": null, + "y_lower_bound": null, + "y_upper_bound": null + }, "grid_4": { "_grid_x": [ -72.05141087733057, @@ -159,5 +846,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank0_.json index f856bf7f..fdcd8db2 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank0_.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,7 +118,672 @@ "initial_time": 0, "time_step_seconds": 3600 }, - "geo_meta": null, + "dimensionality": 1, + "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], + "centerCoords": null, + "config_options": { + "ExactExtract": null, + "actual_output_steps": 71, + "ana_flag": 0, + "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", + "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", + "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", + "aorc_conus_year_url": "{source}/{year}.zarr", + "aws": true, + "aws_obj": { + "attrs": {}, + "coords": { + "spatial_ref": { + "attrs": { + "crs_wkt": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]", + "geographic_crs_name": "WGS 84", + "grid_mapping_name": "latitude_longitude", + "horizontal_datum_name": "World Geodetic System 1984", + "inverse_flattening": 298.257223563, + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "reference_ellipsoid_name": "WGS 84", + "semi_major_axis": 6378137.0, + "semi_minor_axis": 6356752.314245179, + "spatial_ref": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]" + }, + "data": 0, + "dims": [] + }, + "time": { + "attrs": { + "long_name": "verification time generated by wgrib2 function verftime()", + "reference_date": "2013.01.01 00:00:00 UTC", + "reference_time": 1356998400.0, + "reference_time_description": "kind of product unclear, reference date is variable, min found reference date is given", + "reference_time_type": 0, + "time_step": 0.0, + "time_step_setting": "auto" + }, + "data": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", + "dims": [] + }, + "x": { + "attrs": { + "long_name": "longitude", + "units": "degrees_east" + }, + "data": "hash_-5399622063019475031_len_28", + "dims": [ + "x" + ] + }, + "y": { + "attrs": { + "long_name": "latitude", + "units": "degrees_north" + }, + "data": "hash_-1457289664996315734_len_37", + "dims": [ + "y" + ] + } + }, + "data_vars": { + "APCP_surface": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "surface", + "long_name": "Total Precipitation", + "short_name": "APCP_surface", + "units": "kg/m^2" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "DLWRF_surface": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "surface", + "long_name": "Downward Long-Wave Rad. Flux", + "short_name": "DLWRF_surface", + "units": "W/m^2" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "DSWRF_surface": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "surface", + "long_name": "Downward Short-Wave Rad. Flux", + "short_name": "DSWRF_surface", + "units": "W/m^2" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "PRES_surface": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "surface", + "long_name": "Pressure", + "short_name": "PRES_surface", + "units": "Pa" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "SPFH_2maboveground": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "2 m above ground", + "long_name": "Specific Humidity", + "short_name": "SPFH_2maboveground", + "units": "kg/kg" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "TMP_2maboveground": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "2 m above ground", + "long_name": "Temperature", + "short_name": "TMP_2maboveground", + "units": "K" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "UGRD_10maboveground": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "10 m above ground", + "long_name": "U-Component of Wind", + "short_name": "UGRD_10maboveground", + "units": "m/s" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "VGRD_10maboveground": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "10 m above ground", + "long_name": "V-Component of Wind", + "short_name": "VGRD_10maboveground", + "units": "m/s" + }, + "data": null, + "dims": [ + "y", + "x" + ] + } + }, + "dims": { + "x": 28, + "y": 37 + } + }, + "aws_time": null, + "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, + "bmi_time": 10800, + "bmi_time_index": 3, + "cfsv2EnsMember": null, + "cosalpha_var": null, + "currentCustomForceNum": 0, + "currentForceNum": 1, + "current_fcst_cycle": "2013-07-01T00:00:00", + "current_output_date": "2013-07-01T03:00:00", + "current_output_step": 3, + "current_time": "2013-07-01T03:00:00", + "customFcstFreq": [], + "customSuppPcpFreq": null, + "cycle_length_minutes": 4260, + "dScaleParamDirs": [ + "/ngen-app/data" + ], + "e_date_proc": null, + "elemconn_var": "elementConn", + "elemcoords_var": "centerCoords", + "element_id_var": "element_id", + "errFlag": 0, + "errMsg": null, + "fcst_freq": 60, + "fcst_input_horizons": [ + 4260 + ], + "fcst_input_offsets": [ + 0 + ], + "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], + "first_fcst_cycle": "2013-07-01T00:00:00", + "forceTemoralInterp": [ + 0 + ], + "force_count": 27, + "forcing_output": 0, + "future_time": null, + "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "globalNdv": -9999.0, + "grid_meta": null, + "grid_type": "hydrofabric", + "hgt_var": null, + "ignored_border_widths": [ + 0 + ], + "include_lqfrac": 1, + "input_force_dirs": [ + "s3://null" + ], + "input_force_mandatory": [ + 0 + ], + "input_force_types": [ + "GRIB2" + ], + "input_forcings": [ + 12 + ], + "lat_var": null, + "logFile": null, + "logHandle": null, + "lon_var": null, + "look_back": -9999, + "lwBiasCorrectOpt": [ + 0 + ], + "nFcsts": 1, + "nodecoords_var": "nodeCoords", + "num_output_steps": 71, + "num_supp_output_steps": null, + "number_custom_inputs": 0, + "number_inputs": 1, + "number_supp_pcp": 0, + "numelemconn_var": "numElementConn", + "nwmConfig": "AORC", + "nwmVersion": 4.0, + "nwm_domain": null, + "nwm_geogrid": null, + "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", + "nwm_url": null, + "output_freq": 60, + "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, + "precipBiasCorrectOpt": [ + 0 + ], + "precipDownscaleOpt": [ + 0 + ], + "precip_only_flag": false, + "prev_output_date": "2013-07-01T00:00:00", + "process_window": null, + "psfcBiasCorrectOpt": [ + 0 + ], + "psfcDownscaleOpt": [ + 0 + ], + "q2BiasCorrectOpt": [ + 0 + ], + "q2dDownscaleOpt": [ + 0 + ], + "realtime_flag": false, + "refcst_flag": true, + "regrid_opt": [ + 1 + ], + "regrid_opt_supp_pcp": null, + "rqiMethod": null, + "rqiThresh": null, + "runCfsNldasBiasCorrect": false, + "sinalpha_var": null, + "slope_azimuth_var": null, + "slope_var": null, + "spatial_meta": null, + "statusMsg": "Starting BMI finalize()", + "sub_output_freq": null, + "sub_output_hour": null, + "suppTemporalInterp": null, + "supp_input_offsets": null, + "supp_pcp_max_hours": null, + "supp_precip_count": 15, + "supp_precip_dirs": null, + "supp_precip_file_types": [], + "supp_precip_forcings": [], + "supp_precip_mandatory": null, + "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], + "swBiasCorrectOpt": [ + 0 + ], + "swDownscaleOpt": [ + 0 + ], + "t2BiasCorrectOpt": [ + 0 + ], + "t2dDownscaleOpt": [ + 0 + ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, + "useCompression": 0, + "useFloats": 0, + "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, + "weightsDir": null, + "windBiasCorrect": [ + 0 + ] + }, + "cosa_grid": null, + "crs_atts": null, + "dx_meters": null, + "dy_meters": null, + "element_ids": [ + 0, + 1, + 2, + 3 + ], + "element_ids_global": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ], + "elementcoords_global": [ + [ + -72.05141087733058, + 41.67904491423198 + ], + [ + -72.05118973471748, + 41.739612580494466 + ], + [ + -72.0334056658147, + 41.72174218772185 + ], + [ + -72.06530692999942, + 41.77071194478257 + ], + [ + -72.08488656934233, + 41.78930917764987 + ], + [ + -72.04984213921676, + 41.79151575447281 + ], + [ + -72.04916985395522, + 41.82451694207896 + ] + ], + "esmf_ds": null, + "esmf_grid": { + "_area": [ + null, + null + ], + "_coord_sys": null, + "_coords": [ + [ + "hash_351698735832278484_len_364", + "hash_-6066707896656524608_len_364" + ], + [ + [ + -72.05141087733058, + -72.05118973471748, + -72.0334056658147, + -72.06530692999942 + ], + [ + 41.67904491423198, + 41.739612580494466, + 41.72174218772185, + 41.77071194478257 + ] + ] + ], + "_finalized": false, + "_mask": [ + null, + null + ], + "_meta": {}, + "_parametric_dim": 2, + "_rank": 1, + "_size": [ + 364, + 4 + ], + "_size_owned": [ + 364, + 4 + ], + "_spatial_dim": null, + "_struct": {} + }, + "esmf_lat": null, + "esmf_lon": null, + "geogrid_ds": { + "attrs": { + "gridType": "unstructured", + "version": "0.9" + }, + "coords": {}, + "data_vars": { + "centerCoords": { + "attrs": { + "units": "degrees" + }, + "data": [ + [ + -72.05141087733058, + 41.67904491423198 + ], + [ + -72.05118973471748, + 41.739612580494466 + ], + [ + -72.0334056658147, + 41.72174218772185 + ], + [ + -72.06530692999942, + 41.77071194478257 + ], + [ + -72.08488656934233, + 41.78930917764987 + ], + [ + -72.04984213921676, + 41.79151575447281 + ], + [ + -72.04916985395522, + 41.82451694207896 + ] + ], + "dims": [ + "elementCount", + "coordDim" + ] + }, + "elementConn": { + "attrs": { + "long_name": "Node Indices that define the element connectivity" + }, + "data": "hash_5853656558824341831_len_781", + "dims": [ + "connectionCount" + ] + }, + "element_id": { + "attrs": { + "long_name": "False 32-bit catchment IDs use for ESMF mesh generation" + }, + "data": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ], + "dims": [ + "elementCount" + ] + }, + "nodeCoords": { + "attrs": { + "units": "degrees" + }, + "data": null, + "dims": [ + "nodeCount", + "coordDim" + ] + }, + "numElementConn": { + "attrs": { + "long_name": "Number of nodes per element" + }, + "data": [ + 160, + 119, + 95, + 68, + 69, + 173, + 97 + ], + "dims": [ + "elementCount" + ] + } + }, + "dims": { + "connectionCount": 781, + "coordDim": 2, + "elementCount": 7, + "nodeCount": 582 + } + }, + "height": null, + "height_elem": null, + "heights_global": null, + "inds": null, + "lat_bounds": "hash_902249663378761532_len_582", + "latitude_grid": [ + 41.67904491423198, + 41.739612580494466, + 41.72174218772185, + 41.77071194478257 + ], + "latitude_grid_elem": null, + "lon_bounds": "hash_2904886930620536845_len_582", + "longitude_grid": [ + -72.05141087733058, + -72.05118973471748, + -72.0334056658147, + -72.06530692999942 + ], + "longitude_grid_elem": null, + "mesh_inds": [ + 0, + 1, + 2, + 3 + ], + "mesh_inds_elem": null, + "mpi_config": { + "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", + "rank": 0, + "size": 2 + }, + "nodeCoords": null, + "nx_global": 7, + "nx_global_elem": null, + "nx_local": 4, + "nx_local_elem": null, + "ny_global": 7, + "ny_global_elem": null, + "ny_local": 4, + "ny_local_elem": null, + "pet_element_inds": [ + 0, + 1, + 2, + 3 + ], + "sina_grid": null, + "slope": null, + "slope_elem": null, + "slopes_global": null, + "slp_azi": null, + "slp_azi_elem": null, + "slp_azi_global": null, + "spatial_global_atts": null, + "spatial_metadata_exists": false, + "x_coord_atts": null, + "x_coords": null, + "x_lower_bound": null, + "x_upper_bound": null, + "y_coord_atts": null, + "y_coords": null, + "y_lower_bound": null, + "y_upper_bound": null + }, "grid_4": { "_grid_x": [ -72.05141087733057, @@ -153,5 +819,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank1_.json b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank1_.json index e15a50a2..4339fbe6 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank1_.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,7 +118,665 @@ "initial_time": 0, "time_step_seconds": 3600 }, - "geo_meta": null, + "dimensionality": 1, + "geo_meta": { + "approx_centroid_global_xy": [ + -72.05503025291092, + 41.7594933573475 + ], + "centerCoords": null, + "config_options": { + "ExactExtract": null, + "actual_output_steps": 71, + "ana_flag": 0, + "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", + "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", + "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", + "aorc_conus_year_url": "{source}/{year}.zarr", + "aws": true, + "aws_obj": { + "attrs": {}, + "coords": { + "spatial_ref": { + "attrs": { + "crs_wkt": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]", + "geographic_crs_name": "WGS 84", + "grid_mapping_name": "latitude_longitude", + "horizontal_datum_name": "World Geodetic System 1984", + "inverse_flattening": 298.257223563, + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "reference_ellipsoid_name": "WGS 84", + "semi_major_axis": 6378137.0, + "semi_minor_axis": 6356752.314245179, + "spatial_ref": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST],AUTHORITY[\"EPSG\",\"4326\"]]" + }, + "data": 0, + "dims": [] + }, + "time": { + "attrs": { + "long_name": "verification time generated by wgrib2 function verftime()", + "reference_date": "2013.01.01 00:00:00 UTC", + "reference_time": 1356998400.0, + "reference_time_description": "kind of product unclear, reference date is variable, min found reference date is given", + "reference_time_type": 0, + "time_step": 0.0, + "time_step_setting": "auto" + }, + "data": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", + "dims": [] + }, + "x": { + "attrs": { + "long_name": "longitude", + "units": "degrees_east" + }, + "data": "hash_-5399622063019475031_len_28", + "dims": [ + "x" + ] + }, + "y": { + "attrs": { + "long_name": "latitude", + "units": "degrees_north" + }, + "data": "hash_-1457289664996315734_len_37", + "dims": [ + "y" + ] + } + }, + "data_vars": { + "APCP_surface": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "surface", + "long_name": "Total Precipitation", + "short_name": "APCP_surface", + "units": "kg/m^2" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "DLWRF_surface": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "surface", + "long_name": "Downward Long-Wave Rad. Flux", + "short_name": "DLWRF_surface", + "units": "W/m^2" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "DSWRF_surface": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "surface", + "long_name": "Downward Short-Wave Rad. Flux", + "short_name": "DSWRF_surface", + "units": "W/m^2" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "PRES_surface": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "surface", + "long_name": "Pressure", + "short_name": "PRES_surface", + "units": "Pa" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "SPFH_2maboveground": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "2 m above ground", + "long_name": "Specific Humidity", + "short_name": "SPFH_2maboveground", + "units": "kg/kg" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "TMP_2maboveground": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "2 m above ground", + "long_name": "Temperature", + "short_name": "TMP_2maboveground", + "units": "K" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "UGRD_10maboveground": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "10 m above ground", + "long_name": "U-Component of Wind", + "short_name": "UGRD_10maboveground", + "units": "m/s" + }, + "data": null, + "dims": [ + "y", + "x" + ] + }, + "VGRD_10maboveground": { + "attrs": { + "AORC_Contact": "aorc.info@noaa.gov", + "aorc_version": "v1.1", + "crs": "EPSG:4326", + "level": "10 m above ground", + "long_name": "V-Component of Wind", + "short_name": "VGRD_10maboveground", + "units": "m/s" + }, + "data": null, + "dims": [ + "y", + "x" + ] + } + }, + "dims": { + "x": 28, + "y": 37 + } + }, + "aws_time": null, + "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, + "bmi_time": 10800, + "bmi_time_index": 3, + "cfsv2EnsMember": null, + "cosalpha_var": null, + "currentCustomForceNum": 0, + "currentForceNum": 1, + "current_fcst_cycle": "2013-07-01T00:00:00", + "current_output_date": "2013-07-01T03:00:00", + "current_output_step": 3, + "current_time": "2013-07-01T03:00:00", + "customFcstFreq": [], + "customSuppPcpFreq": null, + "cycle_length_minutes": 4260, + "dScaleParamDirs": [ + "/ngen-app/data" + ], + "e_date_proc": null, + "elemconn_var": "elementConn", + "elemcoords_var": "centerCoords", + "element_id_var": "element_id", + "errFlag": 0, + "errMsg": null, + "fcst_freq": 60, + "fcst_input_horizons": [ + 4260 + ], + "fcst_input_offsets": [ + 0 + ], + "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], + "first_fcst_cycle": "2013-07-01T00:00:00", + "forceTemoralInterp": [ + 0 + ], + "force_count": 27, + "forcing_output": 0, + "future_time": null, + "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "globalNdv": -9999.0, + "grid_meta": null, + "grid_type": "hydrofabric", + "hgt_var": null, + "ignored_border_widths": [ + 0 + ], + "include_lqfrac": 1, + "input_force_dirs": [ + "s3://null" + ], + "input_force_mandatory": [ + 0 + ], + "input_force_types": [ + "GRIB2" + ], + "input_forcings": [ + 12 + ], + "lat_var": null, + "logFile": null, + "logHandle": null, + "lon_var": null, + "look_back": -9999, + "lwBiasCorrectOpt": [ + 0 + ], + "nFcsts": 1, + "nodecoords_var": "nodeCoords", + "num_output_steps": 71, + "num_supp_output_steps": null, + "number_custom_inputs": 0, + "number_inputs": 1, + "number_supp_pcp": 0, + "numelemconn_var": "numElementConn", + "nwmConfig": "AORC", + "nwmVersion": 4.0, + "nwm_domain": null, + "nwm_geogrid": null, + "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", + "nwm_url": null, + "output_freq": 60, + "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, + "precipBiasCorrectOpt": [ + 0 + ], + "precipDownscaleOpt": [ + 0 + ], + "precip_only_flag": false, + "prev_output_date": "2013-07-01T00:00:00", + "process_window": null, + "psfcBiasCorrectOpt": [ + 0 + ], + "psfcDownscaleOpt": [ + 0 + ], + "q2BiasCorrectOpt": [ + 0 + ], + "q2dDownscaleOpt": [ + 0 + ], + "realtime_flag": false, + "refcst_flag": true, + "regrid_opt": [ + 1 + ], + "regrid_opt_supp_pcp": null, + "rqiMethod": null, + "rqiThresh": null, + "runCfsNldasBiasCorrect": false, + "sinalpha_var": null, + "slope_azimuth_var": null, + "slope_var": null, + "spatial_meta": null, + "statusMsg": "Starting BMI finalize()", + "sub_output_freq": null, + "sub_output_hour": null, + "suppTemporalInterp": null, + "supp_input_offsets": null, + "supp_pcp_max_hours": null, + "supp_precip_count": 15, + "supp_precip_dirs": null, + "supp_precip_file_types": [], + "supp_precip_forcings": [], + "supp_precip_mandatory": null, + "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], + "swBiasCorrectOpt": [ + 0 + ], + "swDownscaleOpt": [ + 0 + ], + "t2BiasCorrectOpt": [ + 0 + ], + "t2dDownscaleOpt": [ + 0 + ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, + "useCompression": 0, + "useFloats": 0, + "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, + "weightsDir": null, + "windBiasCorrect": [ + 0 + ] + }, + "cosa_grid": null, + "crs_atts": null, + "dx_meters": null, + "dy_meters": null, + "element_ids": [ + 4, + 5, + 6 + ], + "element_ids_global": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ], + "elementcoords_global": [ + [ + -72.05141087733058, + 41.67904491423198 + ], + [ + -72.05118973471748, + 41.739612580494466 + ], + [ + -72.0334056658147, + 41.72174218772185 + ], + [ + -72.06530692999942, + 41.77071194478257 + ], + [ + -72.08488656934233, + 41.78930917764987 + ], + [ + -72.04984213921676, + 41.79151575447281 + ], + [ + -72.04916985395522, + 41.82451694207896 + ] + ], + "esmf_ds": null, + "esmf_grid": { + "_area": [ + null, + null + ], + "_coord_sys": null, + "_coords": [ + [ + "hash_7545881362083397004_len_218", + "hash_-1022784215004640784_len_218" + ], + [ + [ + -72.08488656934233, + -72.04984213921676, + -72.04916985395522 + ], + [ + 41.78930917764987, + 41.79151575447281, + 41.82451694207896 + ] + ] + ], + "_finalized": false, + "_mask": [ + null, + null + ], + "_meta": {}, + "_parametric_dim": 2, + "_rank": 1, + "_size": [ + 278, + 3 + ], + "_size_owned": [ + 218, + 3 + ], + "_spatial_dim": null, + "_struct": {} + }, + "esmf_lat": null, + "esmf_lon": null, + "geogrid_ds": { + "attrs": { + "gridType": "unstructured", + "version": "0.9" + }, + "coords": {}, + "data_vars": { + "centerCoords": { + "attrs": { + "units": "degrees" + }, + "data": [ + [ + -72.05141087733058, + 41.67904491423198 + ], + [ + -72.05118973471748, + 41.739612580494466 + ], + [ + -72.0334056658147, + 41.72174218772185 + ], + [ + -72.06530692999942, + 41.77071194478257 + ], + [ + -72.08488656934233, + 41.78930917764987 + ], + [ + -72.04984213921676, + 41.79151575447281 + ], + [ + -72.04916985395522, + 41.82451694207896 + ] + ], + "dims": [ + "elementCount", + "coordDim" + ] + }, + "elementConn": { + "attrs": { + "long_name": "Node Indices that define the element connectivity" + }, + "data": "hash_5853656558824341831_len_781", + "dims": [ + "connectionCount" + ] + }, + "element_id": { + "attrs": { + "long_name": "False 32-bit catchment IDs use for ESMF mesh generation" + }, + "data": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ], + "dims": [ + "elementCount" + ] + }, + "nodeCoords": { + "attrs": { + "units": "degrees" + }, + "data": null, + "dims": [ + "nodeCount", + "coordDim" + ] + }, + "numElementConn": { + "attrs": { + "long_name": "Number of nodes per element" + }, + "data": [ + 160, + 119, + 95, + 68, + 69, + 173, + 97 + ], + "dims": [ + "elementCount" + ] + } + }, + "dims": { + "connectionCount": 781, + "coordDim": 2, + "elementCount": 7, + "nodeCount": 582 + } + }, + "height": null, + "height_elem": null, + "heights_global": null, + "inds": null, + "lat_bounds": null, + "latitude_grid": [ + 41.78930917764987, + 41.79151575447281, + 41.82451694207896 + ], + "latitude_grid_elem": null, + "lon_bounds": null, + "longitude_grid": [ + -72.08488656934233, + -72.04984213921676, + -72.04916985395522 + ], + "longitude_grid_elem": null, + "mesh_inds": [ + 4, + 5, + 6 + ], + "mesh_inds_elem": null, + "mpi_config": { + "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", + "rank": 1, + "size": 2 + }, + "nodeCoords": null, + "nx_global": 7, + "nx_global_elem": null, + "nx_local": 3, + "nx_local_elem": null, + "ny_global": 7, + "ny_global_elem": null, + "ny_local": 3, + "ny_local_elem": null, + "pet_element_inds": [ + 4, + 5, + 6 + ], + "sina_grid": null, + "slope": null, + "slope_elem": null, + "slopes_global": null, + "slp_azi": null, + "slp_azi_elem": null, + "slp_azi_global": null, + "spatial_global_atts": null, + "spatial_metadata_exists": false, + "x_coord_atts": null, + "x_coords": null, + "x_lower_bound": null, + "x_upper_bound": null, + "y_coord_atts": null, + "y_coords": null, + "y_lower_bound": null, + "y_upper_bound": null + }, "grid_4": { "_grid_x": [ -72.08488656934233, @@ -151,5 +810,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json index 15342a9c..0349bee9 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,6 +118,7 @@ "initial_time": 0, "time_step_seconds": 3600 }, + "dimensionality": 1, "geo_meta": { "approx_centroid_global_xy": [ -72.05503025291092, @@ -135,10 +137,29 @@ "aws_obj": null, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_time": null, "bmi_time_index": 0, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "current_fcst_cycle": null, "current_output_date": null, @@ -164,17 +185,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": null, "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -216,6 +245,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -244,24 +277,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, "statusMsg": "func esmf_mesh_retry finished after 1 attempts.", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -274,9 +312,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 @@ -533,165 +576,6 @@ "mesh_inds_elem": null, "mpi_config": { "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "config_options": { - "ExactExtract": null, - "actual_output_steps": 71, - "ana_flag": 0, - "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", - "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", - "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", - "aorc_conus_year_url": "{source}/{year}.zarr", - "aws": true, - "aws_obj": null, - "aws_time": null, - "b_date_proc": "2013-07-01T00:00:00", - "bmi_time": null, - "bmi_time_index": 0, - "cfsv2EnsMember": null, - "config_path": null, - "cosalpha_var": null, - "current_fcst_cycle": null, - "current_output_date": null, - "current_output_step": null, - "current_time": null, - "customFcstFreq": [], - "customSuppPcpFreq": null, - "cycle_length_minutes": 4260, - "dScaleParamDirs": [ - "/ngen-app/data" - ], - "e_date_proc": null, - "elemconn_var": "elementConn", - "elemcoords_var": "centerCoords", - "element_id_var": "element_id", - "errFlag": 0, - "errMsg": null, - "fcst_freq": 60, - "fcst_input_horizons": [ - 4260 - ], - "fcst_input_offsets": [ - 0 - ], - "fcst_shift": 0, - "first_fcst_cycle": null, - "forceTemoralInterp": [ - 0 - ], - "forcing_output": 0, - "future_time": null, - "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", - "globalNdv": -9999.0, - "grid_meta": null, - "grid_type": "hydrofabric", - "hgt_elem_var": null, - "hgt_var": null, - "ignored_border_widths": [ - 0 - ], - "include_lqfrac": 1, - "input_force_dirs": [ - "s3://null" - ], - "input_force_mandatory": [ - 0 - ], - "input_force_types": [ - "GRIB2" - ], - "input_forcings": [ - 12 - ], - "lat_var": null, - "logFile": null, - "logHandle": null, - "lon_var": null, - "look_back": -9999, - "lwBiasCorrectOpt": [ - 0 - ], - "nFcsts": 1, - "nodecoords_var": "nodeCoords", - "num_output_steps": 71, - "num_supp_output_steps": null, - "number_custom_inputs": 0, - "number_inputs": 1, - "number_supp_pcp": 0, - "numelemconn_var": "numElementConn", - "nwmConfig": "AORC", - "nwmVersion": 4.0, - "nwm_domain": null, - "nwm_geogrid": null, - "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", - "nwm_url": null, - "output_freq": 60, - "paramFlagArray": null, - "precipBiasCorrectOpt": [ - 0 - ], - "precipDownscaleOpt": [ - 0 - ], - "precip_only_flag": false, - "prev_output_date": null, - "process_window": null, - "psfcBiasCorrectOpt": [ - 0 - ], - "psfcDownscaleOpt": [ - 0 - ], - "q2BiasCorrectOpt": [ - 0 - ], - "q2dDownscaleOpt": [ - 0 - ], - "realtime_flag": false, - "refcst_flag": true, - "regrid_opt": [ - 1 - ], - "regrid_opt_supp_pcp": null, - "rqiMethod": null, - "rqiThresh": 1.0, - "runCfsNldasBiasCorrect": false, - "sinalpha_var": null, - "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, - "slope_var": null, - "slope_var_elem": null, - "spatial_meta": null, - "statusMsg": "func esmf_mesh_retry finished after 1 attempts.", - "sub_output_freq": null, - "sub_output_hour": null, - "suppTemporalInterp": null, - "supp_pcp_max_hours": null, - "supp_precip_dirs": null, - "supp_precip_file_types": [], - "supp_precip_forcings": [], - "supp_precip_mandatory": null, - "supp_precip_param_dir": null, - "swBiasCorrectOpt": [ - 0 - ], - "swDownscaleOpt": [ - 0 - ], - "t2BiasCorrectOpt": [ - 0 - ], - "t2dDownscaleOpt": [ - 0 - ], - "useCompression": 0, - "useFloats": 0, - "use_data_at_current_time": true, - "weightsDir": null, - "windBiasCorrect": [ - 0 - ] - }, "rank": 0, "size": 1 }, @@ -772,5 +656,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json index c3476459..1d8b1d20 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,6 +118,7 @@ "initial_time": 0, "time_step_seconds": 3600 }, + "dimensionality": 1, "geo_meta": { "approx_centroid_global_xy": [ -72.05503025291092, @@ -135,10 +137,29 @@ "aws_obj": null, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_time": null, "bmi_time_index": 0, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "current_fcst_cycle": null, "current_output_date": null, @@ -164,17 +185,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": null, "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -216,6 +245,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -244,24 +277,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, "statusMsg": "func esmf_mesh_retry finished after 1 attempts.", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -274,9 +312,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 @@ -341,7 +384,7 @@ "_coords": [ [ "hash_351698735832278484_len_364", - "hash_1673925891551318769_len_364" + "hash_-6066707896656524608_len_364" ], [ [ @@ -490,7 +533,7 @@ "height_elem": null, "heights_global": null, "inds": null, - "lat_bounds": "hash_2117205719057770460_len_582", + "lat_bounds": "hash_902249663378761532_len_582", "latitude_grid": [ 41.67904491423198, 41.73961258049449, @@ -515,165 +558,6 @@ "mesh_inds_elem": null, "mpi_config": { "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "config_options": { - "ExactExtract": null, - "actual_output_steps": 71, - "ana_flag": 0, - "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", - "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", - "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", - "aorc_conus_year_url": "{source}/{year}.zarr", - "aws": true, - "aws_obj": null, - "aws_time": null, - "b_date_proc": "2013-07-01T00:00:00", - "bmi_time": null, - "bmi_time_index": 0, - "cfsv2EnsMember": null, - "config_path": null, - "cosalpha_var": null, - "current_fcst_cycle": null, - "current_output_date": null, - "current_output_step": null, - "current_time": null, - "customFcstFreq": [], - "customSuppPcpFreq": null, - "cycle_length_minutes": 4260, - "dScaleParamDirs": [ - "/ngen-app/data" - ], - "e_date_proc": null, - "elemconn_var": "elementConn", - "elemcoords_var": "centerCoords", - "element_id_var": "element_id", - "errFlag": 0, - "errMsg": null, - "fcst_freq": 60, - "fcst_input_horizons": [ - 4260 - ], - "fcst_input_offsets": [ - 0 - ], - "fcst_shift": 0, - "first_fcst_cycle": null, - "forceTemoralInterp": [ - 0 - ], - "forcing_output": 0, - "future_time": null, - "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", - "globalNdv": -9999.0, - "grid_meta": null, - "grid_type": "hydrofabric", - "hgt_elem_var": null, - "hgt_var": null, - "ignored_border_widths": [ - 0 - ], - "include_lqfrac": 1, - "input_force_dirs": [ - "s3://null" - ], - "input_force_mandatory": [ - 0 - ], - "input_force_types": [ - "GRIB2" - ], - "input_forcings": [ - 12 - ], - "lat_var": null, - "logFile": null, - "logHandle": null, - "lon_var": null, - "look_back": -9999, - "lwBiasCorrectOpt": [ - 0 - ], - "nFcsts": 1, - "nodecoords_var": "nodeCoords", - "num_output_steps": 71, - "num_supp_output_steps": null, - "number_custom_inputs": 0, - "number_inputs": 1, - "number_supp_pcp": 0, - "numelemconn_var": "numElementConn", - "nwmConfig": "AORC", - "nwmVersion": 4.0, - "nwm_domain": null, - "nwm_geogrid": null, - "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", - "nwm_url": null, - "output_freq": 60, - "paramFlagArray": null, - "precipBiasCorrectOpt": [ - 0 - ], - "precipDownscaleOpt": [ - 0 - ], - "precip_only_flag": false, - "prev_output_date": null, - "process_window": null, - "psfcBiasCorrectOpt": [ - 0 - ], - "psfcDownscaleOpt": [ - 0 - ], - "q2BiasCorrectOpt": [ - 0 - ], - "q2dDownscaleOpt": [ - 0 - ], - "realtime_flag": false, - "refcst_flag": true, - "regrid_opt": [ - 1 - ], - "regrid_opt_supp_pcp": null, - "rqiMethod": null, - "rqiThresh": 1.0, - "runCfsNldasBiasCorrect": false, - "sinalpha_var": null, - "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, - "slope_var": null, - "slope_var_elem": null, - "spatial_meta": null, - "statusMsg": "func esmf_mesh_retry finished after 1 attempts.", - "sub_output_freq": null, - "sub_output_hour": null, - "suppTemporalInterp": null, - "supp_pcp_max_hours": null, - "supp_precip_dirs": null, - "supp_precip_file_types": [], - "supp_precip_forcings": [], - "supp_precip_mandatory": null, - "supp_precip_param_dir": null, - "swBiasCorrectOpt": [ - 0 - ], - "swDownscaleOpt": [ - 0 - ], - "t2BiasCorrectOpt": [ - 0 - ], - "t2dDownscaleOpt": [ - 0 - ], - "useCompression": 0, - "useFloats": 0, - "use_data_at_current_time": true, - "weightsDir": null, - "windBiasCorrect": [ - 0 - ] - }, "rank": 0, "size": 2 }, @@ -745,5 +629,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json index 18023934..e2b6c0df 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json @@ -1,4 +1,5 @@ { + "bmi_cfg_file": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -117,6 +118,7 @@ "initial_time": 0, "time_step_seconds": 3600 }, + "dimensionality": 1, "geo_meta": { "approx_centroid_global_xy": [ -72.05503025291092, @@ -135,10 +137,29 @@ "aws_obj": null, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_time": null, "bmi_time_index": 0, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "current_fcst_cycle": null, "current_output_date": null, @@ -164,17 +185,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": null, "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -216,6 +245,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -244,24 +277,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, "statusMsg": "func esmf_mesh_retry finished after 1 attempts.", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -274,9 +312,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 @@ -340,7 +383,7 @@ "_coords": [ [ "hash_7545881362083397004_len_218", - "hash_6855503821947747755_len_218" + "hash_-1022784215004640784_len_218" ], [ [ @@ -509,165 +552,6 @@ "mesh_inds_elem": null, "mpi_config": { "comm": "ERR_NOT_JSON_SERIALIZABLE:TYPE:", - "config_options": { - "ExactExtract": null, - "actual_output_steps": 71, - "ana_flag": 0, - "aorc_alaska_source": "s3://ngwpc-data/AORC/Alaska", - "aorc_alaska_url": "{source}/{year}/{year}{month:02d}/AK_AORC-OWP_{date}.nc4", - "aorc_conus_source": "s3://noaa-nws-aorc-v1-1-1km", - "aorc_conus_year_url": "{source}/{year}.zarr", - "aws": true, - "aws_obj": null, - "aws_time": null, - "b_date_proc": "2013-07-01T00:00:00", - "bmi_time": null, - "bmi_time_index": 0, - "cfsv2EnsMember": null, - "config_path": null, - "cosalpha_var": null, - "current_fcst_cycle": null, - "current_output_date": null, - "current_output_step": null, - "current_time": null, - "customFcstFreq": [], - "customSuppPcpFreq": null, - "cycle_length_minutes": 4260, - "dScaleParamDirs": [ - "/ngen-app/data" - ], - "e_date_proc": null, - "elemconn_var": "elementConn", - "elemcoords_var": "centerCoords", - "element_id_var": "element_id", - "errFlag": 0, - "errMsg": null, - "fcst_freq": 60, - "fcst_input_horizons": [ - 4260 - ], - "fcst_input_offsets": [ - 0 - ], - "fcst_shift": 0, - "first_fcst_cycle": null, - "forceTemoralInterp": [ - 0 - ], - "forcing_output": 0, - "future_time": null, - "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", - "globalNdv": -9999.0, - "grid_meta": null, - "grid_type": "hydrofabric", - "hgt_elem_var": null, - "hgt_var": null, - "ignored_border_widths": [ - 0 - ], - "include_lqfrac": 1, - "input_force_dirs": [ - "s3://null" - ], - "input_force_mandatory": [ - 0 - ], - "input_force_types": [ - "GRIB2" - ], - "input_forcings": [ - 12 - ], - "lat_var": null, - "logFile": null, - "logHandle": null, - "lon_var": null, - "look_back": -9999, - "lwBiasCorrectOpt": [ - 0 - ], - "nFcsts": 1, - "nodecoords_var": "nodeCoords", - "num_output_steps": 71, - "num_supp_output_steps": null, - "number_custom_inputs": 0, - "number_inputs": 1, - "number_supp_pcp": 0, - "numelemconn_var": "numElementConn", - "nwmConfig": "AORC", - "nwmVersion": 4.0, - "nwm_domain": null, - "nwm_geogrid": null, - "nwm_source": "s3://noaa-nwm-retrospective-3-0-pds", - "nwm_url": null, - "output_freq": 60, - "paramFlagArray": null, - "precipBiasCorrectOpt": [ - 0 - ], - "precipDownscaleOpt": [ - 0 - ], - "precip_only_flag": false, - "prev_output_date": null, - "process_window": null, - "psfcBiasCorrectOpt": [ - 0 - ], - "psfcDownscaleOpt": [ - 0 - ], - "q2BiasCorrectOpt": [ - 0 - ], - "q2dDownscaleOpt": [ - 0 - ], - "realtime_flag": false, - "refcst_flag": true, - "regrid_opt": [ - 1 - ], - "regrid_opt_supp_pcp": null, - "rqiMethod": null, - "rqiThresh": 1.0, - "runCfsNldasBiasCorrect": false, - "sinalpha_var": null, - "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, - "slope_var": null, - "slope_var_elem": null, - "spatial_meta": null, - "statusMsg": "func esmf_mesh_retry finished after 1 attempts.", - "sub_output_freq": null, - "sub_output_hour": null, - "suppTemporalInterp": null, - "supp_pcp_max_hours": null, - "supp_precip_dirs": null, - "supp_precip_file_types": [], - "supp_precip_forcings": [], - "supp_precip_mandatory": null, - "supp_precip_param_dir": null, - "swBiasCorrectOpt": [ - 0 - ], - "swDownscaleOpt": [ - 0 - ], - "t2BiasCorrectOpt": [ - 0 - ], - "t2dDownscaleOpt": [ - 0 - ], - "useCompression": 0, - "useFloats": 0, - "use_data_at_current_time": true, - "weightsDir": null, - "windBiasCorrect": [ - 0 - ] - }, "rank": 1, "size": 2 }, @@ -736,5 +620,6 @@ -1 ] }, + "output_path": null, "var_array_lengths": 1 } \ No newline at end of file diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json index bb89f16e..b2edf10e 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json @@ -198,6 +198,26 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -209,8 +229,116 @@ ], "bmi_time": 3600, "bmi_time_index": 1, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -238,17 +366,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -290,6 +426,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -318,24 +458,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -348,9 +493,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json index ed3c2aaa..33914021 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json @@ -198,6 +198,26 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -209,8 +229,116 @@ ], "bmi_time": 7200, "bmi_time_index": 2, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -238,17 +366,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -290,6 +426,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -318,24 +458,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -348,9 +493,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json index 20e3ce1c..d5781251 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json @@ -198,6 +198,26 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -209,8 +229,116 @@ ], "bmi_time": 10800, "bmi_time_index": 3, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -238,17 +366,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -290,6 +426,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -318,24 +458,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -348,9 +493,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json index bb89f16e..b2edf10e 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json @@ -198,6 +198,26 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -209,8 +229,116 @@ ], "bmi_time": 3600, "bmi_time_index": 1, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -238,17 +366,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -290,6 +426,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -318,24 +458,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -348,9 +493,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json index ed3c2aaa..33914021 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json @@ -198,6 +198,26 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -209,8 +229,116 @@ ], "bmi_time": 7200, "bmi_time_index": 2, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -238,17 +366,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -290,6 +426,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -318,24 +458,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -348,9 +493,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json index 20e3ce1c..d5781251 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json @@ -198,6 +198,26 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -209,8 +229,116 @@ ], "bmi_time": 10800, "bmi_time_index": 3, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -238,17 +366,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -290,6 +426,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -318,24 +458,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -348,9 +493,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json index bb89f16e..b2edf10e 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json @@ -198,6 +198,26 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -209,8 +229,116 @@ ], "bmi_time": 3600, "bmi_time_index": 1, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -238,17 +366,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -290,6 +426,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -318,24 +458,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -348,9 +493,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json index ed3c2aaa..33914021 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json @@ -198,6 +198,26 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -209,8 +229,116 @@ ], "bmi_time": 7200, "bmi_time_index": 2, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -238,17 +366,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -290,6 +426,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -318,24 +458,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -348,9 +493,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json index 20e3ce1c..d5781251 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json @@ -198,6 +198,26 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -209,8 +229,116 @@ ], "bmi_time": 10800, "bmi_time_index": 3, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -238,17 +366,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -290,6 +426,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -318,24 +458,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, - "statusMsg": "func esmf_regridobj_call_retry finished after 1 attempts.", + "statusMsg": "End of loop for force_key 12", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -348,9 +493,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json index 147309ce..35b7e0a4 100644 --- a/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json @@ -198,6 +198,26 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -209,8 +229,116 @@ ], "bmi_time": 10800, "bmi_time_index": 3, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -238,17 +366,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -290,6 +426,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -318,24 +458,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, "statusMsg": "Starting BMI finalize()", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -348,9 +493,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json index 147309ce..35b7e0a4 100644 --- a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json @@ -198,6 +198,26 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -209,8 +229,116 @@ ], "bmi_time": 10800, "bmi_time_index": 3, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -238,17 +366,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -290,6 +426,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -318,24 +458,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, "statusMsg": "Starting BMI finalize()", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -348,9 +493,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json index 147309ce..35b7e0a4 100644 --- a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json @@ -198,6 +198,26 @@ }, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -209,8 +229,116 @@ ], "bmi_time": 10800, "bmi_time_index": 3, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "currentCustomForceNum": 0, "currentForceNum": 1, @@ -238,17 +366,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -290,6 +426,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -318,24 +458,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, "statusMsg": "Starting BMI finalize()", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -348,9 +493,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_init_n1_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_init_n1_rank0_.json index cb4682a4..115490a3 100644 --- a/tests/test_data/expected_results/test_expected_config_options_init_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_init_n1_rank0_.json @@ -10,6 +10,26 @@ "aws_obj": null, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -21,8 +41,116 @@ ], "bmi_time": null, "bmi_time_index": 0, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "current_fcst_cycle": null, "current_output_date": null, @@ -48,17 +176,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": null, "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -100,6 +236,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -128,24 +268,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, "statusMsg": "func esmf_mesh_retry finished after 1 attempts.", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -158,9 +303,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_init_n2_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_init_n2_rank0_.json index cb4682a4..115490a3 100644 --- a/tests/test_data/expected_results/test_expected_config_options_init_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_init_n2_rank0_.json @@ -10,6 +10,26 @@ "aws_obj": null, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -21,8 +41,116 @@ ], "bmi_time": null, "bmi_time_index": 0, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "current_fcst_cycle": null, "current_output_date": null, @@ -48,17 +176,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": null, "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -100,6 +236,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -128,24 +268,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, "statusMsg": "func esmf_mesh_retry finished after 1 attempts.", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -158,9 +303,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 diff --git a/tests/test_data/expected_results/test_expected_config_options_init_n2_rank1_.json b/tests/test_data/expected_results/test_expected_config_options_init_n2_rank1_.json index cb4682a4..115490a3 100644 --- a/tests/test_data/expected_results/test_expected_config_options_init_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_config_options_init_n2_rank1_.json @@ -10,6 +10,26 @@ "aws_obj": null, "aws_time": null, "b_date_proc": "2013-07-01T00:00:00", + "bias_correction_properties": { + "Precipitation": [ + 0 + ], + "long-wave radiation": [ + 0 + ], + "short-wave radiation": [ + 0 + ], + "specific humidity": [ + 0 + ], + "surface pressure": [ + 0 + ], + "wind forcings": [ + 0 + ] + }, "bmi_model_values__CAT-ID": [ 1285829030285639, 1285830316458152, @@ -21,8 +41,116 @@ ], "bmi_time": null, "bmi_time_index": 0, + "cfg_bmi": { + "AnAFlag": 0, + "DownscalingParamDirs": [ + "/ngen-app/data" + ], + "ElemConn": "elementConn", + "ElemCoords": "centerCoords", + "ElemID": "element_id", + "ForcingTemporalInterpolation": [ + 0 + ], + "ForecastFrequency": 60, + "ForecastInputHorizons": [ + 4260 + ], + "ForecastInputOffsets": [ + 0 + ], + "ForecastShift": 0, + "GRID_TYPE": "hydrofabric", + "GeogridIn": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/esmf_mesh/gauge_01123000_ESMF_Mesh.nc", + "Geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", + "HGTVAR": "Element_Elevation", + "HumidityBiasCorrection": [ + 0 + ], + "HumidityDownscaling": [ + 0 + ], + "IgnoredBorderWidths": [ + 0 + ], + "InputForcingDirectories": [ + "s3://null" + ], + "InputForcingTypes": [ + "GRIB2" + ], + "InputForcings": [ + 12 + ], + "InputMandatory": [ + 0 + ], + "LookBack": -9999, + "LwBiasCorrection": [ + 0 + ], + "NWM_CONFIG": "AORC", + "NWM_VERSION": 4.0, + "NodeCoords": "nodeCoords", + "NumElemConn": "numElementConn", + "Output": 0, + "OutputFrequency": 60, + "PrecipBiasCorrection": [ + 0 + ], + "PrecipDownscaling": [ + 0 + ], + "PressureBiasCorrection": [ + 0 + ], + "PressureDownscaling": [ + 0 + ], + "RefcstBDateProc": "201307010000", + "RegridOpt": [ + 1 + ], + "RegridOptSuppPcp": [], + "RqiMethod": 0, + "RqiThreshold": 0.9, + "SLOPE": "Element_Slope", + "SLOPE_AZIMUTH": "Element_Slope_Azmuith", + "ScratchDir": "/ngen-app/data/scratch/AORC/", + "ShortwaveDownscaling": [ + 0 + ], + "SpatialMetaIn": "", + "SubOutFreq": 0, + "SubOutputHour": 0, + "SuppPcp": [], + "SuppPcpDirectories": [], + "SuppPcpForcingTypes": [], + "SuppPcpInputOffsets": [], + "SuppPcpMandatory": [], + "SuppPcpParamDir": "", + "SuppPcpTemporalInterpolation": [], + "SwBiasCorrection": [ + 0 + ], + "TemperatureBiasCorrection": [ + 0 + ], + "TemperatureDownscaling": [ + 0 + ], + "WindBiasCorrection": [ + 0 + ], + "cfsEnsNumber": 1, + "compressOutput": 0, + "custom_input_fcst_freq": [], + "floatOutput": 0, + "includeLQFrac": 1, + "initial_time": 0, + "time_step_seconds": 3600 + }, "cfsv2EnsMember": null, - "config_path": null, "cosalpha_var": null, "current_fcst_cycle": null, "current_output_date": null, @@ -48,17 +176,25 @@ 0 ], "fcst_shift": 0, + "file_types": [ + "GRIB1", + "GRIB2", + "NETCDF", + "NETCDF4", + "NWM", + "ZARR" + ], "first_fcst_cycle": null, "forceTemoralInterp": [ 0 ], + "force_count": 27, "forcing_output": 0, "future_time": null, "geopackage": "/workspaces/nwm-rte/src/ngen-forcing/tests/test_data/gpkg/gauge_01123000.gpkg", "globalNdv": -9999.0, "grid_meta": null, "grid_type": "hydrofabric", - "hgt_elem_var": null, "hgt_var": null, "ignored_border_widths": [ 0 @@ -100,6 +236,10 @@ "nwm_url": null, "output_freq": 60, "paramFlagArray": null, + "param_flag": [ + 0 + ], + "perform_downscaling": false, "precipBiasCorrectOpt": [ 0 ], @@ -128,24 +268,29 @@ ], "regrid_opt_supp_pcp": null, "rqiMethod": null, - "rqiThresh": 1.0, + "rqiThresh": null, "runCfsNldasBiasCorrect": false, "sinalpha_var": null, "slope_azimuth_var": null, - "slope_azimuth_var_elem": null, "slope_var": null, - "slope_var_elem": null, "spatial_meta": null, "statusMsg": "func esmf_mesh_retry finished after 1 attempts.", "sub_output_freq": null, "sub_output_hour": null, "suppTemporalInterp": null, + "supp_input_offsets": null, "supp_pcp_max_hours": null, + "supp_precip_count": 15, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], "supp_precip_mandatory": null, "supp_precip_param_dir": null, + "supplemental_precip_file_type_options": [ + "GRIB1", + "GRIB2", + "NETCDF" + ], "swBiasCorrectOpt": [ 0 ], @@ -158,9 +303,14 @@ "t2dDownscaleOpt": [ 0 ], + "try_config_get_except_attr_map": { + "Geopackage": "geopackage", + "SpatialMetaIn": "spatial_meta" + }, "useCompression": 0, "useFloats": 0, "use_data_at_current_time": true, + "user_provided_geogrid_flag": false, "weightsDir": null, "windBiasCorrect": [ 0 From 4ab14e5ff54bc1ce1e4dc7e6178393402af112cf Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 18 Aug 2026 11:48:10 -0400 Subject: [PATCH 091/132] Bolster test utils handling of serialization and deserialization --- tests/test_utils.py | 74 ++++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 43012925..c3cd2634 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -295,6 +295,7 @@ def __init__(self, cfg: TestConfig_Base) -> None: self.keys_to_check = cfg.keys_to_check self.keys_to_exclude = cfg.keys_to_exclude + self.keys_no_hash = cfg.keys_no_hash self.map_old_to_new_var_names = cfg.map_old_to_new_var_names self.test_file_name_prefix = cfg.test_file_name_prefix @@ -323,6 +324,46 @@ def _trim_arrays_to_input_map_output(data_dict: dict) -> None: ) data_dict[key] = value[: len(input_map_output)] + def _apply_transformations(self, data: dict, keys_to_exclude: tuple, keys_no_hash: tuple = (), apply_map: bool = True) -> dict: + """Apply transformations to live BMI data to produce its final JSON representation. + + Used only by deserial_actual() to transform raw BMI state. Expected result files are already + in their final form (transformed + extra_attrs added), so they are read directly without re-transformation. + + The processing order is: + 1. Re-order the keys + 2. Save raw values for keys_no_hash before hashing + 3. Convert long lists to hashes (except keys_no_hash which are excluded) + 4. Restore raw values for keys_no_hash + 5. Remove excluded keys + 6. Map old variable names to new (if apply_map=True and self.map_old_to_new_var_names) + + Args: + data: The deserialized data dictionary + keys_to_exclude: Tuple of keys to exclude from result + keys_no_hash: Tuple of keys that should NOT be hashed (preserved as raw values) + apply_map: Whether to apply variable name mapping (converting earlier pre-refactor namespace to later namespace) + + Returns: + Transformed dictionary + """ + # Order and reverse so private attributes are last + result = OrderedDict(reversed(list(data.items()))) + # Save raw values for keys that should not be hashed + raw_vals_no_hash = { + k: result[k] for k in keys_no_hash if k in result + } + # Convert long lists to hash strings (this may hash keys_no_hash too) + result = convert_long_lists(result, 10) + # Restore raw (unhashed) values for keys_no_hash + result.update(raw_vals_no_hash) + # Remove excluded keys + result = remove_key(dict(result), keys_to_exclude) + # Map old variable names to new if enabled + if apply_map and self.map_old_to_new_var_names: + result = self.map_old_to_new_variable_names(result) + return result + class BMIForcingFixture_Class(BMIForcingFixture): """Test fixture for Class-based tests.""" @@ -340,41 +381,32 @@ def __init__(self, cfg: TestConfig_Base) -> None: self.actual_sub_dir = "test_data/actual_results" self.test_dir = os.path.dirname(os.path.abspath(__file__)) self.extra_attrs: tuple[ClassAttrFetcher] = cfg.extra_attrs - self.keys_no_hash: tuple[str] = cfg.keys_no_hash self.keys_to_exclude_at_init: tuple[str] = cfg.keys_to_exclude_at_init def deserial_actual( self, suffix: str, current_output_step: str = "", write_to_file: bool = True ) -> dict: """Get the actual metadata results as a deserialized dictionary, including any extra_attrs.""" - deserial_actual = json.loads( + data = json.loads( serialize_to_json( copy_and_stringify_functions(self.test_class_as_dict), sort_keys=True ) ) - # order and reverse so private attributes are last - deserial_actual = OrderedDict(reversed(list(deserial_actual.items()))) - # Save raw values for keys that should not be hashed - raw_vals = { - k: deserial_actual[k] for k in self.keys_no_hash if k in deserial_actual - } - deserial_actual = convert_long_lists(deserial_actual, 10) - deserial_actual.update(raw_vals) - deserial_actual = remove_key(dict(deserial_actual), self.keys_to_exclude) + data = self._apply_transformations(data, self.keys_to_exclude, keys_no_hash=self.keys_no_hash, apply_map=True) # Add any extra attributes to the results for ea in self.extra_attrs: - deserial_actual[ea.results_key_name] = ea.get( + data[ea.results_key_name] = ea.get( self, serialize_and_deserialize=True ) - self._trim_arrays_to_input_map_output(deserial_actual) + self._trim_arrays_to_input_map_output(data) if write_to_file: self.write_json( - deserial_actual, + data, self.actual_results_file_path(suffix, current_output_step), ) - return deserial_actual + return data def write_json(self, dictionary_to_write: dict, json_path: str) -> None: """Write the deserialized results to a JSON file.""" @@ -388,16 +420,8 @@ def deserial_expected(self, suffix: str, current_output_step: str = "") -> dict: try: with open(file_path) as f: - deserial_expected = json.load(f) - if self.map_old_to_new_var_names: - deserial_expected = self.map_old_to_new_variable_names( - deserial_expected - ) - self._trim_arrays_to_input_map_output(deserial_expected) - # Remove keys that should be excluded from comparison - deserial_expected = remove_key(deserial_expected, self.keys_to_exclude) - # order and reverse so private attributes are last - return OrderedDict(reversed(list(deserial_expected.items()))) + # Files are already in their final representation (transformed + trimmed), do not re-transform. + return json.load(f) except FileNotFoundError as e: raise FileNotFoundError( f"Could not find {file_path}. Try running the test using OS var {OS_VAR__CREATE_TEST_EXPECT_DATA}=true first to set up the test results expected data." From 2d9b4049a2586ab74479801db8870f19f7a85ff7 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 18 Aug 2026 12:57:27 -0400 Subject: [PATCH 092/132] Format with ruff default line length (88) --- .../NextGen_hyfab_to_ESMF_Mesh.py | 114 +++++++++++------- .../NextGen_Forcings_Engine/bmi_model.py | 22 ++-- .../NextGen_Forcings_Engine/core/config.py | 20 ++- .../core/forcingInputMod.py | 3 +- .../NextGen_Forcings_Engine/core/parallel.py | 8 +- .../NextGen_Forcings_Engine/core/regrid.py | 36 +++--- .../NextGen_Forcings_Engine/esmf_utils.py | 11 +- .../NextGen_Forcings_Engine/model.py | 10 +- tests/bmi_model/test_bmi_model.py | 12 +- tests/geomod/test_geomod.py | 4 +- tests/test_utils.py | 42 ++++--- 11 files changed, 177 insertions(+), 105 deletions(-) diff --git a/ESMF_Mesh_Domain_Configuration_Production/NextGen_hyfab_to_ESMF_Mesh.py b/ESMF_Mesh_Domain_Configuration_Production/NextGen_hyfab_to_ESMF_Mesh.py index c9426a94..b6c39859 100644 --- a/ESMF_Mesh_Domain_Configuration_Production/NextGen_hyfab_to_ESMF_Mesh.py +++ b/ESMF_Mesh_Domain_Configuration_Production/NextGen_hyfab_to_ESMF_Mesh.py @@ -34,7 +34,7 @@ def convert_hyfab_to_esmf(hyfab_gpkg: pathlib.Path, esmf_mesh_output: pathlib.Pa # for orientation properties since there are issues # with geopandas for converting crs and translating # orientation of polygon from original dataset - hyfab_cart = gpd.read_file(hyfab_gpkg, layer='divides') + hyfab_cart = gpd.read_file(hyfab_gpkg, layer="divides") hyfab_cart = hyfab_cart.sort_values(by=["div_id"]).reset_index(drop=True) hyfab = hyfab_cart.to_crs("WGS84") @@ -52,23 +52,25 @@ def convert_hyfab_to_esmf(hyfab_gpkg: pathlib.Path, esmf_mesh_output: pathlib.Pa hyfab_coords[:, 1] = false_ids # Sort data by feature id and reset index - hyfab['element_id'] = false_ids - hyfab_cart['element_id'] = false_ids + hyfab["element_id"] = false_ids + hyfab_cart["element_id"] = false_ids # Get element count element_count = len(hyfab.element_id) # find the number of nodes in first element # based on geometry type - if (hyfab.geometry[0].geom_type == "Polygon"): + if hyfab.geometry[0].geom_type == "Polygon": dup_df = pd.DataFrame([]) - dup_df['node_x'], dup_df['node_y'] = hyfab.geometry[0].exterior.coords.xy - dup_df = dup_df.drop_duplicates(subset=['node_x', 'node_y'], keep='first') + dup_df["node_x"], dup_df["node_y"] = hyfab.geometry[0].exterior.coords.xy + dup_df = dup_df.drop_duplicates(subset=["node_x", "node_y"], keep="first") elem_max_nodes = len(dup_df) else: dup_df = pd.DataFrame([]) - dup_df['node_x'], dup_df['node_y'] = hyfab.geometry[0].geoms._get_geom_item(0).exterior.xy - dup_df = dup_df.drop_duplicates(subset=['node_x', 'node_y'], keep='first') + dup_df["node_x"], dup_df["node_y"] = ( + hyfab.geometry[0].geoms._get_geom_item(0).exterior.xy + ) + dup_df = dup_df.drop_duplicates(subset=["node_x", "node_y"], keep="first") elem_max_nodes = len(dup_df) # Allocate element arrays for center point calculations @@ -85,15 +87,17 @@ def convert_hyfab_to_esmf(hyfab_gpkg: pathlib.Path, esmf_mesh_output: pathlib.Pa # based on geometry type total_num_nodes = 0 for i in range(element_count): - if (hyfab.geometry[i].geom_type == "Polygon"): + if hyfab.geometry[i].geom_type == "Polygon": dup_df = pd.DataFrame([]) - dup_df['node_x'], dup_df['node_y'] = hyfab.geometry[i].exterior.coords.xy - dup_df = dup_df.drop_duplicates(subset=['node_x', 'node_y'], keep='first') + dup_df["node_x"], dup_df["node_y"] = hyfab.geometry[i].exterior.coords.xy + dup_df = dup_df.drop_duplicates(subset=["node_x", "node_y"], keep="first") total_num_nodes += len(dup_df) else: dup_df = pd.DataFrame([]) - dup_df['node_x'], dup_df['node_y'] = hyfab.geometry[i].geoms._get_geom_item(0).exterior.xy - dup_df = dup_df.drop_duplicates(subset=['node_x', 'node_y'], keep='first') + dup_df["node_x"], dup_df["node_y"] = ( + hyfab.geometry[i].geoms._get_geom_item(0).exterior.xy + ) + dup_df = dup_df.drop_duplicates(subset=["node_x", "node_y"], keep="first") total_num_nodes += len(dup_df) # assign current node id and allocate node arrays to extract @@ -108,24 +112,26 @@ def convert_hyfab_to_esmf(hyfab_gpkg: pathlib.Path, esmf_mesh_output: pathlib.Pa # flip node coordinates based on orientation of polygons # from the original cartesian coordinate system for i in range(element_count): - if (hyfab.geometry[i].geom_type == "Polygon"): + if hyfab.geometry[i].geom_type == "Polygon": dup_df = pd.DataFrame([]) - dup_df['node_x'], dup_df['node_y'] = hyfab.geometry[i].exterior.coords.xy - dup_df = dup_df.drop_duplicates(subset=['node_x', 'node_y'], keep='first') + dup_df["node_x"], dup_df["node_y"] = hyfab.geometry[i].exterior.coords.xy + dup_df = dup_df.drop_duplicates(subset=["node_x", "node_y"], keep="first") node_x = dup_df.node_x.values node_y = dup_df.node_y.values ccw = hyfab_cart.geometry[i].exterior.is_ccw else: dup_df = pd.DataFrame([]) - dup_df['node_x'], dup_df['node_y'] = hyfab.geometry[i].geoms._get_geom_item(0).exterior.xy - dup_df = dup_df.drop_duplicates(subset=['node_x', 'node_y'], keep='first') + dup_df["node_x"], dup_df["node_y"] = ( + hyfab.geometry[i].geoms._get_geom_item(0).exterior.xy + ) + dup_df = dup_df.drop_duplicates(subset=["node_x", "node_y"], keep="first") node_x = dup_df.node_x.values node_y = dup_df.node_y.values ccw = hyfab_cart.geometry[i].geoms._get_geom_item(0).exterior.is_ccw num_nodes = len(node_x) element_num_nodes[i] = num_nodes - if (num_nodes > elem_max_nodes): + if num_nodes > elem_max_nodes: elem_max_nodes = num_nodes element_x_coord[i] = hyfab.geometry[i].centroid.coords.xy[0][0] @@ -133,24 +139,34 @@ def convert_hyfab_to_esmf(hyfab_gpkg: pathlib.Path, esmf_mesh_output: pathlib.Pa element_elevation[i] = hyfab.elevation_mean[i] element_slope[i] = hyfab.slope1km_mean[i] - element_slope_azmuith[i] = hyfab.aspect_circmean[i] # NHF aspect is currently in radians, may need to be converted to degrees - - if (ccw): - node_x_coord[node_start:node_start + num_nodes] = np.array(node_x, dtype=np.double) - node_y_coord[node_start:node_start + num_nodes] = np.array(node_y, dtype=np.double) + element_slope_azmuith[i] = hyfab.aspect_circmean[ + i + ] # NHF aspect is currently in radians, may need to be converted to degrees + + if ccw: + node_x_coord[node_start : node_start + num_nodes] = np.array( + node_x, dtype=np.double + ) + node_y_coord[node_start : node_start + num_nodes] = np.array( + node_y, dtype=np.double + ) else: - node_x_coord[node_start:node_start + num_nodes] = np.array(np.concatenate([[node_x[0]], np.flip(node_x[1:])]), dtype=np.double) - node_y_coord[node_start:node_start + num_nodes] = np.array(np.concatenate([[node_y[0]], np.flip(node_y[1:])]), dtype=np.double) + node_x_coord[node_start : node_start + num_nodes] = np.array( + np.concatenate([[node_x[0]], np.flip(node_x[1:])]), dtype=np.double + ) + node_y_coord[node_start : node_start + num_nodes] = np.array( + np.concatenate([[node_y[0]], np.flip(node_y[1:])]), dtype=np.double + ) node_start += num_nodes # Assign node data to pandas dataframe # and calculate the duplicate nodes throughout # the hydrofabric geometry network node_connectivity = pd.DataFrame([]) - node_connectivity['node_x'] = node_x_coord - node_connectivity['node_y'] = node_y_coord + node_connectivity["node_x"] = node_x_coord + node_connectivity["node_y"] = node_y_coord - duplicates = node_connectivity[node_connectivity.duplicated(keep='first')] + duplicates = node_connectivity[node_connectivity.duplicated(keep="first")] # Create array to assign duplicate nodes as # zeroes, while creating unique ids for only @@ -159,25 +175,27 @@ def convert_hyfab_to_esmf(hyfab_gpkg: pathlib.Path, esmf_mesh_output: pathlib.Pa node_id_connectivity = np.empty(len(node_id), dtype=np.int32) node_count = 1 for i in range(len(node_id)): - if (i in duplicates_index): + if i in duplicates_index: node_id_connectivity[i] = 0 else: node_id_connectivity[i] = node_count node_count += 1 # Assign new node id network to dataframe - node_connectivity['node_id'] = node_id_connectivity + node_connectivity["node_id"] = node_id_connectivity # calculate the node id network to include its duplicate ids # for each instance of the node coordinates - ESMF_node_id_connectivity = node_connectivity.groupby(['node_x', 'node_y']).node_id.transform('max') + ESMF_node_id_connectivity = node_connectivity.groupby( + ["node_x", "node_y"] + ).node_id.transform("max") - node_connectivity['node_id_connectivity'] = ESMF_node_id_connectivity.values + node_connectivity["node_id_connectivity"] = ESMF_node_id_connectivity.values node_connectivity_final = node_connectivity.node_id_connectivity.values # Extract only the unique node id network and respective coordinates - node_connectivity = node_connectivity.drop_duplicates('node_id_connectivity') + node_connectivity = node_connectivity.drop_duplicates("node_id_connectivity") node_count = len(node_connectivity) node_x_coord_final = node_connectivity.node_x.values node_y_coord_final = node_connectivity.node_y.values @@ -190,7 +208,9 @@ def convert_hyfab_to_esmf(hyfab_gpkg: pathlib.Path, esmf_mesh_output: pathlib.Pa end_index = 0 for i in range(element_count): end_index += element_num_nodes[i] - elementConn[i, 0:element_num_nodes[i]] = node_connectivity_final[start_index:end_index] + elementConn[i, 0 : element_num_nodes[i]] = node_connectivity_final[ + start_index:end_index + ] start_index = end_index out_dir = os.path.dirname(esmf_mesh_output) @@ -204,9 +224,11 @@ def convert_hyfab_to_esmf(hyfab_gpkg: pathlib.Path, esmf_mesh_output: pathlib.Pa nc = netCDF4.Dataset(temp_path, "w", format="NETCDF4") node_count_dim = nc.createDimension("nodeCount", node_count) elem_count_dim = nc.createDimension("elementCount", element_count) - elem_conn_count_dim = nc.createDimension("connectionCount", len(node_connectivity_final)) + elem_conn_count_dim = nc.createDimension( + "connectionCount", len(node_connectivity_final) + ) node_count_dim = nc.createDimension("coordDim", 2) - node_coords_var = nc.createVariable("nodeCoords", 'f8', ("nodeCount", "coordDim")) + node_coords_var = nc.createVariable("nodeCoords", "f8", ("nodeCount", "coordDim")) node_coords_var.units = "degrees" elem_id = nc.createVariable("element_id", "i4", "elementCount") elem_id.long_name = "False 32-bit catchment IDs use for ESMF mesh generation" @@ -214,7 +236,9 @@ def convert_hyfab_to_esmf(hyfab_gpkg: pathlib.Path, esmf_mesh_output: pathlib.Pa elem_conn_var.long_name = "Node Indices that define the element connectivity" num_elem_conn_var = nc.createVariable("numElementConn", "i", "elementCount") num_elem_conn_var.long_name = "Number of nodes per element" - center_coords_var = nc.createVariable("centerCoords", 'f8', ("elementCount", "coordDim")) + center_coords_var = nc.createVariable( + "centerCoords", "f8", ("elementCount", "coordDim") + ) center_coords_var.units = "degrees" nc.gridType = "unstructured" nc.version = "0.9" @@ -225,7 +249,9 @@ def convert_hyfab_to_esmf(hyfab_gpkg: pathlib.Path, esmf_mesh_output: pathlib.Pa slope_elem_var = nc.createVariable("Element_Slope", "f8", ("elementCount")) slope_elem_var.long_name = "Catchment slope" slope_elem_var.units = "meters" - slope_azi_elem_var = nc.createVariable("Element_Slope_Azmuith", "f8", ("elementCount")) + slope_azi_elem_var = nc.createVariable( + "Element_Slope_Azmuith", "f8", ("elementCount") + ) slope_azi_elem_var.long_name = "Catchment slope azmuith angle" slope_azi_elem_var.units = "Degrees" hgt_elem_var[:] = element_elevation @@ -259,8 +285,14 @@ def convert_hyfab_to_esmf(hyfab_gpkg: pathlib.Path, esmf_mesh_output: pathlib.Pa def get_options(): parser = argparse.ArgumentParser() - parser.add_argument('hyfab_gpkg', type=pathlib.Path, help="Hydrofabric geopackage file pathway") - parser.add_argument("esmf_mesh_output", type=pathlib.Path, help="File pathway to save ESMF netcdf mesh file for hydrofabric") + parser.add_argument( + "hyfab_gpkg", type=pathlib.Path, help="Hydrofabric geopackage file pathway" + ) + parser.add_argument( + "esmf_mesh_output", + type=pathlib.Path, + help="File pathway to save ESMF netcdf mesh file for hydrofabric", + ) return parser.parse_args() diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index c63d0afa..fe393c85 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -71,15 +71,14 @@ try: from ewts.helper import getenv_any from ewts.logger import configure_existing_logger + FORCING_USE_EWTS = True except ImportError: FORCING_USE_EWTS = False -class StdoutStyleFormatter(logging.Formatter): - INFO_FORMAT = ( - "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" - ) +class StdoutStyleFormatter(logging.Formatter): + INFO_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" DETAILED_FORMAT = ( "%(asctime)s %(name)-8s %(levelname)-7s " @@ -94,7 +93,7 @@ def format(self, record): self._style._fmt = self.DETAILED_FORMAT return super().format(record) - + def formatTime(self, record, datefmt=None): dt = datetime.fromtimestamp(record.created, tz=timezone.utc) return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" @@ -111,6 +110,7 @@ def _configure_stdout_logging(): LOG.propagate = False + # If less than 0, then ESMF.__version__ is greater than 8.7.0 if ESMF.version_compare("8.7.0", ESMF.__version__) < 0: manager = ESMF.api.esmpymanager.Manager(endFlag=ESMF.constants.EndAction.KEEP_MPI) @@ -174,7 +174,9 @@ def init_log(self) -> None: configure_existing_logger(LOG) else: _configure_stdout_logging() - LOG.warning("ewts package installed but EWTS_USE_NGEN_BRIDGE not on. Falling back to default logging.") + LOG.warning( + "ewts package installed but EWTS_USE_NGEN_BRIDGE not on. Falling back to default logging." + ) else: _configure_stdout_logging() LOG.info("-" * 30) @@ -283,9 +285,9 @@ def create_esmf_mesh(self) -> None: """Create the ESMF mesh for the model and set ``self._cat_ids`` (later used as BMI variable "CAT-ID").""" if self._mpi_meta.rank == 0: cat_ids = esmf_creation.create_mesh(self._job_meta) - cat_count = np.array([ - len(cat_ids) if self._mpi_meta.rank == 0 else 0 - ], dtype=np.intc) + cat_count = np.array( + [len(cat_ids) if self._mpi_meta.rank == 0 else 0], dtype=np.intc + ) self._mpi_meta.comm.Bcast(cat_count, root=0) if self._mpi_meta.rank != 0: cat_ids = np.empty(cat_count[0], dtype=np.int64) @@ -735,7 +737,7 @@ def get_value_ptr(self, var_name: str) -> NDArray[Any]: # Ensure dtype is float64 (C double), except for CAT-ID if var_name == "CAT-ID": - return arr # allow CAT-ID to pass on whatever the dtype is based on the input data + return arr # allow CAT-ID to pass on whatever the dtype is based on the input data elif arr.dtype != np.float64: LOG.warning( f"[BMI] Array for '{var_name}' has dtype {arr.dtype}, expected float64; converting." diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 868d7d1c..cfdeee2a 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -71,7 +71,7 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No self.nwm_source = "s3://noaa-nwm-retrospective-3-0-pds" self._scratch_dir_has_been_uniquefied = False - + # These must exist (as None) before the properties are accessed self._supp_precip_forcings = None self._b_date_proc = None @@ -89,7 +89,7 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No self._geopackage = None self._geogrid = None self._grid_type = None - + # set list of attributes from consts.py to None early on in the init process. # These are indexed from the consts dictionary. # This must happen before accessing properties like precip_only_flag @@ -144,12 +144,14 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No # Initialize downscaling attributes to None even if downscaling is not performed self.set_attrs(CONFIGOPTIONS["downscaling_attrs_map"], set_none=True) if self.grid_type == "unstructured": - self.set_attrs(CONFIGOPTIONS["downscaling_unstructred_attrs_map"], set_none=True) + self.set_attrs( + CONFIGOPTIONS["downscaling_unstructred_attrs_map"], set_none=True + ) for cfg_bmi_attr, config_options_attr in CONFIGOPTIONS[ "extract_input_variable_set_default_attrs_map" ].items(): - if config_options_attr in ["supp_pcp_max_hours","weightsDir"]: + if config_options_attr in ["supp_pcp_max_hours", "weightsDir"]: default = None else: default = 0 @@ -700,7 +702,9 @@ def number_custom_inputs(self) -> int: @number_custom_inputs.setter def number_custom_inputs(self, value: int) -> None: """This is a read-only computed property based on input_forcings.""" - raise AttributeError(f"number_custom_inputs is read-only (tried to set to: {value})") + raise AttributeError( + f"number_custom_inputs is read-only (tried to set to: {value})" + ) @property def nwm_geogrid(self) -> str: @@ -710,7 +714,11 @@ def nwm_geogrid(self) -> str: @nwm_geogrid.setter def nwm_geogrid(self, value: str) -> None: """Set the pathway to the NWM geogrid file specified by the user in the configuration file. This is used to specify the grid information for regridding NWM input forcings, and is only necessary if the user has chosen to regrid NWM input forcings in the configuration file.""" - if not self.precip_only_flag and self.input_forcings is not None and 27 in self.input_forcings: + if ( + not self.precip_only_flag + and self.input_forcings is not None + and 27 in self.input_forcings + ): self._nwm_geogrid = value else: self._nwm_geogrid = None diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py index 1cb97ef7..49dfef2f 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py @@ -25,6 +25,7 @@ MpiConfig, ) import logging + LOG = logging.getLogger("FORCING") @@ -94,7 +95,7 @@ def _initialize_config_options(self) -> None: Check if the attibute allready exists before setting. """ for key in dir(self.config_options): - val=getattr(self.config_options,key) + val = getattr(self.config_options, key) if ( isinstance(val, list) and len(val) > 0 diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index dd87f7e3..de27c74c 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -384,7 +384,9 @@ def merge_slabs_gatherv( try: self.comm.Allgather([shapes, MPI.INTEGER], [global_shapes, MPI.INTEGER]) except Exception: - self.config_options.errMsg = "Failed all gathering slab shapes at rank" + str(self.rank) + self.config_options.errMsg = ( + "Failed all gathering slab shapes at rank" + str(self.rank) + ) err_handler.log_critical(self.config_options, self) return None @@ -455,7 +457,9 @@ def merge_slabs_gatherv( root=0, ) except Exception: - self.config_options.errMsg = "Failed to Gatherv to rank 0 from rank " + str(self.rank) + self.config_options.errMsg = "Failed to Gatherv to rank 0 from rank " + str( + self.rank + ) err_handler.log_critical(self.config_options, self) return None diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py index 92ff26d8..639904a4 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py @@ -45,19 +45,20 @@ from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( ConfigOptions, ) - from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.geoMod import ( - GeoMeta, - ) - from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.suppPrecipMod import ( - supplemental_precip, - ) from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.forcingInputMod import ( InputForcings, ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.geoMod import ( + GeoMeta, + ) from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import ( MpiConfig, ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.suppPrecipMod import ( + supplemental_precip, + ) import logging + from ..esmf_utils import ( esmf_field_retry, esmf_grid_retry, @@ -9683,7 +9684,10 @@ def regrid_sbcv2_liquid_water_fraction( def regrid_hourly_nbm( - forcings_or_precip:supplemental_precip|InputForcings, config_options:ConfigOptions, wrf_hydro_geo_meta:GeoMeta, mpi_config:MpiConfig + forcings_or_precip: supplemental_precip | InputForcings, + config_options: ConfigOptions, + wrf_hydro_geo_meta: GeoMeta, + mpi_config: MpiConfig, ): """Regrid hourly NBM precipitation. @@ -9739,7 +9743,7 @@ def regrid_hourly_nbm( cmd = f'$WGRIB2 -match "({"|".join(fields)})" -not "prob" -not "ens" {forcings_or_precip.file_in1} -netcdf {nbm_tmp_nc}' else: # Perform a GRIB dump to NetCDF for the precip data. - time_str=f"{forcings_or_precip.fcst_hour1}-{forcings_or_precip.fcst_hour2} hour acc fcst" + time_str = f"{forcings_or_precip.fcst_hour1}-{forcings_or_precip.fcst_hour2} hour acc fcst" fieldnbm_match1 = f'":APCP:surface:{time_str}:"' fieldnbm_match2 = ( f'"{forcings_or_precip.fcst_hour1}-{forcings_or_precip.fcst_hour2}"' @@ -11357,10 +11361,14 @@ def check_regrid_status( ) elif config_options.grid_type == "hydrofabric": input_forcings.regridded_forcings1 = np.full( - [force_count, wrf_hydro_geo_meta.ny_local], np.nan,dtype=np.float32 #NOTE changed to np.full to be deterministic for unit tests. + [force_count, wrf_hydro_geo_meta.ny_local], + np.nan, + dtype=np.float32, # NOTE changed to np.full to be deterministic for unit tests. ) input_forcings.regridded_forcings2 = np.full( - [force_count, wrf_hydro_geo_meta.ny_local], np.nan,dtype=np.float32 #NOTE changed to np.full to be deterministic for unit tests. + [force_count, wrf_hydro_geo_meta.ny_local], + np.nan, + dtype=np.float32, # NOTE changed to np.full to be deterministic for unit tests. ) if mpi_config.rank == 0: @@ -11855,13 +11863,9 @@ def calculate_weights( err_handler.check_program_status(config_options, mpi_config) # Broadcast the forcing nx/ny values - input_forcings.ny_global = mpi_config.broadcast_parameter( - input_forcings.ny_global - ) + input_forcings.ny_global = mpi_config.broadcast_parameter(input_forcings.ny_global) err_handler.check_program_status(config_options, mpi_config) - input_forcings.nx_global = mpi_config.broadcast_parameter( - input_forcings.nx_global - ) + input_forcings.nx_global = mpi_config.broadcast_parameter(input_forcings.nx_global) err_handler.check_program_status(config_options, mpi_config) try: diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/esmf_utils.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/esmf_utils.py index f49b8960..9c0072eb 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/esmf_utils.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/esmf_utils.py @@ -1,7 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any import types +from typing import TYPE_CHECKING, Any import esmpy as ESMF @@ -10,9 +10,14 @@ import shapely from . import retry_utils + if TYPE_CHECKING: - from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ConfigOptions - from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import MpiConfig + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( + ConfigOptions, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import ( + MpiConfig, + ) @retry_utils.retry_w_mpi_context( diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 41fad2b9..4335fea5 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -500,12 +500,10 @@ def __handle_aorc_and_nwm_force_keys( self._bmi.geo_meta, ) elif self._bmi._job_meta.nwm_domain == "PR": - self.source_data_processor = ( - NWMV3PuertoRicoProcessor( - self._bmi._job_meta, - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) + self.source_data_processor = NWMV3PuertoRicoProcessor( + self._bmi._job_meta, + self._bmi._mpi_meta, + self._bmi.geo_meta, ) elif self._bmi._job_meta.nwm_domain == "Alaska": proc_cls = NWMV3AlaskaProcessor diff --git a/tests/bmi_model/test_bmi_model.py b/tests/bmi_model/test_bmi_model.py index 80461061..dc17feb3 100644 --- a/tests/bmi_model/test_bmi_model.py +++ b/tests/bmi_model/test_bmi_model.py @@ -21,8 +21,14 @@ config_file=consts.RETRO_FORCING_CONFIG_FILE__AORC_CONUS, keys_to_check=(), keys_to_exclude=tuple( - set(consts.KEYS_TO_EXCLUDE) | { - "d_program_init", "geogrid", "scratch_dir", "Element_Elevation", "Element_Slope", "Element_Slope_Azmuith", + set(consts.KEYS_TO_EXCLUDE) + | { + "d_program_init", + "geogrid", + "scratch_dir", + "Element_Elevation", + "Element_Slope", + "Element_Slope_Azmuith", "geo_meta.config_options.cfg_bmi", "geo_meta.mpi_config.config_options", "mpi_config.config_options", @@ -30,7 +36,7 @@ ), grid_type=consts.GRID_TYPE, test_file_name_prefix="bmi_model", - extra_attrs=[ClassAttrFetcher("bmi_model_values", "CAT-ID")] + extra_attrs=[ClassAttrFetcher("bmi_model_values", "CAT-ID")], ), ] diff --git a/tests/geomod/test_geomod.py b/tests/geomod/test_geomod.py index 3cbc8b9a..e329be0c 100644 --- a/tests/geomod/test_geomod.py +++ b/tests/geomod/test_geomod.py @@ -27,7 +27,9 @@ ), grid_type=consts.GRID_TYPE, test_file_name_prefix=TEST_FILE_NAME_PREFIX, - extra_attrs=[ClassAttrFetcher("bmi_model_values", "CAT-ID"),] + extra_attrs=[ + ClassAttrFetcher("bmi_model_values", "CAT-ID"), + ], ), ] diff --git a/tests/test_utils.py b/tests/test_utils.py index c3cd2634..f561d65d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -324,12 +324,18 @@ def _trim_arrays_to_input_map_output(data_dict: dict) -> None: ) data_dict[key] = value[: len(input_map_output)] - def _apply_transformations(self, data: dict, keys_to_exclude: tuple, keys_no_hash: tuple = (), apply_map: bool = True) -> dict: + def _apply_transformations( + self, + data: dict, + keys_to_exclude: tuple, + keys_no_hash: tuple = (), + apply_map: bool = True, + ) -> dict: """Apply transformations to live BMI data to produce its final JSON representation. - + Used only by deserial_actual() to transform raw BMI state. Expected result files are already in their final form (transformed + extra_attrs added), so they are read directly without re-transformation. - + The processing order is: 1. Re-order the keys 2. Save raw values for keys_no_hash before hashing @@ -337,22 +343,20 @@ def _apply_transformations(self, data: dict, keys_to_exclude: tuple, keys_no_has 4. Restore raw values for keys_no_hash 5. Remove excluded keys 6. Map old variable names to new (if apply_map=True and self.map_old_to_new_var_names) - + Args: data: The deserialized data dictionary keys_to_exclude: Tuple of keys to exclude from result keys_no_hash: Tuple of keys that should NOT be hashed (preserved as raw values) apply_map: Whether to apply variable name mapping (converting earlier pre-refactor namespace to later namespace) - + Returns: Transformed dictionary """ # Order and reverse so private attributes are last result = OrderedDict(reversed(list(data.items()))) # Save raw values for keys that should not be hashed - raw_vals_no_hash = { - k: result[k] for k in keys_no_hash if k in result - } + raw_vals_no_hash = {k: result[k] for k in keys_no_hash if k in result} # Convert long lists to hash strings (this may hash keys_no_hash too) result = convert_long_lists(result, 10) # Restore raw (unhashed) values for keys_no_hash @@ -392,12 +396,12 @@ def deserial_actual( copy_and_stringify_functions(self.test_class_as_dict), sort_keys=True ) ) - data = self._apply_transformations(data, self.keys_to_exclude, keys_no_hash=self.keys_no_hash, apply_map=True) + data = self._apply_transformations( + data, self.keys_to_exclude, keys_no_hash=self.keys_no_hash, apply_map=True + ) # Add any extra attributes to the results for ea in self.extra_attrs: - data[ea.results_key_name] = ea.get( - self, serialize_and_deserialize=True - ) + data[ea.results_key_name] = ea.get(self, serialize_and_deserialize=True) self._trim_arrays_to_input_map_output(data) @@ -427,9 +431,11 @@ def deserial_expected(self, suffix: str, current_output_step: str = "") -> dict: f"Could not find {file_path}. Try running the test using OS var {OS_VAR__CREATE_TEST_EXPECT_DATA}=true first to set up the test results expected data." ) from e - def _write_expected_file(self, actual_data: dict, suffix: str, current_output_step: str = "") -> None: + def _write_expected_file( + self, actual_data: dict, suffix: str, current_output_step: str = "" + ) -> None: """Write actual data to expected results file for test data generation. - + This is a separate explicit step in the workflow to avoid confusion between expected and actual data. Should only be called when FORCING_PYTEST_WRITE_TEST_EXPECTED_DATA=true. """ @@ -499,8 +505,12 @@ def after_bmi_model_update(self, current_output_step: int) -> None: logging.info("Starting after_bmi_model_update()...") actual = self.deserial_actual("after_update", f"_step_{current_output_step}") if os.environ.get(OS_VAR__CREATE_TEST_EXPECT_DATA, "").lower() == "true": - self._write_expected_file(actual, "after_update", f"_step_{current_output_step}") - expected = self.deserial_expected("after_update", f"_step_{current_output_step}") + self._write_expected_file( + actual, "after_update", f"_step_{current_output_step}" + ) + expected = self.deserial_expected( + "after_update", f"_step_{current_output_step}" + ) self.compare(actual, expected) def after_finalize(self) -> None: From da09922123fadf14be43943d89853c6cd26c1225 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 18 Aug 2026 14:14:55 -0400 Subject: [PATCH 093/132] Adjustments following review --- .../NextGen_hyfab_to_ESMF_Mesh.py | 5 ++--- .../NextGen_Forcings_Engine/bmi_model.py | 4 ++-- .../NextGen_Forcings_Engine/core/parallel.py | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ESMF_Mesh_Domain_Configuration_Production/NextGen_hyfab_to_ESMF_Mesh.py b/ESMF_Mesh_Domain_Configuration_Production/NextGen_hyfab_to_ESMF_Mesh.py index b6c39859..4c6a7d9f 100644 --- a/ESMF_Mesh_Domain_Configuration_Production/NextGen_hyfab_to_ESMF_Mesh.py +++ b/ESMF_Mesh_Domain_Configuration_Production/NextGen_hyfab_to_ESMF_Mesh.py @@ -139,9 +139,8 @@ def convert_hyfab_to_esmf(hyfab_gpkg: pathlib.Path, esmf_mesh_output: pathlib.Pa element_elevation[i] = hyfab.elevation_mean[i] element_slope[i] = hyfab.slope1km_mean[i] - element_slope_azmuith[i] = hyfab.aspect_circmean[ - i - ] # NHF aspect is currently in radians, may need to be converted to degrees + # NHF aspect is currently in radians, may need to be converted to degrees + element_slope_azmuith[i] = hyfab.aspect_circmean[i] if ccw: node_x_coord[node_start : node_start + num_nodes] = np.array( diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index fe393c85..2b321d8c 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -423,8 +423,8 @@ def initialize(self, config_file: str) -> None: """Initialize the model using a configuration file. This function is part of the BMI (Basic Model Interface) specification and is automatically - invoked by the BMI system. When running standalone, call `initialize_with_params()` instead, - which sets additional parameters such as `b_date`, `geogrid`, and `output_path`. + invoked by the BMI system. To override parameters like `b_date`, `geogrid`, and `output_path` + (normally read from the config file), pass them to the constructor. This function is responsible for: - Setting up core model attributes, grids, and MPI communication. diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index de27c74c..e221758a 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -20,6 +20,9 @@ from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( ConfigOptions, ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.geoMod import ( + GriddedGeoMeta, + ) # If MPI was initialized outside of python, # disable initialization/finalization behavior @@ -27,9 +30,6 @@ mpi4py.rc.initialize = False mpi4py.rc.finalize = False -if TYPE_CHECKING: - from .config import ConfigOptions - from .geoMod import GriddedGeoMeta _T = TypeVar("_T") From 3977f4b6fb4eeb6f73d576491559c5ba5e95b47d Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 10:29:54 -0400 Subject: [PATCH 094/132] Fix NameError during handling of class init args introduced during refactor and/or rebasing --- .../NextGen_Forcings_Engine/model.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 4335fea5..66886ab9 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -488,23 +488,11 @@ def __handle_aorc_and_nwm_force_keys( # Flag to indicate the AWS .zarr NWMv3 Forcing file method elif force_key == 27: if self._bmi._job_meta.nwm_domain == "CONUS": - self.source_data_processor = NWMV3ConusProcessor( - self._bmi._job_meta, - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) + proc_cls = NWMV3ConusProcessor elif self._bmi._job_meta.nwm_domain == "Hawaii": - self.source_data_processor = NWMV3HawaiiProcessor( - self._bmi._job_meta, - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) + proc_cls = NWMV3HawaiiProcessor elif self._bmi._job_meta.nwm_domain == "PR": - self.source_data_processor = NWMV3PuertoRicoProcessor( - self._bmi._job_meta, - self._bmi._mpi_meta, - self._bmi.geo_meta, - ) + proc_cls = NWMV3PuertoRicoProcessor elif self._bmi._job_meta.nwm_domain == "Alaska": proc_cls = NWMV3AlaskaProcessor else: From 28043b274238e364b7cbb0f6d26fb8d1ab205691 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 11:34:27 -0400 Subject: [PATCH 095/132] Add short-circuit guard for property perform_downscaling to return False when precip_only_flag is falsy. --- .../NextGen_Forcings_Engine/core/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index cfdeee2a..b5fe943e 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -1188,6 +1188,8 @@ def dScaleParamDirs(self, value: list) -> None: @property def perform_downscaling(self) -> bool: """Determine whether downscaling of input forcings is necessary based on the downscaling options specified by the user for each input forcing in the configuration file.""" + if self.precip_only_flag: + return False if ( 1 in self.q2dDownscaleOpt or 1 in self.swDownscaleOpt From f512c55521b340d977380347f74ac6c536c6f1f9 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 11:45:38 -0400 Subject: [PATCH 096/132] Remove vestigial self.GeoMeta attribute --- .../NextGen_Forcings_Engine/bmi_model.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 2b321d8c..6f3256e0 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -1550,7 +1550,6 @@ def __init__( Initializes the model with default values for time, variables, and grid types. """ super().__init__(b_date, geogrid, output_path) - self.GeoMeta = GriddedGeoMeta def grid_ranks(self) -> list[int]: """Get the grid ranks for the gridded domain.""" @@ -1621,7 +1620,6 @@ def __init__( Initializes the model with default values for time, variables, and grid types. """ super().__init__(b_date, geogrid, output_path) - self.GeoMeta = HydrofabricGeoMeta def grid_ranks(self) -> list[int]: """Get the grid ranks for the hydrofabric domain.""" @@ -1688,7 +1686,6 @@ def __init__( Initializes the model with default values for time, variables, and grid types. """ super().__init__(b_date, geogrid, output_path) - self.GeoMeta = UnstructuredGeoMeta def grid_ranks(self) -> list[int]: """Get the grid ranks for the unstructured domain.""" From c47db3ace58ae1eb7ea32b0bc91fbf6e4ae981b8 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 12:29:19 -0400 Subject: [PATCH 097/132] Add TODOs about potential bugs --- .../NextGen_Forcings_Engine/core/regrid.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py index 639904a4..fa751dc9 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py @@ -350,6 +350,8 @@ def regrid_ak_ext_ana(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co input_forcings.regridded_forcings2_elem = np.empty( [9, wrf_hydro_geo_meta.ny_local_elem], np.float32 ) + # TODO likely a bug, should this be "hydrofabric"? + # Issue might be overridden for hydrofabric case in function `check_regrid_status`. elif config_options.grid_type == "unstructured": input_forcings.regridded_forcings1 = np.empty( [9, wrf_hydro_geo_meta.ny_local], np.float32 @@ -446,6 +448,9 @@ def regrid_ak_ext_ana(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co ] = input_forcings.regridded_forcings2_elem[ input_forcings.input_map_output[force_count], : ] + # TODO likely a bug, "hydrofabric" slicing should access 1 dimension, not 2. + # See `regridded_forcings2 =` for hydrofabric case in function `check_regrid_status`. + # Is AK Extended AnA runnable like this? elif config_options.grid_type == "hydrofabric": try: input_forcings.regridded_forcings2[ From 35ea0f983dbd64e498d0fc443202dd232c5e771d Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 17:26:37 -0400 Subject: [PATCH 098/132] Replace hard-coded supplemental precip count with dynamic length lookup of supp precip forcing list --- .../NextGen_Forcings_Engine/core/config.py | 13 ++++++++----- ...ted_bmi_model_after_update_n1_rank0__step_1.json | 2 +- ...ted_bmi_model_after_update_n1_rank0__step_2.json | 2 +- ...ted_bmi_model_after_update_n1_rank0__step_3.json | 2 +- ...ted_bmi_model_after_update_n2_rank0__step_1.json | 2 +- ...ted_bmi_model_after_update_n2_rank0__step_2.json | 2 +- ...ted_bmi_model_after_update_n2_rank0__step_3.json | 2 +- ...ted_bmi_model_after_update_n2_rank1__step_1.json | 2 +- ...ted_bmi_model_after_update_n2_rank1__step_2.json | 2 +- ...ted_bmi_model_after_update_n2_rank1__step_3.json | 2 +- .../test_expected_bmi_model_finalize_n1_rank0_.json | 2 +- .../test_expected_bmi_model_finalize_n2_rank0_.json | 2 +- .../test_expected_bmi_model_finalize_n2_rank1_.json | 2 +- .../test_expected_bmi_model_init_n1_rank0_.json | 2 +- .../test_expected_bmi_model_init_n2_rank0_.json | 2 +- .../test_expected_bmi_model_init_n2_rank1_.json | 2 +- ...onfig_options_after_update_n1_rank0__step_1.json | 2 +- ...onfig_options_after_update_n1_rank0__step_2.json | 2 +- ...onfig_options_after_update_n1_rank0__step_3.json | 2 +- ...onfig_options_after_update_n2_rank0__step_1.json | 2 +- ...onfig_options_after_update_n2_rank0__step_2.json | 2 +- ...onfig_options_after_update_n2_rank0__step_3.json | 2 +- ...onfig_options_after_update_n2_rank1__step_1.json | 2 +- ...onfig_options_after_update_n2_rank1__step_2.json | 2 +- ...onfig_options_after_update_n2_rank1__step_3.json | 2 +- ..._expected_config_options_finalize_n1_rank0_.json | 2 +- ..._expected_config_options_finalize_n2_rank0_.json | 2 +- ..._expected_config_options_finalize_n2_rank1_.json | 2 +- ...test_expected_config_options_init_n1_rank0_.json | 2 +- ...test_expected_config_options_init_n2_rank0_.json | 2 +- ...test_expected_config_options_init_n2_rank1_.json | 2 +- 31 files changed, 38 insertions(+), 35 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index b5fe943e..d4bb52b2 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -16,6 +16,7 @@ from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.consts import ( CONFIGOPTIONS, FORCINGINPUTMOD, + SUPPPRECIPMOD, ) from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.err_handler import ( err_out_screen, @@ -205,15 +206,17 @@ def cfg_bmi(self, value: dict) -> None: @property def force_count(self) -> int: - """Calculate the number of total possible input forcing options based on the length of the InputForcings list in the consts.py file. This is used for error checking to ensure users specify valid input forcing options in the configuration file.""" + """Calculate the number of total possible input forcing options based on the length of the InputForcings list in consts.py. + This is used for error checking to ensure users specify valid input forcing options in the configuration file. + """ return len(FORCINGINPUTMOD["PRODUCT_NAME"]) @property def supp_precip_count(self) -> int: - """Calculate the number of total possible supplemental precip forcing options based on the length of the SuppPrecipForcings list in the consts.py file. This is used for error checking to ensure users specify valid supplemental precip forcing options in the configuration file.""" - # TODO make this dynamic based on the length of the SUPPPRECIPMOD list in consts.py, but for now hardcoding to 15 since that is the number of options currently available in consts.py and this will avoid any issues with the formatting of the consts.py file causing errors in the program. This is used for error checking to ensure users specify valid supplemental precip forcing options in the configuration file. - # return len(SUPPPRECIPMOD["suppPrecipMod"]["PRODUCT_NAMES"]) - return 15 + """Calculate the number of total possible supplemental precip forcing options based on the length of the Supplemental Precip PRODUCT_NAMES dict in consts.py. + This is used for error checking to ensure users specify valid supplemental precip forcing options in the configuration file. + """ + return len(SUPPPRECIPMOD["PRODUCT_NAMES"]) @property def precip_only_flag(self) -> bool: diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json index 283f9511..8f7df495 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json @@ -479,7 +479,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json index 26e1450f..617df417 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json @@ -479,7 +479,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json index 78e899a3..0617ab38 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json @@ -479,7 +479,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json index 93fc260a..6b4e9f2c 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json @@ -479,7 +479,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json index 50cf0d86..d60330e1 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json @@ -479,7 +479,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json index fe18aacb..3f62699f 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json @@ -479,7 +479,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json index 5f28473b..6ce96b23 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json @@ -479,7 +479,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json index 65e8fc17..79bc3416 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json @@ -479,7 +479,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json index ca8f708a..ad066576 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json @@ -479,7 +479,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n1_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n1_rank0_.json index 5086d888..2e13b37f 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n1_rank0_.json @@ -479,7 +479,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank0_.json index fdcd8db2..c8c8115c 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank0_.json @@ -479,7 +479,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank1_.json b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank1_.json index 4339fbe6..10e05311 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank1_.json @@ -479,7 +479,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json index 0349bee9..3a29d087 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json @@ -289,7 +289,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json index 1d8b1d20..a4920c77 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json @@ -289,7 +289,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json index e2b6c0df..074032c7 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json @@ -289,7 +289,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json index b2edf10e..f4f6b99c 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json @@ -470,7 +470,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json index 33914021..8699d992 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json @@ -470,7 +470,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json index d5781251..58255f00 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json @@ -470,7 +470,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json index b2edf10e..f4f6b99c 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json @@ -470,7 +470,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json index 33914021..8699d992 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json @@ -470,7 +470,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json index d5781251..58255f00 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json @@ -470,7 +470,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json index b2edf10e..f4f6b99c 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json @@ -470,7 +470,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json index 33914021..8699d992 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json @@ -470,7 +470,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json index d5781251..58255f00 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json @@ -470,7 +470,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json index 35b7e0a4..d0d21c4d 100644 --- a/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json @@ -470,7 +470,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json index 35b7e0a4..d0d21c4d 100644 --- a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json @@ -470,7 +470,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json index 35b7e0a4..d0d21c4d 100644 --- a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json @@ -470,7 +470,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_init_n1_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_init_n1_rank0_.json index 115490a3..99420727 100644 --- a/tests/test_data/expected_results/test_expected_config_options_init_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_init_n1_rank0_.json @@ -280,7 +280,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_init_n2_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_init_n2_rank0_.json index 115490a3..99420727 100644 --- a/tests/test_data/expected_results/test_expected_config_options_init_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_init_n2_rank0_.json @@ -280,7 +280,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], diff --git a/tests/test_data/expected_results/test_expected_config_options_init_n2_rank1_.json b/tests/test_data/expected_results/test_expected_config_options_init_n2_rank1_.json index 115490a3..99420727 100644 --- a/tests/test_data/expected_results/test_expected_config_options_init_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_config_options_init_n2_rank1_.json @@ -280,7 +280,7 @@ "suppTemporalInterp": null, "supp_input_offsets": null, "supp_pcp_max_hours": null, - "supp_precip_count": 15, + "supp_precip_count": 16, "supp_precip_dirs": null, "supp_precip_file_types": [], "supp_precip_forcings": [], From 0ca29ce939cdb73482c6d303fe7c15ab7a00e747 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 18:07:19 -0400 Subject: [PATCH 099/132] Add targeted hardening of ConfigOptions attrs to address concerns documented in: https://github.com/NGWPC/ngen-forcing/pull/107 --- .../NextGen_Forcings_Engine/core/config.py | 23 +++++++++++++++- .../historical_forcing.py | 26 +++++++++++++++---- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index d4bb52b2..037eb3f8 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -86,6 +86,7 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No self._ana_flag = None self._look_back = None self._fcst_freq = None + self._fcst_input_horizons = None self._spatial_meta = None self._geopackage = None self._geogrid = None @@ -530,7 +531,18 @@ def fcst_freq(self) -> int: @fcst_freq.setter def fcst_freq(self, value: int) -> None: - """Set the forecast frequency in hours specified by the user in the configuration file. This is used to calculate the processing window for reforecast simulations, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" + """Set the forecast frequency in hours specified by the user in the configuration file. + + This is used to calculate the processing window for reforecast simulations, and is only necessary + if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation. + + NOTE: this property is hardened such that it allows being set one time, but may not be mutated after that initial set. + For rationale, see: https://github.com/NGWPC/ngen-forcing/pull/107 + """ + if self._fcst_freq is not None and self._fcst_freq != value: + raise ValueError( + f"fcst_freq is immutable after initialization. Current: {self._fcst_freq}, Attempted: {value}" + ) self.check_input_values_non_negative([value], "ForecastFrequency") if value > 1440: err_out_screen( @@ -864,6 +876,15 @@ def fcst_input_horizons(self) -> list: @fcst_input_horizons.setter def fcst_input_horizons(self, value: list) -> None: + """Setter for ``fcst_input_horizons``. + + NOTE: this property is hardened such that it allows being set one time, but may not be mutated after that initial set. + For rationale, see: https://github.com/NGWPC/ngen-forcing/pull/107 + """ + if self._fcst_input_horizons is not None and self._fcst_input_horizons != value: + raise ValueError( + f"fcst_input_horizons is immutable after initialization. Current: {self._fcst_input_horizons}, Attempted: {value}" + ) if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "ForecastInputHorizons") self.check_input_values_non_negative(value, "ForecastInputHorizons") diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py index 976e130f..1314055e 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py @@ -53,6 +53,20 @@ def __init__( self.config_options = config_options self.mpi_config = mpi_config self.wrf_hydro_geo_meta = wrf_hydro_geo_meta + self._b_date_proc_initial = None # Cached value to detect mutations + + def _get_b_date_proc_safe(self): + """Hardened accessor for ``config_options.b_date_proc`` which ensures that is does not get mutated. + + For rationale, see: https://github.com/NGWPC/ngen-forcing/pull/107 + """ + if self._b_date_proc_initial is None: + self._b_date_proc_initial = self.config_options.b_date_proc + elif self.config_options.b_date_proc != self._b_date_proc_initial: + raise ValueError( + "b_date_proc was modified after BaseProcessor was created, which is not allowed." + ) + return self.config_options.b_date_proc @cached_property def bounds(self) -> tuple[float, float, float, float]: @@ -116,7 +130,7 @@ def time_min(self) -> np.datetime64: :return: Minimum time as np.datetime64 """ - return np.datetime64(self.config_options.b_date_proc) + np.timedelta64(1, "h") + return np.datetime64(self._get_b_date_proc_safe()) + np.timedelta64(1, "h") @property def datetimes(self) -> pd.DatetimeIndex: @@ -198,10 +212,12 @@ def start_end_datetimes(self) -> dict[pd.Timestamp, pd.Timestamp]: start and end date pairs based on the cache size. :return: Dictionary of start and end dates as pd.Timestamp - TODO for lru_cache / cached_property safety, confirm or enforce that these are never mutated: - self.config_options.b_date_proc - self.config_options.fcst_input_horizons - self.config_options.fcst_freq + NOTE: + ``b_date_proc`` is protected by _get_``b_date_proc_safe()``. + ``fcst_input_horizons`` is protected by its own setter in ConfigOptions. + ``fcst_freq`` is protected by its own setter in ConfigOptions. + For rationale, see: https://github.com/NGWPC/ngen-forcing/pull/107 + """ start_end_datetimes = {} for start, end in self.year_start_stop_dict.values(): From d94cd2d7b92b0d1c37cb29b945ba9cfd0ad1a1dd Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 18:14:51 -0400 Subject: [PATCH 100/132] Remove unused variable --- .../NextGen_Forcings_Engine/historical_forcing.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py index 1314055e..374154bb 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py @@ -657,10 +657,6 @@ def sliced_ds(self) -> xr.Dataset: for var in self.vars: try: with self.timing_block(f"lazy loading {self.dataset_name} data"): - # TODO this object_store var is not used - object_store = obstore.store.from_url( - self.url(var), skip_signature=True - ) datasets.append(self.slice_ds(self.s3_lazy_ds[var])) except Exception as e: LOG.critical( From 30e907a27ed5824321e643c942cee3656e902442 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 21:43:59 -0400 Subject: [PATCH 101/132] Clean up docstrings --- .../NextGen_Forcings_Engine/core/config.py | 752 ++++++++++++++---- 1 file changed, 615 insertions(+), 137 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 037eb3f8..a70f8fa4 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -182,8 +182,16 @@ def post_init(self) -> None: @property def try_config_get_except_attr_map(self) -> dict: - """Get the mapping of configuration variable names to class attribute names for variables that are extracted directly from the configuration file without any additional processing. This is used to control how variables are extracted from the configuration file and assigned to class attributes in a consistent way based on the mapping specified in the consts.py file.""" - # Don't mutate the module-level CONFIGOPTIONS object. Operate on a copy instead (and return that modified copy). + """Get the mapping of configuration variable names to class attribute names + for variables that are extracted directly from the configuration file + without any additional processing. This is used to control how variables are + extracted from the configuration file and assigned to class attributes in a + consistent way based on the mapping specified in the consts.py file. + + Don't mutate the module-level CONFIGOPTIONS object. + Operate on a copy instead (and return that modified copy). + """ + dict_map = CONFIGOPTIONS["try_config_get_except_attr_map"].copy() if self._b_date_proc is not None and "RefcstBDateProc" in dict_map: dict_map.pop("RefcstBDateProc") @@ -207,21 +215,27 @@ def cfg_bmi(self, value: dict) -> None: @property def force_count(self) -> int: - """Calculate the number of total possible input forcing options based on the length of the InputForcings list in consts.py. - This is used for error checking to ensure users specify valid input forcing options in the configuration file. + """Calculate the number of total possible input forcing options based on the + length of the InputForcings list in consts.py. This is used for error checking + to ensure users specify valid input forcing options in the configuration file. """ return len(FORCINGINPUTMOD["PRODUCT_NAME"]) @property def supp_precip_count(self) -> int: - """Calculate the number of total possible supplemental precip forcing options based on the length of the Supplemental Precip PRODUCT_NAMES dict in consts.py. - This is used for error checking to ensure users specify valid supplemental precip forcing options in the configuration file. + """Calculate the number of total possible supplemental precip forcing options + based on the length of the Supplemental Precip PRODUCT_NAMES dict in consts.py. + This is used for error checking to ensure users specify valid supplemental + precip forcing options in the configuration file. """ return len(SUPPPRECIPMOD["PRODUCT_NAMES"]) @property def precip_only_flag(self) -> bool: - """Flag to indicate whether the user has chosen to run the supplemental precip forcings module only, which will trigger some different processing pathways and error checking for certain configuration options.""" + """Flag to indicate whether the user has chosen to run the supplemental precip + forcings module only, which will trigger some different processing pathways and + error checking for certain configuration options. + """ precip_only = False if self.supp_precip_forcings is not None and len(self.supp_precip_forcings) > 0: if int(self.supp_precip_forcings[0]) == 14: @@ -229,7 +243,10 @@ def precip_only_flag(self) -> bool: return precip_only def set_attrs(self, attrs_dict: dict, set_none: bool = False): - """Set the attributes of the class based on the configuration file. This is used to populate the attributes of the class after they have been read in and validated from the configuration file.""" + """Set the attributes of the class based on the configuration file. This is + used to populate the attributes of the class after they have been read in and + validated from the configuration file. + """ for cfg_bmi_attr, config_options_attr in attrs_dict.items(): if set_none: attr = None @@ -238,7 +255,9 @@ def set_attrs(self, attrs_dict: dict, set_none: bool = False): setattr(self, config_options_attr, attr) def set_attrs_use_default(self, attrs_dict: dict): - """Set the attributes of the class based on the configuration file. Set default value to default if not found in config file.""" + """Set the attributes of the class based on the configuration file. Set default + value to default if not found in config file. + """ for cfg_bmi_attr, config_options_attr in attrs_dict.items(): setattr( self, @@ -265,7 +284,9 @@ def extract_input_variable(self, variable_name: str) -> str: ) def extract_input_variable_set_default(self, variable_name: str, default=0) -> str: - """Extract the variable name from the configuration file for a given variable, and set it to a default value if it is not found.""" + """Extract the variable name from the configuration file for a given variable, + and set it to a default value if it is not found. + """ try: variable = self.cfg_bmi[variable_name] except (KeyError, configparser.NoOptionError) as e: @@ -296,20 +317,30 @@ def try_config_get(self, variable_name: str) -> str: def check_number_of_inputs( self, value: list, variable_name: str, input_type: str, number_inputs: int ) -> None: - """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable.""" + """Check that the number of inputs specified by the user in the configuration + file matches the expected number of inputs for a given variable. + """ if len(value) != number_inputs: err_out_screen( f"Number of {variable_name} values must match the number of {input_type} in the configuration file." ) def check_number_of_inputs_forcings(self, value: list, variable_name: str) -> None: - """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable, specifically for input forcings variables which should match the number of input forcing options specified by the user in the configuration file.""" + """Check that the number of inputs specified by the user in the configuration + file matches the expected number of inputs for a given variable, specifically + for input forcings variables which should match the number of input forcing + options specified by the user in the configuration file. + """ return self.check_number_of_inputs( value, variable_name, " InputForcings", self.number_inputs ) def check_number_of_inputs_supp_pcp(self, value: list, variable_name: str) -> None: - """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable, specifically for supplemental precip forcing variables which should match the number of supplemental precip forcing options specified by the user in the configuration file.""" + """Check that the number of inputs specified by the user in the configuration + file matches the expected number of inputs for a given variable, specifically + for supplemental precip forcing variables which should match the number of + supplemental precip forcing options specified by the user in the configuration file. + """ return self.check_number_of_inputs( value, variable_name, " SupplementalPrecipForcings", self.number_supp_pcp ) @@ -317,7 +348,9 @@ def check_number_of_inputs_supp_pcp(self, value: list, variable_name: str) -> No def check_input_values_in_range( self, value: list, variable_name: str, valid_input_options: list ) -> None: - """Check that the input values specified by the user in the configuration file are within a valid range for a given variable.""" + """Check that the input values specified by the user in the configuration file + are within a valid range for a given variable. + """ for val in value: if val not in valid_input_options: err_out_screen( @@ -325,7 +358,9 @@ def check_input_values_in_range( ) def check_input_values_non_negative(self, value: list, variable_name: str) -> None: - """Check that the input values specified by the user in the configuration file are positive for a given variable.""" + """Check that the input values specified by the user in the configuration file + are positive for a given variable. + """ for val in value: if float(val) < 0: err_out_screen( @@ -333,7 +368,9 @@ def check_input_values_non_negative(self, value: list, variable_name: str) -> No ) def check_input_values_positive(self, value: list, variable_name: str) -> None: - """Check that the input values specified by the user in the configuration file are positive for a given variable.""" + """Check that the input values specified by the user in the configuration file + are positive for a given variable. + """ for val in value: if val <= 0: err_out_screen( @@ -374,7 +411,9 @@ def broadcast_new_64bit_uid(self) -> None: @property def supp_precip_forcings(self): - """Choose a set of supplemental precipitation file(s) to layer into the final LDASIN forcing files processed from the options above. The following is a mapping of numeric values to external input native forcing files. + """Choose a set of supplemental precipitation file(s) to layer into the final + LDASIN forcing files processed from the options above. The following is a mapping + of numeric values to external input native forcing files. 1. MRMS GRIB2 hourly radar-only QPE 2. MRMS GRIB2 hourly gage-corrected radar QPE @@ -398,7 +437,11 @@ def supp_precip_forcings(self): @supp_precip_forcings.setter def supp_precip_forcings(self, value: list) -> None: - """Set the list of supplemental precip forcing options specified by the user in the configuration file. This is used to control which supplemental precip forcings are processed and how they are processed based on the other configuration options specified for each supplemental precip forcing.""" + """Set the list of supplemental precip forcing options specified by the user in + the configuration file. This is used to control which supplemental precip + forcings are processed and how they are processed based on the other + configuration options specified for each supplemental precip forcing. + """ if value is not None and len(value) > 0: self.check_input_values_in_range( [int(i) for i in value], @@ -409,12 +452,17 @@ def supp_precip_forcings(self, value: list) -> None: @property def output_freq(self) -> int: - """Get the output frequency in minutes specified by the user in the configuration file. This is used to control the output frequency of the processed forcings, and is necessary for both realtime and reforecast simulations.""" + """Get the output frequency in minutes specified by the user in the + configuration file. This is used to control the output frequency of the + processed forcings, and is necessary for both realtime and reforecast simulations. + """ return self._output_freq @output_freq.setter def output_freq(self, value: int) -> None: - """Specify the output frequency in minutes. Note that any frequencies at higher intervals than what if provided as input will entail input forcing data being temporally interpolated. + """Specify the output frequency in minutes. Note that any frequencies at higher + intervals than what if provided as input will entail input forcing data being + temporally interpolated. Example- OutputFrequency: 60 """ @@ -423,14 +471,22 @@ def output_freq(self, value: int) -> None: @property def sub_output_hour(self) -> int: - """Get the sub-daily output hour specified by the user in the configuration file. This is used to control the output frequency of the processed forcings for sub-daily output frequencies, and is only necessary if the user has chosen a sub-daily output frequency in the configuration file.""" + """Get the sub-daily output hour specified by the user in the configuration file. + This is used to control the output frequency of the processed forcings for + sub-daily output frequencies, and is only necessary if the user has chosen a + sub-daily output frequency in the configuration file. + """ return self._sub_output_hour @sub_output_hour.setter def sub_output_hour(self, value: int) -> None: """Sub output hour. - New variable currently for NWMv3.1 operations to properly ingest GFS 13km forecast data that outputs various frequencies throughout the forecast cycle lifetime. This variable will properly account for reading time slices of the forecast cycle. Currently only needed for GFS 13km operational configuration. Otherwise, set this value to 0. + New variable currently for NWMv3.1 operations to properly ingest GFS 13km + forecast data that outputs various frequencies throughout the forecast cycle + lifetime. This variable will properly account for reading time slices of the + forecast cycle. Currently only needed for GFS 13km operational configuration. + Otherwise, set this value to 0. Example- SubOutputHour: 0 """ @@ -441,14 +497,23 @@ def sub_output_hour(self, value: int) -> None: @property def sub_output_freq(self) -> int: - """Calculate the sub-daily output frequency in minutes based on the output frequency and sub-daily output hour specified by the user in the configuration file. This is used to control the output frequency of the processed forcings for sub-daily output frequencies, and is only necessary if the user has chosen a sub-daily output frequency in the configuration file.""" + """Calculate the sub-daily output frequency in minutes based on the output + frequency and sub-daily output hour specified by the user in the configuration + file. This is used to control the output frequency of the processed forcings for + sub-daily output frequencies, and is only necessary if the user has chosen a + sub-daily output frequency in the configuration file. + """ return self._sub_output_freq @sub_output_freq.setter def sub_output_freq(self, value: int) -> None: """Sub output frequency. - New variable currently for NWMv3.1 operations to properly ingest GFS 13km forecast data that outputs various frequencies throughout the forecast cycle lifetime. This variable will properly account for reading time slices of the forecast cycle. Currently only needed for GFS 13km operational configuration. Otherwise, set this value to 0. + New variable currently for NWMv3.1 operations to properly ingest GFS 13km + forecast data that outputs various frequencies throughout the forecast cycle + lifetime. This variable will properly account for reading time slices of the + forecast cycle. Currently only needed for GFS 13km operational configuration. + Otherwise, set this value to 0. Example- SubOutputFreq: 0 """ @@ -462,7 +527,10 @@ def sub_output_freq(self, value: int) -> None: @property def scratch_dir(self) -> str: - """Specify a scratch directory that will be used for storage of temporary files. These files will be removed automatically by the program. at the end of the BMI instance. However, this directory will also store the output forcing file if requested by the user as well (will not be deleted in this instance). + """Specify a scratch directory that will be used for storage of temporary files. + These files will be removed automatically by the program. at the end of the BMI + instance. However, this directory will also store the output forcing file if + requested by the user as well (will not be deleted in this instance). Example- ScratchDir: "./ScratchDir """ @@ -470,13 +538,20 @@ def scratch_dir(self) -> str: @scratch_dir.setter def scratch_dir(self, value: str) -> None: - """Set the pathway to the scratch directory specified by the user in the configuration file. This is used to control where intermediate files are written during processing, and is necessary for both realtime and reforecast simulations.""" + """Set the pathway to the scratch directory specified by the user in the + configuration file. This is used to control where intermediate files are written + during processing, and is necessary for both realtime and reforecast simulations. + """ self.make_scratch_dir(value) self._scratch_dir = value @property def useCompression(self) -> int: - """Flag to activate scale_factor / add_offset byte packing in the output files. 0 - Deactivate compression 1 - Activate compression, Only applicable in this instance when you request a netcdf output forcing file (Output: 1). Otherwise, just set to 0. + """Flag to activate scale_factor / add_offset byte packing in the output files. + 0 - Deactivate compression + 1 - Activate compression + Only applicable in this instance when you request a netcdf output forcing file + (Output: 1). Otherwise, just set to 0. Example- compressOutput: 0 """ @@ -484,7 +559,11 @@ def useCompression(self) -> int: @useCompression.setter def useCompression(self, value: int) -> None: - """Set the flag for whether to use compression when writing output files specified by the user in the configuration file. This is used to control whether output files are compressed, which can save disk space but may increase processing time.""" + """Set the flag for whether to use compression when writing output files + specified by the user in the configuration file. This is used to control whether + output files are compressed, which can save disk space but may increase + processing time. + """ if value is None: value = 0 self.check_input_values_in_range([value], "compressOutput", [0, 1]) @@ -492,7 +571,9 @@ def useCompression(self, value: int) -> None: @property def ana_flag(self) -> int: - """If this is AnA run, set AnAFlag to 1, otherwise 0. Setting this flag will change the behavior of some Bias Correction routines as the ForecastInputOffsets options. + """If this is AnA run, set AnAFlag to 1, otherwise 0. Setting this flag will + change the behavior of some Bias Correction routines as the ForecastInputOffsets + options. Example- AnAFlag: 1 """ @@ -500,14 +581,22 @@ def ana_flag(self) -> int: @ana_flag.setter def ana_flag(self, value: int) -> None: - """Set the flag for whether to include the analysis time step in the output files specified by the user in the configuration file. This is used to control whether the analysis time step is included in the output files, which can be useful for certain applications but may not be necessary for all users.""" + """Set the flag for whether to include the analysis time step in the output + files specified by the user in the configuration file. This is used to control + whether the analysis time step is included in the output files, which can be + useful for certain applications but may not be necessary for all users. + """ value = int(value) self.check_input_values_in_range([value], "AnAFlag", [0, 1]) self._ana_flag = value @property def look_back(self) -> int: - """Specify a lookback period in minutes to process data. This is required if you are only processing an AnA operational configuration. This value should specify how far back you need to look in time from your "RefcstBDateProc" start date that you specified. In this instance, that start date will be your actual end date. If no LookBack specified, please specify -9999. + """Specify a lookback period in minutes to process data. This is required if + you are only processing an AnA operational configuration. This value should + specify how far back you need to look in time from your "RefcstBDateProc" start + date that you specified. In this instance, that start date will be your actual + end date. If no LookBack specified, please specify -9999. Example- LookBack: 180 """ @@ -515,7 +604,11 @@ def look_back(self) -> int: @look_back.setter def look_back(self, value: int) -> None: - """Set the look back window in hours specified by the user in the configuration file. This is used to calculate the processing window for reforecast simulations, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" + """Set the look back window in hours specified by the user in the configuration + file. This is used to calculate the processing window for reforecast simulations, + and is only necessary if the user is running a reforecast simulation with a + specified processing window rather than a realtime simulation. + """ if value <= 0 and value != -9999: err_out_screen("Please specify a positive LookBack or -9999 for realtime.") # NOTE: Side effect (calculate_lookback_window) removed - now called in post_init() @@ -523,7 +616,9 @@ def look_back(self, value: int) -> None: @property def fcst_freq(self) -> int: - """Specify a forecast frequency in minutes. This value specifies how often to generate a set of forecast forcings. If generating hourly retrospective forcings, specify this value to be 60. + """Specify a forecast frequency in minutes. This value specifies how often to + generate a set of forecast forcings. If generating hourly retrospective forcings, + specify this value to be 60. Example- ForecastFrequency: 60 """ @@ -533,10 +628,12 @@ def fcst_freq(self) -> int: def fcst_freq(self, value: int) -> None: """Set the forecast frequency in hours specified by the user in the configuration file. - This is used to calculate the processing window for reforecast simulations, and is only necessary - if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation. + This is used to calculate the processing window for reforecast simulations, and + is only necessary if the user is running a reforecast simulation with a + specified processing window rather than a realtime simulation. - NOTE: this property is hardened such that it allows being set one time, but may not be mutated after that initial set. + NOTE: this property is hardened such that it allows being set one time, but may + not be mutated after that initial set. For rationale, see: https://github.com/NGWPC/ngen-forcing/pull/107 """ if self._fcst_freq is not None and self._fcst_freq != value: @@ -552,7 +649,11 @@ def fcst_freq(self, value: int) -> None: @property def spatial_meta(self): - """Specify the optional land spatial metadata file. If found, coordinate projection information and coordinate will be translated from to the final output file. This variable is only a special case if the user is specifying the original WRF-Hydro domain from earlier NWM versions. Otherwise, just leave the one blank (''). + """Specify the optional land spatial metadata file. If found, coordinate + projection information and coordinate will be translated from to the final + output file. This variable is only a special case if the user is specifying the + original WRF-Hydro domain from earlier NWM versions. Otherwise, just leave the + one blank (''). Example- SpatialMetaIn: ./GEOGRID_LDASOUT_Spatial_Metadata_CONUS.nc """ @@ -560,7 +661,10 @@ def spatial_meta(self): @spatial_meta.setter def spatial_meta(self, value: str) -> None: - """Set the spatial metadata options specified by the user in the configuration file. This is used to control how spatial metadata is handled during processing, and is necessary for both realtime and reforecast simulations.""" + """Set the spatial metadata options specified by the user in the configuration + file. This is used to control how spatial metadata is handled during processing, + and is necessary for both realtime and reforecast simulations. + """ if len(value) == 0: # No spatial metadata file found. value = None @@ -573,7 +677,14 @@ def spatial_meta(self, value: str) -> None: @property def b_date_proc(self) -> str: - """If running an operational configuration in realtime or just using a retrospective dataset (NWM, AORC, ERA5), this will be the defined start date for the NextGen Forcing Engine BMI which is assumed to be the beginning of the forecast cycle (i.e. hour 0) or just the start date of the retrospective dataset. From there the first time step will be hour 1 from the start date specified here. If you're running an AnA configuration however, this variable becomes the end date of the simulation and the "LookBack" value specified above will be how far back you look in time for the AnA operational configuration. + """If running an operational configuration in realtime or just using a + retrospective dataset (NWM, AORC, ERA5), this will be the defined start date for + the NextGen Forcing Engine BMI which is assumed to be the beginning of the + forecast cycle (i.e. hour 0) or just the start date of the retrospective dataset. + From there the first time step will be hour 1 from the start date specified here. + If you're running an AnA configuration however, this variable becomes the end + date of the simulation and the "LookBack" value specified above will be how far + back you look in time for the AnA operational configuration. Example- RefcstBDateProc: 202210071400 """ @@ -581,7 +692,9 @@ def b_date_proc(self) -> str: @b_date_proc.setter def b_date_proc(self, value: str | datetime) -> None: - """Set the beginning date of processing for reforecast simulations. This is used to calculate the processing window for reforecast simulations.""" + """Set the beginning date of processing for reforecast simulations. This is used + to calculate the processing window for reforecast simulations. + """ if value is None: self._b_date_proc = None return @@ -606,7 +719,11 @@ def b_date_proc(self, value: str | datetime) -> None: @property def realtime_flag(self) -> bool: - """Flag to indicate whether the user has chosen to run a realtime simulation, which will trigger some different processing pathways and error checking for certain configuration options, and will also control how the processing window is calculated.""" + """Flag to indicate whether the user has chosen to run a realtime simulation, + which will trigger some different processing pathways and error checking for + certain configuration options, and will also control how the processing window + is calculated. + """ if self.look_back == -9999: value = False elif self.b_date_proc == -9999: @@ -618,7 +735,11 @@ def realtime_flag(self) -> bool: @property def refcst_flag(self) -> bool: - """Flag to indicate whether the user has chosen to run a reforecast simulation, which will trigger some different processing pathways and error checking for certain configuration options, and will also control how the processing window is calculated.""" + """Flag to indicate whether the user has chosen to run a reforecast simulation, + which will trigger some different processing pathways and error checking for + certain configuration options, and will also control how the processing window + is calculated. + """ if self.look_back == -9999: return True elif self.b_date_proc == -9999: @@ -628,17 +749,26 @@ def refcst_flag(self) -> bool: @property def geopackage(self) -> str: - """Get the pathway to the geopackage file to be used for processing. This is used to specify the grid information for regridding input forcings, and is only necessary if the user is running a simulation that requires regridding of input forcings.""" + """Get the pathway to the geopackage file to be used for processing. This is + used to specify the grid information for regridding input forcings, and is only + necessary if the user is running a simulation that requires regridding of input + forcings. + """ return self._geopackage @geopackage.setter def geopackage(self, value: str) -> None: - """Set the pathway to the geopackage file to be used for processing. This is used to specify the grid information for regridding input forcings, and is only necessary if the user is running a simulation that requires regridding of input forcings.""" + """Set the pathway to the geopackage file to be used for processing. This is + used to specify the grid information for regridding input forcings, and is only + necessary if the user is running a simulation that requires regridding of input + forcings. + """ self._geopackage = value @property def geogrid(self) -> str: - """Specify a geogrid file (e.g. latitude, longitude, mesh connectivity, elevation, slope) that defines domain to which the forcings are being processed to. + """Specify a geogrid file (e.g. latitude, longitude, mesh connectivity, + elevation, slope) that defines domain to which the forcings are being processed to. Example- GeogridIn: ./geo_em_CONUS.nc """ @@ -646,7 +776,11 @@ def geogrid(self) -> str: @geogrid.setter def geogrid(self, value: str) -> None: - """Set the pathway to the geogrid file to be used for processing. This is used to specify the grid information for regridding input forcings, and is only necessary if the user is running a simulation that requires regridding of input forcings.""" + """Set the pathway to the geogrid file to be used for processing. This is used + to specify the grid information for regridding input forcings, and is only + necessary if the user is running a simulation that requires regridding of input + forcings. + """ # If user provided geogrid, use it as-is if self.user_provided_geogrid_flag: self._geogrid = value @@ -677,12 +811,20 @@ def try_make_dir(self, directory: str, optional_str: str = "") -> None: @property def input_forcings(self) -> list: - """Get the list of input forcing options specified by the user in the configuration file. This is used to control which input forcings are processed and how they are processed based on the other configuration options specified for each input forcing.""" + """Get the list of input forcing options specified by the user in the + configuration file. This is used to control which input forcings are processed + and how they are processed based on the other configuration options specified + for each input forcing. + """ return self._input_forcings @input_forcings.setter def input_forcings(self, value: list) -> None: - """Set the list of input forcing options specified by the user in the configuration file. This is used to control which input forcings are processed and how they are processed based on the other configuration options specified for each input forcing.""" + """Set the list of input forcing options specified by the user in the + configuration file. This is used to control which input forcings are processed + and how they are processed based on the other configuration options specified + for each input forcing. + """ if value is not None and not self.precip_only_flag: self.check_input_values_in_range( value, "InputForcings", list(range(1, self.force_count + 1)) @@ -691,7 +833,11 @@ def input_forcings(self, value: list) -> None: @property def number_inputs(self) -> int: - """Calculate the number of input forcing options specified by the user in the configuration file. This is used for error checking to ensure users specify valid input forcing options in the configuration file, and to control the flow of the program based on how many input forcings are being processed.""" + """Calculate the number of input forcing options specified by the user in the + configuration file. This is used for error checking to ensure users specify + valid input forcing options in the configuration file, and to control the flow + of the program based on how many input forcings are being processed. + """ if self.input_forcings is None: return 0 if not self.precip_only_flag: @@ -704,7 +850,11 @@ def number_inputs(self) -> int: @property def number_custom_inputs(self) -> int: - """Calculate the number of custom input forcing options specified by the user in the configuration file. This is used to control the flow of the program based on how many custom input forcings are being processed, since custom input forcings require some different processing pathways.""" + """Calculate the number of custom input forcing options specified by the user + in the configuration file. This is used to control the flow of the program based + on how many custom input forcings are being processed, since custom input + forcings require some different processing pathways. + """ if not self.precip_only_flag: count = 0 for force_opt in self.input_forcings: @@ -723,12 +873,20 @@ def number_custom_inputs(self, value: int) -> None: @property def nwm_geogrid(self) -> str: - """Only for the NWM v3 retorspective forcing module option (27) that requires the geo_em_NWM_DOMAIN.nc file as input for the NextGen Forcings Engine to properly setup up the ESMF grid object for the NWM forcing files since that information is not readily available in the NWM v3 retrospective forcing files.""" + """Only for the NWM v3 retorspective forcing module option (27) that requires + the geo_em_NWM_DOMAIN.nc file as input for the NextGen Forcings Engine to + properly setup up the ESMF grid object for the NWM forcing files since that + information is not readily available in the NWM v3 retrospective forcing files. + """ return self._nwm_geogrid @nwm_geogrid.setter def nwm_geogrid(self, value: str) -> None: - """Set the pathway to the NWM geogrid file specified by the user in the configuration file. This is used to specify the grid information for regridding NWM input forcings, and is only necessary if the user has chosen to regrid NWM input forcings in the configuration file.""" + """Set the pathway to the NWM geogrid file specified by the user in the + configuration file. This is used to specify the grid information for regridding + NWM input forcings, and is only necessary if the user has chosen to regrid NWM + input forcings in the configuration file. + """ if ( not self.precip_only_flag and self.input_forcings is not None @@ -740,12 +898,17 @@ def nwm_geogrid(self, value: str) -> None: @property def input_force_types(self) -> list: - """Get the list of input forcing file types specified by the user in the configuration file. This is used to control how input forcings are read in and processed based on the file type specified for each input forcing in the configuration file.""" + """Get the list of input forcing file types specified by the user in the + configuration file. This is used to control how input forcings are read in and + processed based on the file type specified for each input forcing in the + configuration file. + """ return self._input_force_types @input_force_types.setter def input_force_types(self, value: list) -> None: - """Specify the file type for each forcing (comma separated). Valid types are GRIB1, GRIB2, NETCDF, and NETCDF4. + """Specify the file type for each forcing (comma separated). + Valid types are GRIB1, GRIB2, NETCDF, and NETCDF4. Example- InputForcingTypes: [GRIB2,GRIB2]\ """ @@ -760,12 +923,19 @@ def input_force_types(self, value: list) -> None: @property def file_types(self): - """Get the list of input forcing file types specified by the user in the configuration file. This is used to control how input forcings are read in and processed based on the file type specified for each input forcing in the configuration file.""" + """Get the list of input forcing file types specified by the user in the + configuration file. This is used to control how input forcings are read in and + processed based on the file type specified for each input forcing in the + configuration file. + """ return CONFIGOPTIONS["file_types"] @property def input_force_dirs(self) -> list: - """Get the list of input forcing directories specified by the user in the configuration file. This is used to control where input forcings are read in from for each input forcing specified by the user in the configuration file.""" + """Get the list of input forcing directories specified by the user in the + configuration file. This is used to control where input forcings are read in + from for each input forcing specified by the user in the configuration file. + """ if self._input_force_dirs: return self._input_force_dirs else: @@ -773,7 +943,10 @@ def input_force_dirs(self) -> list: @input_force_dirs.setter def input_force_dirs(self, value: list) -> None: - """Specify the input directories for each forcing product. If a user has the ability to connect to the AWS servers and they specify configuration #12 (CONUS AORC data) or configuration #27 (NWM retrospective forcing data) then this specific configuration input can be left as a blank string (""). + """Specify the input directories for each forcing product. If a user has the + ability to connect to the AWS servers and they specify configuration #12 + (CONUS AORC data) or configuration #27 (NWM retrospective forcing data) then + this specific configuration input can be left as a blank string (""). Example- InputForcingDirectories: [./GFS,./NDFD] """ @@ -796,12 +969,20 @@ def input_force_dirs(self, value: list) -> None: @property def input_force_mandatory(self) -> list: - """Get the list of input forcing mandatory flags specified by the user in the configuration file. This is used to control whether the program should raise an error if input forcings for a given forecast cycle are not found for each input forcing specified by the user in the configuration file.""" + """Get the list of input forcing mandatory flags specified by the user in the + configuration file. This is used to control whether the program should raise an + error if input forcings for a given forecast cycle are not found for each input + forcing specified by the user in the configuration file. + """ return self._input_force_mandatory @input_force_mandatory.setter def input_force_mandatory(self, value: list) -> None: - """Specify whether the input forcings listed above are mandatory, or optional. This is important for layering contingencies if a product is missing, but forcing files are still desired. 0 - Not mandatory, 1 - Mandatory. NOTE!!! If no files are found for any products, code will error out indicating the final field is all missing values. + """Specify whether the input forcings listed above are mandatory, or optional. + This is important for layering contingencies if a product is missing, but forcing + files are still desired. 0 - Not mandatory, 1 - Mandatory. + NOTE!!! If no files are found for any products, code will error out indicating + the final field is all missing values. Example- InputMandatory: [1,1] """ @@ -812,12 +993,20 @@ def input_force_mandatory(self, value: list) -> None: @property def customSuppPcpFreq(self) -> int: - """Get the custom supplemental precip output frequency specified by the user in the configuration file. This is used to control the output frequency of supplemental precip forcings if the user has chosen to run the supplemental precip forcings module only.""" + """Get the custom supplemental precip output frequency specified by the user in + the configuration file. This is used to control the output frequency of + supplemental precip forcings if the user has chosen to run the supplemental + precip forcings module only. + """ return self._customSuppPcpFreq @customSuppPcpFreq.setter def customSuppPcpFreq(self, value: int) -> None: - """Set the custom supplemental precip output frequency specified by the user in the configuration file. This is used to control the output frequency of supplemental precip forcings if the user has chosen to run the supplemental precip forcings module only.""" + """Set the custom supplemental precip output frequency specified by the user in + the configuration file. This is used to control the output frequency of + supplemental precip forcings if the user has chosen to run the supplemental + precip forcings module only. + """ if self.precip_only_flag: self.check_input_values_non_negative([value], "customSuppPcpFreq") self._customSuppPcpFreq = value @@ -826,7 +1015,14 @@ def customSuppPcpFreq(self, value: int) -> None: @property def fcst_shift(self) -> int: - """Forecast cycles are determined by splitting up a day by equal ForecastFrequency interval. If there is a desire to shift the cycles to a different time step, ForecastShift will shift forecast cycles ahead by a determined set of minutes. For example, ForecastFrequency of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, a ForecastShift of 1 hour will produce forecast cycles at 01, 07, 13, and 18 UTC. NOTE - This is only used by the realtime instance to calculate forecast cycles accordingly. Re-forecasts will use the beginning and ending dates specified in conjunction with the forecast frequency to determine forecast cycle dates. + """Forecast cycles are determined by splitting up a day by equal ForecastFrequency + interval. If there is a desire to shift the cycles to a different time step, + ForecastShift will shift forecast cycles ahead by a determined set of minutes. + For example, ForecastFrequency of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. + However, a ForecastShift of 1 hour will produce forecast cycles at 01, 07, 13, and 18 UTC. + NOTE - This is only used by the realtime instance to calculate forecast cycles accordingly. + Re-forecasts will use the beginning and ending dates specified in conjunction + with the forecast frequency to determine forecast cycle dates. Example- ForecastShift: 0 """ @@ -856,19 +1052,33 @@ def fcst_shift(self, value: int) -> None: @property def nFcsts(self): - """Get the number of forecasts to issue for a reforecast simulation based on the forecast shift and the processing window specified by the user in the configuration file. This is used to control how many forecast time steps are output for a reforecast simulation, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" + """Get the number of forecasts to issue for a reforecast simulation based on the + forecast shift and the processing window specified by the user in the + configuration file. This is used to control how many forecast time steps are + output for a reforecast simulation, and is only necessary if the user is running + a reforecast simulation with a specified processing window rather than a + realtime simulation. + """ return self._nFcsts @nFcsts.setter def nFcsts(self, value: int) -> None: - """Set the number of forecasts to issue for a reforecast simulation based on the forecast shift and the processing window specified by the user in the configuration file. This is used to control how many forecast time steps are output for a reforecast simulation, and is only necessary if the user is running a reforecast simulation with a specified processing window rather than a realtime simulation.""" + """Set the number of forecasts to issue for a reforecast simulation based on the + forecast shift and the processing window specified by the user in the + configuration file. This is used to control how many forecast time steps are + output for a reforecast simulation, and is only necessary if the user is running + a reforecast simulation with a specified processing window rather than a realtime + simulation. + """ if value is None: value = 1 self._nFcsts = value @property def fcst_input_horizons(self) -> list: - """Specify how much (in minutes) of each input forcing is desires for each forecast cycle. See documentation for examples. The length of this array must match the input forcing choices. + """Specify how much (in minutes) of each input forcing is desires for each + forecast cycle. See documentation for examples. The length of this array must + match the input forcing choices. - Example- ForecastInputHorizons: [60, 60] """ @@ -878,7 +1088,8 @@ def fcst_input_horizons(self) -> list: def fcst_input_horizons(self, value: list) -> None: """Setter for ``fcst_input_horizons``. - NOTE: this property is hardened such that it allows being set one time, but may not be mutated after that initial set. + NOTE: this property is hardened such that it allows being set one time, but may + not be mutated after that initial set. For rationale, see: https://github.com/NGWPC/ngen-forcing/pull/107 """ if self._fcst_input_horizons is not None and self._fcst_input_horizons != value: @@ -897,7 +1108,10 @@ def fcst_input_horizons(self, value: list) -> None: @property def fcst_input_offsets(self): - """Option for applying an offset to input forcings to use a different forecasted interval. For example, a user may wish to use 4-5 hour forecasted fields from an NWP grid from one of their input forcings. In that instance the offset would be 4 hours, but 0 for other remaining forcings. + """Option for applying an offset to input forcings to use a different forecasted + interval. For example, a user may wish to use 4-5 hour forecasted fields from an + NWP grid from one of their input forcings. In that instance the offset would be + 4 hours, but 0 for other remaining forcings. Example- ForecastInputOffsets: [0, 0] """ @@ -912,7 +1126,9 @@ def fcst_input_offsets(self, value: list) -> None: @property def cycle_length_minutes(self) -> int: - """Get the forecast cycle length in minutes, which is calculated based on the maximum of the forecast input horizons specified by the user in the configuration file. + """Get the forecast cycle length in minutes, which is calculated based on the + maximum of the forecast input horizons specified by the user in the configuration + file. Ensure the number maximum cycle length is an equal divider of the output time step specified by the user. """ @@ -925,7 +1141,10 @@ def cycle_length_minutes(self) -> int: @property def num_output_steps(self) -> int: - """Calculate the number of output time steps per forecast cycle based on the forecast cycle length and the output frequency specified by the user in the configuration file.""" + """Calculate the number of output time steps per forecast cycle based on the + forecast cycle length and the output frequency specified by the user in the + configuration file. + """ if self.sub_output_hour is None: num_steps = int(self.cycle_length_minutes / self.output_freq) else: @@ -941,13 +1160,21 @@ def num_output_steps(self) -> int: @property def num_supp_output_steps(self) -> int: - """Calculate the number of supplemental precip output time steps per forecast cycle based on the forecast cycle length and the custom supplemental precip output frequency specified by the user in the configuration file.""" + """Calculate the number of supplemental precip output time steps per forecast + cycle based on the forecast cycle length and the custom supplemental precip + output frequency specified by the user in the configuration file. + """ if self.precip_only_flag: return int(self.cycle_length_minutes / self.customSuppPcpFreq) @property def actual_output_steps(self) -> int: - """Calculate the actual number of output time steps per forecast cycle based on whether the user has chosen to run a reforecast simulation with a specified processing window, which will only output time steps for which input forcings are available based on the processing window and forecast time horizons specified by the user in the configuration file.""" + """Calculate the actual number of output time steps per forecast cycle based on + whether the user has chosen to run a reforecast simulation with a specified + processing window, which will only output time steps for which input forcings + are available based on the processing window and forecast time horizons + specified by the user in the configuration file. + """ if self.ana_flag: return np.int32(self.nFcsts) else: @@ -955,7 +1182,9 @@ def actual_output_steps(self) -> int: @property def grid_type(self) -> str: - """Tells the NextGen Forcings Engine BMI which grid type the engine is initalizing as a BMI instance. This is a required field and the proper string values should be "gridded", "hydrofabric", or "unstructured". + """Tells the NextGen Forcings Engine BMI which grid type the engine is + initalizing as a BMI instance. This is a required field and the proper string + values should be "gridded", "hydrofabric", or "unstructured". Example- GRID_TYPE: "gridded" """ @@ -963,7 +1192,11 @@ def grid_type(self) -> str: @grid_type.setter def grid_type(self, value: str) -> None: - """Set the grid type specified by the user in the configuration file. This is used to control how the program reads in and processes the geogrid information for regridding input forcings based on the grid type specified by the user in the configuration file.""" + """Set the grid type specified by the user in the configuration file. This is + used to control how the program reads in and processes the geogrid information + for regridding input forcings based on the grid type specified by the user in + the configuration file. + """ self.check_input_values_in_range( [value.lower()], "GRID_TYPE", ["gridded", "unstructured", "hydrofabric"] ) @@ -971,7 +1204,16 @@ def grid_type(self, value: str) -> None: @property def lon_var(self) -> str: - """Naming convention of the longitude variable within the "GeogridIn" file the user has specified. Variable naming convention ONLY for gridded domain configurations. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. In the case for "gridded" domain configuration options and a user specifying downscaling options while only specifying a height variable feature on the grid, this netcdf variable (LONVAR) is then EXPECTED to contain a netcdf metadata attribute called "dx" that specifies the grid spacing in the longtiudinal direction. Otherwise, it will throw an error and not be able to calculate the slope and tilt of each grid cell. + """Naming convention of the longitude variable within the "GeogridIn" file the + user has specified. Variable naming convention ONLY for gridded domain + configurations. This is required so the NextGen Forcings Engine BMI can + dyanmically initialize the domain geogrid as an ESMF regridding object. In the + case for "gridded" domain configuration options and a user specifying downscaling + options while only specifying a height variable feature on the grid, this netcdf + variable (LONVAR) is then EXPECTED to contain a netcdf metadata attribute called + "dx" that specifies the grid spacing in the longtiudinal direction. Otherwise, + it will throw an error and not be able to calculate the slope and tilt of each + grid cell. Example- LONVAR: "XLONG_M" """ @@ -980,7 +1222,16 @@ def lon_var(self) -> str: @property def lat_var(self) -> str: - """Naming convention of the latitude variable within the "GeogridIn" file the user has specified. Variable naming convention ONLY for gridded domain configurations. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. In the case for "gridded" domain configuration options and a user specifying downscaling options while only specifying a height variable feature on the grid, this netcdf variable (LATVAR) is then EXPECTED to contain a netcdf metadata attribute called "dy" that specifies the grid spacing in the latitudinal direction. Otherwise, it will throw an error and not be able to calculate the slope and tilt of each grid cell. + """Naming convention of the latitude variable within the "GeogridIn" file the + user has specified. Variable naming convention ONLY for gridded domain + configurations. This is required so the NextGen Forcings Engine BMI can + dyanmically initialize the domain geogrid as an ESMF regridding object. In the + case for "gridded" domain configuration options and a user specifying + downscaling options while only specifying a height variable feature on the grid, + this netcdf variable (LATVAR) is then EXPECTED to contain a netcdf metadata + attribute called "dy" that specifies the grid spacing in the latitudinal + direction. Otherwise, it will throw an error and not be able to calculate the + slope and tilt of each grid cell. Example- LATVAR: "XLAT_M" """ @@ -989,7 +1240,10 @@ def lat_var(self) -> str: @property def nodecoords_var(self) -> str: - """Naming convention of the node coordinates variable within the "GeogridIn" file the user has specified for ONLY an unstructured mesh or the NextGen hydrofabric. This is a 2-D array stating the latitude and longitude coordinates for all the nodes in the mesh. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. + """Naming convention of the node coordinates variable within the "GeogridIn" + file the user has specified for ONLY an unstructured mesh or the NextGen hydrofabric. + This is a 2-D array stating the latitude and longitude coordinates for all the nodes in the mesh. + This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. Example- NodeCoods: "nodecoords" """ @@ -998,7 +1252,10 @@ def nodecoords_var(self) -> str: @property def elemcoords_var(self) -> str: - """Naming convention of the element coordinates variable within the "GeogridIn" file the user has specified for ONLY an unstructured mesh or the NextGen hydrofabric. This is a 2-D array stating the latitude and longitude coordinates for all the elements in the mesh. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. + """Naming convention of the element coordinates variable within the "GeogridIn" + file the user has specified for ONLY an unstructured mesh or the NextGen hydrofabric. + This is a 2-D array stating the latitude and longitude coordinates for all the elements in the mesh. + This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. Example- ElemCoods: "elemcoords" """ @@ -1007,7 +1264,10 @@ def elemcoords_var(self) -> str: @property def elemconn_var(self) -> str: - """Naming convention of the element connectivity variable within the "GeogridIn" file the user has specified for ONLY an unstructured mesh or the NextGen hydrofabric. This is a 2-D array stating the node ids for each element connecting the entire mesh structure. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. + """Naming convention of the element connectivity variable within the "GeogridIn" + file the user has specified for ONLY an unstructured mesh or the NextGen hydrofabric. + This is a 2-D array stating the node ids for each element connecting the entire mesh structure. + This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. Example- ElemConn: "elemconn" """ @@ -1016,7 +1276,10 @@ def elemconn_var(self) -> str: @property def numelemconn_var(self) -> str: - """Naming convention of the number of nodes per element variable within the "GeogridIn" file the user has specified for ONLY an unstructured mesh or the NextGen hydrofabric. This is a 1-D array stating the how many nodes are connecting each element within the unstructured mesh. This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. + """Naming convention of the number of nodes per element variable within the "GeogridIn" + file the user has specified for ONLY an unstructured mesh or the NextGen hydrofabric. + This is a 1-D array stating the how many nodes are connecting each element within the unstructured mesh. + This is required so the NextGen Forcings Engine BMI can dyanmically initialize the domain geogrid as an ESMF regridding object. Example- NumElemConn: "numelemconn" """ @@ -1025,7 +1288,10 @@ def numelemconn_var(self) -> str: @property def element_id_var(self) -> str: - """Naming convention of the element id variable within the "GeogridIn" file the user has specified for ONLY the NextGen hydrofabric. This is a 1-D array stating the catchment id numeric naming convention within the "divides" geopackage layer of a given NextGen hydrofabric file. This variable is required in order for the NextGen Forcings Engine to properly advertise the element ids of the unstructured mesh linked to the NextGen hydrofabric catchment ids. + """Naming convention of the element id variable within the "GeogridIn" + file the user has specified for ONLY the NextGen hydrofabric. + This is a 1-D array stating the catchment id numeric naming convention within the "divides" geopackage layer of a given NextGen hydrofabric file. + This variable is required in order for the NextGen Forcings Engine to properly advertise the element ids of the unstructured mesh linked to the NextGen hydrofabric catchment ids. Example- ElemID: "element_ids" """ @@ -1034,7 +1300,8 @@ def element_id_var(self) -> str: @property def ignored_border_widths(self) -> list: - """Border width (in grid cells) to ignore for each input dataset. NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output. + """Border width (in grid cells) to ignore for each input dataset. + NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output. Example- IgnoredBorderWidths: [0,10] """ @@ -1042,7 +1309,10 @@ def ignored_border_widths(self) -> list: @ignored_border_widths.setter def ignored_border_widths(self, value: list) -> None: - """Set the list of ignored border widths specified by the user in the configuration file. This is used to control how the program processes input forcings based on the ignored border widths specified for each input forcing in the configuration file.""" + """Set the list of ignored border widths specified by the user in the configuration file. + This is used to control how the program processes input forcings based on the + ignored border widths specified for each input forcing in the configuration file. + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "IgnoredBorderWidths") self.check_input_values_non_negative(value, "IgnoredBorderWidths") @@ -1050,7 +1320,11 @@ def ignored_border_widths(self, value: list) -> None: @property def regrid_opt(self): - """Choose regridding options for each input forcing files being used. Options available are: 1 - ESMF Bilinear, 2 - ESMF Nearest Neighbor, 3 - ESMF Conservative Bilinear. + """Choose regridding options for each input forcing files being used. + Options available are: + 1 - ESMF Bilinear, + 2 - ESMF Nearest Neighbor, + 3 - ESMF Conservative Bilinear. Example- RegridOpt: [1,1] """ @@ -1058,7 +1332,10 @@ def regrid_opt(self): @regrid_opt.setter def regrid_opt(self, value: list) -> None: - """Set the list of regridding options specified by the user in the configuration file. This is used to control how input forcings are regridded based on the regridding option specified for each input forcing in the configuration file.""" + """Set the list of regridding options specified by the user in the configuration file. + This is used to control how input forcings are regridded based on the regridding + option specified for each input forcing in the configuration file. + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "RegridOpt") self.check_input_values_in_range(value, "RegridOpt", [1, 2, 3]) @@ -1068,12 +1345,20 @@ def regrid_opt(self, value: list) -> None: @property def weightsDir(self) -> str: - """Get the pathway to the ESMF weights directory specified by the user in the configuration file. This is used to control where the program looks for ESMF weights files if the user has chosen to use pre-generated ESMF weights files for regridding input forcings in the configuration file.""" + """Get the pathway to the ESMF weights directory specified by the user in the configuration file. + This is used to control where the program looks for ESMF weights files if the + user has chosen to use pre-generated ESMF weights files for regridding input + forcings in the configuration file. + """ return self._weightsDir @weightsDir.setter def weightsDir(self, value: str) -> None: - """Set the pathway to the ESMF weights directory specified by the user in the configuration file. This is used to control where the program looks for ESMF weights files if the user has chosen to use pre-generated ESMF weights files for regridding input forcings in the configuration file.""" + """Set the pathway to the ESMF weights directory specified by the user in the configuration file. + This is used to control where the program looks for ESMF weights files if the + user has chosen to use pre-generated ESMF weights files for regridding input + forcings in the configuration file. + """ if not self.precip_only_flag: if value is not None and not os.path.exists(value): err_out_screen( @@ -1083,12 +1368,24 @@ def weightsDir(self, value: str) -> None: @property def forceTemoralInterp(self) -> list: - """Get the list of forcing temporal interpolation options specified by the user in the configuration file. This is used to control how input forcings are temporally interpolated based on the temporal interpolation option specified for each input forcing in the configuration file.""" + """Get the list of forcing temporal interpolation options specified by the user in the configuration file. + This is used to control how input forcings are temporally interpolated based on + the temporal interpolation option specified for each input forcing in the + configuration file. + """ return self._forceTemoralInterp @forceTemoralInterp.setter def forceTemoralInterp(self, value: list) -> None: - """Specify an temporal interpolation for the forcing variables. Interpolation will be done between the two neighboring input forcing states that exist. If only one nearest state exist (I.E. only a state forward in time, or behind), then that state will be used as a "nearest neighbor". NOTE - All input options here must be of the same length of the input forcing number. Also note all temporal interpolation occurs BEFORE downscaling and bias correction. 0 - No temporal interpolation. 1 - Nearest Neighbor, 2 - Linear weighted, average. + """Specify an temporal interpolation for the forcing variables. Interpolation + will be done between the two neighboring input forcing states that exist. + If only one nearest state exist (I.E. only a state forward in time, or behind), + then that state will be used as a "nearest neighbor". + NOTE - All input options here must be of the same length of the input forcing number. + Also note all temporal interpolation occurs BEFORE downscaling and bias correction. + 0 - No temporal interpolation. + 1 - Nearest Neighbor, + 2 - Linear weighted average. Example- ForcingTemporalInterpolation: [0,0] """ @@ -1101,7 +1398,11 @@ def forceTemoralInterp(self, value: list) -> None: @property def t2dDownscaleOpt(self) -> list: - """Specify a temperature downscaling method: 0 - No downscaling, 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation to the WRF-Hydro elevation, 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain (only NWM), 3 - Use a dynamic lapse rate calculated at each timstep. + """Specify a temperature downscaling method: + 0 - No downscaling, + 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation to the WRF-Hydro elevation, + 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain (only NWM), + 3 - Use a dynamic lapse rate calculated at each timstep. Example- TemperatureDownscaling: [3, 3] """ @@ -1109,7 +1410,11 @@ def t2dDownscaleOpt(self) -> list: @t2dDownscaleOpt.setter def t2dDownscaleOpt(self, value: list) -> None: - """Set the list of temperature downscaling options specified by the user in the configuration file. This is used to control how temperature input forcings are downscaled based on the temperature downscaling option specified for each input forcing in the configuration file.""" + """Set the list of temperature downscaling options specified by the user in the configuration file. + This is used to control how temperature input forcings are downscaled based on + the temperature downscaling option specified for each input forcing in the + configuration file. + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "TemperatureDownscaling") self.check_input_values_in_range(value, "TemperatureDownscaling", [0, 1, 2]) @@ -1122,7 +1427,9 @@ def t2dDownscaleOpt(self, value: list) -> None: @property def psfcDownscaleOpt(self) -> list: - """Specify a surface pressure downscaling method: 0 - No downscaling, 1 - Use input elevation and WRF-Hydro elevation to downscale surface pressure. + """Specify a surface pressure downscaling method: + 0 - No downscaling, + 1 - Use input elevation and WRF-Hydro elevation to downscale surface pressure. Example- PressureDownscaling: [1, 1] """ @@ -1130,7 +1437,10 @@ def psfcDownscaleOpt(self) -> list: @psfcDownscaleOpt.setter def psfcDownscaleOpt(self, value: list) -> None: - """Set the list of pressure downscaling options specified by the user in the configuration file. This is used to control how pressure input forcings are downscaled based on the pressure downscaling option specified for each input forcing in the configuration file.""" + """Set the list of pressure downscaling options specified by the user in the configuration file. + This is used to control how pressure input forcings are downscaled based on the + pressure downscaling option specified for each input forcing in the configuration file. + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "PressureDownscaling") self.check_input_values_in_range(value, "PressureDownscaling", [0, 1]) @@ -1138,7 +1448,9 @@ def psfcDownscaleOpt(self, value: list) -> None: @property def swDownscaleOpt(self) -> list: - """Specify a shortwave radiation downscaling routine. 0 - No downscaling, 1 - Run a topographic adjustment using the WRF-Hydro elevation. + """Specify a shortwave radiation downscaling routine. + 0 - No downscaling, + 1 - Run a topographic adjustment using the WRF-Hydro elevation. Example- ShortwaveDownscaling: [1, 1] """ @@ -1154,7 +1466,10 @@ def swDownscaleOpt(self, value: list) -> None: @property def q2dDownscaleOpt(self) -> list: - """Specify a specific humidity downscaling routine. 0 - No downscaling, 1 - Use regridded humidity, along with downscaled temperature/pressure to extrapolate a downscaled surface specific humidty. + """Specify a specific humidity downscaling routine. + 0 - No downscaling, + 1 - Use regridded humidity, along with downscaled temperature/pressure + to extrapolate a downscaled surface specific humidty. Example- HumidityDownscaling: [1, 1] """ @@ -1162,7 +1477,10 @@ def q2dDownscaleOpt(self) -> list: @q2dDownscaleOpt.setter def q2dDownscaleOpt(self, value: list) -> None: - """Set the list of humidity downscaling options specified by the user in the configuration file. This is used to control how humidity input forcings are downscaled based on the humidity downscaling option specified for each input forcing in the configuration file.""" + """Set the list of humidity downscaling options specified by the user in the configuration file. + This is used to control how humidity input forcings are downscaled based on the + humidity downscaling option specified for each input forcing in the configuration file. + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "HumidityDownscaling") self.check_input_values_in_range(value, "HumidityDownscaling", [0, 1]) @@ -1170,7 +1488,9 @@ def q2dDownscaleOpt(self, value: list) -> None: @property def precipDownscaleOpt(self) -> list: - """Specify a precipitation downscaling routine. 0 - No downscaling, 1 - NWM mountain mapper downscaling using monthly PRISM climo. + """Specify a precipitation downscaling routine. + 0 - No downscaling, + 1 - NWM mountain mapper downscaling using monthly PRISM climo. Example- PrecipDownscaling: [0, 0] """ @@ -1178,7 +1498,10 @@ def precipDownscaleOpt(self) -> list: @precipDownscaleOpt.setter def precipDownscaleOpt(self, value: list) -> None: - """Set the list of precipitation downscaling options specified by the user in the configuration file. This is used to control how precipitation input forcings are downscaled based on the precipitation downscaling option specified for each input forcing in the configuration file.""" + """Set the list of precipitation downscaling options specified by the user in the configuration file. + This is used to control how precipitation input forcings are downscaled based on + the precipitation downscaling option specified for each input forcing in the configuration file. + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "PrecipDownscaling") self.check_input_values_in_range(value, "PrecipDownscaling", [0, 1]) @@ -1191,7 +1514,9 @@ def precipDownscaleOpt(self, value: list) -> None: @property def dScaleParamDirs(self) -> list: - """Specify the input parameter directory containing necessary downscaling grids. This is ONLY needed for the original NWM WRF-Hydro domain. Otherwise, just point it to a random directory and it will be ignored. + """Specify the input parameter directory containing necessary downscaling grids. + This is ONLY needed for the original NWM WRF-Hydro domain. + Otherwise, just point it to a random directory and it will be ignored. Example- DownscalingParamDirs: ["./forcingParam/AnA", "./forcingParam/AnA"] """ @@ -1199,7 +1524,11 @@ def dScaleParamDirs(self) -> list: @dScaleParamDirs.setter def dScaleParamDirs(self, value: list) -> None: - """Set the list of downscaling parameter directories specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for each input forcing based on the downscaling parameter directory specified for each input forcing in the configuration file.""" + """Set the list of downscaling parameter directories specified by the user in the configuration file. + This is used to control where the program looks for downscaling parameter + files for each input forcing based on the downscaling parameter directory + specified for each input forcing in the configuration file. + """ self.check_number_of_inputs_forcings(value, "DownscalingParamDirs") for dirTmp in range(0, len(value)): dir_path = value[dirTmp] @@ -1211,7 +1540,9 @@ def dScaleParamDirs(self, value: list) -> None: @property def perform_downscaling(self) -> bool: - """Determine whether downscaling of input forcings is necessary based on the downscaling options specified by the user for each input forcing in the configuration file.""" + """Determine whether downscaling of input forcings is necessary based on the + downscaling options specified by the user for each input forcing in the configuration file. + """ if self.precip_only_flag: return False if ( @@ -1227,7 +1558,12 @@ def perform_downscaling(self) -> bool: @property def t2BiasCorrectOpt(self) -> list: - """Specify a temperature bias correction method. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION), 3 - NCAR parametric GFS bias correction, 4 - NCAR parametric HRRR bias correction. + """Specify a temperature bias correction method. + 0 - No bias correction, + 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, + 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION), + 3 - NCAR parametric GFS bias correction, + 4 - NCAR parametric HRRR bias correction. Example- TemperatureBiasCorrection: [0, 4] """ @@ -1235,7 +1571,11 @@ def t2BiasCorrectOpt(self) -> list: @t2BiasCorrectOpt.setter def t2BiasCorrectOpt(self, value: list) -> None: - """Set the list of temperature bias correction options specified by the user in the configuration file. This is used to control how temperature input forcings are bias corrected based on the temperature bias correction option specified for each input forcing in the configuration file.""" + """Set the list of temperature bias correction options specified by the user in the configuration file. + This is used to control how temperature input forcings are bias corrected based + on the temperature bias correction option specified for each input forcing in + the configuration file. + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "TemperatureBiasCorrection") self.check_input_values_in_range( @@ -1245,7 +1585,9 @@ def t2BiasCorrectOpt(self, value: list) -> None: @property def psfcBiasCorrectOpt(self) -> list: - """Specify a surface pressure bias correction method. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY. + """Specify a surface pressure bias correction method. + 0 - No bias correction, + 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY. Example- PressureBiasCorrection: [0,0] """ @@ -1253,7 +1595,11 @@ def psfcBiasCorrectOpt(self) -> list: @psfcBiasCorrectOpt.setter def psfcBiasCorrectOpt(self, value: list) -> None: - """Set the list of pressure bias correction options specified by the user in the configuration file. This is used to control how pressure input forcings are bias corrected based on the pressure bias correction option specified for each input forcing in the configuration file.""" + """Set the list of pressure bias correction options specified by the user in the configuration file. + This is used to control how pressure input forcings are bias corrected based on + the pressure bias correction option specified for each input forcing in the + configuration file. + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "PressureBiasCorrection") self.check_input_values_in_range(value, "PressureBiasCorrection", [0, 1]) @@ -1261,7 +1607,10 @@ def psfcBiasCorrectOpt(self, value: list) -> None: @property def q2BiasCorrectOpt(self): - """Specify a specific humidity bias correction method. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). + """Specify a specific humidity bias correction method. + 0 - No bias correction, + 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, + 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). Example- HumidityBiasCorrection: [0,0] """ @@ -1269,7 +1618,11 @@ def q2BiasCorrectOpt(self): @q2BiasCorrectOpt.setter def q2BiasCorrectOpt(self, value): - """Set the list of humidity bias correction options specified by the user in the configuration file. This is used to control how humidity input forcings are bias corrected based on the humidity bias correction option specified for each input forcing in the configuration file.""" + """Set the list of humidity bias correction options specified by the user in the configuration file. + This is used to control how humidity input forcings are bias corrected based on + the humidity bias correction option specified for each input forcing in the + configuration file. + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "HumidityBiasCorrection") self.check_input_values_in_range(value, "HumidityBiasCorrection", [0, 1, 2]) @@ -1277,7 +1630,12 @@ def q2BiasCorrectOpt(self, value): @property def windBiasCorrect(self): - """Specify a wind bias correction. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION), 3 - NCAR parametric GFS bias correction, 4 - NCAR parametric HRRR bias correction. + """Specify a wind bias correction. + 0 - No bias correction, + 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, + 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION), + 3 - NCAR parametric GFS bias correction, + 4 - NCAR parametric HRRR bias correction. Example- WindBiasCorrection: [0, 4] """ @@ -1285,7 +1643,10 @@ def windBiasCorrect(self): @windBiasCorrect.setter def windBiasCorrect(self, value): - """Set the list of wind bias correction options specified by the user in the configuration file. This is used to control how wind input forcings are bias corrected based on the wind bias correction option specified for each input forcing in the configuration file.""" + """Set the list of wind bias correction options specified by the user in the configuration file. + This is used to control how wind input forcings are bias corrected based on the + wind bias correction option specified for each input forcing in the configuration file. + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "WindBiasCorrection") self.check_input_values_in_range( @@ -1295,7 +1656,10 @@ def windBiasCorrect(self, value): @property def swBiasCorrectOpt(self) -> list: - """Specify a bias correction for incoming short wave radiation flux. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). + """Specify a bias correction for incoming short wave radiation flux. + 0 - No bias correction, + 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, + 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). Example- SwBiasCorrection: [0, 2] """ @@ -1311,7 +1675,11 @@ def swBiasCorrectOpt(self, value: list) -> None: @property def lwBiasCorrectOpt(self) -> list: - """Specify a bias correction for incoming long wave radiation flux. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION), 3 - NCAR parametric GFS bias correction. + """Specify a bias correction for incoming long wave radiation flux. + 0 - No bias correction, + 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY, + 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION), + 3 - NCAR parametric GFS bias correction. Example- LwBiasCorrection: [0, 2] """ @@ -1319,7 +1687,11 @@ def lwBiasCorrectOpt(self) -> list: @lwBiasCorrectOpt.setter def lwBiasCorrectOpt(self, value: list) -> None: - """Set the list of longwave radiation bias correction options specified by the user in the configuration file. This is used to control how longwave radiation input forcings are bias corrected based on the longwave radiation bias correction option specified for each input forcing in the configuration file.""" + """Set the list of longwave radiation bias correction options specified by the user in the configuration file. + This is used to control how longwave radiation input forcings are bias corrected + based on the longwave radiation bias correction option specified for each input + forcing in the configuration file. + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "LwBiasCorrection") self.check_input_values_in_range(value, "LwBiasCorrection", [0, 1, 2, 3, 4]) @@ -1327,7 +1699,9 @@ def lwBiasCorrectOpt(self, value: list) -> None: @property def precipBiasCorrectOpt(self): - """Specify a bias correction for precipitation. 0 - No bias correction, 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY. + """Specify a bias correction for precipitation. + 0 - No bias correction, + 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY. Example- PrecipBiasCorrection: [0, 0] """ @@ -1335,7 +1709,11 @@ def precipBiasCorrectOpt(self): @precipBiasCorrectOpt.setter def precipBiasCorrectOpt(self, value): - """Set the list of precipitation bias correction options specified by the user in the configuration file. This is used to control how precipitation input forcings are bias corrected based on the precipitation bias correction option specified for each input forcing in the configuration file.""" + """Set the list of precipitation bias correction options specified by the user in the configuration file. + This is used to control how precipitation input forcings are bias corrected + based on the precipitation bias correction option specified for each input + forcing in the configuration file. + """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "PrecipBiasCorrection") self.check_input_values_in_range(value, "PrecipBiasCorrection", [0, 1]) @@ -1343,9 +1721,14 @@ def precipBiasCorrectOpt(self, value): @property def bias_correction_properties(self) -> dict: - """Get the dictionary of bias correction properties specified by the user in the configuration file. This is used to control how input forcings are bias corrected based on the bias correction options specified for each input forcing in the configuration file.""" + """Get the dictionary of bias correction properties specified by the user in the configuration file. + This is used to control how input forcings are bias corrected based on the bias + correction options specified for each input forcing in the configuration file. + + # TODO "surface temperature" was excluded from this consideration in the orignal code (5/7/2026 pre-refactor). Should it actually be included? + """ return { - # "surface temperature": self.t2BiasCorrectOpt, #NOTE surface temperature was excluded from this consideration in the orignal code (5/7/2026 pre-refactor). Should it actually be included? + # "surface temperature": self.t2BiasCorrectOpt, "surface pressure": self.psfcBiasCorrectOpt, "specific humidity": self.q2BiasCorrectOpt, "wind forcings": self.windBiasCorrect, @@ -1356,7 +1739,12 @@ def bias_correction_properties(self) -> dict: @property def runCfsNldasBiasCorrect(self) -> bool: - """Get the flag for whether to run the NWM-specific bias correction of CFSv2 input forcings specified by the user in the configuration file. This is used to control whether the NWM-specific bias correction of CFSv2 input forcings is run based on whether the user has chosen to run this bias correction in the configuration file.""" + """Get the flag for whether to run the NWM-specific bias correction of CFSv2 + input forcings specified by the user in the configuration file. This is used to + control whether the NWM-specific bias correction of CFSv2 input forcings is run + based on whether the user has chosen to run this bias correction in the + configuration file. + """ run_cfs_nldas_bias_correct = False for bias_option in self.bias_correction_properties.values(): for opt in bias_option: @@ -1382,19 +1770,33 @@ def runCfsNldasBiasCorrect(self) -> bool: @property def number_supp_pcp(self) -> int: - """Get the number of supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how many supplemental precipitation input forcings are processed based on the number of supplemental precipitation input forcings specified in the configuration file.""" + """Get the number of supplemental precipitation input forcings specified by the + user in the configuration file. This is used to control how many supplemental + precipitation input forcings are processed based on the number of supplemental + precipitation input forcings specified in the configuration file. + """ if self.supp_precip_forcings is None: return 0 return len(self.supp_precip_forcings) @property def supp_precip_file_types(self) -> list: - """Get the list of supplemental precipitation input forcing file types specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are read in and processed based on the file types specified for each supplemental precipitation input forcing in the configuration file.""" + """Get the list of supplemental precipitation input forcing file types + specified by the user in the configuration file. This is used to control how + supplemental precipitation input forcing files are read in and processed based + on the file types specified for each supplemental precipitation input forcing + in the configuration file. + """ return self._supp_precip_file_types @supp_precip_file_types.setter def supp_precip_file_types(self, value: list) -> None: - """Set the list of supplemental precipitation input forcing file types specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are read in and processed based on the file types specified for each supplemental precipitation input forcing in the configuration file.""" + """Set the list of supplemental precipitation input forcing file types + specified by the user in the configuration file. This is used to control how + supplemental precipitation input forcing files are read in and processed based + on the file types specified for each supplemental precipitation input forcing + in the configuration file. + """ if value is not None: value = [stype.strip() for stype in value] if value == [""]: @@ -1409,12 +1811,20 @@ def supp_precip_file_types(self, value: list) -> None: @property def supplemental_precip_file_type_options(self) -> list: - """Get the list of valid supplemental precipitation input forcing file types that can be specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are read in and processed based on the file types specified for each supplemental precipitation input forcing in the configuration file.""" + """Get the list of valid supplemental precipitation input forcing file types + that can be specified by the user in the configuration file. This is used to + control how supplemental precipitation input forcing files are read in and + processed based on the file types specified for each supplemental precipitation + input forcing in the configuration file. + """ return ["GRIB1", "GRIB2", "NETCDF"] @property def rqiMethod(self) -> int | list[int] | None: - """Optional RQI method for radar-based data. 0 - Do not use any RQI filtering. Use all radar-based estimates. 1 - Use hourly MRMS Radar Quality Index grids, 2 - Use NWM monthly climatology grids (NWM only!!!!). + """Optional RQI method for radar-based data. 0 - Do not use any RQI filtering. + Use all radar-based estimates. + 1 - Use hourly MRMS Radar Quality Index grids, + 2 - Use NWM monthly climatology grids (NWM only!!!!). Example- RqiMethod: 2 """ @@ -1438,7 +1848,9 @@ def rqiMethod(self) -> int | list[int] | None: @property def rqiThresh(self) -> float | list[float] | None: - """Optional RQI threshold to be used to mask out. Currently used for MRMS products. Please choose a value from 0.0-1.0. Associated radar quality index files will be expected from MRMS data. + """Optional RQI threshold to be used to mask out. Currently used for MRMS products. + Please choose a value from 0.0-1.0. Associated radar quality index files will + be expected from MRMS data. Example- RqiThreshold: 0.9 """ @@ -1466,7 +1878,11 @@ def rqiThresh(self) -> float | list[float] | None: @property def supp_precip_mandatory(self): - """Specify whether the Supplemental Precips listed above are mandatory, or optional. This is important for layering contingencies if a product is missing, but forcing files are still desired. 0 - Not mandatory, 1 - Mandatory. + """Specify whether the Supplemental Precips listed above are mandatory, or optional. + This is important for layering contingencies if a product is missing, but + forcing files are still desired. + 0 - Not mandatory, + 1 - Mandatory. Example- SuppPcpMandatory: [0, 0, 0] """ @@ -1474,7 +1890,13 @@ def supp_precip_mandatory(self): @supp_precip_mandatory.setter def supp_precip_mandatory(self, value): - """Set the list of flags for whether each supplemental precipitation input forcing specified by the user in the configuration file is mandatory or optional. This is used to control whether an error is raised if supplemental precipitation input forcing files are not found for each supplemental precipitation input forcing based on whether the user has specified each supplemental precipitation input forcing as mandatory or optional in the configuration file.""" + """Set the list of flags for whether each supplemental precipitation input + forcing specified by the user in the configuration file is mandatory or optional. + This is used to control whether an error is raised if supplemental precipitation + input forcing files are not found for each supplemental precipitation input + forcing based on whether the user has specified each supplemental precipitation + input forcing as mandatory or optional in the configuration file. + """ if self.number_supp_pcp > 0: self.check_input_values_in_range(value, "SuppPcpMandatory", [0, 1]) self._supp_precip_mandatory = value @@ -1483,7 +1905,11 @@ def supp_precip_mandatory(self, value): @property def regrid_opt_supp_pcp(self): - """Specify regridding options for the supplemental precipitation products. Options available are: 1 - ESMF Bilinear, 2 - ESMF Nearest Neighbor, 3 - ESMF Conservative Bilinear. + """Specify regridding options for the supplemental precipitation products. + Options available are: + 1 - ESMF Bilinear, + 2 - ESMF Nearest Neighbor, + 3 - ESMF Conservative Bilinear. Example- RegridOptSuppPcp: [1, 1, 1] """ @@ -1491,7 +1917,12 @@ def regrid_opt_supp_pcp(self): @regrid_opt_supp_pcp.setter def regrid_opt_supp_pcp(self, value): - """Set the list of regridding options for supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how supplemental precipitation input forcings are regridded based on the regridding option specified for each supplemental precipitation input forcing in the configuration file.""" + """Set the list of regridding options for supplemental precipitation input + forcings specified by the user in the configuration file. This is used to + control how supplemental precipitation input forcings are regridded based on + the regridding option specified for each supplemental precipitation input + forcing in the configuration file. + """ if self.number_supp_pcp > 0: self.check_input_values_in_range(value, "RegridOptSuppPcp", [1, 2, 3]) self._regrid_opt_supp_pcp = value @@ -1508,7 +1939,13 @@ def suppTemporalInterp(self): @suppTemporalInterp.setter def suppTemporalInterp(self, value): - """Set the list of flags for whether temporal interpolation of supplemental precipitation input forcings specified by the user in the configuration file is performed or not. This is used to control whether temporal interpolation of supplemental precipitation input forcings is performed based on whether the user has chosen to perform temporal interpolation for each supplemental precipitation input forcing in the configuration file.""" + """Set the list of flags for whether temporal interpolation of supplemental + precipitation input forcings specified by the user in the configuration file is + performed or not. This is used to control whether temporal interpolation of + supplemental precipitation input forcings is performed based on whether the + user has chosen to perform temporal interpolation for each supplemental + precipitation input forcing in the configuration file. + """ if self.number_supp_pcp > 0: self.check_input_values_in_range( value, "SuppPcpTemporalInterpolation", [0, 1, 2] @@ -1519,12 +1956,22 @@ def suppTemporalInterp(self, value): @property def supp_pcp_max_hours(self): - """Get the list of maximum forecast hours for supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how supplemental precipitation input forcings are processed based on the maximum forecast hour specified for each supplemental precipitation input forcing in the configuration file.""" + """Get the list of maximum forecast hours for supplemental precipitation input + forcings specified by the user in the configuration file. This is used to + control how supplemental precipitation input forcings are processed based on the + maximum forecast hour specified for each supplemental precipitation input + forcing in the configuration file. + """ return self._supp_pcp_max_hours @supp_pcp_max_hours.setter def supp_pcp_max_hours(self, value): - """Set the list of maximum forecast hours for supplemental precipitation input forcings specified by the user in the configuration file. This is used to control how supplemental precipitation input forcings are processed based on the maximum forecast hour specified for each supplemental precipitation input forcing in the configuration file.""" + """Set the list of maximum forecast hours for supplemental precipitation input + forcings specified by the user in the configuration file. This is used to control + how supplemental precipitation input forcings are processed based on the maximum + forecast hour specified for each supplemental precipitation input forcing in the + configuration file. + """ if self.number_supp_pcp > 0: if isinstance(value, list): self.check_number_of_inputs_supp_pcp(value, "SuppPcpMaxHours") @@ -1536,7 +1983,8 @@ def supp_pcp_max_hours(self, value): @property def supp_input_offsets(self): - """In AnA runs, this value is the offset from the available forecast and 00z. For example, if forecast are available at 06z and 18z, set this value to 6. + """In AnA runs, this value is the offset from the available forecast and 00z. + For example, if forecast are available at 06z and 18z, set this value to 6. Example- SuppPcpInputOffsets = [0, 0, 0] """ @@ -1544,7 +1992,12 @@ def supp_input_offsets(self): @supp_input_offsets.setter def supp_input_offsets(self, value): - """Set the list of time offsets to apply to supplemental precipitation input forcing files specified by the user in the configuration file. This is used to control how supplemental precipitation input forcing files are processed based on the time offset specified for each supplemental precipitation input forcing in the configuration file.""" + """Set the list of time offsets to apply to supplemental precipitation input + forcing files specified by the user in the configuration file. This is used to + control how supplemental precipitation input forcing files are processed based + on the time offset specified for each supplemental precipitation input forcing + in the configuration file. + """ if self.number_supp_pcp > 0: self.check_number_of_inputs_supp_pcp(value, "SuppPcpInputOffsets") self._supp_input_offsets = value @@ -1561,7 +2014,13 @@ def supp_precip_dirs(self): @supp_precip_dirs.setter def supp_precip_dirs(self, value): - """Set the list of pathways to the supplemental precipitation input forcing directories specified by the user in the configuration file. This is used to control where the program looks for supplemental precipitation input forcing files for each supplemental precipitation input forcing based on the directory specified for each supplemental precipitation input forcing in the configuration file.""" + """Set the list of pathways to the supplemental precipitation input forcing + directories specified by the user in the configuration file. This is used to + control where the program looks for supplemental precipitation input forcing + files for each supplemental precipitation input forcing based on the directory + specified for each supplemental precipitation input forcing in the configuration + file. + """ if self.number_supp_pcp > 0: self.check_number_of_inputs_supp_pcp(value, "SuppPcpDirectories") # Loop through and ensure all supp pcp directories exist. Also strip out any whitespace @@ -1583,7 +2042,10 @@ def supp_precip_dirs(self, value): @property def supp_precip_param_dir(self): - """Specify an optional directory that contains supplemental precipitation parameter fields, I.E monthly RQI climatology. This is ONLY needed for the original NWM WRF-Hydro domain. Otherwise, just point it to a random directory and it will be ignored. + """Specify an optional directory that contains supplemental precipitation + parameter fields, I.E monthly RQI climatology. + This is ONLY needed for the original NWM WRF-Hydro domain. + Otherwise, just point it to a random directory and it will be ignored. Example- SuppPcpParamDir: ['./forcingParam/AnA','./forcingParam/AnA','./forcingParam/AnA'] """ @@ -1591,7 +2053,12 @@ def supp_precip_param_dir(self): @supp_precip_param_dir.setter def supp_precip_param_dir(self, value): - """Set the directory where downscaling parameters for supplemental precipitation input forcings are stored specified by the user in the configuration file. This is used to control where the program looks for downscaling parameter files for supplemental precipitation input forcings based on the directory specified for supplemental precipitation input forcings in the configuration file.""" + """Set the directory where downscaling parameters for supplemental precipitation + input forcings are stored specified by the user in the configuration file. + This is used to control where the program looks for downscaling parameter files + for supplemental precipitation input forcings based on the directory specified + for supplemental precipitation input forcings in the configuration file. + """ if self.number_supp_pcp > 0: self.try_make_dir(value, " SuppPcpParamDir") self._supp_precip_param_dir = value @@ -1600,7 +2067,11 @@ def supp_precip_param_dir(self, value): @property def cfsv2EnsMember(self): - """Set the CFSv2 ensemble member to process specified by the user in the configuration file. This is used to control which CFSv2 ensemble member is processed for CFSv2 input forcings based on the ensemble member specified in the configuration file.""" + """Set the CFSv2 ensemble member to process specified by the user in the + configuration file. This is used to control which CFSv2 ensemble member is + processed for CFSv2 input forcings based on the ensemble member specified in the + configuration file. + """ value = None if not self.precip_only_flag: # Read in Ensemble information @@ -1616,12 +2087,19 @@ def cfsv2EnsMember(self): @property def customFcstFreq(self): - """Get the custom forecast frequency in minutes specified by the user in the configuration file. This is used to control how often forecasts are issued based on the custom forecast frequency specified in the configuration file.""" + """Get the custom forecast frequency in minutes specified by the user in the + configuration file. This is used to control how often forecasts are issued + based on the custom forecast frequency specified in the configuration file. + """ return self._customFcstFreq @customFcstFreq.setter def customFcstFreq(self, value): - """Options for specifying custom input NetCDF forcing files (in minutes). Choose the input frequency of files that are being processed. I.E., are the input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the length of custom input frequencies to match the number of custom NetCDF inputs selected above in the Logistics section. + """Options for specifying custom input NetCDF forcing files (in minutes). Choose + the input frequency of files that are being processed. I.E., are the input files + every 15 minutes, 60 minutes, 3-hours, etc. Please specify the length of custom + input frequencies to match the number of custom NetCDF inputs selected above in + the Logistics section. Example- custom_input_fcst_freq: [] """ From ddb37fd006c5c9d391ede52001899663322c6fe9 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 21:59:15 -0400 Subject: [PATCH 102/132] Fix key names for "NWM_Geogrid" and "RqiThreshold" --- .../NextGen_Forcings_Engine/core/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index a70f8fa4..0fd59c10 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -136,7 +136,7 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No set_none=True, ) if 27 in self.input_forcings: - self.nwm_geogrid = self.extract_input_variable("NWMGeogridIn") + self.nwm_geogrid = self.extract_input_variable("NWM_Geogrid") if self.perform_downscaling: self.set_attrs(CONFIGOPTIONS["downscaling_attrs_map"]) @@ -1860,11 +1860,11 @@ def rqiThresh(self) -> float | list[float] | None: # Read in RQI threshold to apply to radar products. if supp_opt in (1, 2, 7, 10, 11, 12): # Returns None if key missing (not configured for this product) - value = self.cfg_bmi.get("RqiThresh") + value = self.cfg_bmi.get("RqiThreshold") if value is not None: # Validate if type(value) is list: - self.check_number_of_inputs_supp_pcp(value, "RqiThresh") + self.check_number_of_inputs_supp_pcp(value, "RqiThreshold") elif type(value) in (int, float, type(None)): # Support configuration file representing this with a single scalar value, apply to all value = [value] * self.number_supp_pcp From dcd396cb155bcc0a08100c216ae940c73bb575ec Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 22:00:23 -0400 Subject: [PATCH 103/132] Re-add GRIB2_CFS to file_types list --- .../NextGen_Forcings_Engine/core/consts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py index 077048b3..91a182a5 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py @@ -1245,7 +1245,7 @@ 0: timeInterpMod.no_interpolation_supp_pcp, 1: timeInterpMod.nearest_neighbor_supp_pcp, 2: timeInterpMod.weighted_average_supp_pcp, - } + }, } MODEL = { @@ -1396,5 +1396,5 @@ "GeogridIn": "geogrid", "SpatialMetaIn": "spatial_meta", }, - "file_types": ["GRIB1", "GRIB2", "NETCDF", "NETCDF4", "NWM", "ZARR"], + "file_types": ["GRIB1", "GRIB2", "NETCDF", "NETCDF4", "NWM", "ZARR", "GRIB2_CFS"], } From ca02ea18f5966cc2ee7ff019c66a7f5af6c3a506 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 22:09:10 -0400 Subject: [PATCH 104/132] Fix key names "HGTVAR" and "HGTVAR_ELEM" --- .../NextGen_Forcings_Engine/core/consts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py index 91a182a5..f04b7913 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py @@ -1376,12 +1376,12 @@ "COSALPHA": "cosalpha_var", "SLOPE": "slope_var", "SLOPE_AZIMUTH": "slope_azimuth_var", - "HGT": "hgt_var", + "HGTVAR": "hgt_var", }, "downscaling_unstructred_attrs_map": { "SLOPE_ELEM": "slope_var_elem", "SLOPE_AZIMUTH_ELEM": "slope_azimuth_var_elem", - "HGT_ELEM": "hgt_elem_var", + "HGTVAR_ELEM": "hgt_elem_var", }, "extract_input_variable_set_default_attrs_map": { "includeLQFrac": "include_lqfrac", From c0c770e1304d847e82a0c8c9b66fa35a9c38b5c3 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 22:13:09 -0400 Subject: [PATCH 105/132] Fix type for cfsv2EnsMember validation. Add defensive type checking to validation methods. --- .../NextGen_Forcings_Engine/core/config.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 0fd59c10..302ef6d7 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -320,6 +320,10 @@ def check_number_of_inputs( """Check that the number of inputs specified by the user in the configuration file matches the expected number of inputs for a given variable. """ + if not isinstance(value, (list, tuple)): + raise TypeError( + f"Expected list or tuple for `value`, got type {type(value)}" + ) if len(value) != number_inputs: err_out_screen( f"Number of {variable_name} values must match the number of {input_type} in the configuration file." @@ -351,6 +355,10 @@ def check_input_values_in_range( """Check that the input values specified by the user in the configuration file are within a valid range for a given variable. """ + if not isinstance(value, (list, tuple)): + raise TypeError( + f"Expected list or tuple for `value`, got type {type(value)}" + ) for val in value: if val not in valid_input_options: err_out_screen( @@ -2081,7 +2089,7 @@ def cfsv2EnsMember(self): if opt_tmp == 7: value = self.extract_input_variable("cfsEnsNumber") self.check_input_values_in_range( - value, "cfsEnsNumber", [1, 2, 3, 4] + [value], "cfsEnsNumber", [1, 2, 3, 4] ) return value From 39fb626464f5fa8c1b76c505477cea14053ca064 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 22:17:09 -0400 Subject: [PATCH 106/132] Fix order of validation call when building supp_precip_dirs for the Extended AnA case --- .../NextGen_Forcings_Engine/core/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 302ef6d7..c9371572 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -2030,7 +2030,6 @@ def supp_precip_dirs(self, value): file. """ if self.number_supp_pcp > 0: - self.check_number_of_inputs_supp_pcp(value, "SuppPcpDirectories") # Loop through and ensure all supp pcp directories exist. Also strip out any whitespace # or new line characters. for dirTmp in range(0, len(value)): @@ -2038,12 +2037,14 @@ def supp_precip_dirs(self, value): self.try_make_dir(value[dirTmp], " supp pcp") # Special case for ExtAnA where we treat comma separated stage IV, MRMS data as one SuppPcp input + # NOTE: the length check must happen after this join, since ExtAnA may supply 2 dirs for 1 SuppPcp product. if 11 in self.supp_precip_forcings or 12 in self.supp_precip_forcings: if len(self.supp_precip_forcings) != 1: err_out_screen( "CONUS or Alaska Stage IV/MRMS SuppPcp option is only supported as a standalone option" ) value = [",".join(value)] + self.check_number_of_inputs_supp_pcp(value, "SuppPcpDirectories") self._supp_precip_dirs = value else: self._supp_precip_dirs = None From 5bfaff9e1124a68d0e2750129f1375d81e6d45a2 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 23:02:22 -0400 Subject: [PATCH 107/132] Handle None fcst_input_horizons for precip-only configs --- .../NextGen_Forcings_Engine/core/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index c9371572..09b87ef2 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -1108,7 +1108,7 @@ def fcst_input_horizons(self, value: list) -> None: self.check_number_of_inputs_forcings(value, "ForecastInputHorizons") self.check_input_values_non_negative(value, "ForecastInputHorizons") else: - if len(self.fcst_input_horizons) != 1: + if value is not None and len(value) != 1: err_out_screen( "Please specify ForecastInputHorizon values for each corresponding input forcings for forecasts." ) From cb66dd3296c62db588486c5d4898005ccc480c18 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 23:03:55 -0400 Subject: [PATCH 108/132] Add length assertions for SuppPcpMandatory, RegridOptSuppPcp, SuppPcpTemporalInterpolation --- .../NextGen_Forcings_Engine/core/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 09b87ef2..2f978115 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -1906,6 +1906,7 @@ def supp_precip_mandatory(self, value): input forcing as mandatory or optional in the configuration file. """ if self.number_supp_pcp > 0: + self.check_number_of_inputs_supp_pcp(value, "SuppPcpMandatory") self.check_input_values_in_range(value, "SuppPcpMandatory", [0, 1]) self._supp_precip_mandatory = value else: @@ -1932,6 +1933,7 @@ def regrid_opt_supp_pcp(self, value): forcing in the configuration file. """ if self.number_supp_pcp > 0: + self.check_number_of_inputs_supp_pcp(value, "RegridOptSuppPcp") self.check_input_values_in_range(value, "RegridOptSuppPcp", [1, 2, 3]) self._regrid_opt_supp_pcp = value else: @@ -1955,6 +1957,7 @@ def suppTemporalInterp(self, value): precipitation input forcing in the configuration file. """ if self.number_supp_pcp > 0: + self.check_number_of_inputs_supp_pcp(value, "SuppPcpTemporalInterpolation") self.check_input_values_in_range( value, "SuppPcpTemporalInterpolation", [0, 1, 2] ) From 4aee11e7684a52139e8031cfedf07a5c82bbea20 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 25 Aug 2026 23:13:55 -0400 Subject: [PATCH 109/132] Initialize more vars (required for certain workflows) --- .../NextGen_Forcings_Engine/core/config.py | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 2f978115..5c992626 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -91,6 +91,24 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No self._geopackage = None self._geogrid = None self._grid_type = None + # Backing vars for setters that guard on precip_only_flag and do not unconditionally assign + self._fcst_input_offsets = None + self._ignored_border_widths = None + self._regrid_opt = None + self._weightsDir = None + self._forceTemoralInterp = None + self._t2dDownscaleOpt = None + self._psfcDownscaleOpt = None + self._swDownscaleOpt = None + self._q2dDownscaleOpt = None + self._precipDownscaleOpt = None + self._t2BiasCorrectOpt = None + self._psfcBiasCorrectOpt = None + self._q2BiasCorrectOpt = None + self._windBiasCorrect = None + self._swBiasCorrectOpt = None + self._lwBiasCorrectOpt = None + self._precipBiasCorrectOpt = None # set list of attributes from consts.py to None early on in the init process. # These are indexed from the consts dictionary. @@ -1512,13 +1530,15 @@ def precipDownscaleOpt(self, value: list) -> None: """ if not self.precip_only_flag: self.check_number_of_inputs_forcings(value, "PrecipDownscaling") - self.check_input_values_in_range(value, "PrecipDownscaling", [0, 1]) - count = 0 - for opt in value: - if opt == 1: - self.param_flag[count] = 1 - count += 1 - self._precipDownscaleOpt = value + self.check_input_values_in_range(value, "PrecipDownscaling", [0, 1]) + count = 0 + for opt in value: + if opt == 1: + self.param_flag[count] = 1 + count += 1 + self._precipDownscaleOpt = value + else: + self._precipDownscaleOpt = None @property def dScaleParamDirs(self) -> list: From 9227b6b857978dd400ade9d9b6cd34463e4d3b49 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 09:05:57 -0400 Subject: [PATCH 110/132] Better handling of SuppPcpMaxHours list --- .../NextGen_Forcings_Engine/core/config.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 5c992626..d765da97 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -2177,9 +2177,26 @@ def nwm_url(self): @property def use_data_at_current_time(self): - """Determine if supplemental precipitation data can be used at the current output time.""" + """Determine if supplemental precipitation data can be used at the current output time. + + TODO: This property has no product context; it asserts that all supp pcp products share + the same value for ``SuppPcpMaxHours`` in the config file . Needs revisiting if + support is needed for differing values per-product. + """ if self.supp_pcp_max_hours: hrs_since_start = self.current_output_date - self.current_fcst_cycle - return hrs_since_start <= timedelta(hours=self.supp_pcp_max_hours) + if isinstance(self.supp_pcp_max_hours, list): + if len(set(self.supp_pcp_max_hours)) != 1: + raise ValueError( + f"use_data_at_current_time does not support differing supp_pcp_max_hours per product: {self.supp_pcp_max_hours}" + ) + hours = self.supp_pcp_max_hours[0] + elif isinstance(self.supp_pcp_max_hours, (int, float)): + hours = self.supp_pcp_max_hours + else: + raise TypeError( + f"Unexpected type for supp_pcp_max_hours: {type(self.supp_pcp_max_hours)}" + ) + return hrs_since_start <= timedelta(hours=hours) else: return True From 7f973ef0373eff22206c17d2ef06de237430a312 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 09:14:51 -0400 Subject: [PATCH 111/132] spatial_meta setter: handle None --- .../NextGen_Forcings_Engine/core/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index d765da97..2e93069b 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -691,6 +691,10 @@ def spatial_meta(self, value: str) -> None: file. This is used to control how spatial metadata is handled during processing, and is necessary for both realtime and reforecast simulations. """ + if value is None: + raise TypeError( + "spatial_meta setter received None; pass '' to indicate no spatial metadata file." + ) if len(value) == 0: # No spatial metadata file found. value = None From e412d87b4224ee9bfcff3bcb6679cdb8fca2d334 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 11:23:04 -0400 Subject: [PATCH 112/132] runCfsNldasBiasCorrect: add short-circuit for "precip only" case --- .../NextGen_Forcings_Engine/core/config.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 2e93069b..3904fbf1 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -1776,8 +1776,14 @@ def runCfsNldasBiasCorrect(self) -> bool: control whether the NWM-specific bias correction of CFSv2 input forcings is run based on whether the user has chosen to run this bias correction in the configuration file. + + NOTE: Returns False immediately when precip_only_flag is truthy because + bias correction is never configured for precip-only runs. In that case the + correction backing vars are None and iterating them would raise TypeError. """ run_cfs_nldas_bias_correct = False + if self.precip_only_flag: + return run_cfs_nldas_bias_correct for bias_option in self.bias_correction_properties.values(): for opt in bias_option: if opt == 1: From 7b28b3d249cf598d5b6486ccc9f8895b3f612e41 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 11:25:38 -0400 Subject: [PATCH 113/132] Validate that supp precip time offsets are non-negative --- .../NextGen_Forcings_Engine/core/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 3904fbf1..b3abb3fa 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -2041,6 +2041,7 @@ def supp_input_offsets(self, value): """ if self.number_supp_pcp > 0: self.check_number_of_inputs_supp_pcp(value, "SuppPcpInputOffsets") + self.check_input_values_non_negative(value, "SuppPcpInputOffsets") self._supp_input_offsets = value else: self._supp_input_offsets = None From be9825198078a0a39ffc4afa79663eba49b3a7c1 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 11:26:42 -0400 Subject: [PATCH 114/132] Minor improvements: avoid setting _regrid_opt unnecessarily; return None directly instead of implicitly. --- .../NextGen_Forcings_Engine/core/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index b3abb3fa..e1ac72dc 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -94,7 +94,6 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No # Backing vars for setters that guard on precip_only_flag and do not unconditionally assign self._fcst_input_offsets = None self._ignored_border_widths = None - self._regrid_opt = None self._weightsDir = None self._forceTemoralInterp = None self._t2dDownscaleOpt = None @@ -968,8 +967,7 @@ def input_force_dirs(self) -> list: """ if self._input_force_dirs: return self._input_force_dirs - else: - None + return None @input_force_dirs.setter def input_force_dirs(self, value: list) -> None: From 440450114e47ef3b1cf80dc822d4a1acebe2e4c3 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 11:52:00 -0400 Subject: [PATCH 115/132] To match original behavior, assert that output_freq and fcst_freq are positive, not just non-negative. i.e. disallow value 0. --- .../NextGen_Forcings_Engine/core/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index e1ac72dc..a2ef83b9 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -491,7 +491,7 @@ def output_freq(self, value: int) -> None: Example- OutputFrequency: 60 """ - self.check_input_values_non_negative([value], "OutputFrequency") + self.check_input_values_positive([value], "OutputFrequency") self._output_freq = value @property @@ -665,7 +665,7 @@ def fcst_freq(self, value: int) -> None: raise ValueError( f"fcst_freq is immutable after initialization. Current: {self._fcst_freq}, Attempted: {value}" ) - self.check_input_values_non_negative([value], "ForecastFrequency") + self.check_input_values_positive([value], "ForecastFrequency") if value > 1440: err_out_screen( "Only forecast cycles of daily or sub-daily are supported at this time" From 29de25dbf5f390e93d24f73933b752dbc4e6e70d Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 11:52:45 -0400 Subject: [PATCH 116/132] Add backing var for _input_force_types to avoid potential crash under certain config combinations. --- .../NextGen_Forcings_Engine/core/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index a2ef83b9..c8cb4896 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -108,6 +108,7 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No self._swBiasCorrectOpt = None self._lwBiasCorrectOpt = None self._precipBiasCorrectOpt = None + self._input_force_types = None # set list of attributes from consts.py to None early on in the init process. # These are indexed from the consts dictionary. From bfa47b026f98e919e9a4632c1f5aa21a0b59b1b9 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 14:05:32 -0400 Subject: [PATCH 117/132] Handle a precip-only edge case restriction on configuration --- .../NextGen_Forcings_Engine/core/config.py | 23 ++++++++++++------- .../NextGen_Forcings_Engine/core/consts.py | 2 +- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index c8cb4896..65847afb 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -109,6 +109,7 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No self._lwBiasCorrectOpt = None self._precipBiasCorrectOpt = None self._input_force_types = None + self._dScaleParamDirs = None # set list of attributes from consts.py to None early on in the init process. # These are indexed from the consts dictionary. @@ -1559,15 +1560,21 @@ def dScaleParamDirs(self, value: list) -> None: This is used to control where the program looks for downscaling parameter files for each input forcing based on the downscaling parameter directory specified for each input forcing in the configuration file. + + NOTE: The guard on ``precip_only_flag`` is because DownscalingParamDirs is omitted + from precip-only configurations. """ - self.check_number_of_inputs_forcings(value, "DownscalingParamDirs") - for dirTmp in range(0, len(value)): - dir_path = value[dirTmp] - if not os.path.isdir(dir_path): - err_out_screen( - f"Unable to locate parameter directory: {os.path.abspath(dir_path)}" - ) - self._dScaleParamDirs = value + if not self.precip_only_flag: + self.check_number_of_inputs_forcings(value, "DownscalingParamDirs") + for dirTmp in range(0, len(value)): + dir_path = value[dirTmp] + if not os.path.isdir(dir_path): + err_out_screen( + f"Unable to locate parameter directory: {os.path.abspath(dir_path)}" + ) + self._dScaleParamDirs = value + else: + self._dScaleParamDirs = None @property def perform_downscaling(self) -> bool: diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py index f04b7913..cc99f943 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/consts.py @@ -1336,7 +1336,6 @@ "ForecastFrequency": "fcst_freq", "ForecastShift": "fcst_shift", "GRID_TYPE": "grid_type", - "DownscalingParamDirs": "dScaleParamDirs", "SuppPcpDirectories": "supp_precip_dirs", "SuppPcpMandatory": "supp_precip_mandatory", "RegridOptSuppPcp": "regrid_opt_supp_pcp", @@ -1370,6 +1369,7 @@ "InputForcingDirectories": "input_force_dirs", # np "InputMandatory": "input_force_mandatory", # np "custom_input_fcst_freq": "customFcstFreq", # np + "DownscalingParamDirs": "dScaleParamDirs", # np }, "downscaling_attrs_map": { "SINALPHA": "sinalpha_var", From 63a336f1cd1657cd1531b2919ddc986f1841053c Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 14:06:04 -0400 Subject: [PATCH 118/132] Fix attribute access typo --- .../NextGen_Forcings_Engine/bmi_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 6f3256e0..6739cd69 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -1578,7 +1578,7 @@ def set_var_names(self) -> None: # will support a BMI field for liquid fraction of precipitation self._output_var_names = BMI_MODEL["_output_var_names"] self._var_name_units_map = BMI_MODEL["_var_name_units_map"] - if self.config_options.include_lqfrac == 1: + if self._job_meta.include_lqfrac == 1: self._output_var_names += ["LQFRAC_ELEMENT"] self._var_name_units_map |= { "LQFRAC_ELEMENT": ["Liquid Fraction of Precipitation", "%"] From f7b964e491d88d6bd860cf96827e1da7555382b9 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 14:58:42 -0400 Subject: [PATCH 119/132] Raise error if non-hydrofabric discretization type (grid_type) is used, with error message and docstrings pointing to pending code where pending attempts are made to support these alternate discretization types. --- .../NextGen_Forcings_Engine/bmi_model.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 6739cd69..7e652b62 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -1,3 +1,12 @@ +"""BMI model implementation for the forcing engine. + +TODO: to support other discretization types ("gridded" and "unstructured"), use self.GeoMeta(...). +Other changes may be necessary than just this to enable those discretization types. +See these PRs for pending code that attempts to address this: + https://github.com/NGWPC/ngen-forcing/pull/202 -- Adjustments to Support Gridded Forcing + https://github.com/NGWPC/ngen-forcing/pull/212 -- Coastal Forcing +""" + # Need these for BMI # This is needed for get_var_bytes import gc @@ -255,8 +264,18 @@ def _mpi_meta(self, value: MpiConfig) -> None: @property def geo_meta(self) -> GeoMeta: - """Return the geospatial metadata object.""" + """Return the geospatial metadata object. + + TODO: to support other discretization types ("gridded" and "unstructured"), use self.GeoMeta(...). + Other changes may be necessary than just this to enable those discretization types. + See these PRs for pending code that attempts to address this: + https://github.com/NGWPC/ngen-forcing/pull/202 -- Adjustments to Support Gridded Forcing + https://github.com/NGWPC/ngen-forcing/pull/212 -- Coastal Forcing + """ if self._geo_meta is None: + assert self._job_meta.grid_type == "hydrofabric", ( + f"Only 'hydrofabric' grid type is currently supported; got '{self._job_meta.grid_type}'. See docstrings for discretization types." + ) self._geo_meta = HydrofabricGeoMeta(self._job_meta, self._mpi_meta) return self._geo_meta From edd6a5ed48eb8f952a95995dfe22588807279cc3 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 16:17:15 -0400 Subject: [PATCH 120/132] Fix crashing method calls --- .../NextGen_Forcings_Engine/bmi_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 7e652b62..6d2c3798 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -659,13 +659,13 @@ def get_value(self, var_name: str, dest: NDArray[Any]) -> NDArray[Any]: LOG.debug( f"[BMI get_value] Special case: 'grid:ids', grid_type: {self._job_meta.grid_type}" ) - dest[:] = self.grid_ids(self) + dest[:] = self.grid_ids() elif var_name == "grid:ranks": LOG.debug( f"[BMI get_value] Special case: 'grid:ranks', grid_type: {self._job_meta.grid_type}" ) - dest[:] = self.grid_ranks(self) + dest[:] = self.grid_ranks() else: src = self.get_value_ptr(var_name) LOG.debug( From ed9b65f4334e90e0df8b762236e7fbc0b9dcc025 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 16:23:36 -0400 Subject: [PATCH 121/132] Fix log calls --- .../NextGen_Forcings_Engine/bmi_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 6d2c3798..75e3d4b8 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -395,7 +395,7 @@ def init_output_obj(self) -> None: try: self._output_obj = ioMod.OutputObj(self._job_meta, self.geo_meta) except Exception as e: - err_handler.err_out_screen_para(self._job_meta, self._mpi_meta) + err_handler.err_out_screen_para(self._job_meta.errMsg, self._mpi_meta) err_handler.check_program_status(self._job_meta, self._mpi_meta) def init_input_forcing_mod(self) -> None: @@ -412,7 +412,7 @@ def init_input_forcing_mod(self) -> None: self._job_meta, self.geo_meta, self._mpi_meta ) except Exception as e: - err_handler.err_out_screen_para(self._job_meta, self._mpi_meta) + err_handler.err_out_screen_para(self._job_meta.errMsg, self._mpi_meta) err_handler.check_program_status(self._job_meta, self._mpi_meta) def init_supp_pcp_mod(self) -> None: @@ -741,7 +741,7 @@ def get_value_ptr(self, var_name: str) -> NDArray[Any]: LOG.error("Output variable names:") for var in self._output_var_names: LOG.error(f" - {var}") - LOG.error("Grid type: {self._grid_type}") + LOG.error(f"Grid type: {self._grid_type}") raise UnknownBMIVariable(f"No known variable in BMI model: '{var_name}'") arr = self._values[var_name] From bf0ad5ab6825269488bd29542981ff79f068a679 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 26 Aug 2026 17:42:52 -0400 Subject: [PATCH 122/132] Tests data: Update expected json to add "GRIB2_CFS" to "file_types" list --- .../test_expected_bmi_model_after_update_n1_rank0__step_1.json | 3 ++- .../test_expected_bmi_model_after_update_n1_rank0__step_2.json | 3 ++- .../test_expected_bmi_model_after_update_n1_rank0__step_3.json | 3 ++- .../test_expected_bmi_model_after_update_n2_rank0__step_1.json | 3 ++- .../test_expected_bmi_model_after_update_n2_rank0__step_2.json | 3 ++- .../test_expected_bmi_model_after_update_n2_rank0__step_3.json | 3 ++- .../test_expected_bmi_model_after_update_n2_rank1__step_1.json | 3 ++- .../test_expected_bmi_model_after_update_n2_rank1__step_2.json | 3 ++- .../test_expected_bmi_model_after_update_n2_rank1__step_3.json | 3 ++- .../test_expected_bmi_model_finalize_n1_rank0_.json | 3 ++- .../test_expected_bmi_model_finalize_n2_rank0_.json | 3 ++- .../test_expected_bmi_model_finalize_n2_rank1_.json | 3 ++- .../test_expected_bmi_model_init_n1_rank0_.json | 3 ++- .../test_expected_bmi_model_init_n2_rank0_.json | 3 ++- .../test_expected_bmi_model_init_n2_rank1_.json | 3 ++- ..._expected_config_options_after_update_n1_rank0__step_1.json | 3 ++- ..._expected_config_options_after_update_n1_rank0__step_2.json | 3 ++- ..._expected_config_options_after_update_n1_rank0__step_3.json | 3 ++- ..._expected_config_options_after_update_n2_rank0__step_1.json | 3 ++- ..._expected_config_options_after_update_n2_rank0__step_2.json | 3 ++- ..._expected_config_options_after_update_n2_rank0__step_3.json | 3 ++- ..._expected_config_options_after_update_n2_rank1__step_1.json | 3 ++- ..._expected_config_options_after_update_n2_rank1__step_2.json | 3 ++- ..._expected_config_options_after_update_n2_rank1__step_3.json | 3 ++- .../test_expected_config_options_finalize_n1_rank0_.json | 3 ++- .../test_expected_config_options_finalize_n2_rank0_.json | 3 ++- .../test_expected_config_options_finalize_n2_rank1_.json | 3 ++- .../test_expected_config_options_init_n1_rank0_.json | 3 ++- .../test_expected_config_options_init_n2_rank0_.json | 3 ++- .../test_expected_config_options_init_n2_rank1_.json | 3 ++- 30 files changed, 60 insertions(+), 30 deletions(-) diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json index 8f7df495..2b70124f 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_1.json @@ -381,7 +381,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json index 617df417..282a7701 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_2.json @@ -381,7 +381,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json index 0617ab38..afbf352e 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n1_rank0__step_3.json @@ -381,7 +381,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json index 6b4e9f2c..cb9e2264 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_1.json @@ -381,7 +381,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json index d60330e1..5fa8aaf7 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_2.json @@ -381,7 +381,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json index 3f62699f..bf864533 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank0__step_3.json @@ -381,7 +381,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json index 6ce96b23..1b925e98 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_1.json @@ -381,7 +381,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json index 79bc3416..9413f394 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_2.json @@ -381,7 +381,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json index ad066576..ed0ca0b1 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_after_update_n2_rank1__step_3.json @@ -381,7 +381,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n1_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n1_rank0_.json index 2e13b37f..26d5d6de 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n1_rank0_.json @@ -381,7 +381,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank0_.json index c8c8115c..5b6f4494 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank0_.json @@ -381,7 +381,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank1_.json b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank1_.json index 10e05311..a8a04795 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_finalize_n2_rank1_.json @@ -381,7 +381,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json index 3a29d087..f2773ee7 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_init_n1_rank0_.json @@ -191,7 +191,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": null, "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json index a4920c77..fdf9309c 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank0_.json @@ -191,7 +191,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": null, "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json index 074032c7..54811e7f 100644 --- a/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_bmi_model_init_n2_rank1_.json @@ -191,7 +191,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": null, "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json index f4f6b99c..78aff667 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_1.json @@ -372,7 +372,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json index 8699d992..b383c4c4 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_2.json @@ -372,7 +372,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json index 58255f00..5e19ba8f 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n1_rank0__step_3.json @@ -372,7 +372,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json index f4f6b99c..78aff667 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_1.json @@ -372,7 +372,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json index 8699d992..b383c4c4 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_2.json @@ -372,7 +372,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json index 58255f00..5e19ba8f 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank0__step_3.json @@ -372,7 +372,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json index f4f6b99c..78aff667 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_1.json @@ -372,7 +372,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json index 8699d992..b383c4c4 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_2.json @@ -372,7 +372,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json index 58255f00..5e19ba8f 100644 --- a/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json +++ b/tests/test_data/expected_results/test_expected_config_options_after_update_n2_rank1__step_3.json @@ -372,7 +372,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json index d0d21c4d..ff46860e 100644 --- a/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_finalize_n1_rank0_.json @@ -372,7 +372,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json index d0d21c4d..ff46860e 100644 --- a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank0_.json @@ -372,7 +372,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json index d0d21c4d..ff46860e 100644 --- a/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_config_options_finalize_n2_rank1_.json @@ -372,7 +372,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": "2013-07-01T00:00:00", "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_init_n1_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_init_n1_rank0_.json index 99420727..b054ac32 100644 --- a/tests/test_data/expected_results/test_expected_config_options_init_n1_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_init_n1_rank0_.json @@ -182,7 +182,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": null, "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_init_n2_rank0_.json b/tests/test_data/expected_results/test_expected_config_options_init_n2_rank0_.json index 99420727..b054ac32 100644 --- a/tests/test_data/expected_results/test_expected_config_options_init_n2_rank0_.json +++ b/tests/test_data/expected_results/test_expected_config_options_init_n2_rank0_.json @@ -182,7 +182,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": null, "forceTemoralInterp": [ diff --git a/tests/test_data/expected_results/test_expected_config_options_init_n2_rank1_.json b/tests/test_data/expected_results/test_expected_config_options_init_n2_rank1_.json index 99420727..b054ac32 100644 --- a/tests/test_data/expected_results/test_expected_config_options_init_n2_rank1_.json +++ b/tests/test_data/expected_results/test_expected_config_options_init_n2_rank1_.json @@ -182,7 +182,8 @@ "NETCDF", "NETCDF4", "NWM", - "ZARR" + "ZARR", + "GRIB2_CFS" ], "first_fcst_cycle": null, "forceTemoralInterp": [ From d54eb322fc458ed7ea4c5c75207880a39a7e5ad1 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 18 Aug 2026 17:31:45 -0400 Subject: [PATCH 123/132] Refactor layeringMod.layer_final_forcings into abstract classes by discretization type --- .../core/layeringMod.py | 255 ++++++++++++------ .../NextGen_Forcings_Engine/model.py | 1 - 2 files changed, 175 insertions(+), 81 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py index 42cad986..056091b2 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py @@ -1,12 +1,169 @@ -"""Layering module for implementing various layering schemes in the WRF-Hydro forcing engine. +"""Layering module for implementing various layering schemes. -Future functionality may include blenidng, etc. +Key Concepts +------------ +force_idx : int + Index into the forcing product array. See function ``layer_final_forcings`` for additional information. + +attr_suffix : str + Suffix appended to attribute names to access different array variants. + Empty string for standard arrays; '_elem' for element-based arrays in unstructured grids. + +Future functionality may include blending, etc. """ +from abc import ABC, abstractmethod +from typing import Any + import numpy as np +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( + ConfigOptions, +) +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.forcingInputMod import ( + InputForcings, +) +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.ioMod import ( + OutputObj, +) + + +class _LayeringMod(ABC): + """Abstract Class for layering of forcing grids""" + + def __init__( + self, + output_obj, + input_forcings: InputForcings, + config_options: ConfigOptions, + ): + self.output_obj = output_obj + self.input_forcings = input_forcings + self.config_options = config_options + + @abstractmethod + def get_slice(self, obj: Any, force_idx: int) -> Any: + """Abstract method: Using bracket syntax, return a slice of an object based on its forcing index.""" + raise NotImplementedError + + @abstractmethod + def set_slice(self, obj: Any, force_idx: int, value: Any) -> None: + """Abstract method: Using bracket syntax, set the value of a slice of an object based on its forcing index.""" + raise NotImplementedError + + @abstractmethod + def apply_layering(self, force_idx: int) -> None: + """Abstract method: Apply the layering logic (this is the primary function of this class).""" + raise NotImplementedError + + def layerIn(self, force_idx: int, attr_suffix: str = "") -> Any: + """Return an input dataset used for layering.""" + return self.get_slice( + getattr(self.input_forcings, f"final_forcings{attr_suffix}"), force_idx + ) + + def indSet(self, force_idx: int, attr_suffix: str = "") -> Any: + """Return the indices of the input dataset that are not equal to the global no-data value (a non-no-data mask).""" + return np.where( + self.layerIn(force_idx, attr_suffix) != self.config_options.globalNdv + ) + + def update_output_local(self, force_idx: int, attr_suffix: str = "") -> None: + """Apply layering logic to update the output grid with input forcing data. + + This is the primary business logic of the layering module. It retrieves input forcing data, + applies validity checks (using globalNdv as the no-data value), and updates the output grid + with valid data. Special handling is provided for ERA5 data. + + Parameters + ---------- + force_idx : int + Index of the forcing product to layer. + attr_suffix : str, optional + Suffix to append to attribute names (e.g., '_elem' for element arrays). + Default is empty string, which accesses standard arrays. + This is leveraged by the Unstructured discretization type. + + Notes + ----- + - Uses `get_slice()` and `set_slice()` to support different grid discretizations (gridded, unstructured, hydrofabric). + - For ERA5 with forcing keys [12, 21], uses `regridded_mask_AORC` (or `regridded_mask_elem_AORC` for elem variants) to determine valid cells. + - For other cases, uses global no-data value (`globalNdv`) to identify valid data. + """ + output_tmp = self.get_slice( + getattr(self.output_obj, f"output_local{attr_suffix}"), force_idx + ) + layerIn = self.layerIn(force_idx, attr_suffix) + + if ( + self.input_forcings.product_name == "ERA5" + and [12, 21] in self.config_options.input_forcings + ): + mask = getattr(self.input_forcings, f"regridded_mask{attr_suffix}_AORC") + output_tmp[np.where(mask == 0)] = layerIn[np.where(mask == 0)] + else: + indSet = self.indSet(force_idx, attr_suffix) + output_tmp[indSet] = layerIn[indSet] + + self.set_slice( + getattr(self.output_obj, f"output_local{attr_suffix}"), + force_idx, + output_tmp, + ) + -def layer_final_forcings(OutputObj, input_forcings, ConfigOptions, MpiConfig): +class _LayeringMod_Gridded(_LayeringMod): + """Implementation of abstract class _LayeringMod for Gridded discretization""" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def get_slice(self, obj: Any, force_idx: int) -> Any: + return obj[force_idx, :, :] + + def set_slice(self, obj: Any, force_idx: int, value: Any) -> None: + obj[force_idx, :, :] = value + + def apply_layering(self, force_idx: int) -> None: + self.update_output_local(force_idx) + + +class _LayeringMod_Unstructured(_LayeringMod): + """Implementation of abstract class _LayeringMod for Unstructured discretization""" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def get_slice(self, obj: Any, force_idx: int) -> Any: + return obj[force_idx, :] + + def set_slice(self, obj: Any, force_idx: int, value: Any) -> None: + obj[force_idx, :] = value + + def apply_layering(self, force_idx: int) -> None: + self.update_output_local(force_idx) + self.update_output_local(force_idx, "_elem") + + +class _LayeringMod_Hydrofabric(_LayeringMod): + """Implementation of abstract class _LayeringMod for Hydrofabric discretization""" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def get_slice(self, obj: Any, force_idx: int) -> Any: + return obj[force_idx, :] + + def set_slice(self, obj: Any, force_idx: int, value: Any) -> None: + obj[force_idx, :] = value + + def apply_layering(self, force_idx: int) -> None: + self.update_output_local(force_idx) + + +def layer_final_forcings( + output_obj: OutputObj, input_forcings: InputForcings, config_options: ConfigOptions +): """Layer input forcings onto the output grid. Function to perform basic layering of input forcings as they are processed. The logic @@ -15,10 +172,9 @@ def layer_final_forcings(OutputObj, input_forcings, ConfigOptions, MpiConfig): for this timestep, forcings are placed onto the output grid by shear brute replacement. However, this only occurs where valid data exists. Supplemental precipitation will be layered in separately. - :param OutputObj: + :param output_obj: :param input_forcings: - :param ConfigOptions: - :param MpiConfig: + :param config_options: :return: """ # Loop through the 8(or 9) forcing products to layer in: @@ -32,82 +188,21 @@ def layer_final_forcings(OutputObj, input_forcings, ConfigOptions, MpiConfig): # 7.) Surface incoming shortwave radiation flux (W/m^2) # 8.) Liquid fraction of precipitation ([0..1]) - force_count = 9 if ConfigOptions.include_lqfrac else 8 + if config_options.grid_type == "gridded": + factory = _LayeringMod_Gridded + elif config_options.grid_type == "unstructured": + factory = _LayeringMod_Unstructured + elif config_options.grid_type == "hydrofabric": + factory = _LayeringMod_Hydrofabric + else: + raise ValueError( + f"Unexpected discretization type / grid type: {config_options.grid_type}" + ) + layering_mod = factory(output_obj, input_forcings, config_options) + force_count = 9 if config_options.include_lqfrac else 8 for force_idx in range(0, force_count): if force_idx in input_forcings.input_map_output: - if ConfigOptions.grid_type == "gridded": - outLayerCurrent = OutputObj.output_local[force_idx, :, :] - layerIn = input_forcings.final_forcings[force_idx, :, :] - if ( - input_forcings.product_name == "ERA5" - and [12, 21] in ConfigOptions.input_forcings - ): - outLayerCurrent[ - np.where(input_forcings.regridded_mask_AORC == 0) - ] = layerIn[np.where(input_forcings.regridded_mask_AORC == 0)] - OutputObj.output_local[force_idx, :, :] = outLayerCurrent - else: - indSet = np.where(layerIn != ConfigOptions.globalNdv) - outLayerCurrent[indSet] = layerIn[indSet] - OutputObj.output_local[force_idx, :, :] = outLayerCurrent - - # Reset for next iteration and memory efficiency. - indSet = None - elif ConfigOptions.grid_type == "unstructured": - outLayerCurrent = OutputObj.output_local[force_idx, :] - layerIn = input_forcings.final_forcings[force_idx, :] - if ( - input_forcings.product_name == "ERA5" - and [12, 21] in ConfigOptions.input_forcings - ): - outLayerCurrent[ - np.where(input_forcings.regridded_mask_AORC == 0) - ] = layerIn[np.where(input_forcings.regridded_mask_AORC == 0)] - OutputObj.output_local[force_idx, :] = outLayerCurrent - else: - indSet = np.where(layerIn != ConfigOptions.globalNdv) - outLayerCurrent[indSet] = layerIn[indSet] - OutputObj.output_local[force_idx, :] = outLayerCurrent - - outLayerCurrent_elem = OutputObj.output_local_elem[force_idx, :] - layerIn_elem = input_forcings.final_forcings_elem[force_idx, :] - if ( - input_forcings.product_name == "ERA5" - and [12, 21] in ConfigOptions.input_forcings - ): - outLayerCurrent_elem[ - np.where(input_forcings.regridded_mask_elem_AORC == 0) - ] = layerIn_elem[ - np.where(input_forcings.regridded_mask_elem_AORC == 0) - ] - OutputObj.output_local_elem[force_idx, :] = outLayerCurrent_elem - else: - indSet_elem = np.where(layerIn_elem != ConfigOptions.globalNdv) - outLayerCurrent_elem[indSet_elem] = layerIn_elem[indSet_elem] - OutputObj.output_local_elem[force_idx, :] = outLayerCurrent_elem - - # Reset for next iteration and memory efficiency. - indSet = None - indSet_elem = None - elif ConfigOptions.grid_type == "hydrofabric": - outLayerCurrent = OutputObj.output_local[force_idx, :] - layerIn = input_forcings.final_forcings[force_idx, :] - if ( - input_forcings.product_name == "ERA5" - and [12, 21] in ConfigOptions.input_forcings - ): - outLayerCurrent[ - np.where(input_forcings.regridded_mask_AORC == 0) - ] = layerIn[np.where(input_forcings.regridded_mask_AORC == 0)] - OutputObj.output_local[force_idx, :] = outLayerCurrent - else: - indSet = np.where(layerIn != ConfigOptions.globalNdv) - outLayerCurrent[indSet] = layerIn[indSet] - OutputObj.output_local[force_idx, :] = outLayerCurrent - # Reset for next iteration and memory efficiency. - indSet = None - - # MpiConfig.comm.barrier() + layering_mod.apply_layering(force_idx) def layer_supplemental_forcing( diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index 66886ab9..ea7338f5 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -417,7 +417,6 @@ def loop_through_forcing_products( self._bmi._output_obj, input_forcings, self._bmi._job_meta, - self._bmi._mpi_meta, ) self.check_program_status() From 45c18c5d392bd0c3177fd187483b4147992cc81d Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Wed, 19 Aug 2026 11:01:38 -0400 Subject: [PATCH 124/132] Refactor layeringMod.layer_supplemental_forcing into abstract classes by discretization type --- .../core/layeringMod.py | 202 ++++++++++++------ .../NextGen_Forcings_Engine/model.py | 1 - 2 files changed, 137 insertions(+), 66 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py index 056091b2..521aeaf8 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py @@ -26,6 +26,9 @@ from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.ioMod import ( OutputObj, ) +from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.suppPrecipMod import ( + SupplementalPrecip, +) class _LayeringMod(ABC): @@ -205,81 +208,150 @@ def layer_final_forcings( layering_mod.apply_layering(force_idx) -def layer_supplemental_forcing( - OutputObj, supplemental_precip, ConfigOptions, MpiConfig -): - """Layer in supplemental precipitation where valid values exist. +class _LayeringModSupplemental(ABC): + def __init__( + self, + output_obj: OutputObj, + supplemental_precip: SupplementalPrecip, + config_options: ConfigOptions, + ): + self.output_obj = output_obj + self.supplemental_precip = supplemental_precip + self.config_options = config_options - Function to layer in supplemental precipitation where we have valid values. Any pixel - cells that contain missing values will not be layered in, and background input forcings - will be used instead. - :param OutputObj: - :param supplemental_precip: - :param ConfigOptions: - :param MpiConfig: - :return: - """ - if ConfigOptions.grid_type == "gridded": - indSet = np.where( - supplemental_precip.final_supp_precip != ConfigOptions.globalNdv + @abstractmethod + def get_slice(self, obj: Any) -> Any: + """Abstract method: Using bracket syntax, return a slice of an object based on its forcing index.""" + raise NotImplementedError + + @abstractmethod + def set_slice(self, obj: Any, value: Any) -> None: + """Abstract method: Using bracket syntax, set the value of a slice of an object based on its forcing index.""" + raise NotImplementedError + + @abstractmethod + def apply_layering(self) -> None: + """Abstract method: Apply the layering logic (this is the primary function of this class).""" + raise NotImplementedError + + def indSet(self, attr_suffix: str = ""): + return np.where( + getattr(self.supplemental_precip, f"final_supp_precip{attr_suffix}") + != self.config_options.globalNdv + ) + + def layerIn(self, attr_suffix: str = ""): + return getattr(self.supplemental_precip, f"final_supp_precip{attr_suffix}") + + def layerOut(self, attr_suffix: str = ""): + return self.get_slice( + getattr(self.output_obj, f"output_local{attr_suffix}"), + self.supplemental_precip.output_var_idx, ) - layerIn = supplemental_precip.final_supp_precip - layerOut = OutputObj.output_local[supplemental_precip.output_var_idx, :, :] + + def update_output_local(self, attr_suffix: str = "") -> None: + indSet = self.indSet(attr_suffix) + layerIn = self.layerIn(attr_suffix) + layerOut = self.layerOut(attr_suffix) + # NOTE original TODO comment below was for "gridded" discretization. Unknown intent: # TODO: review test layering for ExtAnA calculation to replace FE QPE with MPE RAINRATE # If this isn't sufficient, replace QPE with MPE here: # if supplemental_precip.keyValue == 11: - # ConfigOptions.statusMsg = "Performing ExtAnA calculation" - # err_handler.log_msg(ConfigOptions, MpiConfig) + # config_options.statusMsg = "Performing ExtAnA calculation" + # err_handler.log_msg(config_options, MpiConfig) if len(indSet[0]) != 0: layerOut[indSet] = layerIn[indSet] - else: - # We have all missing data for the supplemental precip for this step. - layerOut = layerOut + # NOTE original TODO comment below was for all discretizations ("gridded", "unstructured", and "hydrofabric"). Unknown intent. # TODO: test that even does anything...?s - OutputObj.output_local[supplemental_precip.output_var_idx, :, :] = layerOut - elif ConfigOptions.grid_type == "unstructured": - indSet = np.where( - supplemental_precip.final_supp_precip != ConfigOptions.globalNdv + self.set_slice( + getattr(self.output_obj, f"output_local{attr_suffix}"), + self.supplemental_precip.output_var_idx, + layerOut, ) - layerIn = supplemental_precip.final_supp_precip - layerOut = OutputObj.output_local[supplemental_precip.output_var_idx, :] - if len(indSet[0]) != 0: - layerOut[indSet] = layerIn[indSet] - else: - # We have all missing data for the supplemental precip for this step. - layerOut = layerOut - # TODO: test that even does anything...?s - OutputObj.output_local[supplemental_precip.output_var_idx, :] = layerOut - indSet_elem = np.where( - supplemental_precip.final_supp_precip_elem != ConfigOptions.globalNdv - ) - layerIn_elem = supplemental_precip.final_supp_precip_elem - layerOut_elem = OutputObj.output_local_elem[ - supplemental_precip.output_var_idx, : - ] +class _LayeringModSupplemental_Gridded(_LayeringModSupplemental): + """Implementation of abstract class _LayeringModSupplemental for Gridded discretization. + Slicing is 3-dimensional. + Primary business logic executes once (no extra "_elem" call). + """ - if len(indSet_elem[0]) != 0: - layerOut_elem[indSet_elem] = layerIn_elem[indSet_elem] - else: - # We have all missing data for the supplemental precip for this step. - layerOut_elem = layerOut_elem - # TODO: test that even does anything...?s - OutputObj.output_local_elem[supplemental_precip.output_var_idx, :] = ( - layerOut_elem - ) - elif ConfigOptions.grid_type == "hydrofabric": - indSet = np.where( - supplemental_precip.final_supp_precip != ConfigOptions.globalNdv - ) - layerIn = supplemental_precip.final_supp_precip - layerOut = OutputObj.output_local[supplemental_precip.output_var_idx, :] + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) - if len(indSet[0]) != 0: - layerOut[indSet] = layerIn[indSet] - else: - # We have all missing data for the supplemental precip for this step. - layerOut = layerOut - # TODO: test that even does anything...?s - OutputObj.output_local[supplemental_precip.output_var_idx, :] = layerOut + def get_slice(self, obj: Any, first_dim_idx: int) -> Any: + return obj[first_dim_idx, :, :] + + def set_slice(self, obj: Any, first_dim_idx: int, value: Any) -> None: + obj[first_dim_idx, :, :] = value + + def apply_layering(self) -> None: + self.update_output_local() + + +class _LayeringModSupplemental_Unstructured(_LayeringModSupplemental): + """Implementation of abstract class _LayeringModSupplemental for Unstructured discretization. + Slicing is 2-dimensional. + Primary business logic executes twice (with extra "_elem" call). + """ + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def get_slice(self, obj: Any, first_dim_idx: int) -> Any: + return obj[first_dim_idx, :] + + def set_slice(self, obj: Any, first_dim_idx: int, value: Any) -> None: + obj[first_dim_idx, :] = value + + def apply_layering(self) -> None: + self.update_output_local() + self.update_output_local("_elem") + + +class _LayeringModSupplemental_Hydrofabric(_LayeringModSupplemental): + """Implementation of abstract class _LayeringModSupplemental for Unstructured discretization. + Slicing is 2-dimensional. + Primary business logic executes once (no extra "_elem" call). + """ + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def get_slice(self, obj: Any, first_dim_idx: int) -> Any: + return obj[first_dim_idx, :] + + def set_slice(self, obj: Any, first_dim_idx: int, value: Any) -> None: + obj[first_dim_idx, :] = value + + def apply_layering(self) -> None: + self.update_output_local() + + +def layer_supplemental_forcing( + output_obj: OutputObj, + supplemental_precip: SupplementalPrecip, + config_options: ConfigOptions, +): + """Layer in supplemental precipitation where valid values exist. + + Function to layer in supplemental precipitation where we have valid values. Any pixel + cells that contain missing values will not be layered in, and background input forcings + will be used instead. + :param output_obj: + :param supplemental_precip: + :param config_options: + :return: + """ + if config_options.grid_type == "gridded": + factory = _LayeringModSupplemental_Gridded + elif config_options.grid_type == "unstructured": + factory = _LayeringModSupplemental_Unstructured + elif config_options.grid_type == "hydrofabric": + factory = _LayeringModSupplemental_Hydrofabric + else: + raise ValueError( + f"Unexpected discretization type / grid type: {config_options.grid_type}" + ) + layering_mod_supp = factory(output_obj, supplemental_precip, config_options) + layering_mod_supp.apply_layering() diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index ea7338f5..769f4a58 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -567,7 +567,6 @@ def __process_supp_precip_key( self._bmi._output_obj, self._bmi._supp_pcp_mod[supp_pcp_key], self._bmi._job_meta, - self._bmi._mpi_meta, ) self.check_program_status() From 01ec525e6aaef2a7f84efe50c853c5d2772750fe Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 21 Aug 2026 20:30:39 -0400 Subject: [PATCH 125/132] Docstrings and type hints --- .../core/layeringMod.py | 84 ++++++++++++------- 1 file changed, 56 insertions(+), 28 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py index 521aeaf8..d6cd30c3 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py @@ -12,6 +12,7 @@ Future functionality may include blending, etc. """ +import numbers from abc import ABC, abstractmethod from typing import Any @@ -36,22 +37,22 @@ class _LayeringMod(ABC): def __init__( self, - output_obj, + output_obj: Any, input_forcings: InputForcings, config_options: ConfigOptions, - ): + ) -> None: self.output_obj = output_obj self.input_forcings = input_forcings self.config_options = config_options @abstractmethod - def get_slice(self, obj: Any, force_idx: int) -> Any: - """Abstract method: Using bracket syntax, return a slice of an object based on its forcing index.""" + def get_slice(self, obj: np.ndarray, force_idx: int) -> np.ndarray: + """Abstract method: Using bracket syntax, return a slice of an array based on its forcing index.""" raise NotImplementedError @abstractmethod - def set_slice(self, obj: Any, force_idx: int, value: Any) -> None: - """Abstract method: Using bracket syntax, set the value of a slice of an object based on its forcing index.""" + def set_slice(self, obj: np.ndarray, force_idx: int, value: numbers.Real) -> None: + """Abstract method: Using bracket syntax, set the value of a slice of an array based on its forcing index.""" raise NotImplementedError @abstractmethod @@ -59,13 +60,13 @@ def apply_layering(self, force_idx: int) -> None: """Abstract method: Apply the layering logic (this is the primary function of this class).""" raise NotImplementedError - def layerIn(self, force_idx: int, attr_suffix: str = "") -> Any: + def layerIn(self, force_idx: int, attr_suffix: str = "") -> np.ndarray: """Return an input dataset used for layering.""" return self.get_slice( getattr(self.input_forcings, f"final_forcings{attr_suffix}"), force_idx ) - def indSet(self, force_idx: int, attr_suffix: str = "") -> Any: + def indSet(self, force_idx: int, attr_suffix: str = "") -> np.ndarray: """Return the indices of the input dataset that are not equal to the global no-data value (a non-no-data mask).""" return np.where( self.layerIn(force_idx, attr_suffix) != self.config_options.globalNdv @@ -74,6 +75,11 @@ def indSet(self, force_idx: int, attr_suffix: str = "") -> Any: def update_output_local(self, force_idx: int, attr_suffix: str = "") -> None: """Apply layering logic to update the output grid with input forcing data. + This contains much of the primary business logic and comments + from the the original function ``layer_final_forcings``. + Original code: + https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py#L9 + This is the primary business logic of the layering module. It retrieves input forcing data, applies validity checks (using globalNdv as the no-data value), and updates the output grid with valid data. Special handling is provided for ERA5 data. @@ -121,10 +127,10 @@ class _LayeringMod_Gridded(_LayeringMod): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - def get_slice(self, obj: Any, force_idx: int) -> Any: + def get_slice(self, obj: np.ndarray, force_idx: int) -> np.ndarray: return obj[force_idx, :, :] - def set_slice(self, obj: Any, force_idx: int, value: Any) -> None: + def set_slice(self, obj: np.ndarray, force_idx: int, value: numbers.Real) -> None: obj[force_idx, :, :] = value def apply_layering(self, force_idx: int) -> None: @@ -137,10 +143,10 @@ class _LayeringMod_Unstructured(_LayeringMod): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - def get_slice(self, obj: Any, force_idx: int) -> Any: + def get_slice(self, obj: np.ndarray, force_idx: int) -> np.ndarray: return obj[force_idx, :] - def set_slice(self, obj: Any, force_idx: int, value: Any) -> None: + def set_slice(self, obj: np.ndarray, force_idx: int, value: numbers.Real) -> None: obj[force_idx, :] = value def apply_layering(self, force_idx: int) -> None: @@ -154,10 +160,10 @@ class _LayeringMod_Hydrofabric(_LayeringMod): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - def get_slice(self, obj: Any, force_idx: int) -> Any: + def get_slice(self, obj: np.ndarray, force_idx: int) -> np.ndarray: return obj[force_idx, :] - def set_slice(self, obj: Any, force_idx: int, value: Any) -> None: + def set_slice(self, obj: np.ndarray, force_idx: int, value: numbers.Real) -> None: obj[force_idx, :] = value def apply_layering(self, force_idx: int) -> None: @@ -166,7 +172,7 @@ def apply_layering(self, force_idx: int) -> None: def layer_final_forcings( output_obj: OutputObj, input_forcings: InputForcings, config_options: ConfigOptions -): +) -> None: """Layer input forcings onto the output grid. Function to perform basic layering of input forcings as they are processed. The logic @@ -179,6 +185,11 @@ def layer_final_forcings( :param input_forcings: :param config_options: :return: + + This sets up and calls ``_LayeringMod.apply_layering`` which contains much of the + primary business logic and comments from the the original function ``layer_final_forcings``. + Original code: + https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py#L9 """ # Loop through the 8(or 9) forcing products to layer in: # 0.) U-Wind (m/s) @@ -214,18 +225,18 @@ def __init__( output_obj: OutputObj, supplemental_precip: SupplementalPrecip, config_options: ConfigOptions, - ): + ) -> None: self.output_obj = output_obj self.supplemental_precip = supplemental_precip self.config_options = config_options @abstractmethod - def get_slice(self, obj: Any) -> Any: + def get_slice(self, obj: np.ndarray) -> np.ndarray: """Abstract method: Using bracket syntax, return a slice of an object based on its forcing index.""" raise NotImplementedError @abstractmethod - def set_slice(self, obj: Any, value: Any) -> None: + def set_slice(self, obj: np.ndarray, value: numbers.Real) -> None: """Abstract method: Using bracket syntax, set the value of a slice of an object based on its forcing index.""" raise NotImplementedError @@ -234,22 +245,28 @@ def apply_layering(self) -> None: """Abstract method: Apply the layering logic (this is the primary function of this class).""" raise NotImplementedError - def indSet(self, attr_suffix: str = ""): + def indSet(self, attr_suffix: str = "") -> np.ndarray: return np.where( getattr(self.supplemental_precip, f"final_supp_precip{attr_suffix}") != self.config_options.globalNdv ) - def layerIn(self, attr_suffix: str = ""): + def layerIn(self, attr_suffix: str = "") -> np.ndarray: return getattr(self.supplemental_precip, f"final_supp_precip{attr_suffix}") - def layerOut(self, attr_suffix: str = ""): + def layerOut(self, attr_suffix: str = "") -> np.ndarray: return self.get_slice( getattr(self.output_obj, f"output_local{attr_suffix}"), self.supplemental_precip.output_var_idx, ) def update_output_local(self, attr_suffix: str = "") -> None: + """This contains much of the primary business logic and comments from the the original function ``layer_supplemental_forcing``. + + Original code: + https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py#L113 + """ + indSet = self.indSet(attr_suffix) layerIn = self.layerIn(attr_suffix) layerOut = self.layerOut(attr_suffix) @@ -279,10 +296,12 @@ class _LayeringModSupplemental_Gridded(_LayeringModSupplemental): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - def get_slice(self, obj: Any, first_dim_idx: int) -> Any: + def get_slice(self, obj: np.ndarray, first_dim_idx: int) -> np.ndarray: return obj[first_dim_idx, :, :] - def set_slice(self, obj: Any, first_dim_idx: int, value: Any) -> None: + def set_slice( + self, obj: np.ndarray, first_dim_idx: int, value: numbers.Real + ) -> None: obj[first_dim_idx, :, :] = value def apply_layering(self) -> None: @@ -298,10 +317,12 @@ class _LayeringModSupplemental_Unstructured(_LayeringModSupplemental): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - def get_slice(self, obj: Any, first_dim_idx: int) -> Any: + def get_slice(self, obj: np.ndarray, first_dim_idx: int) -> np.ndarray: return obj[first_dim_idx, :] - def set_slice(self, obj: Any, first_dim_idx: int, value: Any) -> None: + def set_slice( + self, obj: np.ndarray, first_dim_idx: int, value: numbers.Real + ) -> None: obj[first_dim_idx, :] = value def apply_layering(self) -> None: @@ -318,10 +339,12 @@ class _LayeringModSupplemental_Hydrofabric(_LayeringModSupplemental): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - def get_slice(self, obj: Any, first_dim_idx: int) -> Any: + def get_slice(self, obj: np.ndarray, first_dim_idx: int) -> np.ndarray: return obj[first_dim_idx, :] - def set_slice(self, obj: Any, first_dim_idx: int, value: Any) -> None: + def set_slice( + self, obj: np.ndarray, first_dim_idx: int, value: numbers.Real + ) -> None: obj[first_dim_idx, :] = value def apply_layering(self) -> None: @@ -332,7 +355,7 @@ def layer_supplemental_forcing( output_obj: OutputObj, supplemental_precip: SupplementalPrecip, config_options: ConfigOptions, -): +) -> None: """Layer in supplemental precipitation where valid values exist. Function to layer in supplemental precipitation where we have valid values. Any pixel @@ -342,6 +365,11 @@ def layer_supplemental_forcing( :param supplemental_precip: :param config_options: :return: + + This sets up and calls ``_LayeringModSupplemental.apply_layering`` which contains much of the + primary business logic and comments from the the original function ``layer_supplemental_forcing``. + Original code: + https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py#L113 """ if config_options.grid_type == "gridded": factory = _LayeringModSupplemental_Gridded From 8104c05cfbc382f623e1b20327b6b5ab102ff69c Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 21 Aug 2026 20:45:34 -0400 Subject: [PATCH 126/132] Docstrings --- .../core/layeringMod.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py index d6cd30c3..33306c51 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py @@ -61,13 +61,13 @@ def apply_layering(self, force_idx: int) -> None: raise NotImplementedError def layerIn(self, force_idx: int, attr_suffix: str = "") -> np.ndarray: - """Return an input dataset used for layering.""" + """Property-like. Return an input dataset used for layering (named from original codebase).""" return self.get_slice( getattr(self.input_forcings, f"final_forcings{attr_suffix}"), force_idx ) def indSet(self, force_idx: int, attr_suffix: str = "") -> np.ndarray: - """Return the indices of the input dataset that are not equal to the global no-data value (a non-no-data mask).""" + """Property-like. Return the indices of the input dataset that are not equal to the global no-data value (a non-no-data mask). Named from the original codebase.""" return np.where( self.layerIn(force_idx, attr_suffix) != self.config_options.globalNdv ) @@ -128,12 +128,15 @@ def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) def get_slice(self, obj: np.ndarray, force_idx: int) -> np.ndarray: + """Using bracket syntax, return a slice of an array based on its forcing index.""" return obj[force_idx, :, :] def set_slice(self, obj: np.ndarray, force_idx: int, value: numbers.Real) -> None: + """Using bracket syntax, set the value of a slice of an array based on its forcing index.""" obj[force_idx, :, :] = value def apply_layering(self, force_idx: int) -> None: + """Apply the layering logic (this is the primary function of this class).""" self.update_output_local(force_idx) @@ -144,12 +147,15 @@ def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) def get_slice(self, obj: np.ndarray, force_idx: int) -> np.ndarray: + """Using bracket syntax, return a slice of an array based on its forcing index.""" return obj[force_idx, :] def set_slice(self, obj: np.ndarray, force_idx: int, value: numbers.Real) -> None: + """Using bracket syntax, set the value of a slice of an array based on its forcing index.""" obj[force_idx, :] = value def apply_layering(self, force_idx: int) -> None: + """Apply the layering logic (this is the primary function of this class).""" self.update_output_local(force_idx) self.update_output_local(force_idx, "_elem") @@ -161,12 +167,15 @@ def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) def get_slice(self, obj: np.ndarray, force_idx: int) -> np.ndarray: + """Using bracket syntax, return a slice of an array based on its forcing index.""" return obj[force_idx, :] def set_slice(self, obj: np.ndarray, force_idx: int, value: numbers.Real) -> None: + """Using bracket syntax, set the value of a slice of an array based on its forcing index.""" obj[force_idx, :] = value def apply_layering(self, force_idx: int) -> None: + """Apply the layering logic (this is the primary function of this class).""" self.update_output_local(force_idx) @@ -232,12 +241,12 @@ def __init__( @abstractmethod def get_slice(self, obj: np.ndarray) -> np.ndarray: - """Abstract method: Using bracket syntax, return a slice of an object based on its forcing index.""" + """Abstract method: Using bracket syntax, return a slice of an array based on its forcing index.""" raise NotImplementedError @abstractmethod def set_slice(self, obj: np.ndarray, value: numbers.Real) -> None: - """Abstract method: Using bracket syntax, set the value of a slice of an object based on its forcing index.""" + """Abstract method: Using bracket syntax, set the value of a slice of an array based on its forcing index.""" raise NotImplementedError @abstractmethod @@ -246,15 +255,18 @@ def apply_layering(self) -> None: raise NotImplementedError def indSet(self, attr_suffix: str = "") -> np.ndarray: + """Property-like. Return the indices of the input dataset that are not equal to the global no-data value (a non-no-data mask). Named from the original codebase.""" return np.where( getattr(self.supplemental_precip, f"final_supp_precip{attr_suffix}") != self.config_options.globalNdv ) def layerIn(self, attr_suffix: str = "") -> np.ndarray: + """Property-like. Return an input dataset used for layering (named from original codebase).""" return getattr(self.supplemental_precip, f"final_supp_precip{attr_suffix}") def layerOut(self, attr_suffix: str = "") -> np.ndarray: + """Property-like method. Return the layerOut array (named from original codebase).""" return self.get_slice( getattr(self.output_obj, f"output_local{attr_suffix}"), self.supplemental_precip.output_var_idx, @@ -297,14 +309,17 @@ def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) def get_slice(self, obj: np.ndarray, first_dim_idx: int) -> np.ndarray: + """Using bracket syntax, return a slice of an array based on its forcing index.""" return obj[first_dim_idx, :, :] def set_slice( self, obj: np.ndarray, first_dim_idx: int, value: numbers.Real ) -> None: + """Using bracket syntax, set the value of a slice of an array based on its forcing index.""" obj[first_dim_idx, :, :] = value def apply_layering(self) -> None: + """Apply the layering logic (this is the primary function of this class).""" self.update_output_local() @@ -318,14 +333,17 @@ def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) def get_slice(self, obj: np.ndarray, first_dim_idx: int) -> np.ndarray: + """Using bracket syntax, return a slice of an array based on its forcing index.""" return obj[first_dim_idx, :] def set_slice( self, obj: np.ndarray, first_dim_idx: int, value: numbers.Real ) -> None: + """Using bracket syntax, set the value of a slice of an array based on its forcing index.""" obj[first_dim_idx, :] = value def apply_layering(self) -> None: + """Apply the layering logic (this is the primary function of this class).""" self.update_output_local() self.update_output_local("_elem") @@ -340,14 +358,17 @@ def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) def get_slice(self, obj: np.ndarray, first_dim_idx: int) -> np.ndarray: + """Using bracket syntax, return a slice of an array based on its forcing index.""" return obj[first_dim_idx, :] def set_slice( self, obj: np.ndarray, first_dim_idx: int, value: numbers.Real ) -> None: + """Using bracket syntax, set the value of a slice of an array based on its forcing index.""" obj[first_dim_idx, :] = value def apply_layering(self) -> None: + """Apply the layering logic (this is the primary function of this class).""" self.update_output_local() From 9cd9e15a0926816c54eefda82f139520ff6e76b5 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 24 Aug 2026 13:00:56 -0400 Subject: [PATCH 127/132] Add TYPE_CHECKING guard for imports that are purely for type hints --- .../core/layeringMod.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py index 33306c51..4f75f38a 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py @@ -12,24 +12,27 @@ Future functionality may include blending, etc. """ +from __future__ import annotations + import numbers from abc import ABC, abstractmethod -from typing import Any +from typing import TYPE_CHECKING, Any import numpy as np -from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( - ConfigOptions, -) -from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.forcingInputMod import ( - InputForcings, -) -from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.ioMod import ( - OutputObj, -) -from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.suppPrecipMod import ( - SupplementalPrecip, -) +if TYPE_CHECKING: + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( + ConfigOptions, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.forcingInputMod import ( + InputForcings, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.ioMod import ( + OutputObj, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.suppPrecipMod import ( + SupplementalPrecip, + ) class _LayeringMod(ABC): From 7cf9a7ab789b1d3ec2ea7e4d93143c1fb5144f46 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 24 Aug 2026 13:09:26 -0400 Subject: [PATCH 128/132] Replace camelCase with snake_case --- .../core/layeringMod.py | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py index 4f75f38a..2060c4ac 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py @@ -63,16 +63,16 @@ def apply_layering(self, force_idx: int) -> None: """Abstract method: Apply the layering logic (this is the primary function of this class).""" raise NotImplementedError - def layerIn(self, force_idx: int, attr_suffix: str = "") -> np.ndarray: - """Property-like. Return an input dataset used for layering (named from original codebase).""" + def layer_in(self, force_idx: int, attr_suffix: str = "") -> np.ndarray: + """Property-like. Return an input dataset used for layering (named ``layerIn`` in original codebase).""" return self.get_slice( getattr(self.input_forcings, f"final_forcings{attr_suffix}"), force_idx ) - def indSet(self, force_idx: int, attr_suffix: str = "") -> np.ndarray: - """Property-like. Return the indices of the input dataset that are not equal to the global no-data value (a non-no-data mask). Named from the original codebase.""" + def indices_set(self, force_idx: int, attr_suffix: str = "") -> np.ndarray: + """Property-like. Return the indices of the input dataset that are not equal to the global no-data value (a non-no-data mask). Named ``indSet`` in the original codebase.""" return np.where( - self.layerIn(force_idx, attr_suffix) != self.config_options.globalNdv + self.layer_in(force_idx, attr_suffix) != self.config_options.globalNdv ) def update_output_local(self, force_idx: int, attr_suffix: str = "") -> None: @@ -105,17 +105,17 @@ def update_output_local(self, force_idx: int, attr_suffix: str = "") -> None: output_tmp = self.get_slice( getattr(self.output_obj, f"output_local{attr_suffix}"), force_idx ) - layerIn = self.layerIn(force_idx, attr_suffix) + layer_in = self.layer_in(force_idx, attr_suffix) if ( self.input_forcings.product_name == "ERA5" and [12, 21] in self.config_options.input_forcings ): mask = getattr(self.input_forcings, f"regridded_mask{attr_suffix}_AORC") - output_tmp[np.where(mask == 0)] = layerIn[np.where(mask == 0)] + output_tmp[np.where(mask == 0)] = layer_in[np.where(mask == 0)] else: - indSet = self.indSet(force_idx, attr_suffix) - output_tmp[indSet] = layerIn[indSet] + indices_set = self.indices_set(force_idx, attr_suffix) + output_tmp[indices_set] = layer_in[indices_set] self.set_slice( getattr(self.output_obj, f"output_local{attr_suffix}"), @@ -257,19 +257,19 @@ def apply_layering(self) -> None: """Abstract method: Apply the layering logic (this is the primary function of this class).""" raise NotImplementedError - def indSet(self, attr_suffix: str = "") -> np.ndarray: - """Property-like. Return the indices of the input dataset that are not equal to the global no-data value (a non-no-data mask). Named from the original codebase.""" + def indices_set(self, attr_suffix: str = "") -> np.ndarray: + """Property-like. Return the indices of the input dataset that are not equal to the global no-data value (a non-no-data mask). Named ``indSet`` in the original codebase.""" return np.where( getattr(self.supplemental_precip, f"final_supp_precip{attr_suffix}") != self.config_options.globalNdv ) - def layerIn(self, attr_suffix: str = "") -> np.ndarray: - """Property-like. Return an input dataset used for layering (named from original codebase).""" + def layer_in(self, attr_suffix: str = "") -> np.ndarray: + """Property-like. Return an input dataset used for layering (named ``layerIn`` in original codebase).""" return getattr(self.supplemental_precip, f"final_supp_precip{attr_suffix}") - def layerOut(self, attr_suffix: str = "") -> np.ndarray: - """Property-like method. Return the layerOut array (named from original codebase).""" + def layer_out(self, attr_suffix: str = "") -> np.ndarray: + """Property-like method. Return the ``layer_out`` array (named ``layerOut`` in original codebase).""" return self.get_slice( getattr(self.output_obj, f"output_local{attr_suffix}"), self.supplemental_precip.output_var_idx, @@ -282,23 +282,23 @@ def update_output_local(self, attr_suffix: str = "") -> None: https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py#L113 """ - indSet = self.indSet(attr_suffix) - layerIn = self.layerIn(attr_suffix) - layerOut = self.layerOut(attr_suffix) + indices_set = self.indices_set(attr_suffix) + layer_in = self.layer_in(attr_suffix) + layer_out = self.layer_out(attr_suffix) # NOTE original TODO comment below was for "gridded" discretization. Unknown intent: # TODO: review test layering for ExtAnA calculation to replace FE QPE with MPE RAINRATE # If this isn't sufficient, replace QPE with MPE here: # if supplemental_precip.keyValue == 11: # config_options.statusMsg = "Performing ExtAnA calculation" # err_handler.log_msg(config_options, MpiConfig) - if len(indSet[0]) != 0: - layerOut[indSet] = layerIn[indSet] + if len(indices_set[0]) != 0: + layer_out[indices_set] = layer_in[indices_set] # NOTE original TODO comment below was for all discretizations ("gridded", "unstructured", and "hydrofabric"). Unknown intent. # TODO: test that even does anything...?s self.set_slice( getattr(self.output_obj, f"output_local{attr_suffix}"), self.supplemental_precip.output_var_idx, - layerOut, + layer_out, ) From 31ae48d7a233a25b9b79890cbc36eadb18fe447b Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 24 Aug 2026 14:31:00 -0400 Subject: [PATCH 129/132] Improve class name self-references to allow for potential future inheritance --- .../NextGen_Forcings_Engine/bmi_model.py | 2 +- .../NextGen_Forcings_Engine/core/config.py | 2 +- .../NextGen_Forcings_Engine/core/forcingInputMod.py | 8 ++++---- .../NextGen_Forcings_Engine/core/geoMod.py | 8 ++++---- .../NextGen_Forcings_Engine/core/suppPrecipMod.py | 10 +++++----- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 75e3d4b8..895ac14c 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -453,7 +453,7 @@ def initialize(self, config_file: str) -> None: :raises RuntimeError: If the configuration file is invalid or missing. """ self._config_file = config_file - for attr in BMI_MODEL[self.__class__.__base__.__name__]: + for attr in BMI_MODEL[__class__.__name__]: setattr(self, attr, None) self._model = NWMv3ForcingEngineModel(self) self.init_log() diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 65847afb..a6c302ab 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -114,7 +114,7 @@ def __init__(self, cfg_bmi: dict, b_date: str = None, geogrid: str = None) -> No # set list of attributes from consts.py to None early on in the init process. # These are indexed from the consts dictionary. # This must happen before accessing properties like precip_only_flag - for attr in CONFIGOPTIONS[self.__class__.__name__]: + for attr in CONFIGOPTIONS[__class__.__name__]: setattr(self, attr, None) self.broadcast_new_64bit_uid() diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py index 49dfef2f..ed59de57 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py @@ -66,7 +66,7 @@ def __init__( # set list of attibutes from consts.py to None. # These are indexed from the consts dictionary using the class name - for attr in FORCINGINPUTMOD[self.__class__.__base__.__name__]: + for attr in FORCINGINPUTMOD[__class__.__name__]: setattr(self, attr, None) self._initialize_config_options() @@ -313,7 +313,7 @@ def __init__( """ super().__init__(idx, config_options, geo_meta, mpi_config, custom_count) - for attr in FORCINGINPUTMOD[self.__class__.__name__]: + for attr in FORCINGINPUTMOD[__class__.__name__]: setattr(self, attr, None) @property @@ -443,7 +443,7 @@ def __init__( """ super().__init__(idx, config_options, geo_meta, mpi_config, custom_count) - for attr in FORCINGINPUTMOD[self.__class__.__name__]: + for attr in FORCINGINPUTMOD[__class__.__name__]: setattr(self, attr, None) @property @@ -551,7 +551,7 @@ def __init__( """ super().__init__(idx, config_options, geo_meta, mpi_config, custom_count) - for attr in FORCINGINPUTMOD[self.__class__.__name__]: + for attr in FORCINGINPUTMOD[__class__.__name__]: setattr(self, attr, None) @property diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py index c219e2b0..7f6dd909 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py @@ -120,7 +120,7 @@ def __init__(self, config_options: ConfigOptions, mpi_config: MpiConfig) -> None """Initialize GeoMeta class variables.""" self.config_options = config_options self.mpi_config = mpi_config - for attr in GEOMOD[self.__class__.__base__.__name__]: + for attr in GEOMOD[__class__.__name__]: setattr(self, attr, None) @cached_property @@ -272,7 +272,7 @@ def __init__(self, config_options: ConfigOptions, mpi_config: MpiConfig) -> None :return: """ super().__init__(config_options, mpi_config) - for attr in GEOMOD[self.__class__.__name__]: + for attr in GEOMOD[__class__.__name__]: setattr(self, attr, None) @broadcast @@ -829,7 +829,7 @@ def __init__(self, config_options: ConfigOptions, mpi_config: MpiConfig): :return: """ super().__init__(config_options, mpi_config) - for attr in GEOMOD[self.__class__.__name__]: + for attr in GEOMOD[__class__.__name__]: setattr(self, attr, None) @cached_property @@ -984,7 +984,7 @@ def __init__(self, config_options: ConfigOptions, mpi_config: MpiConfig) -> None :return: """ super().__init__(config_options, mpi_config) - for attr in GEOMOD[self.__class__.__name__]: + for attr in GEOMOD[__class__.__name__]: setattr(self, attr, None) @broadcast diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py index 8a2c929b..29f08020 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py @@ -50,7 +50,7 @@ def __init__(self, idx: int, config_options: ConfigOptions, geo_meta: GeoMeta): self.config_options = config_options self.geo_meta = geo_meta - for attr in SUPPPRECIPMOD[self.__class__.__base__.__name__]: + for attr in SUPPPRECIPMOD[__class__.__name__]: setattr(self, attr, None) self._initialize_config_options() @@ -73,7 +73,7 @@ def _initialize_config_options(self) -> None: For each attribute in SUPPPRECIPMOD["SupplementalPrecip"], check if the same-named attribute exists in config_options as a list and set it on self. """ - for attr in SUPPPRECIPMOD[self.__class__.__base__.__name__]: + for attr in SUPPPRECIPMOD[__class__.__name__]: if hasattr(self.config_options, attr): val = getattr(self.config_options, attr) if isinstance(val, list) and len(val) > 0: @@ -316,7 +316,7 @@ def __init__( ) -> None: """Initialize SupplementalPrecipGridded. Any subclass-specific attr names are sourced from SUPPPRECIPMOD[classname] in consts.py.""" super().__init__(idx, config_options, geo_meta) - for attr in SUPPPRECIPMOD[self.__class__.__name__]: + for attr in SUPPPRECIPMOD[__class__.__name__]: setattr(self, attr, None) @property @@ -361,7 +361,7 @@ def __init__( ) -> None: """Initialize SupplementalPrecipHydrofabric. Any subclass-specific attr names are sourced from SUPPPRECIPMOD[classname] in consts.py.""" super().__init__(idx, config_options, geo_meta) - for attr in SUPPPRECIPMOD[self.__class__.__name__]: + for attr in SUPPPRECIPMOD[__class__.__name__]: setattr(self, attr, None) @property @@ -404,7 +404,7 @@ def __init__( ) -> None: """Initialize SupplementalPrecipUnstructured. Any subclass-specific attr names are sourced from SUPPPRECIPMOD[classname] in consts.py.""" super().__init__(idx, config_options, geo_meta) - for attr in SUPPPRECIPMOD[self.__class__.__name__]: + for attr in SUPPPRECIPMOD[__class__.__name__]: setattr(self, attr, None) @property From 2bddeeaa59f49ee66c7c59967c5b8d2f5f16bb45 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 21 Aug 2026 15:55:02 -0400 Subject: [PATCH 130/132] Refactor timeInterpMod.py --- .../core/timeInterpMod.py | 1328 ++++++++--------- 1 file changed, 592 insertions(+), 736 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py index f5aa8150..0e018303 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py @@ -1,770 +1,514 @@ -"""Temporal interpolation input forcings to the current output timestep.""" +"""Temporal interpolation input forcings to the current output timestep. + +This file was refactored in August/September 2026, with the general goal of preserving existing business logic +while making it more DRY, abstracting shared logic, and aligning with PEP8 on docstrings, type hints, and symbol names. +""" + +from __future__ import annotations + +import numbers +from typing import TYPE_CHECKING, Any import numpy as np +if TYPE_CHECKING: + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( + ConfigOptions, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.forcingInputMod import ( + InputForcings, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import ( + MpiConfig, + ) + from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.suppPrecipMod import ( + SupplementalPrecip, + ) + from . import err_handler -def no_interpolation(input_forcings, ConfigOptions, MpiConfig): - """No temporal interpolation. +def arr_set_scalar(arr: np.ndarray, scalar: numbers.Real, ndim: int): + """Use numpy slice(None) syntax to modify the array in-place by assigning any number of dimensions to the provided scalar value.""" + if not 0 <= ndim <= 99: + raise ValueError(f"Unexpected value for ndim: {ndim}") + dims_tup = (slice(None),) * ndim + arr[dims_tup] = scalar - Function for simply setting the final regridded fields to the - input forcings that are from the next input forcing frequency. - :param input_forcings: - :param ConfigOptions: - :param MpiConfig: - :return: - """ - if ConfigOptions.grid_type == "gridded": - # Check to make sure we have valid grids. - if input_forcings.regridded_forcings2 is None: - input_forcings.final_forcings[:, :, :] = ConfigOptions.globalNdv - else: - input_forcings.final_forcings[:, :, :] = input_forcings.regridded_forcings2[ - :, :, : - ] - elif ConfigOptions.grid_type == "unstructured": - # Check to make sure we have valid grids. - if input_forcings.regridded_forcings2 is None: - input_forcings.final_forcings[:, :] = ConfigOptions.globalNdv - else: - input_forcings.final_forcings[:, :] = input_forcings.regridded_forcings2[ - :, : - ] - # Check to make sure we have valid grids for elements. - if input_forcings.regridded_forcings2_elem is None: - input_forcings.final_forcings_elem[:, :] = ConfigOptions.globalNdv - else: - input_forcings.final_forcings_elem[:, :] = ( - input_forcings.regridded_forcings2_elem[:, :] - ) - elif ConfigOptions.grid_type == "hydrofabric": - # Check to make sure we have valid grids. - if input_forcings.regridded_forcings2 is None: - input_forcings.final_forcings[:, :] = ConfigOptions.globalNdv - else: - input_forcings.final_forcings[:, :] = input_forcings.regridded_forcings2[ - :, : - ] +def arr_set_arr(arr_1: np.ndarray, arr_2: np.ndarray, ndim: int): + if not 0 <= ndim <= 99: + raise ValueError(f"Unexpected value for ndim: {ndim}") + dims_tup = (slice(None),) * ndim + arr_1[dims_tup] = arr_2[dims_tup] -def no_interpolation_supp_pcp(supplemental_precip, ConfigOptions, MpiConfig): - """No temporal interpolation for supplemental precipitation. - Function for simply setting the final regridded supplemental precipitation - to the supplemental precipitation grids from the next precip frequency that - is available. - :param supplemental_precip: - :param ConfigOptions: - :param MpiConfig: - :return: - """ - if ConfigOptions.grid_type == "gridded": - if supplemental_precip.regridded_precip2 is not None: - supplemental_precip.final_supp_precip[:, :] = ( - supplemental_precip.regridded_precip2[:, :] - ) - else: - # We have missing files. - supplemental_precip.final_supp_precip[:, :] = ConfigOptions.globalNdv - elif ConfigOptions.grid_type == "unstructured": - if supplemental_precip.regridded_precip2 is not None: - supplemental_precip.final_supp_precip[:] = ( - supplemental_precip.regridded_precip2[:] - ) - else: - # We have missing files. - supplemental_precip.final_supp_precip[:] = ConfigOptions.globalNdv - if supplemental_precip.regridded_precip2_elem is not None: - supplemental_precip.final_supp_precip_elem[:] = ( - supplemental_precip.regridded_precip2_elem[:] - ) - else: - # We have missing files. - supplemental_precip.final_supp_precip_elem[:] = ConfigOptions.globalNdv - elif ConfigOptions.grid_type == "hydrofabric": - if supplemental_precip.regridded_precip2 is not None: - supplemental_precip.final_supp_precip[:] = ( - supplemental_precip.regridded_precip2[:] - ) - else: - # We have missing files. - supplemental_precip.final_supp_precip[:] = ConfigOptions.globalNdv +class _TimeInterp: + """Class for handling temporal interpolation of forcing arrays. + Some nested attrs are reassigned to top-level class attrs for readability / noise reduction + in the methods that use them. -def nearest_neighbor(input_forcings, ConfigOptions, MpiConfig): - """Nearest neighbor temporal interpolation. + The methods of this private class contain the actual business logic of the public functions of timeInterpMod which use this class. - Function for setting the current output regridded forcings to the nearest - input forecast step. - :param input_forcings: - :param ConfigOptions: - :param MpiConfig: - :return: - """ - # If we are running CFSv2 with bias correction, bypass as temporal interpolation is done - # internally (NWM-only). - if ( - ConfigOptions.runCfsNldasBiasCorrect - and input_forcings.product_name == "CFSv2_6Hr_Global_GRIB2" - ): - if MpiConfig.rank == 0: - ConfigOptions.statusMsg = "Bypassing temporal interpolation routine due to NWM bias correction for CFSv2" - err_handler.log_msg(ConfigOptions, MpiConfig) - return + For docstrings explaining the business logic and domain-level synopses, see the docstrings of the public calling functions. - # Calculate the difference between the current output timestep, - # and the previous input forecast output step. - dtFromPrevious = ConfigOptions.current_output_date - input_forcings.fcst_date1 + A common pattern among the methods, to keep the discretization methods DRY ("gridded", "unstructured", "hydrofabric") is to + parameterize the dimensionality of the array reading and writing behavior and set those parameters at the top of each method. + Also, the unstructured discretization adds "*_elem" attr interactions, so this has been parameterized into a Boolean "also_elem" + that conditionally (when True) causes those _elem behaviors to be executed. - # Calculate the difference between the current output timesetp, - # and the next forecast output step. - dtFromNext = ConfigOptions.current_output_date - input_forcings.fcst_date2 + This class was written during a refactoring effort in 2026. At the time of writing the refactor, + the vast majority of the code in each method is taken verbatim from the original function, but with + parameterization added as explained above, to keep things DRY among the discretization types. + """ - if ConfigOptions.grid_type == "gridded": - if abs(dtFromNext.total_seconds()) <= abs(dtFromPrevious.total_seconds()): - # Default to the regridded states from the next forecast output step. - if input_forcings.regridded_forcings2 is None: - input_forcings.final_forcings[:, :, :] = ConfigOptions.globalNdv - else: - input_forcings.final_forcings[:, :, :] = ( - input_forcings.regridded_forcings2[:, :, :] - ) - else: - # Default to the regridded states from the previous forecast output - # step. - if input_forcings.regridded_forcings1 is None: - input_forcings.final_forcings[:, :, :] = ConfigOptions.globalNdv - else: - input_forcings.final_forcings[:, :, :] = ( - input_forcings.regridded_forcings1[:, :, :] - ) - elif ConfigOptions.grid_type == "unstructured": - if abs(dtFromNext.total_seconds()) <= abs(dtFromPrevious.total_seconds()): - # Default to the regridded states from the next forecast output step. - if input_forcings.regridded_forcings2 is None: - input_forcings.final_forcings[:, :] = ConfigOptions.globalNdv - else: - input_forcings.final_forcings[:, :] = ( - input_forcings.regridded_forcings2[:, :] - ) + def __init__( + self, + input_forcings: InputForcings, + supplemental_precip: SupplementalPrecip, + config_options: ConfigOptions, + mpi_config: MpiConfig, + ): + self.input_forcings = input_forcings + self.supplemental_precip = supplemental_precip + self.config_options = config_options + self.mpi_config = mpi_config + + self.globalNdv = self.config_options.globalNdv + + if input_forcings is not None: + self.final_forcings = self.input_forcings.final_forcings + self.regridded_forcings2 = self.input_forcings.regridded_forcings2 + self.regridded_forcings1 = self.input_forcings.regridded_forcings1 + # _elem equivalents + self.final_forcings_elem = self.input_forcings.final_forcings_elem + self.regridded_forcings2_elem = self.input_forcings.regridded_forcings2_elem + self.regridded_forcings1_elem = self.input_forcings.regridded_forcings1_elem + + if supplemental_precip is not None: + self.final_supp_precip = self.supplemental_precip.final_supp_precip + self.regridded_precip2 = self.supplemental_precip.regridded_precip2 + self.regridded_precip1 = self.supplemental_precip.regridded_precip1 + # _elem equivalents + self.final_supp_precip_elem = ( + self.supplemental_precip.final_supp_precip_elem + ) + self.regridded_precip2_elem = ( + self.supplemental_precip.regridded_precip2_elem + ) + self.regridded_precip1_elem = ( + self.supplemental_precip.regridded_precip1_elem + ) + + def _no_interpolation(self) -> None: + """Perform the business logic of the *input forcings* 'no_interpolation' option. + + See the docstring of the public calling function ``no_interpolation`` for details. + Business logic and comments from original code: + https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py#L8-L48 + """ + ndim, also_elem = { + "gridded": (3, False), + "unstructured": (2, True), + "hydrofabric": (2, False), + }[self.config_options.grid_type] + + # Check to make sure we have valid grids. + if self.regridded_forcings2 is None: + arr_set_scalar(self.final_forcings, self.globalNdv, ndim) + if also_elem: + arr_set_scalar(self.final_forcings_elem, self.globalNdv, ndim) else: - # Default to the regridded states from the previous forecast output - # step. - if input_forcings.regridded_forcings1 is None: - input_forcings.final_forcings[:, :] = ConfigOptions.globalNdv - else: - input_forcings.final_forcings[:, :] = ( - input_forcings.regridded_forcings1[:, :] + arr_set_arr(self.final_forcings, self.regridded_forcings2, ndim) + if also_elem: + arr_set_arr( + self.final_forcings_elem, self.regridded_forcings2_elem, ndim ) - if abs(dtFromNext.total_seconds()) <= abs(dtFromPrevious.total_seconds()): - # Default to the regridded states from the next forecast output step. - if input_forcings.regridded_forcings2_elem is None: - input_forcings.final_forcings_elem[:, :] = ConfigOptions.globalNdv - else: - input_forcings.final_forcings_elem[:, :] = ( - input_forcings.regridded_forcings2_elem[:, :] + + def _no_interpolation_supp_pcp(self): + """Perform the business logic of the *supplemental precipitation* 'no_interpolation' option. + + See the docstring of the public calling function ``no_interpolation_supp_pcp`` for details. + Business logic and comments from original code: + https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py#L51-L92 + """ + ndim, also_elem = { + "gridded": (2, False), + "unstructured": (1, True), + "hydrofabric": (1, False), + }[self.config_options.grid_type] + + if self.regridded_precip2 is not None: + arr_set_arr(self.final_supp_precip, self.regridded_precip2, ndim) + if also_elem: + arr_set_arr( + self.final_supp_precip_elem, self.regridded_precip2_elem, ndim ) else: - # Default to the regridded states from the previous forecast output - # step. - if input_forcings.regridded_forcings1_elem is None: - input_forcings.final_forcings_elem[:, :] = ConfigOptions.globalNdv - else: - input_forcings.final_forcings_elem[:, :] = ( - input_forcings.regridded_forcings1_elem[:, :] - ) - elif ConfigOptions.grid_type == "hydrofabric": + # We have missing files. + arr_set_scalar(self.final_supp_precip, self.globalNdv, ndim) + if also_elem: + arr_set_scalar(self.final_supp_precip_elem, self.globalNdv, ndim) + + def _nearest_neighbor(self): + """Perform the business logic of the *input forcings* 'nearest_neighbor' option. + + See the docstring of the public calling function ``nearest_neighbor`` for details. + Business logic and comments from original code: + https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py#L95-L194 + """ + ndim, also_elem = { + "gridded": (3, False), + "unstructured": (2, True), + "hydrofabric": (2, False), + }[self.config_options.grid_type] + + # If we are running CFSv2 with bias correction, bypass as temporal interpolation is done + # internally (NWM-only). + if ( + self.config_options.runCfsNldasBiasCorrect + and self.input_forcings.product_name == "CFSv2_6Hr_Global_GRIB2" + ): + if self.mpi_config.rank == 0: + self.config_options.statusMsg = "Bypassing temporal interpolation routine due to NWM bias correction for CFSv2" + err_handler.log_msg(self.config_options, self.mpi_config) + return + + # Calculate the difference between the current output timestep, + # and the previous input forecast output step. + dtFromPrevious = ( + self.config_options.current_output_date - self.input_forcings.fcst_date1 + ) + + # Calculate the difference between the current output timesetp, + # and the next forecast output step. + dtFromNext = ( + self.config_options.current_output_date - self.input_forcings.fcst_date2 + ) + if abs(dtFromNext.total_seconds()) <= abs(dtFromPrevious.total_seconds()): # Default to the regridded states from the next forecast output step. - if input_forcings.regridded_forcings2 is None: - input_forcings.final_forcings[:, :] = ConfigOptions.globalNdv + if self.regridded_forcings2 is None: + arr_set_scalar(self.final_forcings, self.globalNdv, ndim) + if also_elem: + arr_set_scalar(self.final_forcings_elem, self.globalNdv, ndim) else: - input_forcings.final_forcings[:, :] = ( - input_forcings.regridded_forcings2[:, :] - ) + arr_set_arr(self.final_forcings, self.regridded_forcings2, ndim) + if also_elem: + arr_set_arr( + self.final_forcings_elem, self.regridded_forcings2_elem, ndim + ) else: # Default to the regridded states from the previous forecast output # step. - if input_forcings.regridded_forcings1 is None: - input_forcings.final_forcings[:, :] = ConfigOptions.globalNdv + if self.regridded_forcings1 is None: + arr_set_scalar(self.final_forcings, self.globalNdv, ndim) + if also_elem: + arr_set_scalar(self.final_forcings_elem, self.globalNdv, ndim) else: - input_forcings.final_forcings[:, :] = ( - input_forcings.regridded_forcings1[:, :] - ) - + arr_set_arr(self.final_forcings, self.regridded_forcings1, ndim) + if also_elem: + arr_set_arr( + self.final_forcings_elem, self.regridded_forcings1_elem, ndim + ) -def nearest_neighbor_supp_pcp(supplemental_precip, ConfigOptions, MpiConfig): - """Nearest neighbor temporal interpolation for supplemental precipitation. + def _nearest_neighbor_supp_pcp(self): + """Perform the business logic of the *supplemental precipitation* 'nearest_neighbor' option. + + TODO review the calculation for ``dtFromPrevious``: + In the original business logic here, it used ``dtFromPrevious = ...current_output_step - ...pcp_date1``. + In other functions, the original business logic used ``dtFromPrevious = ...current_output_date - ...pcp_date1``. + Both were preserved during the refactor, but it is not clear whether this was a typo in this function, maybe it was + intending to use ``current_output_date`` instead of ``current_output_step``? + + See the docstring of the public calling function ``nearest_neighbor_supp_pcp`` for details. + Business logic and comments from original code: + https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py#L197-L279 + """ + ndim, also_elem = { + "gridded": (2, False), + "unstructured": (1, True), + "hydrofabric": (1, False), + }[self.config_options.grid_type] + + if self.regridded_precip2 is None or self.regridded_precip1 is None: + return - Function for setting the current output regridded supplemental precipitation - to the nearest supplemental precipitation input step. - :param supplemental_precip: - :param ConfigOptions: - :param MpiConfig: - :return: - """ - if ( - supplemental_precip.regridded_precip2 is not None - and supplemental_precip.regridded_precip1 is not None - ): # Calculate the difference between the current ouptut timestep, # and the previous supplemental input step. dtFromPrevious = ( - ConfigOptions.current_output_step - supplemental_precip.pcp_date1 + self.config_options.current_output_step - self.supplemental_precip.pcp_date1 ) # Calculate the difference between the current output timestep, # and the next supplemental input step. - dtFromNext = ConfigOptions.current_output_date - supplemental_precip.pcp_date2 + dtFromNext = ( + self.config_options.current_output_date - self.supplemental_precip.pcp_date2 + ) - if ConfigOptions.grid_type == "gridded": - if abs(dtFromNext.total_seconds()) <= abs(dtFromPrevious.total_seconds()): - # Default to the regridded states from the next forecast output step. - supplemental_precip.final_supp_precip[:, :] = ( - supplemental_precip.regridded_precip2[:, :] - ) - else: - # Default to the regridded states from the previous forecast output - # step. - supplemental_precip.final_supp_precip[:, :] = ( - supplemental_precip.regridded_precip1[:, :] - ) - elif ConfigOptions.grid_type == "unstructured": - if abs(dtFromNext.total_seconds()) <= abs(dtFromPrevious.total_seconds()): - # Default to the regridded states from the next forecast output step. - supplemental_precip.final_supp_precip[:] = ( - supplemental_precip.regridded_precip2[:] - ) - else: - # Default to the regridded states from the previous forecast output - # step. - supplemental_precip.final_supp_precip[:] = ( - supplemental_precip.regridded_precip1[:] - ) - if abs(dtFromNext.total_seconds()) <= abs(dtFromPrevious.total_seconds()): - # Default to the regridded states from the next forecast output step. - supplemental_precip.final_supp_precip_elem[:] = ( - supplemental_precip.regridded_precip2_elem[:] - ) - else: - # Default to the regridded states from the previous forecast output - # step. - supplemental_precip.final_supp_precip_elem[:] = ( - supplemental_precip.regridded_precip1_elem[:] - ) - elif ConfigOptions.grid_type == "hydrofabric": - if abs(dtFromNext.total_seconds()) <= abs(dtFromPrevious.total_seconds()): - # Default to the regridded states from the next forecast output step. - supplemental_precip.final_supp_precip[:] = ( - supplemental_precip.regridded_precip2[:] - ) - else: - # Default to the regridded states from the previous forecast output - # step. - supplemental_precip.final_supp_precip[:] = ( - supplemental_precip.regridded_precip1[:] + if abs(dtFromNext.total_seconds()) <= abs(dtFromPrevious.total_seconds()): + # Default to the regridded states from the next forecast output step. + arr_set_arr(self.final_supp_precip, self.regridded_precip2, ndim) + if also_elem: + arr_set_arr( + self.final_supp_precip_elem, self.regridded_precip2_elem, ndim ) - else: - if ConfigOptions.grid_type == "gridded": - # We have missing files. - supplemental_precip.final_supp_precip[:, :] = ConfigOptions.globalNdv - elif ConfigOptions.grid_type == "unstructured": - # We have missing files. - supplemental_precip.final_supp_precip[:] = ConfigOptions.globalNdv - supplemental_precip.final_supp_precip_elem[:] = ConfigOptions.globalNdv - elif ConfigOptions.grid_type == "hydrofabric": - # We have missing files. - supplemental_precip.final_supp_precip[:] = ConfigOptions.globalNdv - - -def weighted_average(input_forcings, ConfigOptions, MpiConfig): - """Weighted average temporal interpolation for supplemental precipitation. + # Default to the regridded states from the previous forecast output + # step. + arr_set_arr(self.final_supp_precip, self.regridded_precip1, ndim) + if also_elem: + arr_set_arr( + self.final_supp_precip_elem, self.regridded_precip1_elem, ndim + ) - Function for setting the current output regridded fields as a weighted - average between the previous output step and the next output step. - :param input_forcings: - :param ConfigOptions: - :param MpiConfig: - :return: - """ - # Check to make sure we have valid grids. - if ConfigOptions.grid_type == "gridded": - if input_forcings.regridded_forcings2 is None: - input_forcings.final_forcings[:, :, :] = ConfigOptions.globalNdv - return - if input_forcings.regridded_forcings1 is None: - input_forcings.final_forcings[:, :, :] = ConfigOptions.globalNdv + def _weighted_average(self): + """Perform the business logic of the *input forcings* 'weighted_average' option. + + TODO Review how each ``if ... is None`` case returns early, which is from the original business logic. + It is unclear whether this was intentional, but it was preserved in the 2026 refactor. + + NOTE in original code, for the "unstructured" case: + ``ind1Ndv_elem`` and ``ind2Ndv_elem`` were getting assigned but never used. + That may have been a bug, since the _elem logic path was applying the ``ind1Ndv`` mask. + During the 2026 refactor, it was assumed that the intent was to actually use ``ind1Ndv_elem`` + and ``ind2Ndv_elem`` after setting them, so the refactored code does use them. + + See the docstring of the public calling function ``weighted_average`` for details. + Business logic and comments from original code: + https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py#L282-L459 + """ + ndim, also_elem = { + "gridded": (3, False), + "unstructured": (2, True), + "hydrofabric": (2, False), + }[self.config_options.grid_type] + + if self.regridded_forcings2 is None: + arr_set_scalar(self.final_forcings, self.globalNdv, ndim) return - # Check to make sure we have valid grids. - elif ConfigOptions.grid_type == "unstructured": - if input_forcings.regridded_forcings2 is None: - input_forcings.final_forcings[:, :] = ConfigOptions.globalNdv + if self.regridded_forcings1 is None: + arr_set_scalar(self.final_forcings, self.globalNdv, ndim) return - if input_forcings.regridded_forcings1 is None: - input_forcings.final_forcings[:, :] = ConfigOptions.globalNdv - return - if input_forcings.regridded_forcings2_elem is None: - input_forcings.final_forcings_elem[:, :] = ConfigOptions.globalNdv - return - if input_forcings.regridded_forcings1_elem is None: - input_forcings.final_forcings_elem[:, :] = ConfigOptions.globalNdv - return - # Check to make sure we have valid grids. - elif ConfigOptions.grid_type == "hydrofabric": - if input_forcings.regridded_forcings2 is None: - input_forcings.final_forcings[:, :] = ConfigOptions.globalNdv - return - if input_forcings.regridded_forcings1 is None: - input_forcings.final_forcings[:, :] = ConfigOptions.globalNdv + if also_elem: + if self.regridded_forcings2_elem is None: + arr_set_scalar(self.final_forcings_elem, self.globalNdv, ndim) + return + if self.regridded_forcings1_elem is None: + arr_set_scalar(self.final_forcings_elem, self.globalNdv, ndim) + return + + # If we are running CFSv2 with bias correction, bypass as temporal interpolation is done + # internally (NWM-only). + if ( + self.config_options.runCfsNldasBiasCorrect + and self.input_forcings.product_name == "CFSv2_6Hr_Global_GRIB2" + ): + if self.mpi_config.rank == 0: + self.config_options.statusMsg = "Bypassing temporal interpolation routine due to NWM bias correction for CFSv2" + err_handler.log_msg(self.config_options, self.mpi_config) return - # If we are running CFSv2 with bias correction, bypass as temporal interpolation is done - # internally (NWM-only). - if ( - ConfigOptions.runCfsNldasBiasCorrect - and input_forcings.product_name == "CFSv2_6Hr_Global_GRIB2" - ): - if MpiConfig.rank == 0: - ConfigOptions.statusMsg = "Bypassing temporal interpolation routine due to NWM bias correction for CFSv2" - err_handler.log_msg(ConfigOptions, MpiConfig) - return - if ConfigOptions.grid_type == "gridded": # Calculate the difference between the current output timestep, # and the previous input forecast output step. Use this to calculate a fraction # of the previous forcing output to use in the final output for this step. - dtFromPrevious = ConfigOptions.current_output_date - input_forcings.fcst_date1 - weight1 = 1 - ( - abs(dtFromPrevious.total_seconds()) / (input_forcings.outFreq * 60.0) - ) - - # Calculate the difference between the current output timesetp, - # and the next forecast output step. Use this to calculate a fraction of - # the next forcing output to use in the final output for this step. - dtFromNext = ConfigOptions.current_output_date - input_forcings.fcst_date2 - weight2 = 1 - ( - abs(dtFromNext.total_seconds()) / (input_forcings.outFreq * 60.0) - ) - - # Calculate where we have missing data in either the previous or next forcing dataset. - ind1Ndv = np.where( - input_forcings.regridded_forcings1 == ConfigOptions.globalNdv - ) - ind2Ndv = np.where( - input_forcings.regridded_forcings2 == ConfigOptions.globalNdv - ) - - input_forcings.final_forcings[:, :, :] = ( - input_forcings.regridded_forcings1[:, :, :] * weight1 - + input_forcings.regridded_forcings2[:, :, :] * weight2 + dtFromPrevious = ( + self.config_options.current_output_date - self.input_forcings.fcst_date1 ) - - # Set any pixel cells that were missing for either window to missing value. - input_forcings.final_forcings[ind1Ndv] = ConfigOptions.globalNdv - input_forcings.final_forcings[ind2Ndv] = ConfigOptions.globalNdv - - # Reset for memory efficiency. - ind1Ndv = None - ind2Ndv = None - - elif ConfigOptions.grid_type == "unstructured": - # Calculate the difference between the current output timestep, - # and the previous input forecast output step. Use this to calculate a fraction - # of the previous forcing output to use in the final output for this step. - dtFromPrevious = ConfigOptions.current_output_date - input_forcings.fcst_date1 weight1 = 1 - ( - abs(dtFromPrevious.total_seconds()) / (input_forcings.outFreq * 60.0) + abs(dtFromPrevious.total_seconds()) / (self.input_forcings.outFreq * 60.0) ) # Calculate the difference between the current output timesetp, # and the next forecast output step. Use this to calculate a fraction of # the next forcing output to use in the final output for this step. - dtFromNext = ConfigOptions.current_output_date - input_forcings.fcst_date2 - weight2 = 1 - ( - abs(dtFromNext.total_seconds()) / (input_forcings.outFreq * 60.0) - ) - - # Calculate where we have missing data in either the previous or next forcing dataset. - ind1Ndv = np.where( - input_forcings.regridded_forcings1 == ConfigOptions.globalNdv + dtFromNext = ( + self.config_options.current_output_date - self.input_forcings.fcst_date2 ) - ind2Ndv = np.where( - input_forcings.regridded_forcings2 == ConfigOptions.globalNdv - ) - ind1Ndv_elem = np.where( - input_forcings.regridded_forcings1_elem == ConfigOptions.globalNdv - ) - ind2Ndv_elem = np.where( - input_forcings.regridded_forcings2_elem == ConfigOptions.globalNdv - ) - - input_forcings.final_forcings[:, :] = ( - input_forcings.regridded_forcings1[:, :] * weight1 - + input_forcings.regridded_forcings2[:, :] * weight2 - ) - input_forcings.final_forcings_elem[:, :] = ( - input_forcings.regridded_forcings1_elem[:, :] * weight1 - + input_forcings.regridded_forcings2_elem[:, :] * weight2 - ) - - # Set any pixel cells that were missing for either window to missing value. - input_forcings.final_forcings[ind1Ndv] = ConfigOptions.globalNdv - input_forcings.final_forcings[ind2Ndv] = ConfigOptions.globalNdv - input_forcings.final_forcings_elem[ind1Ndv] = ConfigOptions.globalNdv - input_forcings.final_forcings_elem[ind2Ndv] = ConfigOptions.globalNdv - - # Reset for memory efficiency. - ind1Ndv = None - ind2Ndv = None - ind1Ndv_elem = None - ind2Ndv_elem = None - - elif ConfigOptions.grid_type == "hydrofabric": - # Calculate the difference between the current output timestep, - # and the previous input forecast output step. Use this to calculate a fraction - # of the previous forcing output to use in the final output for this step. - dtFromPrevious = ConfigOptions.current_output_date - input_forcings.fcst_date1 - weight1 = 1 - ( - abs(dtFromPrevious.total_seconds()) / (input_forcings.outFreq * 60.0) - ) - - # Calculate the difference between the current output timesetp, - # and the next forecast output step. Use this to calculate a fraction of - # the next forcing output to use in the final output for this step. - dtFromNext = ConfigOptions.current_output_date - input_forcings.fcst_date2 weight2 = 1 - ( - abs(dtFromNext.total_seconds()) / (input_forcings.outFreq * 60.0) + abs(dtFromNext.total_seconds()) / (self.input_forcings.outFreq * 60.0) ) # Calculate where we have missing data in either the previous or next forcing dataset. - ind1Ndv = np.where( - input_forcings.regridded_forcings1 == ConfigOptions.globalNdv - ) - ind2Ndv = np.where( - input_forcings.regridded_forcings2 == ConfigOptions.globalNdv - ) - - input_forcings.final_forcings[:, :] = ( - input_forcings.regridded_forcings1[:, :] * weight1 - + input_forcings.regridded_forcings2[:, :] * weight2 + ind1Ndv = np.where(self.regridded_forcings1 == self.globalNdv) + ind2Ndv = np.where(self.regridded_forcings2 == self.globalNdv) + if also_elem: + ind1Ndv_elem = np.where(self.regridded_forcings1_elem == self.globalNdv) + ind2Ndv_elem = np.where(self.regridded_forcings2_elem == self.globalNdv) + + arr_set_arr( + self.final_forcings, + ( + self.regridded_forcings1[(slice(None),) * ndim] * weight1 + + self.regridded_forcings2[(slice(None),) * ndim] * weight2 + ), + ndim, ) + if also_elem: + arr_set_arr( + self.final_forcings_elem, + ( + self.regridded_forcings1_elem[(slice(None),) * ndim] * weight1 + + self.regridded_forcings2_elem[(slice(None),) * ndim] * weight2 + ), + ndim, + ) # Set any pixel cells that were missing for either window to missing value. - input_forcings.final_forcings[ind1Ndv] = ConfigOptions.globalNdv - input_forcings.final_forcings[ind2Ndv] = ConfigOptions.globalNdv + self.final_forcings[ind1Ndv] = self.globalNdv + self.final_forcings[ind2Ndv] = self.globalNdv + if also_elem: + self.final_forcings_elem[ind1Ndv_elem] = self.globalNdv + self.final_forcings_elem[ind2Ndv_elem] = self.globalNdv # Reset for memory efficiency. ind1Ndv = None ind2Ndv = None + if also_elem: + ind1Ndv_elem = None + ind2Ndv_elem = None + def __calc_weights_for_supp_pcp(self) -> tuple[float | None, float | None]: + """Calculate weights for supplemental precip weighting. -def weighted_average_supp_pcp(supplemental_precip, ConfigOptions, MpiConfig): - """Weighted average temporal interpolation for supplemental precipitation. + Return (None, None) unless both the current and previous are non-None + (for either the non-_elem case or the _elem case).) - Function for setting the current output regridded supplemental precipitation fields - as an average between the previous and next input supplemental precipitation timesteps. - :param supplemental_precip: - :param ConfigOptions: - :param MpiConfig: - :return: - """ - if ConfigOptions.grid_type == "gridded": - if ( - supplemental_precip.regridded_precip2 is not None - and supplemental_precip.regridded_precip1 is not None - ): - # Calculate the difference between the current output timestep, - # and the previous input supp pcp step. Use this to calculate a fraction - # of the previous supp pcp to use in the final output for this step. - dtFromPrevious = ( - ConfigOptions.current_output_date - supplemental_precip.pcp_date1 - ) - weight1 = 1 - ( - abs(dtFromPrevious.total_seconds()) - / (supplemental_precip.input_frequency * 60.0) - ) + For ``weight1``: + Calculate the difference between the current output timestep, + and the previous input supp pcp step. Use this to calculate a fraction + of the previous supp pcp to use in the final output for this step. - # Calculate the difference between the current output timesetp, - # and the next input supp pcp step. Use this to calculate a fraction of - # the next forcing supp pcp to use in the final output for this step. - dtFromNext = ( - ConfigOptions.current_output_date - supplemental_precip.pcp_date2 - ) - weight2 = 1 - ( - abs(dtFromNext.total_seconds()) - / (supplemental_precip.input_frequency * 60.0) - ) + For ``weight2``: + Calculate the difference between the current output timesetp, + and the next input supp pcp step. Use this to calculate a fraction of + the next forcing supp pcp to use in the final output for this step. - # Calculate where we have missing data in either the previous or next forcing dataset. - ind1Ndv = np.where( - supplemental_precip.regridded_precip1 == ConfigOptions.globalNdv - ) - ind2Ndv = np.where( - supplemental_precip.regridded_precip2 == ConfigOptions.globalNdv - ) + Business logic and comments from original code (duplicated among the discretization types): + https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py#L473-L497 + """ - supplemental_precip.final_supp_precip[:, :] = ( - supplemental_precip.regridded_precip1[:, :] * weight1 - + supplemental_precip.regridded_precip2[:, :] * weight2 + if not ( + (self.regridded_precip2 is not None and self.regridded_precip1 is not None) + or ( + self.regridded_precip2_elem is not None + and self.regridded_precip1_elem is not None ) - - # Set any pixel cells that were missing for either window to missing value. - supplemental_precip.final_supp_precip[ind1Ndv] = ConfigOptions.globalNdv - supplemental_precip.final_supp_precip[ind2Ndv] = ConfigOptions.globalNdv - - # Reset for memory efficiency. - ind1Ndv = None - ind2Ndv = None - else: - # We have missing files. - supplemental_precip.final_supp_precip[:, :] = ConfigOptions.globalNdv - elif ConfigOptions.grid_type == "unstructured": - if ( - supplemental_precip.regridded_precip2 is not None - and supplemental_precip.regridded_precip1 is not None ): - # Calculate the difference between the current output timestep, - # and the previous input supp pcp step. Use this to calculate a fraction - # of the previous supp pcp to use in the final output for this step. - dtFromPrevious = ( - ConfigOptions.current_output_date - supplemental_precip.pcp_date1 - ) - weight1 = 1 - ( - abs(dtFromPrevious.total_seconds()) - / (supplemental_precip.input_frequency * 60.0) - ) - - # Calculate the difference between the current output timesetp, - # and the next input supp pcp step. Use this to calculate a fraction of - # the next forcing supp pcp to use in the final output for this step. - dtFromNext = ( - ConfigOptions.current_output_date - supplemental_precip.pcp_date2 - ) - weight2 = 1 - ( - abs(dtFromNext.total_seconds()) - / (supplemental_precip.input_frequency * 60.0) - ) + return None, None - # Calculate where we have missing data in either the previous or next forcing dataset. - ind1Ndv = np.where( - supplemental_precip.regridded_precip1 == ConfigOptions.globalNdv - ) - ind2Ndv = np.where( - supplemental_precip.regridded_precip2 == ConfigOptions.globalNdv - ) + # weight1 + dtFromPrevious = ( + self.config_options.current_output_date - self.supplemental_precip.pcp_date1 + ) + weight1 = 1 - ( + abs(dtFromPrevious.total_seconds()) + / (self.supplemental_precip.input_frequency * 60.0) + ) - supplemental_precip.final_supp_precip[:] = ( - supplemental_precip.regridded_precip1[:] * weight1 - + supplemental_precip.regridded_precip2[:] * weight2 - ) + # weight2 + dtFromNext = ( + self.config_options.current_output_date - self.supplemental_precip.pcp_date2 + ) + weight2 = 1 - ( + abs(dtFromNext.total_seconds()) + / (self.supplemental_precip.input_frequency * 60.0) + ) - # Set any pixel cells that were missing for either window to missing value. - supplemental_precip.final_supp_precip[ind1Ndv] = ConfigOptions.globalNdv - supplemental_precip.final_supp_precip[ind2Ndv] = ConfigOptions.globalNdv + return weight1, weight2 - # Reset for memory efficiency. - ind1Ndv = None - ind2Ndv = None + def _weighted_average_supp_pcp( + self, weight1: float | None, weight2: float | None, attr_suffix: str = "" + ): + """Perform the business logic of the *supplemental precip* 'weighted_average' option. + + ``attr_suffix`` can be empty string or "_elem". The "_elem" choices is only supported for the "unstructured" discretization. + + In the original code, for the "unstructured" discretization, the calculations of ``weight1`` and ``weight2`` were identical + between the non-_elem and the _elem logic paths. So when this was refactored, that block was moved to a shared inner function + ``_calc_weights``. This was decorated with lru_cache for the "unstructured" where it is called twice. + + Business logic and comments from original code: + https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py#L462-L676 + """ + ndim = { + "gridded": 2, + "unstructured": 1, + "hydrofabric": 1, + }[self.config_options.grid_type] + + if attr_suffix == "": + pass + elif attr_suffix == "_elem": + if self.config_options.grid_type != "unstructured": + raise ValueError( + "attr_suffix '_elem' is only supported for the 'unstructured' grid type." + ) else: - # We have missing files. - supplemental_precip.final_supp_precip[:] = ConfigOptions.globalNdv + raise ValueError(f"Unexpected attr_suffix: {repr(attr_suffix)}") if ( - supplemental_precip.regridded_precip2_elem is not None - and supplemental_precip.regridded_precip1_elem is not None + getattr(self, f"regridded_precip2{attr_suffix}") is not None + and getattr(self, f"regridded_precip1{attr_suffix}") is not None ): - # Calculate the difference between the current output timestep, - # and the previous input supp pcp step. Use this to calculate a fraction - # of the previous supp pcp to use in the final output for this step. - dtFromPrevious = ( - ConfigOptions.current_output_date - supplemental_precip.pcp_date1 - ) - weight1 = 1 - ( - abs(dtFromPrevious.total_seconds()) - / (supplemental_precip.input_frequency * 60.0) - ) - - # Calculate the difference between the current output timesetp, - # and the next input supp pcp step. Use this to calculate a fraction of - # the next forcing supp pcp to use in the final output for this step. - dtFromNext = ( - ConfigOptions.current_output_date - supplemental_precip.pcp_date2 - ) - weight2 = 1 - ( - abs(dtFromNext.total_seconds()) - / (supplemental_precip.input_frequency * 60.0) - ) - # Calculate where we have missing data in either the previous or next forcing dataset. ind1Ndv = np.where( - supplemental_precip.regridded_precip1_elem == ConfigOptions.globalNdv + getattr(self, f"regridded_precip1{attr_suffix}") == self.globalNdv ) ind2Ndv = np.where( - supplemental_precip.regridded_precip2_elem == ConfigOptions.globalNdv + getattr(self, f"regridded_precip2{attr_suffix}") == self.globalNdv ) - supplemental_precip.final_supp_precip_elem[:] = ( - supplemental_precip.regridded_precip1_elem[:] * weight1 - + supplemental_precip.regridded_precip2_elem[:] * weight2 + arr_set_arr( + getattr(self, f"final_supp_precip{attr_suffix}"), + ( + ( + getattr(self, f"regridded_precip1{attr_suffix}")[ + (slice(None),) * ndim + ] + * weight1 + ) + + ( + getattr(self, f"regridded_precip2{attr_suffix}")[ + (slice(None),) * ndim + ] + * weight2 + ) + ), + ndim, ) # Set any pixel cells that were missing for either window to missing value. - supplemental_precip.final_supp_precip_elem[ind1Ndv] = ( - ConfigOptions.globalNdv - ) - supplemental_precip.final_supp_precip_elem[ind2Ndv] = ( - ConfigOptions.globalNdv - ) + getattr(self, f"final_supp_precip{attr_suffix}")[ind1Ndv] = self.globalNdv + getattr(self, f"final_supp_precip{attr_suffix}")[ind2Ndv] = self.globalNdv # Reset for memory efficiency. ind1Ndv = None ind2Ndv = None else: # We have missing files. - supplemental_precip.final_supp_precip_elem[:] = ConfigOptions.globalNdv - - elif ConfigOptions.grid_type == "hydrofabric": - if ( - supplemental_precip.regridded_precip2 is not None - and supplemental_precip.regridded_precip1 is not None - ): - # Calculate the difference between the current output timestep, - # and the previous input supp pcp step. Use this to calculate a fraction - # of the previous supp pcp to use in the final output for this step. - dtFromPrevious = ( - ConfigOptions.current_output_date - supplemental_precip.pcp_date1 - ) - weight1 = 1 - ( - abs(dtFromPrevious.total_seconds()) - / (supplemental_precip.input_frequency * 60.0) - ) - - # Calculate the difference between the current output timesetp, - # and the next input supp pcp step. Use this to calculate a fraction of - # the next forcing supp pcp to use in the final output for this step. - dtFromNext = ( - ConfigOptions.current_output_date - supplemental_precip.pcp_date2 - ) - weight2 = 1 - ( - abs(dtFromNext.total_seconds()) - / (supplemental_precip.input_frequency * 60.0) + arr_set_scalar( + getattr(self, f"final_supp_precip{attr_suffix}"), self.globalNdv, ndim ) - # Calculate where we have missing data in either the previous or next forcing dataset. - ind1Ndv = np.where( - supplemental_precip.regridded_precip1 == ConfigOptions.globalNdv - ) - ind2Ndv = np.where( - supplemental_precip.regridded_precip2 == ConfigOptions.globalNdv - ) - - supplemental_precip.final_supp_precip[:] = ( - supplemental_precip.regridded_precip1[:] * weight1 - + supplemental_precip.regridded_precip2[:] * weight2 - ) - - # Set any pixel cells that were missing for either window to missing value. - supplemental_precip.final_supp_precip[ind1Ndv] = ConfigOptions.globalNdv - supplemental_precip.final_supp_precip[ind2Ndv] = ConfigOptions.globalNdv - - # Reset for memory efficiency. - ind1Ndv = None - ind2Ndv = None - else: - # We have missing files. - supplemental_precip.final_supp_precip[:] = ConfigOptions.globalNdv - - -def gfs_pcp_time_interp(input_forcings, ConfigOptions, MpiConfig): - """Calculate instantaneous precipitation rate from GFS average rates. - - Function that will calculate an instantaneous precipitation rate, representative - of the latest forecast GFS hour, or range of GFS forecast hours. This is - done as GFS has a quirky way of outputting precipitation rates. - :param input_forcings: - :param ConfigOptions: - :param MpiConfig: - :return: instPcpGlobal - """ - # There is a chain of logic we will follow here for GFS data. - # Forecast Hours <= 120: - # 1.) For first hour in every six hour period, the avg precip rate - # will be treated as the instantaneous rate. No processing - # needed. So 0-1, 6-7, 12-13, etc. - # 2.) For remaining hours, we need to calculate the difference between - # this avg precip rate, and the previous one. So, - # [(0-4)-(0-3)] gives us a precipitation rate for hour 4. - # Forecast Hours > 120: - # 1.) For the first three hours of every six hour period, we - # will treat the avg precip rate coming in as the instantaneous - # value at hour 3. This is because there are no values for - # individual hours within this horizon. So, 120-123, 126-129, etc - # 2.) For other three hours, we need to calculate the difference - # between the previous three hourly average rate, and the average - # rate for this entire six hour time frame. Why is this the case - # with GFS? Ask someone at NCEP....... - # [(120-126)-(120-123)] gives us a precipitation rate representative - # of the second three hour period. It's not necessarily instantaneous, - # but we will treat it as such. - - if ConfigOptions.grid_type == "gridded": - if input_forcings.fcst_hour2 <= 120: - if input_forcings.fcst_hour2 % 6 == 1: - # We are on the first hour of a six-hour period. We can treat - # the precipitation rate as instantaneous for this hour. - instPcpGlobal = input_forcings.globalPcpRate2 - total1 = None - total2 = None - else: - # We need to calculate the difference from the previous - # avg window to get an instantaneous value for this hour. - total1 = input_forcings.globalPcpRate1 * ( - 3600.0 * (input_forcings.fcst_hour1 % 6) - ) - if input_forcings.fcst_hour2 % 6 == 0: - # We have a 0-6, 6-12, 12-18 avg rate.... - total2 = input_forcings.globalPcpRate2 * (3600.0 * 6) - else: - # We have 0-5, 0-4, etc - total2 = input_forcings.globalPcpRate2 * ( - 3600.0 * (input_forcings.fcst_hour2 % 6) - ) - instPcpGlobal = (total2 - total1) / 3600.0 - # Reset variables to free up memory - total1 = None - total2 = None - else: - # We are in Situation #2 which currently runs out until the end of the - # end of the GFS forecast cycle of 384 hours. - if input_forcings.fcst_hour2 % 6 == 3: - # We are on the first 3 hours of a six hour period. Simply treat the average - # precipitation rate for this time period as the instantaneous precipitation - # rate. - instPcpGlobal = input_forcings.globalPcpRate2 - total2 = None - total1 = None - else: - total1 = input_forcings.globalPcpRate1 * (3600.0 * 3.0) - total2 = input_forcings.globalPcpRate2 * (3600.0 * 6.0) - instPcpGlobal = (total2 - total1) / (3600.0 * 3.0) - # Reset variables to free up memory. - total1 = None - total2 = None - # Return the interpolated grid back to the regridding program. + def _gfs_pcp_time_interp(self) -> Any | tuple[Any, Any]: + """Perform the business logic of the GFS time interpolation. - # Set any negative values to 0.0 - instPcpGlobal[np.where(instPcpGlobal < 0.0)] = 0.0 + Business logic and comments from original code: + https://github.com/NGWPC/ngen-forcing/blob/a0f217f06a0045d9f139bfa14abe711fc6f248b0/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py#L679-L881 + """ + also_elem = { + "gridded": False, + "unstructured": True, + "hydrofabric": False, + }[self.config_options.grid_type] - return instPcpGlobal - - elif ConfigOptions.grid_type == "unstructured": - if input_forcings.fcst_hour2 <= 120: - if input_forcings.fcst_hour2 % 6 == 1: + if self.input_forcings.fcst_hour2 <= 120: + if self.input_forcings.fcst_hour2 % 6 == 1: # We are on the first hour of a six-hour period. We can treat # the precipitation rate as instantaneous for this hour. - instPcpGlobal = input_forcings.globalPcpRate2 - instPcpGlobal_elem = input_forcings.globalPcpRate2_elem + instPcpGlobal = self.input_forcings.globalPcpRate2 total1 = None total2 = None total1_elem = None @@ -772,27 +516,32 @@ def gfs_pcp_time_interp(input_forcings, ConfigOptions, MpiConfig): else: # We need to calculate the difference from the previous # avg window to get an instantaneous value for this hour. - total1 = input_forcings.globalPcpRate1 * ( - 3600.0 * (input_forcings.fcst_hour1 % 6) - ) - total1_elem = input_forcings.globalPcpRate1_elem * ( - 3600.0 * (input_forcings.fcst_hour1 % 6) + total1 = self.input_forcings.globalPcpRate1 * ( + 3600.0 * (self.input_forcings.fcst_hour1 % 6) ) - if input_forcings.fcst_hour2 % 6 == 0: + if also_elem: + total1_elem = self.input_forcings.globalPcpRate1_elem * ( + 3600.0 * (self.input_forcings.fcst_hour1 % 6) + ) + if self.input_forcings.fcst_hour2 % 6 == 0: # We have a 0-6, 6-12, 12-18 avg rate.... - total2 = input_forcings.globalPcpRate2 * (3600.0 * 6) - total2_elem = input_forcings.globalPcpRate2_elem * (3600.0 * 6) + total2 = self.input_forcings.globalPcpRate2 * (3600.0 * 6) + if also_elem: + total2_elem = self.input_forcings.globalPcpRate2_elem * ( + 3600.0 * 6 + ) else: # We have 0-5, 0-4, etc - total2 = input_forcings.globalPcpRate2 * ( - 3600.0 * (input_forcings.fcst_hour2 % 6) - ) - total2_elem = input_forcings.globalPcpRate2_elem * ( - 3600.0 * (input_forcings.fcst_hour2 % 6) + total2 = self.input_forcings.globalPcpRate2 * ( + 3600.0 * (self.input_forcings.fcst_hour2 % 6) ) - + if also_elem: + total2_elem = self.input_forcings.globalPcpRate2_elem * ( + 3600.0 * (self.input_forcings.fcst_hour2 % 6) + ) instPcpGlobal = (total2 - total1) / 3600.0 - instPcpGlobal_elem = (total2_elem - total1_elem) / 3600.0 + if also_elem: + instPcpGlobal_elem = (total2_elem - total1_elem) / 3600.0 # Reset variables to free up memory total1 = None total2 = None @@ -801,81 +550,188 @@ def gfs_pcp_time_interp(input_forcings, ConfigOptions, MpiConfig): else: # We are in Situation #2 which currently runs out until the end of the # end of the GFS forecast cycle of 384 hours. - if input_forcings.fcst_hour2 % 6 == 3: + if self.input_forcings.fcst_hour2 % 6 == 3: # We are on the first 3 hours of a six hour period. Simply treat the average # precipitation rate for this time period as the instantaneous precipitation # rate. - instPcpGlobal = input_forcings.globalPcpRate2 - instPcpGlobal_elem = input_forcings.globalPcpRate2_elem + instPcpGlobal = self.input_forcings.globalPcpRate2 + if also_elem: + instPcpGlobal_elem = self.input_forcings.globalPcpRate2_elem total2 = None total1 = None total1_elem = None total2_elem = None else: - total1 = input_forcings.globalPcpRate1 * (3600.0 * 3.0) - total2 = input_forcings.globalPcpRate2 * (3600.0 * 6.0) - total1_elem = input_forcings.globalPcpRate1_elem * (3600.0 * 3.0) - total2_elem = input_forcings.globalPcpRate2_elem * (3600.0 * 6.0) + total1 = self.input_forcings.globalPcpRate1 * (3600.0 * 3.0) + total2 = self.input_forcings.globalPcpRate2 * (3600.0 * 6.0) + if also_elem: + total1_elem = self.input_forcings.globalPcpRate1_elem * ( + 3600.0 * 3.0 + ) + total2_elem = self.input_forcings.globalPcpRate2_elem * ( + 3600.0 * 6.0 + ) instPcpGlobal = (total2 - total1) / (3600.0 * 3.0) - instPcpGlobal_elem = (total2_elem - total1_elem) / (3600.0 * 3.0) + if also_elem: + instPcpGlobal_elem = (total2_elem - total1_elem) / (3600.0 * 3.0) # Reset variables to free up memory. total1 = None total2 = None total1_elem = None total2_elem = None # Return the interpolated grid back to the regridding program. + # Set any negative values to 0.0 instPcpGlobal[np.where(instPcpGlobal < 0.0)] = 0.0 - instPcpGlobal_elem[np.where(instPcpGlobal_elem < 0.0)] = 0.0 - - return instPcpGlobal, instPcpGlobal_elem + if also_elem: + instPcpGlobal_elem[np.where(instPcpGlobal_elem < 0.0)] = 0.0 - elif ConfigOptions.grid_type == "hydrofabric": - if input_forcings.fcst_hour2 <= 120: - if input_forcings.fcst_hour2 % 6 == 1: - # We are on the first hour of a six-hour period. We can treat - # the precipitation rate as instantaneous for this hour. - instPcpGlobal = input_forcings.globalPcpRate2 - total1 = None - total2 = None - else: - # We need to calculate the difference from the previous - # avg window to get an instantaneous value for this hour. - total1 = input_forcings.globalPcpRate1 * ( - 3600.0 * (input_forcings.fcst_hour1 % 6) - ) - if input_forcings.fcst_hour2 % 6 == 0: - # We have a 0-6, 6-12, 12-18 avg rate.... - total2 = input_forcings.globalPcpRate2 * (3600.0 * 6) - else: - # We have 0-5, 0-4, etc - total2 = input_forcings.globalPcpRate2 * ( - 3600.0 * (input_forcings.fcst_hour2 % 6) - ) - instPcpGlobal = (total2 - total1) / 3600.0 - # Reset variables to free up memory - total1 = None - total2 = None + if not also_elem: + return instPcpGlobal else: - # We are in Situation #2 which currently runs out until the end of the - # end of the GFS forecast cycle of 384 hours. - if input_forcings.fcst_hour2 % 6 == 3: - # We are on the first 3 hours of a six hour period. Simply treat the average - # precipitation rate for this time period as the instantaneous precipitation - # rate. - instPcpGlobal = input_forcings.globalPcpRate2 - total2 = None - total1 = None - else: - total1 = input_forcings.globalPcpRate1 * (3600.0 * 3.0) - total2 = input_forcings.globalPcpRate2 * (3600.0 * 6.0) - instPcpGlobal = (total2 - total1) / (3600.0 * 3.0) - # Reset variables to free up memory. - total1 = None - total2 = None - # Return the interpolated grid back to the regridding program. + return instPcpGlobal, instPcpGlobal_elem - # Set any negative values to 0.0 - instPcpGlobal[np.where(instPcpGlobal < 0.0)] = 0.0 - return instPcpGlobal +def no_interpolation( + input_forcings: InputForcings, config_options: ConfigOptions, mpi_config: MpiConfig +): + """No temporal interpolation. + + Function for simply setting the final regridded fields to the + input forcings that are from the next input forcing frequency. + :param input_forcings: + :param config_options: + :param mpi_config: + :return: + """ + _TimeInterp(input_forcings, None, config_options, mpi_config)._no_interpolation() + + +def no_interpolation_supp_pcp( + supplemental_precip: SupplementalPrecip, + config_options: ConfigOptions, + mpi_config: MpiConfig, +): + """No temporal interpolation for supplemental precipitation. + + Function for simply setting the final regridded supplemental precipitation + to the supplemental precipitation grids from the next precip frequency that + is available. + :param supplemental_precip: + :param config_options: + :param mpi_config: + :return: + """ + _TimeInterp( + None, supplemental_precip, config_options, mpi_config + )._no_interpolation_supp_pcp() + + +def nearest_neighbor( + input_forcings: InputForcings, config_options: ConfigOptions, mpi_config: MpiConfig +): + """Nearest neighbor temporal interpolation. + + Function for setting the current output regridded forcings to the nearest + input forecast step. + :param input_forcings: + :param config_options: + :param mpi_config: + :return: + """ + _TimeInterp(input_forcings, None, config_options, mpi_config)._nearest_neighbor() + + +def nearest_neighbor_supp_pcp( + supplemental_precip: SupplementalPrecip, + config_options: ConfigOptions, + mpi_config: MpiConfig, +): + """Nearest neighbor temporal interpolation for supplemental precipitation. + + Function for setting the current output regridded supplemental precipitation + to the nearest supplemental precipitation input step. + :param supplemental_precip: + :param config_options: + :param mpi_config: + :return: + """ + _TimeInterp( + None, supplemental_precip, config_options, mpi_config + )._nearest_neighbor_supp_pcp() + + +def weighted_average( + input_forcings: InputForcings, config_options: ConfigOptions, mpi_config: MpiConfig +): + """Weighted average temporal interpolation for supplemental precipitation. + + Function for setting the current output regridded fields as a weighted + average between the previous output step and the next output step. + :param input_forcings: + :param config_options: + :param mpi_config: + :return: + """ + _TimeInterp(input_forcings, None, config_options, mpi_config)._weighted_average() + + +def weighted_average_supp_pcp( + supplemental_precip: SupplementalPrecip, + config_options: ConfigOptions, + mpi_config: MpiConfig, +): + """Weighted average temporal interpolation for supplemental precipitation. + + Function for setting the current output regridded supplemental precipitation fields + as an average between the previous and next input supplemental precipitation timesteps. + :param supplemental_precip: + :param config_options: + :param mpi_config: + :return: + """ + interpolator = _TimeInterp(None, supplemental_precip, config_options, mpi_config) + + weight1, weight2 = interpolator.__calc_weights_for_supp_pcp() + + interpolator._weighted_average_supp_pcp(weight1, weight2) + if config_options.grid_type == "unstructured": + interpolator._weighted_average_supp_pcp(weight1, weight2, "_elem") + + +def gfs_pcp_time_interp( + input_forcings: InputForcings, config_options: ConfigOptions, mpi_config: MpiConfig +) -> Any | tuple[Any, Any]: + """Calculate instantaneous precipitation rate from GFS average rates. + + Function that will calculate an instantaneous precipitation rate, representative + of the latest forecast GFS hour, or range of GFS forecast hours. This is + done as GFS has a quirky way of outputting precipitation rates. + :param input_forcings: + :param config_options: + :param mpi_config: + :return: instPcpGlobal + """ + # There is a chain of logic we will follow here for GFS data. + # Forecast Hours <= 120: + # 1.) For first hour in every six hour period, the avg precip rate + # will be treated as the instantaneous rate. No processing + # needed. So 0-1, 6-7, 12-13, etc. + # 2.) For remaining hours, we need to calculate the difference between + # this avg precip rate, and the previous one. So, + # [(0-4)-(0-3)] gives us a precipitation rate for hour 4. + # Forecast Hours > 120: + # 1.) For the first three hours of every six hour period, we + # will treat the avg precip rate coming in as the instantaneous + # value at hour 3. This is because there are no values for + # individual hours within this horizon. So, 120-123, 126-129, etc + # 2.) For other three hours, we need to calculate the difference + # between the previous three hourly average rate, and the average + # rate for this entire six hour time frame. Why is this the case + # with GFS? Ask someone at NCEP....... + # [(120-126)-(120-123)] gives us a precipitation rate representative + # of the second three hour period. It's not necessarily instantaneous, + # but we will treat it as such. + return _TimeInterp( + input_forcings, None, config_options, mpi_config + )._gfs_pcp_time_interp() From 3efe9e8c61cf53b7af7ecdd15298ec080e7080bf Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Thu, 27 Aug 2026 16:47:14 -0400 Subject: [PATCH 131/132] Add README for 2026 refactor --- README_refactor_2026.md | 165 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 README_refactor_2026.md diff --git a/README_refactor_2026.md b/README_refactor_2026.md new file mode 100644 index 00000000..0f2ee0ec --- /dev/null +++ b/README_refactor_2026.md @@ -0,0 +1,165 @@ +# NGWPC Forcing Engine: Refactoring, Restructuring, and Codebase Improvement Summary + +**January 2026 to August 2026** + +## Overview + +This document summarizes the refactoring and code improvement efforts applied to the Python modules of the NextGen forcing engine (`ngen-forcing` repository) from January 2026 to August 2026. These efforts are distinct from new capabilities and feature additions. The work has improved the readability, maintainability, and extensibility of the codebase. + +The improvements fall into five categories: + +1. **Detailed Golden File Tests & Python Debugger Configurations** +2. **Formatting, Linting, Type-Hinting, and Style** +3. **Code Restructuring** +4. **Removing Deprecated / Unused Code and Data** +5. **Bug Discovery and Resolution** + +These refactorings and code improvements are not exhaustive — some files have received more attention than others. However, the new patterns implemented can be continued. + +### Primary List of Affected Files + +Not every file was refactored. Here is a list of files most affected: + +- [`NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py`](https://github.com/NGWPC/ngen-forcing/blob/NGWPC-7625_PI_10_ngen_forcing_refactor/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py) +- [`NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py`](https://github.com/NGWPC/ngen-forcing/blob/NGWPC-7625_PI_10_ngen_forcing_refactor/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py) +- [`NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py`](https://github.com/NGWPC/ngen-forcing/blob/NGWPC-7625_PI_10_ngen_forcing_refactor/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py) +- [`NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py`](https://github.com/NGWPC/ngen-forcing/blob/NGWPC-7625_PI_10_ngen_forcing_refactor/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py) +- [`NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py`](https://github.com/NGWPC/ngen-forcing/blob/NGWPC-7625_PI_10_ngen_forcing_refactor/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py) +- [`NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py`](https://github.com/NGWPC/ngen-forcing/blob/NGWPC-7625_PI_10_ngen_forcing_refactor/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py) -- `regrid.py` was not extensively refactored, but initial headway was made and patterns were implemented which could be continued. Significant amounts of duplicated logic remain, more DRYification efforts are needed. +- [`NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py`](https://github.com/NGWPC/ngen-forcing/blob/NGWPC-7625_PI_10_ngen_forcing_refactor/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py) +- [`NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py`](https://github.com/NGWPC/ngen-forcing/blob/NGWPC-7625_PI_10_ngen_forcing_refactor/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py) +- [`NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py`](https://github.com/NGWPC/ngen-forcing/blob/NGWPC-7625_PI_10_ngen_forcing_refactor/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py) +- [`NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py`](https://github.com/NGWPC/ngen-forcing/blob/NGWPC-7625_PI_10_ngen_forcing_refactor/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py) + +### Pull Requests Related to Tests, Refactoring, and Repo Cleanup + +| PR | Date | Title | Primary Files | Refactor Gist | +|----|------|-------|---------------|------| +| [#55](https://github.com/NGWPC/ngen-forcing/pull/55) | 2026-01-09 | Regridding Weights: unique cache file names, refactor | regrid.py, parallel.py | Refactored `calculate_weights` into smaller functions | +| [#65](https://github.com/NGWPC/ngen-forcing/pull/65) | 2026-01-29 | Remove the data and binary files from the coastal code | *(Repo Cleanup)* | Removed coastal binary and data files that had been moved to the nwm-coastal repository. | +| [#91](https://github.com/NGWPC/ngen-forcing/pull/91) | 2026-02-11 | Formatting | *(Multiple)* | Applied Ruff auto-formatting across the codebase. | +| [#98](https://github.com/NGWPC/ngen-forcing/pull/98) | 2026-02-13 | regrid.py Further Formatting and Direct Log Calls | regrid.py | Additional formatting and improved `err_handler` log calls. | +| [#99](https://github.com/NGWPC/ngen-forcing/pull/99) | 2026-02-13 | regrid.py: Partials | regrid.py | Introduced the `Partials` class to DRYify repeated log and error-handling call patterns. | +| [#100](https://github.com/NGWPC/ngen-forcing/pull/100) | 2026-02-13 | Add new OS utils for DRY file handling | os_utils.py, regrid.py | Created shared `os_utils.py` module for DRY file operations (remove, close, symlink). | +| [#113](https://github.com/NGWPC/ngen-forcing/pull/113) | 2026-03-10 | Test for regrid.py | regrid.py, tests | Added golden file tests for regrid operations. | +| [#125](https://github.com/NGWPC/ngen-forcing/pull/125) | 2026-04-07 | Tests for GeoMeta and InputForcings | tests | Added golden file tests for GeoMeta prior to refactoring. | +| [#105](https://github.com/NGWPC/ngen-forcing/pull/105) | 2026-04-08 | GeoMeta Linting | geoMod.py | Applied linting and formatting to `geoMod.py`. | +| [#126](https://github.com/NGWPC/ngen-forcing/pull/126) | 2026-04-08 | Modularize GeoMeta | geoMod.py | Broke out monolithic functions into smaller modular units. | +| [#127](https://github.com/NGWPC/ngen-forcing/pull/127) | 2026-04-08 | Refactor InputForcings (Part 1) | forcingInputMod.py, geoMod.py | Refactored `forcingInputMod.py` with OOP inheritance for discretization types. | +| [#128](https://github.com/NGWPC/ngen-forcing/pull/128) | 2026-04-08 | Refactor InputForcings (Part 2) | forcingInputMod.py | Continued OOP refactor of `forcingInputMod.py` with extracted constants. | +| [#101](https://github.com/NGWPC/ngen-forcing/pull/101) | 2026-04-08 | Refactor GeoMetaWrfHydro | geoMod.py | Restructured WRF/geo/hydro metadata handling with OOP inheritance. | +| [#137](https://github.com/NGWPC/ngen-forcing/pull/137) | 2026-04-16 | Refactor GeoMeta and InputForcing + Tests | geoMod.py, forcingInputMod.py | Further refactoring of GeoMeta and InputForcing with accompanying tests. | +| [#145](https://github.com/NGWPC/ngen-forcing/pull/145) | 2026-04-25 | Repo cleanup | *(repo cleanup)* | General repository cleanup of unused files and artifacts. | +| [#147](https://github.com/NGWPC/ngen-forcing/pull/147) | 2026-05-06 | Added bmi_model tests | *(tests)* | Added golden file tests for `BmiModel` class attributes. | +| [#148](https://github.com/NGWPC/ngen-forcing/pull/148) | 2026-05-08 | Add Test for CONUS Standard AnA and Refactor Test Configuration | *(tests)* | Added AnA test configuration and refactored test framework for extensibility. | +| [#196](https://github.com/NGWPC/ngen-forcing/pull/196) | 2026-07-13 | Remove streamflow scripts | *(repo cleanup)* | Removed streamflow scripts that were moved to `nwm-data-assimilation`. | +| [#197](https://github.com/NGWPC/ngen-forcing/pull/197) | 2026-07-14 | Run flynt on downscale.py | downscale.py | Converted string formatting to f-strings via flynt. | +| [#198](https://github.com/NGWPC/ngen-forcing/pull/198) | 2026-07-14 | Run flynt on bias_correction.py | bias_correction.py | Converted string formatting to f-strings via flynt. | +| [#199](https://github.com/NGWPC/ngen-forcing/pull/199) | 2026-07-14 | Run flynt on forecast_mod.py | bias_correction.py, forecastMod.py | Converted string formatting to f-strings via flynt. | +| [#200](https://github.com/NGWPC/ngen-forcing/pull/200) | 2026-07-14 | Add basic type hints to downscale.py | downscale.py | Added type hints and renamed args for PEP 8 consistency. | +| [#207](https://github.com/NGWPC/ngen-forcing/pull/207) | 2026-07-30 | Update Test Data for NHF 1.2.2 and Latest Forcing Class Structures | *(tests)* | Updated golden file test data for NHF 1.2.2 hydrofabric and latest class structures. | +| [#134](https://github.com/NGWPC/ngen-forcing/pull/134) | 2026-08-13 | Refactor of Supplemental Precipitation Mod (suppPrecipMod.py) | suppPrecipMod.py, consts.py | Refactored into OOP inheritance with parent/child classes per discretization type. | +| [#201](https://github.com/NGWPC/ngen-forcing/pull/201) | 2026-08-13 | Refactor parallel.py (NGWPC-10583) | parallel.py | Simplified MPI methods, added docs, replaced `atexit` with explicit cleanup. | +| [#191](https://github.com/NGWPC/ngen-forcing/pull/191) | 2026-08-26 | Refactor: bmi_model.py, config.py, model.py | bmi_model.py, config.py, model.py | bmi_model.py: decomposed initialization into focused setup methods with property access. config.py: split config parsing into property setters with validation. model.py: broke run loop into named steps and extracted dispatch logic. | +| [#224](https://github.com/NGWPC/ngen-forcing/pull/224) | 2026-08-26 | Refactor layeringMod.py | layeringMod.py | Established abstract parent class with discretization-specific children to replaced if/elif blocks. | +| [#226](https://github.com/NGWPC/ngen-forcing/pull/226) | 2026-08-26 | Refactor timeInterpMod.py | timeInterpMod.py | Reorganized top-level functions into a `_TimeInterp` class with smaller methods. | + +# Summary of Improvements by Category + +## 1. Detailed Golden File Tests & Python Debugger Configurations + +The tests originally inherited in the `ngen-forcing` repository asserted that the BMI interface methods would run without throwing an exception, but did not provide a mechanism for inspecting the results of the processes nor for confirming that changes to the code were not causing unexpected changes in the outputs. + +The new golden file tests added in 2026 perform a serialization (to disk) of various class attribute structures. The serialization operations occur at initialization, at timesteps 0, 1, and 2, and at finalization. This is a minimal amount of timesteps needed to reach certain parts of the flow. + +These dumped JSON files are committed to the repository as "expected" data -- commonly called "golden files". With these files in place. When the tests run in a normal fashion, they serialize the equivalent "actual" data structures to JSON on disk, and then confirm that the two are equivalent. + +Some numerical tolerance is accounted for, and there are convenient ways to exclude certain keys from the set of keys that are dumped and checked. For structures that are too large, they are replaced by a string that contains metadata: the length of the object and a hash of its values. + +The tests are structured with care for convenient and idiomatic extensibility going forward, to increase coverage as needed. + +### Testing Environment + +**NOTE**: The example commands listed in [tests/README.md](tests/README.md) assume the user is running in the Dev Container environment provided by the `nwm-rte` repository. + +### Coverage and Limitations of Tests + +Timesteps 0, 1, and 2 are covered. + +The forcing configurations covered include default configurations for Short Range and Analysis & Assimilation realizations, as well as AORC usage for historical realizations. + +The tests currently only cover the **Hydrofabric** discretization type (not the **Gridded** or **Unstructured** discretization types). + +While the current golden file tests do not provide 100% coverage of every manner in which the forcing engine can be run, they provide a reliable baseline for confirming that code changes produce numerically identical results, with configurable tolerance (absolute and relative). + +### Python Debugger + +Python debugger configurations were added which allow developers to execute `run_bmi_model.py` with `debugpy`, with either one MPI process or 2 MPI processes. + +The `-n 2` (2 MPI processes) configuration is particularly important for being able to supervise the call stack and variables' states of each MPI rank in real time. Placing a breakpoint at a particular line in the debugger causes both ranks to pause at the same location. Without this capability, it can be particularly difficult to debug MPI-related nuances, especially considering how MPI rank 0 naturally has different logic paths than other MPI ranks. + +## 2. Formatting, Linting, Type-Hinting, and Style + +Automatic formatting was applied to improve consistency and readability of the code. **Ruff** was the primary tool applied for automatic formatting. **Flynt** was also used to replace existing antipattern string composition approaches with modern Python f-strings. + +Variables were renamed to follow general PEP 8 guidelines on case. + +Type hints were added. Some circular imports were avoided by placing type-only imports behind `if TYPE_CHECKING:` guards, which prevent the imports from executing at runtime while still allowing static type checkers and IDEs to resolve the references. + +## 3. Code Restructuring + +### 3.1 Long Monolithic Functions Broken into Smaller Units + +Long monolithic functions were decomposed into smaller units with improved (reduced) scope. + +**Example — `config.py`:** The original `read_config()` method was spanned over 1000 lines. In the refactored code, configuration parsing is handled by individual property setters, each responsible for validating and storing a single configuration attribute. + - [Before](https://github.com/NGWPC/ngen-forcing/blob/02e47e64555b7704e9f00c03bdac07fbb127ca2c/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py#L18) · [After](https://github.com/NGWPC/ngen-forcing/blob/2bddeeaa59f49ee66c7c59967c5b8d2f5f16bb45/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py#L31) + +**Example — `timeInterpMod.py`:** The original module had long top-level functions, each containing `if` / `elif` / `else` conditional branches to handle the 3 discretization types, of which the majority of the business logic was replicated among those 3. The refactored module organizes these into a `_TimeInterp` class with shared business logic and smaller methods. + - [Before](https://github.com/NGWPC/ngen-forcing/blob/02e47e64555b7704e9f00c03bdac07fbb127ca2c/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py#L9) · [After](https://github.com/NGWPC/ngen-forcing/blob/2bddeeaa59f49ee66c7c59967c5b8d2f5f16bb45/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/timeInterpMod.py#L46) + +### 3.2 OOP Inheritance for Discretization Types + +Long `if`/`elif`/`elif` conditional flows were replaced with OOP class inheritance designs. The 3 main discretization modes — **Hydrofabric**, **Gridded**, and **Unstructured** — have been separated into individual classes that inherit from a common parent class. The abstract `_LayeringMod` parent class has three concrete children: `_LayeringMod_Gridded`, `_LayeringMod_Unstructured`, and `_LayeringMod_Hydrofabric`. + +Examples: `suppPrecipMod.py` and `forcingInputMod.py`. + + - `layeringMod.py`: [Before](https://github.com/NGWPC/ngen-forcing/blob/02e47e64555b7704e9f00c03bdac07fbb127ca2c/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py#L8) · [After](https://github.com/NGWPC/ngen-forcing/blob/2bddeeaa59f49ee66c7c59967c5b8d2f5f16bb45/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/layeringMod.py#L38) + - `suppPrecipMod.py`: [Before](https://github.com/NGWPC/ngen-forcing/blob/02e47e64555b7704e9f00c03bdac07fbb127ca2c/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py#L12) · [After](https://github.com/NGWPC/ngen-forcing/blob/2bddeeaa59f49ee66c7c59967c5b8d2f5f16bb45/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/suppPrecipMod.py#L28) + +### 3.3 Extraction of Constants to `consts.py` + +Configuration values for the various forcing input data sources, configuration modes, and magic numbers were moved into a new shared file, `consts.py`. This centralizes these large objects that previously were scattered across multiple files, mixed-in with business logic. + +### 3.4 Shared Utilities + +Duplicated logic was replaced with shared utilities, for example the new file [NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/os_utils.py](https://github.com/NGWPC/ngen-forcing/blob/2bddeeaa59f49ee66c7c59967c5b8d2f5f16bb45/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/os_utils.py#L23) for DRY file handling. + +### 3.5 `regrid.py`: Improvements and Remaining Work + +`regrid.py` is one of the largest files in the codebase. It was not fully refactored between January and August 2026, but it did receive some improvements. Significant refactoring is still needed to reduce duplicated logic and convert the flow into a more human-readable state. + +Here are some of the improvements that were made: + +- **Auto-formatting** via Ruff and flynt for consistent style and modern f-strings. +- **Logic refactoring in `calculate_weights`**: The regridding steps at the bottom of `calculate_weights` were refactored into smaller functions and common logic among branching flow paths was consolidated. +- **DRY OS operations and log calls:** Shared logic is now used for OS operations (file creation, removal, symbolic links) via `os_utils`, replacing scattered inline implementations. +- **`Partials` class:** A class of `functools.partial` objects was defined to share logic for the types of log and error-handling calls that go through the `err_handler` module. These calls affect program state and can cause intentional exits, so centralizing them reduces the risk of inconsistent error handling across the many regridding functions, and usage of the partials causes a significant reduction in total lines of code across the file. + - [Example](https://github.com/NGWPC/ngen-forcing/blob/2bddeeaa59f49ee66c7c59967c5b8d2f5f16bb45/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py#L85) + +### 3.6 Setters, Getters, and Properties + +In some cases, explicit property setters/getters were defined. These allow the implementation code to be more readable by colocating data validation logic with the definition of the attribute. In some cases these were leveraged to make properties read-only after initial set, or "hardened", for safety. + +## 4. Removal of Deprecated Code and Data + +- Coastal-related files were moved to the dedicated `nwm-coastal` repository. +- Streamflow scripts were moved to the `nwm-data-assimilation` repository. +- `forecastMod.py` was deprecated. This file appears to be unused by the current ngen stack; it now raises a `NotImplementedError` if another codebase attempts to import it or run it. + +## 5. Bug Discovery and Resolution + +Some inherited bugs and potential bugs (requiring further investigation) were discovered during the refactoring work. Some were resolved, while others were tagged with TODO comments. + +Example: `perform_downscaling` used `[1] in list` instead of `1 in list`: + - [Before](https://github.com/NGWPC/ngen-forcing/blob/02e47e64555b7704e9f00c03bdac07fbb127ca2c/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py#L921) · [After](https://github.com/NGWPC/ngen-forcing/blob/2bddeeaa59f49ee66c7c59967c5b8d2f5f16bb45/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py#L1580) From 66004b691b63a8e5480f8a7139deabba930c25b3 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Thu, 27 Aug 2026 17:07:53 -0400 Subject: [PATCH 132/132] Revise README_refactor_2026.md --- README_refactor_2026.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_refactor_2026.md b/README_refactor_2026.md index 0f2ee0ec..5d96cbbd 100644 --- a/README_refactor_2026.md +++ b/README_refactor_2026.md @@ -48,7 +48,7 @@ Not every file was refactored. Here is a list of files most affected: | [#127](https://github.com/NGWPC/ngen-forcing/pull/127) | 2026-04-08 | Refactor InputForcings (Part 1) | forcingInputMod.py, geoMod.py | Refactored `forcingInputMod.py` with OOP inheritance for discretization types. | | [#128](https://github.com/NGWPC/ngen-forcing/pull/128) | 2026-04-08 | Refactor InputForcings (Part 2) | forcingInputMod.py | Continued OOP refactor of `forcingInputMod.py` with extracted constants. | | [#101](https://github.com/NGWPC/ngen-forcing/pull/101) | 2026-04-08 | Refactor GeoMetaWrfHydro | geoMod.py | Restructured WRF/geo/hydro metadata handling with OOP inheritance. | -| [#137](https://github.com/NGWPC/ngen-forcing/pull/137) | 2026-04-16 | Refactor GeoMeta and InputForcing + Tests | geoMod.py, forcingInputMod.py | Further refactoring of GeoMeta and InputForcing with accompanying tests. | +| [#137](https://github.com/NGWPC/ngen-forcing/pull/137) | 2026-04-16 | Refactor GeoMeta, InputForcings, NWMv3_Forcing_Engine_BMI_model, + Tests | geoMod.py, forcingInputMod.py, bmi_model.py | Further refactoring of GeoMeta and InputForcings with accompanying tests. Also establish OOP class inheritance structure for bmi_model.py. | | [#145](https://github.com/NGWPC/ngen-forcing/pull/145) | 2026-04-25 | Repo cleanup | *(repo cleanup)* | General repository cleanup of unused files and artifacts. | | [#147](https://github.com/NGWPC/ngen-forcing/pull/147) | 2026-05-06 | Added bmi_model tests | *(tests)* | Added golden file tests for `BmiModel` class attributes. | | [#148](https://github.com/NGWPC/ngen-forcing/pull/148) | 2026-05-08 | Add Test for CONUS Standard AnA and Refactor Test Configuration | *(tests)* | Added AnA test configuration and refactored test framework for extensibility. |