diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 88d4a783..e89d7a3e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -5,7 +5,7 @@ on: push: branches: [ master, dev-pdaf-* ] pull_request: - branches: [ master, stages-2025-pdaf ] + branches: [ master ] jobs: eclm_build_job: @@ -28,6 +28,11 @@ jobs: parflow_dir: "parflow", model_opts: "eCLM ParFlowGPU" } + - { + name: "ICON standalone", + use_oasis: "False", + model_opts: "ICON" + } # - { # name: "CLM3.5-PDAF", # use_oasis: "False", @@ -63,6 +68,7 @@ jobs: VER_HYPRE: 2.33.0 VER_ECCODES: 2.40.0 VER_OASIS: tsmp-patches-v0.1 + VER_ICON_UPSTREAM: release-2026.04-public steps: - uses: actions/checkout@v4 @@ -321,7 +327,9 @@ jobs: echo "VER_ParFlow=${VER_ParFlow}" >> $GITHUB_OUTPUT fi - if [[ "${{ matrix.config.name }}" == *"ICON"* ]]; then + if [[ "${{ matrix.config.name }}" == "ICON standalone" ]]; then + echo "${{ env.VER_ICON_UPSTREAM }} => VER_ICON_UPSTREAM" + elif [[ "${{ matrix.config.name }}" == *"ICON"* ]]; then VER_ICON=$(cat model_versions | grep -w "icon" | cut -d' ' -f1) echo "${VER_ICON} => VER_ICON" echo "VER_ICON=${VER_ICON}" >> $GITHUB_OUTPUT @@ -402,6 +410,30 @@ jobs: run: | pwd && git submodule update --init --force . + # + # ICON Standalone + # + - if: matrix.config.name == 'ICON standalone' + name: Restore cached ICON standalone ${{ env.VER_ICON_UPSTREAM }} + uses: actions/cache/restore@v4 + id: cache-icon-standalone-restore + with: + path: ${{ env.TSMP2_ROOT }}/models/icon-model + key: ${{ matrix.config.name }}_icon-${{ env.VER_ICON_UPSTREAM }} + + - if: matrix.config.name == 'ICON standalone' && steps.cache-icon-standalone-restore.outputs.cache-hit != 'true' + name: Update ICON standalone ${{ env.VER_ICON_UPSTREAM }} + working-directory: ${{ env.TSMP2_ROOT }}/models + run: | + pwd && git clone -b ${{ env.VER_ICON_UPSTREAM }} --recursive https://gitlab.dkrz.de/icon/icon-model.git + + - if: matrix.config.name == 'ICON standalone' && steps.cache-icon-standalone-restore.outputs.cache-hit != 'true' + name: Cache ICON standalone ${{ env.VER_ICON_UPSTREAM }} + uses: actions/cache/save@v4 + with: + path: ${{ env.TSMP2_ROOT }}/models/icon-model + key: ${{ matrix.config.name }}_icon-${{ env.VER_ICON_UPSTREAM }} + # # Pre-build checks # @@ -413,6 +445,10 @@ jobs: echo "Entering 'oasis3-mct'" echo "$(git -C oasis3-mct describe --tags --always)" fi + if [[ "${{ matrix.config.name }}" == "ICON standalone" ]]; then + echo "Entering 'models/icon-model'" + echo "$(git -C models/icon-model describe --tags --always)" + fi - name: Check TSMP2 dependencies working-directory: ${{ env.DEPENDENCIES_ROOT }} @@ -432,6 +468,9 @@ jobs: if [[ "${{ matrix.config.use_oasis }}" == "True" ]]; then MODEL_OPTS="${MODEL_OPTS} --OASIS_SRC ${TSMP2_ROOT}/oasis3-mct" fi + if [[ "${{ matrix.config.name }}" == "ICON standalone" ]]; then + MODEL_OPTS="${MODEL_OPTS} --ICON_SRC ${TSMP2_ROOT}/models/icon-model" + fi BUILD_TSMP2_CMD="./build_tsmp2.sh ${MODEL_OPTS} --no_update" echo $BUILD_TSMP2_CMD && echo "" eval $BUILD_TSMP2_CMD diff --git a/CMakeLists.txt b/CMakeLists.txt index 3d2dc124..90ad5fe9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) # set default of components to off option(ICON "Compile ICON within TSMP2 framework" OFF) +option(ICONGPU "Compile ICONGPU within TSMP2 framework" OFF) option(COSMO "Compile COSMO within TSMP2 framework" OFF) option(eCLM "Compile eCLM within TSMP2 framework" OFF) option(CLM3.5 "Compile CLM3.5 within TSMP2 framework" OFF) @@ -45,16 +46,21 @@ if (BUILD_OASIS) list(APPEND MODEL_DEPENDENCIES OASIS3_MCT) endif() -if (${ICON}) +if (${ICON} OR ${ICONGPU}) + if (${ICON}) + set (ICON_ID "ICON") + else() + set (ICON_ID "ICONGPU") + endif() if(NOT DEFINED ICON_SRC) set(ICON_SRC "${CMAKE_SOURCE_DIR}/models/icon") endif() include(BuildICON) list(APPEND COMPONENT_MODELS "ICON") if ("${MODEL_ID}" STREQUAL "") - set(MODEL_ID "ICON") + set(MODEL_ID "${ICON_ID}") else() - set(MODEL_ID "${MODEL_ID}-ICON") + set(MODEL_ID "${MODEL_ID}-${ICON_ID}") endif() endif() diff --git a/build_tsmp2.sh b/build_tsmp2.sh index b0e65260..9d07bc3c 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -20,6 +20,7 @@ function help_tsmp2() { echo "Component models:" echo "" echo " icon Compile with ICON atmosphere model." + echo " iconGPU Compile with ICON atmosphere model (GPU-enabled)." echo " eclm Compile with eCLM land surface model." echo " parflow Compile with ParFlow subsurface model." echo " parflowGPU Compile with ParFlow subsurface model (GPU-enabled)" @@ -48,6 +49,7 @@ function help_tsmp2() { echo "" echo " ./build_tsmp2.sh icon eclm parflow" echo " ./build_tsmp2.sh eclm parflowGPU" + echo " ./build_tsmp2.sh iconGPU" echo " ./build_tsmp2.sh icon eclm" echo " ./build_tsmp2.sh eclm pdaf" echo "" @@ -64,7 +66,7 @@ if [ "${component}" = "y" ];then model_id+="-${cmake_name}" fi # model_id cmake_comp_str+=" -D${cmake_name}=ON" - if [[ $cmake_name = @(ICON|eCLM|ParFlow|ParFlowGPU|COSMO|CLM3.5) ]]; then + if [[ $cmake_name = @(ICON|ICONGPU|eCLM|ParFlow|ParFlowGPU|COSMO|CLM3.5) ]]; then model_count=$(( $model_count + 1 )) fi # cmake_name fi # component @@ -121,7 +123,8 @@ while [[ "$#" -gt 0 ]]; do -q|--quiet) quiet=y;; -v|--verbose) verbose_makefile=y;; --version) echo "$0 version 0.2.0"; exit 0;; - --icon|icon) icon=y;; + --icon|icon) icon=y iconCPU=y iconCMakeModelID="ICON";; + icongpu) icon=y iconGPU=y iconCMakeModelID="ICONGPU";; --eclm|eclm) eclm=y;; --parflow|parflow) parflow=y parflowCPU=y parflowCMakeModelID="ParFlow";; --parflowgpu|parflowgpu) parflow=y parflowGPU=y parflowCMakeModelID="ParFlowGPU";; @@ -156,7 +159,7 @@ cmake_comp_str="" message "Setting model-id and component string..." # fun set_component shell_name cmake_name -set_component icon "ICON" +set_component icon $iconCMakeModelID set_component eclm "eCLM" set_component parflow $parflowCMakeModelID set_component cosmo "COSMO" @@ -175,6 +178,11 @@ if [[ "${parflowCPU}" == "y" && "${parflowGPU}" == "y" ]];then exit 1 fi +if [[ "${iconCPU}" == "y" && "${iconGPU}" == "y" ]];then + echo "ABORT: Building icon and iconGPU at the same time is not supported." + exit 1 +fi + ## CONCATENATE SOURCE CODE STRING message "Setting component source dir..." cmake_compsrc_str="" diff --git a/cmake/BuildICON.cmake b/cmake/BuildICON.cmake index 82619e5b..12867dbe 100644 --- a/cmake/BuildICON.cmake +++ b/cmake/BuildICON.cmake @@ -19,6 +19,23 @@ if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM") set(ICON_CFLAGS "-gdwarf-4 -qno-opt-dynamic-align -ftz -march=native") set(ICON_FCFLAGS "-gdwarf-4 -march=native -pc64 -fp-model source -traceback -qno-opt-dynamic-align -no-fma") +elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") + set(ICON_FCFLAGS "-Mrecursive -Mallocatable=03 -Mstack_arrays") + if(${ICONGPU}) + string(APPEND ICON_FCFLAGS " -Minfo=accel,inline -acc=gpu,verystrict -gpu=cc90") + include(CheckLanguage) + check_language(CUDA) + if(CMAKE_CUDA_COMPILER) + enable_language(CUDA) + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -arch=sm_90") + else() + message(FATAL_ERROR "BuildICON.cmake: Cannot find CUDA library in the system.") + endif() + else() + string(APPEND ICON_FCFLAGS " -Minfo=inline") + set(CMAKE_CUDA_COMPILER "") + set(CMAKE_CUDA_FLAGS "") + endif() endif() set(ICON_ECRAD_FCFLAGS "-D__ECRAD_LITTLE_ENDIAN") @@ -46,8 +63,10 @@ if (CMAKE_MESSAGE_LOG_LEVEL STREQUAL "DEBUG") set(HDF5_FIND_DEBUG "TRUE") endif() set(HDF5_PREFER_PARALLEL "TRUE") -find_package(HDF5 REQUIRED COMPONENTS Fortran HL) -list(APPEND ICON_LIBS "${HDF5_Fortran_HL_LIBRARIES}") +set(HDF5_USE_STATIC_LIBRARIES "TRUE") +set(HDF5_NO_FIND_PACKAGE_CONFIG_FILE "TRUE") +find_package(HDF5 REQUIRED COMPONENTS C) +list(APPEND ICON_LIBS "${HDF5_LIBRARIES}") # libXML2 - XML parsing library find_package(LibXml2 REQUIRED) @@ -79,8 +98,9 @@ string(PREPEND ICON_FCFLAGS "-I${NetCDF_F90_ROOT}/include ") find_package(ZLIB REQUIRED) list(APPEND ICON_LIBS "${ZLIB_LIBRARIES}") + # Enable/disable model-specific features -list(APPEND EXTRA_CONFIG_ARGS --enable-parallel-netcdf --enable-openmp --disable-ocean --disable-jsbach --disable-coupling --enable-ecrad --disable-mpi-checks --disable-rte-rrtmgp) +list(APPEND EXTRA_CONFIG_ARGS --enable-parallel-netcdf --disable-ocean --disable-jsbach --disable-coupling --enable-ecrad --disable-mpi-checks --disable-rte-rrtmgp) # Coupling-specific options if( ${eCLM} OR ${CLM3.5} OR ${ParFlow} OR ${ParFlowGPU} ) @@ -90,6 +110,20 @@ if( ${eCLM} OR ${CLM3.5} OR ${ParFlow} OR ${ParFlowGPU} ) list(APPEND EXTRA_CONFIG_ARGS --enable-oascoupling) endif() +# GPU-specific options +if(${ICONGPU}) + list(APPEND ICON_LIBS "-c++libs -nvmalloc -cuda") + list(APPEND EXTRA_CONFIG_ARGS --enable-gpu=openacc --enable-mpi-gpu --enable-cuda-graphs --enable-pgi-inlib) +else() + list(APPEND EXTRA_CONFIG_ARGS --enable-openmp) +endif() +if(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") + # Use of NVHPC toolchain implies nvc++ is being used; hence + # it is necessary to link to the C++ stdlib. + list(APPEND ICON_LIBS "-lstdc++") + list(APPEND EXTRA_CONFIG_ARGS --enable-realloc-buf ) +endif() + # Assemble linker options list(JOIN ICON_LIBS " " ICON_LIBS) @@ -100,8 +134,10 @@ ExternalProject_Add(ICON CONFIGURE_COMMAND ${ICON_SRC}/configure CC=${CMAKE_C_COMPILER} FC=${CMAKE_Fortran_COMPILER} + CUDACXX=${CMAKE_CUDA_COMPILER} CFLAGS=${ICON_CFLAGS} FCFLAGS=${ICON_FCFLAGS} + CUDAFLAGS=${CMAKE_CUDA_FLAGS} LDFLAGS=${ICON_LDFLAGS} ICON_ECRAD_FCFLAGS=${ICON_ECRAD_FCFLAGS} LIBS=${ICON_LIBS} diff --git a/env/jsc.2026.nvhpc.openmpi b/env/jsc.2026.nvhpc.openmpi new file mode 100644 index 00000000..81961028 --- /dev/null +++ b/env/jsc.2026.nvhpc.openmpi @@ -0,0 +1,66 @@ +# ----------------------------------------------------------------------------------------- +# Loads NVHPC+OpenMPI build environment for TSMP2. +# This environment is tailored for JURECA [1] and JUPITER [2] supercomputers. +# +# [1] https://apps.fz-juelich.de/jsc/software/jureca/index.xhtml +# [2] https://apps.fz-juelich.de/jsc/software/jupiter/index.xhtml +# +# Usage: source jsc.2026.nvhpc.openmpi +# ----------------------------------------------------------------------------------------- + +# Load compilers and MPI library +module --force purge +module use $OTHERSTAGES +module load Stages/2026 +module load nvidia-compilers/25.9-CUDA-13 +module load OpenMPI +module load ScaLAPACK + +# Basic scripting and build tools +module load Python +module load CMake +module load git + +# Storage libraries +module load HDF5 +module load netCDF/4.9.3 +module load netCDF-Fortran/4.6.2 +module load PnetCDF/1.14.1 + +# ParFlow additional libraries +module load CUDA +module load UCX-settings/RC-CUDA +module load Hypre +module load Umpire +module load SUNDIALS + +# TODO: Verify these values +if [[ $SYSTEMNAME == "jupiter" ]]; then + export CUDAARCHS="90" +else + export CUDAARCHS="80" +fi +export CMAKE_CUDA_RUNTIME_LIBRARY="Shared" + +# ICON additional libraries +module load ecCodes + +# Set default MPI compilers +export CC=mpicc +export FC=mpifort +export CXX=nvc++ +export CUDACXX=nvcc +export MPI_HOME=$EBROOTOPENMPI +export UMPIRE_ROOT=$EBROOTUMPIRE +export SUNDIALS_ROOT=$EBROOTSUNDIALS + +# Display compiler settings +module list +echo "=========================== COMPILER SETTINGS =======================" +echo " Machine: ${SYSTEMNAME} on Stages/$STAGE" +echo " MPI lib: OpenMPI v$EBVERSIONOPENMPI" +echo " C: $($CC --version | head -n 2 | tail -n 1)" +echo " C++: $($CXX --version | head -n 2 | tail -n 1)" +echo " Fortran: $($FC --version | head -n 2 | tail -n 1)" +echo "======================================================================" +