From e337bccd855c02efc6499e1e96f72f75041df953 Mon Sep 17 00:00:00 2001 From: Ritvik Rao Date: Tue, 2 Jun 2026 13:35:28 -0500 Subject: [PATCH] New build for reconverse --- CMakeLists.txt | 28 ++++-- README.md | 95 ++++++++++++++++--- buildcmake | 8 ++ cmake/converse.cmake | 20 ++-- src/arch/reconverse-darwin-arm8/conv-mach.h | 68 +++++++++++++ src/arch/reconverse-darwin-arm8/conv-mach.sh | 10 ++ src/arch/reconverse-linux-x86_64/conv-mach.h | 62 ++++++++++++ src/arch/reconverse-linux-x86_64/conv-mach.sh | 10 ++ src/libs/ck-libs/metis/GKlib/GKlib.h | 4 +- src/scripts/charmc | 2 +- src/util/charmrun-src/CMakeLists.txt | 12 ++- src/util/charmrun-src/charmrun.C | 2 + src/util/sockRoutines.C | 1 + src/xlat-i/xi-util.h | 4 +- 14 files changed, 290 insertions(+), 36 deletions(-) create mode 100644 src/arch/reconverse-darwin-arm8/conv-mach.h create mode 100644 src/arch/reconverse-darwin-arm8/conv-mach.sh create mode 100644 src/arch/reconverse-linux-x86_64/conv-mach.h create mode 100644 src/arch/reconverse-linux-x86_64/conv-mach.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 32544d8897..d4800e035a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,7 +107,7 @@ else() endif() # Build options -set(NETWORK "netlrts" CACHE STRING "Target network layer, can be one of mpi,multicore,netlrts,gni,ofi,pami,pamilrts,ucx,verbs") +set(NETWORK "netlrts" CACHE STRING "Target network layer, can be one of mpi,multicore,netlrts,reconverse,gni,ofi,pami,pamilrts,ucx,verbs") set(TARGET "charm++" CACHE STRING "Target build type, can be one of charm++,AMPI,LIBS,charm4py,ChaNGa,everylb,all-test") set(ARCH "" CACHE STRING "Target architecture, can be one of i386,x86_64,arm7,arm8,ppc64le") # By default, detect arch we are running on. @@ -266,6 +266,10 @@ else() set(CMK_TASKQUEUE 0) endif() +if(${NETWORK} STREQUAL "reconverse") + set(RECONVERSE ON) +endif() + if(${RECONVERSE}) set(CMK_RECONVERSE 1) else() @@ -290,8 +294,10 @@ else() set(CMK_SMP 0) endif() -if(${NETWORK} STREQUAL "multicore") +if(${NETWORK} STREQUAL "multicore" OR ${NETWORK} STREQUAL "reconverse") set(CMK_SMP 1) +endif() +if(${NETWORK} STREQUAL "multicore") set(CMK_MULTICORE 1) else() set(CMK_MULTICORE 0) @@ -327,7 +333,7 @@ IF(JPEG_FOUND) set(CMK_USE_LIBJPEG 1) ENDIF() -if(NETWORK STREQUAL "pami") +if(NETWORK STREQUAL "pami" OR NETWORK STREQUAL "reconverse") set(CMK_USE_LRTS 0) else() set(CMK_USE_LRTS 1) @@ -786,7 +792,9 @@ if(CMK_COMPILER STREQUAL "msvc") endif() # Header files -configure_file(src/arch/${GDIR}/conv-common.h include/ COPYONLY) +if(EXISTS src/arch/${GDIR}/conv-common.h) + configure_file(src/arch/${GDIR}/conv-common.h include/ COPYONLY) +endif() if(EXISTS src/arch/${GDIR}/conv-common.sh) configure_file(src/arch/${GDIR}/conv-common.sh include/ COPYONLY) endif() @@ -880,7 +888,9 @@ include(cmake/hwloc.cmake) # Converse include(cmake/converse.cmake) -add_subdirectory(src/QuickThreads) +if(NOT RECONVERSE) + add_subdirectory(src/QuickThreads) +endif() # add_subdirectory(src/util/boost-context) # add_subdirectory(src/conv-core) @@ -1019,9 +1029,11 @@ else() if(RECONVERSE) target_link_libraries(ckhello PRIVATE reconverse) endif() - add_dependencies(ckhello ck ckqt conv-static - converse ckmain - moduleNDMeshStreamer modulecompletion) + set(ckhello_deps ck converse ckmain moduleNDMeshStreamer modulecompletion) + if(NOT RECONVERSE) + list(APPEND ckhello_deps ckqt conv-static) + endif() + add_dependencies(ckhello ${ckhello_deps}) endif() # Create conv-mach-opt.sh diff --git a/README.md b/README.md index 45f79bd70c..98d4426fb6 100644 --- a/README.md +++ b/README.md @@ -92,19 +92,21 @@ of the `` below. (Note: this isn't a complete list. Run `./build` for a complete listing) -| Charm++ Version | OS | Communication | Default Compiler | -|---------------------------|---------|---------------|---------------------------------------| -| `netlrts-linux-x86_64` | Linux | UDP | GNU compiler | -| `netlrts-darwin-x86_64` | macOS | UDP | Clang C++ compiler | -| `netlrts-win-x86_64` | Windows | UDP | MS Visual C++ | -| `mpi-linux-x86_64` | Linux | MPI | GNU compiler | -| `multicore-linux-x86_64` | Linux | Shared memory | GNU compiler | -| `multicore-darwin-x86_64` | macOS | Shared memory | Clang C++ compiler | -| `gni-crayxc` | Linux | GNI | CC (whatever PrgEnv module is loaded) | -| `gni-crayxe` | Linux | GNI | CC (whatever PrgEnv module is loaded) | -| `verbs-linux-x86_64` | Linux | IB Verbs | GNU compiler | -| `ofi-linux-x86_64` | Linux | OFI | GNU compiler | -| `ucx-linux-x86_64` | Linux | UCX | GNU compiler | +| Charm++ Version | OS | Communication | Default Compiler | +|-------------------------------|---------|----------------|---------------------------------------| +| `netlrts-linux-x86_64` | Linux | UDP | GNU compiler | +| `netlrts-darwin-x86_64` | macOS | UDP | Clang C++ compiler | +| `netlrts-win-x86_64` | Windows | UDP | MS Visual C++ | +| `mpi-linux-x86_64` | Linux | MPI | GNU compiler | +| `multicore-linux-x86_64` | Linux | Shared memory | GNU compiler | +| `multicore-darwin-x86_64` | macOS | Shared memory | Clang C++ compiler | +| `reconverse-linux-x86_64` | Linux | LCI/LCW | GNU compiler | +| `reconverse-darwin-arm8` | macOS | LCI/LCW | Clang C++ compiler | +| `gni-crayxc` | Linux | GNI | CC (whatever PrgEnv module is loaded) | +| `gni-crayxe` | Linux | GNI | CC (whatever PrgEnv module is loaded) | +| `verbs-linux-x86_64` | Linux | IB Verbs | GNU compiler | +| `ofi-linux-x86_64` | Linux | OFI | GNU compiler | +| `ucx-linux-x86_64` | Linux | UCX | GNU compiler | To choose ``, your choice is determined by two options: @@ -119,6 +121,12 @@ To choose ``, your choice is determined by two options: but performance is often worse than using the machine's direct calls referenced above. * `multicore-`: Charm++ communicates using shared memory within a single node. A version of Charm++ built with this option will not run on more than a single node. + * `reconverse-`: Charm++ uses the Reconverse communication layer backed by LCI or LCW. + Reconverse is an alternative to the standard Converse layer and is always built in SMP mode + (there is no separate communication thread; LCI handles progress internally). It supports + single-node runs with no backend, multi-node runs via LCI (recommended for performance), + or multi-node runs via LCW (an MPI-based fallback). See + [Building with Reconverse](#building-with-reconverse) below. 2. Your operating system/architecture: @@ -182,6 +190,67 @@ for more information: Supported options: common cuda flang gfortran ifort local nolb omp ooc papi perftools persistent pgf90 pxshm smp syncft sysvshm tcp tsan +## Building with Reconverse + +The `reconverse-` triplets are always SMP (no `-smp` suffix needed) and always +enable the Reconverse communication layer. They are built with the standard +`./build` script like any other version. + +### Quick start + +On Linux x86\_64: + + $ ./build charm++ reconverse-linux-x86_64 + +On macOS ARM64: + + $ ./build charm++ reconverse-darwin-arm8 + +This fetches Reconverse (and optionally LCI) from their upstream repositories +and builds everything in a single step. The resulting build directory is named +after the triplet (e.g. `reconverse-linux-x86_64/`). + +### Communication backends + +Reconverse supports three backends, selected at runtime: + +* **None** (default for single-process runs): no inter-node communication library + is required. This is suitable for development and testing on a single machine. +* **LCI** (recommended for multi-node): a lightweight communication interface + designed for low-latency active-message communication. LCI is fetched and + built automatically unless a pre-installed copy is available. +* **LCW**: an MPI-based wrapper backend. Useful as a fallback on systems where + LCI is not available or supported. + +At runtime, the backend is selected automatically based on what was compiled in. +To force a specific backend, pass `+backend lci` or `+backend lcw` on the +command line. + +### Fetching LCI automatically + +LCI is fetched by default for reconverse triplets. To use a pre-installed copy +of LCI instead, set `LCI_ROOT` in your environment before building: + + $ LCI_ROOT=/path/to/lci ./build charm++ reconverse-linux-x86_64 + +To fetch a specific Reconverse tag or use a local checkout: + + $ ./build charm++ reconverse-linux-x86_64 --with-fetch-reconverse-tag=v1.2.3 + $ ./build charm++ reconverse-linux-x86_64 --with-fetch-reconverse-dir=/path/to/reconverse + +### Build options + +The same options available for other versions work with reconverse builds: + +* `-j` — parallel build with N jobs (e.g. `-j8`) +* `--with-production` — enable production optimizations +* `--enable-tracing` — enable Projections performance tracing +* `--destination=` — place the build in a custom directory + +For example: + + $ ./build charm++ reconverse-linux-x86_64 -j8 --with-production + ## Building the Source If you have downloaded a binary version of Charm++, you can skip diff --git a/buildcmake b/buildcmake index 777c295172..bc23ba3a42 100755 --- a/buildcmake +++ b/buildcmake @@ -608,6 +608,14 @@ fi my_os=$(echo "$actual_triplet" | cut -d '-' -f2) +# reconverse network type implies reconverse communication layer and inherent SMP. +# CMK_SMP is forced to 1 by CMakeLists.txt for reconverse; we do not pass -DSMP=1 +# here because that would trigger the opt-file loop to source conv-mach-smp.sh, +# which does not exist for reconverse (SMP flags are baked into conv-mach.h directly). +if [[ $opt_network == "reconverse" ]]; then + opt_reconverse=1 +fi + if [[ $opt_network == "mpi" && $my_os == "win" ]]; then echo "Warning: CMake build is not supported with MPI on Windows, running ./buildold instead." sleep 2 diff --git a/cmake/converse.cmake b/cmake/converse.cmake index 2c0c4f34b8..2e42678988 100644 --- a/cmake/converse.cmake +++ b/cmake/converse.cmake @@ -279,12 +279,14 @@ endforeach() # target_include_directories(converse PRIVATE src/util) # for sockRoutines.C # target_include_directories(converse PRIVATE src/conv-core src/util/topomanager src/ck-ldb src/ck-perf src/ck-cp) -# conv-static -add_library(conv-static OBJECT src/conv-core/conv-static.c) -add_dependencies(reconverse conv-static) -add_dependencies(charm_cxx_utils conv-static) -add_custom_command(TARGET charm_cxx_utils - POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/conv-static.dir/src/conv-core/conv-static.c.o ${CMAKE_BINARY_DIR}/lib/conv-static.o - VERBATIM -) +# conv-static: not needed for reconverse builds (reconverse provides its own converse layer) +if(NOT RECONVERSE) + add_library(conv-static OBJECT src/conv-core/conv-static.c) + add_dependencies(reconverse conv-static) + add_dependencies(charm_cxx_utils conv-static) + add_custom_command(TARGET charm_cxx_utils + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/conv-static.dir/src/conv-core/conv-static.c.o ${CMAKE_BINARY_DIR}/lib/conv-static.o + VERBATIM + ) +endif() diff --git a/src/arch/reconverse-darwin-arm8/conv-mach.h b/src/arch/reconverse-darwin-arm8/conv-mach.h new file mode 100644 index 0000000000..cbd16c66f7 --- /dev/null +++ b/src/arch/reconverse-darwin-arm8/conv-mach.h @@ -0,0 +1,68 @@ +#ifndef _CONV_MACH_H +#define _CONV_MACH_H + +#define CMK_ARM 1 +#define CMK_64BIT 1 +#define CMK_SMP 1 + +#undef CMK_IMMEDIATE_MSG +#define CMK_IMMEDIATE_MSG 0 + +#define CMK_ASYNC_NOT_NEEDED 1 +#define CMK_ASYNC_USE_FIOASYNC_AND_FIOSETOWN 0 +#define CMK_ASYNC_USE_FIOASYNC_AND_SIOCSPGRP 0 +#define CMK_ASYNC_USE_FIOSSAIOSTAT_AND_FIOSSAIOOWN 0 +#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN 1 + +#define CMK_GETPAGESIZE_AVAILABLE 0 + +#define CMK_MALLOC_USE_GNU_MALLOC 0 +#define CMK_MALLOC_USE_OS_BUILTIN 1 +#define CMK_MALLOC_USE_GNUOLD_MALLOC 0 + +#undef CMK_MEMORY_BUILD_GNU_HOOKS +#define CMK_MEMORY_BUILD_GNU_HOOKS 0 +#define CMK_MEMORY_PAGESIZE 4096 +#define CMK_MEMORY_PROTECTABLE 0 + + +#define CMK_SSH_IS_A_COMMAND 1 +#define CMK_SSH_NOT_NEEDED 0 + +#define CMK_SHARED_VARS_UNAVAILABLE 0 +#define CMK_SHARED_VARS_POSIX_THREADS_SMP 1 + +#define CMK_THREADS_USE_CONTEXT 0 +#define CMK_THREADS_USE_JCONTEXT 0 +#define CMK_THREADS_USE_FCONTEXT 1 +#define CMK_THREADS_USE_PTHREADS 0 +#define CMK_THREADS_ARE_WIN32_FIBERS 0 + +#define CMK_SIGNAL_NOT_NEEDED 1 +#define CMK_SIGNAL_USE_SIGACTION 0 +#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART 0 + +#define CMK_THREADS_REQUIRE_NO_CPV 0 +#define CMK_THREADS_COPY_STACK 0 + +#define CMK_TIMER_USE_GETRUSAGE 1 +#define CMK_TIMER_USE_SPECIAL 0 +#define CMK_TIMER_USE_TIMES 0 + + +#define CMK_DEBUG_MODE 0 +#define CMK_WEB_MODE 1 + +#define CMK_LBDB_ON 1 + +#define CMK_STACKSIZE_DEFAULT 65536 + +#define CMK_USE_KQUEUE 1 + +#define CMK_NOT_USE_TLS_THREAD 1 + +/* #define CMK_PCQUEUE_LOCK 1 */ + +#define CMK_CONVERSE_MPI 0 + +#endif diff --git a/src/arch/reconverse-darwin-arm8/conv-mach.sh b/src/arch/reconverse-darwin-arm8/conv-mach.sh new file mode 100644 index 0000000000..2e9ec37aaf --- /dev/null +++ b/src/arch/reconverse-darwin-arm8/conv-mach.sh @@ -0,0 +1,10 @@ +. $CHARMINC/cc-clang.sh +. $CHARMINC/conv-mach-darwin.sh + +CMK_DEFS="$CMK_DEFS -D_REENTRANT" + +# Reconverse provides its own threading; remove the QuickThreads library +# injected by conv-mach-darwin.sh +CMK_LIBS="${CMK_LIBS//-lckqt/}" + +CMK_SMP="1" diff --git a/src/arch/reconverse-linux-x86_64/conv-mach.h b/src/arch/reconverse-linux-x86_64/conv-mach.h new file mode 100644 index 0000000000..dd3c5f7b9a --- /dev/null +++ b/src/arch/reconverse-linux-x86_64/conv-mach.h @@ -0,0 +1,62 @@ +#ifndef _CONV_MACH_H +#define _CONV_MACH_H + +#define CMK_AMD64 1 +#define CMK_64BIT 1 +#define CMK_SMP 1 + +#undef CMK_IMMEDIATE_MSG +#define CMK_IMMEDIATE_MSG 0 + +#define CMK_ASYNC_NOT_NEEDED 1 +#define CMK_ASYNC_USE_FIOASYNC_AND_FIOSETOWN 0 +#define CMK_ASYNC_USE_FIOASYNC_AND_SIOCSPGRP 0 +#define CMK_ASYNC_USE_FIOSSAIOSTAT_AND_FIOSSAIOOWN 0 +#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN 0 + +#define CMK_DLL_CC "g++ -shared -O3 -o " + +#define CMK_GETPAGESIZE_AVAILABLE 1 + +#define CMK_MALLOC_USE_GNU_MALLOC 0 +#define CMK_MALLOC_USE_OS_BUILTIN 1 + +#define CMK_MEMORY_PAGESIZE 4096 +#define CMK_MEMORY_PROTECTABLE 0 + + +#define CMK_SSH_IS_A_COMMAND 1 +#define CMK_SSH_NOT_NEEDED 0 + +#define CMK_SHARED_VARS_UNAVAILABLE 0 +#define CMK_SHARED_VARS_POSIX_THREADS_SMP 1 + +#define CMK_THREADS_USE_CONTEXT 0 +#define CMK_THREADS_USE_FCONTEXT 1 +#define CMK_THREADS_USE_PTHREADS 0 +#define CMK_THREADS_ARE_WIN32_FIBERS 0 + +#define CMK_SIGNAL_NOT_NEEDED 1 +#define CMK_SIGNAL_USE_SIGACTION 0 +#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART 0 + +#define CMK_THREADS_REQUIRE_NO_CPV 0 +#define CMK_THREADS_COPY_STACK 0 + +#define CMK_TIMER_USE_RDTSC 0 +#define CMK_TIMER_USE_GETRUSAGE 1 +#define CMK_TIMER_USE_SPECIAL 0 +#define CMK_TIMER_USE_TIMES 0 + + +#define CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT 0 +#define CMK_WHEN_PROCESSOR_IDLE_USLEEP 1 + +#define CMK_DEBUG_MODE 0 +#define CMK_WEB_MODE 1 + +#define CMK_LBDB_ON 1 + +#define CMK_CONVERSE_MPI 0 + +#endif diff --git a/src/arch/reconverse-linux-x86_64/conv-mach.sh b/src/arch/reconverse-linux-x86_64/conv-mach.sh new file mode 100644 index 0000000000..431753f02d --- /dev/null +++ b/src/arch/reconverse-linux-x86_64/conv-mach.sh @@ -0,0 +1,10 @@ +. $CHARMINC/cc-gcc.sh + +CMK_DEFS="$CMK_DEFS -D_REENTRANT" + +CMK_XIOPTS='' +CMK_LIBS="-lpthread $CMK_LIBS" + +CMK_QT='generic64-light' + +CMK_SMP='1' diff --git a/src/libs/ck-libs/metis/GKlib/GKlib.h b/src/libs/ck-libs/metis/GKlib/GKlib.h index 769d4ed4c6..787b899cd8 100644 --- a/src/libs/ck-libs/metis/GKlib/GKlib.h +++ b/src/libs/ck-libs/metis/GKlib/GKlib.h @@ -11,7 +11,9 @@ #ifndef _GKLIB_H_ #define _GKLIB_H_ 1 -#include "conv-config.h" +//#include "conv-config.h" +#include "charm-config.h" +#include "converse.h" #define GKMSPACE diff --git a/src/scripts/charmc b/src/scripts/charmc index 7863c144ae..84ef8fd5fa 100755 --- a/src/scripts/charmc +++ b/src/scripts/charmc @@ -1688,7 +1688,7 @@ fi ALL_LIBS+=($CMK_SYSLIBS) -if [[ "$BUILD_SHARE" = '0' && "$USER_INITIATED_SHARED" = '0' ]] +if [[ "$BUILD_SHARE" = '0' && "$USER_INITIATED_SHARED" = '0' && -f "$CHARMLIB/conv-static.o" ]] then ALL_LIBS+=("$CHARMLIB/conv-static.o") fi diff --git a/src/util/charmrun-src/CMakeLists.txt b/src/util/charmrun-src/CMakeLists.txt index 95a3ccf389..6430a31e09 100644 --- a/src/util/charmrun-src/CMakeLists.txt +++ b/src/util/charmrun-src/CMakeLists.txt @@ -12,6 +12,12 @@ target_link_libraries(charmrun PRIVATE -seq) target_include_directories(charmrun PRIVATE ../../conv-ccs ..) # for ccs-auth.c sockRoutines.c -add_dependencies(charmrun ck converse reconverse moduleNDMeshStreamer ckmain modulecompletion conv-static) -add_dependencies(charmd ck converse moduleNDMeshStreamer ckmain modulecompletion conv-static) -add_dependencies(charmd_faceless ck converse moduleNDMeshStreamer ckmain modulecompletion conv-static) +if(RECONVERSE) + add_dependencies(charmrun ck converse reconverse moduleNDMeshStreamer ckmain modulecompletion) + add_dependencies(charmd ck converse moduleNDMeshStreamer ckmain modulecompletion) + add_dependencies(charmd_faceless ck converse moduleNDMeshStreamer ckmain modulecompletion) +else() + add_dependencies(charmrun ck converse reconverse moduleNDMeshStreamer ckmain modulecompletion conv-static) + add_dependencies(charmd ck converse moduleNDMeshStreamer ckmain modulecompletion conv-static) + add_dependencies(charmd_faceless ck converse moduleNDMeshStreamer ckmain modulecompletion conv-static) +endif() diff --git a/src/util/charmrun-src/charmrun.C b/src/util/charmrun-src/charmrun.C index 9403096137..d199014559 100644 --- a/src/util/charmrun-src/charmrun.C +++ b/src/util/charmrun-src/charmrun.C @@ -1,3 +1,5 @@ +#include "conv-autoconfig.h" +#include "conv-mach.h" #include "converse.h" #include "sockRoutines.h" diff --git a/src/util/sockRoutines.C b/src/util/sockRoutines.C index f800bde302..e05739c333 100644 --- a/src/util/sockRoutines.C +++ b/src/util/sockRoutines.C @@ -1,3 +1,4 @@ +#include "conv-autoconfig.h" #include "sockRoutines.h" #ifndef CMK_NO_SOCKETS /*<- for ASCI Red*/ diff --git a/src/xlat-i/xi-util.h b/src/xlat-i/xi-util.h index 3df91b2152..a9eec89a5d 100644 --- a/src/xlat-i/xi-util.h +++ b/src/xlat-i/xi-util.h @@ -7,7 +7,9 @@ #include #ifndef XI_LIBRARY -#include "conv-config.h" +//#include "conv-config.h" +#include "charm-config.h" +#include "converse.h" #endif #include #include