Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 20 additions & 8 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ else()
endif()

# Build options
set(NETWORK "netlrts" CACHE STRING "Target network layer, can be one of mpi,multicore,netlrts,gni,ofi,pami,pamilrts,ucx,verbs")
set(NETWORK "netlrts" CACHE STRING "Target network layer, can be one of mpi,multicore,netlrts,reconverse,gni,ofi,pami,pamilrts,ucx,verbs")
set(TARGET "charm++" CACHE STRING "Target build type, can be one of charm++,AMPI,LIBS,charm4py,ChaNGa,everylb,all-test")
set(ARCH "" CACHE STRING "Target architecture, can be one of i386,x86_64,arm7,arm8,ppc64le") # By default, detect arch we are running on.

Expand Down Expand Up @@ -266,6 +266,10 @@ else()
set(CMK_TASKQUEUE 0)
endif()

if(${NETWORK} STREQUAL "reconverse")
set(RECONVERSE ON)
endif()

if(${RECONVERSE})
set(CMK_RECONVERSE 1)
else()
Expand All @@ -290,8 +294,10 @@ else()
set(CMK_SMP 0)
endif()

if(${NETWORK} STREQUAL "multicore")
if(${NETWORK} STREQUAL "multicore" OR ${NETWORK} STREQUAL "reconverse")
set(CMK_SMP 1)
endif()
if(${NETWORK} STREQUAL "multicore")
set(CMK_MULTICORE 1)
else()
set(CMK_MULTICORE 0)
Expand Down Expand Up @@ -327,7 +333,7 @@ IF(JPEG_FOUND)
set(CMK_USE_LIBJPEG 1)
ENDIF()

if(NETWORK STREQUAL "pami")
if(NETWORK STREQUAL "pami" OR NETWORK STREQUAL "reconverse")
set(CMK_USE_LRTS 0)
else()
set(CMK_USE_LRTS 1)
Expand Down Expand Up @@ -786,7 +792,9 @@ if(CMK_COMPILER STREQUAL "msvc")
endif()

# Header files
configure_file(src/arch/${GDIR}/conv-common.h include/ COPYONLY)
if(EXISTS src/arch/${GDIR}/conv-common.h)
configure_file(src/arch/${GDIR}/conv-common.h include/ COPYONLY)
endif()
if(EXISTS src/arch/${GDIR}/conv-common.sh)
configure_file(src/arch/${GDIR}/conv-common.sh include/ COPYONLY)
endif()
Expand Down Expand Up @@ -880,7 +888,9 @@ include(cmake/hwloc.cmake)

# Converse
include(cmake/converse.cmake)
add_subdirectory(src/QuickThreads)
if(NOT RECONVERSE)
add_subdirectory(src/QuickThreads)
endif()
# add_subdirectory(src/util/boost-context)
# add_subdirectory(src/conv-core)

Expand Down Expand Up @@ -1019,9 +1029,11 @@ else()
if(RECONVERSE)
target_link_libraries(ckhello PRIVATE reconverse)
endif()
add_dependencies(ckhello ck ckqt conv-static
converse ckmain
moduleNDMeshStreamer modulecompletion)
set(ckhello_deps ck converse ckmain moduleNDMeshStreamer modulecompletion)
if(NOT RECONVERSE)
list(APPEND ckhello_deps ckqt conv-static)
endif()
add_dependencies(ckhello ${ckhello_deps})
endif()

# Create conv-mach-opt.sh
Expand Down
95 changes: 82 additions & 13 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,19 +92,21 @@ of the `<options>` below.

(Note: this isn't a complete list. Run `./build` for a complete listing)

| Charm++ Version | OS | Communication | Default Compiler |
|---------------------------|---------|---------------|---------------------------------------|
| `netlrts-linux-x86_64` | Linux | UDP | GNU compiler |
| `netlrts-darwin-x86_64` | macOS | UDP | Clang C++ compiler |
| `netlrts-win-x86_64` | Windows | UDP | MS Visual C++ |
| `mpi-linux-x86_64` | Linux | MPI | GNU compiler |
| `multicore-linux-x86_64` | Linux | Shared memory | GNU compiler |
| `multicore-darwin-x86_64` | macOS | Shared memory | Clang C++ compiler |
| `gni-crayxc` | Linux | GNI | CC (whatever PrgEnv module is loaded) |
| `gni-crayxe` | Linux | GNI | CC (whatever PrgEnv module is loaded) |
| `verbs-linux-x86_64` | Linux | IB Verbs | GNU compiler |
| `ofi-linux-x86_64` | Linux | OFI | GNU compiler |
| `ucx-linux-x86_64` | Linux | UCX | GNU compiler |
| Charm++ Version | OS | Communication | Default Compiler |
|-------------------------------|---------|----------------|---------------------------------------|
| `netlrts-linux-x86_64` | Linux | UDP | GNU compiler |
| `netlrts-darwin-x86_64` | macOS | UDP | Clang C++ compiler |
| `netlrts-win-x86_64` | Windows | UDP | MS Visual C++ |
| `mpi-linux-x86_64` | Linux | MPI | GNU compiler |
| `multicore-linux-x86_64` | Linux | Shared memory | GNU compiler |
| `multicore-darwin-x86_64` | macOS | Shared memory | Clang C++ compiler |
| `reconverse-linux-x86_64` | Linux | LCI/LCW | GNU compiler |
| `reconverse-darwin-arm8` | macOS | LCI/LCW | Clang C++ compiler |
| `gni-crayxc` | Linux | GNI | CC (whatever PrgEnv module is loaded) |
| `gni-crayxe` | Linux | GNI | CC (whatever PrgEnv module is loaded) |
| `verbs-linux-x86_64` | Linux | IB Verbs | GNU compiler |
| `ofi-linux-x86_64` | Linux | OFI | GNU compiler |
| `ucx-linux-x86_64` | Linux | UCX | GNU compiler |


To choose `<version>`, your choice is determined by two options:
Expand All @@ -119,6 +121,12 @@ To choose `<version>`, your choice is determined by two options:
but performance is often worse than using the machine's direct calls referenced above.
* `multicore-`: Charm++ communicates using shared memory within a single node. A version of
Charm++ built with this option will not run on more than a single node.
* `reconverse-`: Charm++ uses the Reconverse communication layer backed by LCI or LCW.
Reconverse is an alternative to the standard Converse layer and is always built in SMP mode
(there is no separate communication thread; LCI handles progress internally). It supports
single-node runs with no backend, multi-node runs via LCI (recommended for performance),
or multi-node runs via LCW (an MPI-based fallback). See
[Building with Reconverse](#building-with-reconverse) below.


2. Your operating system/architecture:
Expand Down Expand Up @@ -182,6 +190,67 @@ for more information:
Supported options: common cuda flang gfortran ifort local nolb omp ooc papi perftools persistent pgf90 pxshm smp syncft sysvshm tcp tsan


## Building with Reconverse

The `reconverse-` triplets are always SMP (no `-smp` suffix needed) and always
enable the Reconverse communication layer. They are built with the standard
`./build` script like any other version.

### Quick start

On Linux x86\_64:

$ ./build charm++ reconverse-linux-x86_64

On macOS ARM64:

$ ./build charm++ reconverse-darwin-arm8

This fetches Reconverse (and optionally LCI) from their upstream repositories
and builds everything in a single step. The resulting build directory is named
after the triplet (e.g. `reconverse-linux-x86_64/`).

### Communication backends

Reconverse supports three backends, selected at runtime:

* **None** (default for single-process runs): no inter-node communication library
is required. This is suitable for development and testing on a single machine.
* **LCI** (recommended for multi-node): a lightweight communication interface
designed for low-latency active-message communication. LCI is fetched and
built automatically unless a pre-installed copy is available.
* **LCW**: an MPI-based wrapper backend. Useful as a fallback on systems where
LCI is not available or supported.

At runtime, the backend is selected automatically based on what was compiled in.
To force a specific backend, pass `+backend lci` or `+backend lcw` on the
command line.

### Fetching LCI automatically

LCI is fetched by default for reconverse triplets. To use a pre-installed copy
of LCI instead, set `LCI_ROOT` in your environment before building:

$ LCI_ROOT=/path/to/lci ./build charm++ reconverse-linux-x86_64

To fetch a specific Reconverse tag or use a local checkout:

$ ./build charm++ reconverse-linux-x86_64 --with-fetch-reconverse-tag=v1.2.3
$ ./build charm++ reconverse-linux-x86_64 --with-fetch-reconverse-dir=/path/to/reconverse

### Build options

The same options available for other versions work with reconverse builds:

* `-j<N>` — parallel build with N jobs (e.g. `-j8`)
* `--with-production` — enable production optimizations
* `--enable-tracing` — enable Projections performance tracing
* `--destination=<dir>` — place the build in a custom directory

For example:

$ ./build charm++ reconverse-linux-x86_64 -j8 --with-production

## Building the Source

If you have downloaded a binary version of Charm++, you can skip
Expand Down
8 changes: 8 additions & 0 deletions buildcmake
Original file line number Diff line number Diff line change
Expand Up @@ -608,6 +608,14 @@ fi

my_os=$(echo "$actual_triplet" | cut -d '-' -f2)

# reconverse network type implies reconverse communication layer and inherent SMP.
# CMK_SMP is forced to 1 by CMakeLists.txt for reconverse; we do not pass -DSMP=1
# here because that would trigger the opt-file loop to source conv-mach-smp.sh,
# which does not exist for reconverse (SMP flags are baked into conv-mach.h directly).
if [[ $opt_network == "reconverse" ]]; then
opt_reconverse=1
fi

if [[ $opt_network == "mpi" && $my_os == "win" ]]; then
echo "Warning: CMake build is not supported with MPI on Windows, running ./buildold instead."
sleep 2
Expand Down
20 changes: 11 additions & 9 deletions cmake/converse.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -279,12 +279,14 @@ endforeach()
# target_include_directories(converse PRIVATE src/util) # for sockRoutines.C
# target_include_directories(converse PRIVATE src/conv-core src/util/topomanager src/ck-ldb src/ck-perf src/ck-cp)

# conv-static
add_library(conv-static OBJECT src/conv-core/conv-static.c)
add_dependencies(reconverse conv-static)
add_dependencies(charm_cxx_utils conv-static)
add_custom_command(TARGET charm_cxx_utils
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/conv-static.dir/src/conv-core/conv-static.c.o ${CMAKE_BINARY_DIR}/lib/conv-static.o
VERBATIM
)
# conv-static: not needed for reconverse builds (reconverse provides its own converse layer)
if(NOT RECONVERSE)
add_library(conv-static OBJECT src/conv-core/conv-static.c)
add_dependencies(reconverse conv-static)
add_dependencies(charm_cxx_utils conv-static)
add_custom_command(TARGET charm_cxx_utils
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/conv-static.dir/src/conv-core/conv-static.c.o ${CMAKE_BINARY_DIR}/lib/conv-static.o
VERBATIM
)
endif()
68 changes: 68 additions & 0 deletions src/arch/reconverse-darwin-arm8/conv-mach.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#ifndef _CONV_MACH_H
#define _CONV_MACH_H

#define CMK_ARM 1
#define CMK_64BIT 1
#define CMK_SMP 1

#undef CMK_IMMEDIATE_MSG
#define CMK_IMMEDIATE_MSG 0

#define CMK_ASYNC_NOT_NEEDED 1
#define CMK_ASYNC_USE_FIOASYNC_AND_FIOSETOWN 0
#define CMK_ASYNC_USE_FIOASYNC_AND_SIOCSPGRP 0
#define CMK_ASYNC_USE_FIOSSAIOSTAT_AND_FIOSSAIOOWN 0
#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN 1

#define CMK_GETPAGESIZE_AVAILABLE 0

#define CMK_MALLOC_USE_GNU_MALLOC 0
#define CMK_MALLOC_USE_OS_BUILTIN 1
#define CMK_MALLOC_USE_GNUOLD_MALLOC 0

#undef CMK_MEMORY_BUILD_GNU_HOOKS
#define CMK_MEMORY_BUILD_GNU_HOOKS 0
#define CMK_MEMORY_PAGESIZE 4096
#define CMK_MEMORY_PROTECTABLE 0


#define CMK_SSH_IS_A_COMMAND 1
#define CMK_SSH_NOT_NEEDED 0

#define CMK_SHARED_VARS_UNAVAILABLE 0
#define CMK_SHARED_VARS_POSIX_THREADS_SMP 1

#define CMK_THREADS_USE_CONTEXT 0
#define CMK_THREADS_USE_JCONTEXT 0
#define CMK_THREADS_USE_FCONTEXT 1
#define CMK_THREADS_USE_PTHREADS 0
#define CMK_THREADS_ARE_WIN32_FIBERS 0

#define CMK_SIGNAL_NOT_NEEDED 1
#define CMK_SIGNAL_USE_SIGACTION 0
#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART 0

#define CMK_THREADS_REQUIRE_NO_CPV 0
#define CMK_THREADS_COPY_STACK 0

#define CMK_TIMER_USE_GETRUSAGE 1
#define CMK_TIMER_USE_SPECIAL 0
#define CMK_TIMER_USE_TIMES 0


#define CMK_DEBUG_MODE 0
#define CMK_WEB_MODE 1

#define CMK_LBDB_ON 1

#define CMK_STACKSIZE_DEFAULT 65536

#define CMK_USE_KQUEUE 1

#define CMK_NOT_USE_TLS_THREAD 1

/* #define CMK_PCQUEUE_LOCK 1 */

#define CMK_CONVERSE_MPI 0

#endif
10 changes: 10 additions & 0 deletions src/arch/reconverse-darwin-arm8/conv-mach.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
. $CHARMINC/cc-clang.sh
. $CHARMINC/conv-mach-darwin.sh

CMK_DEFS="$CMK_DEFS -D_REENTRANT"

# Reconverse provides its own threading; remove the QuickThreads library
# injected by conv-mach-darwin.sh
CMK_LIBS="${CMK_LIBS//-lckqt/}"

CMK_SMP="1"
62 changes: 62 additions & 0 deletions src/arch/reconverse-linux-x86_64/conv-mach.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#ifndef _CONV_MACH_H
#define _CONV_MACH_H

#define CMK_AMD64 1
#define CMK_64BIT 1
#define CMK_SMP 1

#undef CMK_IMMEDIATE_MSG
#define CMK_IMMEDIATE_MSG 0

#define CMK_ASYNC_NOT_NEEDED 1
#define CMK_ASYNC_USE_FIOASYNC_AND_FIOSETOWN 0
#define CMK_ASYNC_USE_FIOASYNC_AND_SIOCSPGRP 0
#define CMK_ASYNC_USE_FIOSSAIOSTAT_AND_FIOSSAIOOWN 0
#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN 0

#define CMK_DLL_CC "g++ -shared -O3 -o "

#define CMK_GETPAGESIZE_AVAILABLE 1

#define CMK_MALLOC_USE_GNU_MALLOC 0
#define CMK_MALLOC_USE_OS_BUILTIN 1

#define CMK_MEMORY_PAGESIZE 4096
#define CMK_MEMORY_PROTECTABLE 0


#define CMK_SSH_IS_A_COMMAND 1
#define CMK_SSH_NOT_NEEDED 0

#define CMK_SHARED_VARS_UNAVAILABLE 0
#define CMK_SHARED_VARS_POSIX_THREADS_SMP 1

#define CMK_THREADS_USE_CONTEXT 0
#define CMK_THREADS_USE_FCONTEXT 1
#define CMK_THREADS_USE_PTHREADS 0
#define CMK_THREADS_ARE_WIN32_FIBERS 0

#define CMK_SIGNAL_NOT_NEEDED 1
#define CMK_SIGNAL_USE_SIGACTION 0
#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART 0

#define CMK_THREADS_REQUIRE_NO_CPV 0
#define CMK_THREADS_COPY_STACK 0

#define CMK_TIMER_USE_RDTSC 0
#define CMK_TIMER_USE_GETRUSAGE 1
#define CMK_TIMER_USE_SPECIAL 0
#define CMK_TIMER_USE_TIMES 0


#define CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT 0
#define CMK_WHEN_PROCESSOR_IDLE_USLEEP 1

#define CMK_DEBUG_MODE 0
#define CMK_WEB_MODE 1

#define CMK_LBDB_ON 1

#define CMK_CONVERSE_MPI 0

#endif
10 changes: 10 additions & 0 deletions src/arch/reconverse-linux-x86_64/conv-mach.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
. $CHARMINC/cc-gcc.sh

CMK_DEFS="$CMK_DEFS -D_REENTRANT"

CMK_XIOPTS=''
CMK_LIBS="-lpthread $CMK_LIBS"

CMK_QT='generic64-light'

CMK_SMP='1'
4 changes: 3 additions & 1 deletion src/libs/ck-libs/metis/GKlib/GKlib.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@
#ifndef _GKLIB_H_
#define _GKLIB_H_ 1

#include "conv-config.h"
//#include "conv-config.h"
#include "charm-config.h"
#include "converse.h"

#define GKMSPACE

Expand Down
Loading
Loading