Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 48 additions & 17 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,16 +1,36 @@
cmake_minimum_required (VERSION 3.10)
cmake_minimum_required(VERSION 3.15)

project(RaBitQLib LANGUAGES CXX)

set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)

option(RABITQ_BUILD_SAMPLES "Build C++ sample programs" ON)
option(RABITQ_BUILD_TESTS "Build C++ tests" OFF)
option(RABITQ_BUILD_PYTHON_BINDINGS "Build Python bindings" OFF)
option(
RABITQ_ENABLE_NATIVE_OPTIMIZATION
"Optimize generic code for the build machine"
ON
)

include_directories(${PROJECT_SOURCE_DIR}/include)
find_package(OpenMP REQUIRED)

set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
add_library(rabitq_compile_options INTERFACE)
target_compile_features(rabitq_compile_options INTERFACE cxx_std_17)
target_compile_options(rabitq_compile_options INTERFACE
$<$<AND:$<CONFIG:Release>,$<COMPILE_LANG_AND_ID:CXX,Clang,GNU>>:-Ofast>
$<$<AND:$<BOOL:${RABITQ_ENABLE_NATIVE_OPTIMIZATION}>,$<COMPILE_LANG_AND_ID:CXX,Clang,GNU>>:-march=native>
)
target_link_libraries(rabitq_compile_options INTERFACE OpenMP::OpenMP_CXX)

SET(CMAKE_CXX_FLAGS "-Wall -Ofast -Wextra -lrt -march=native -fpic -fopenmp -ftree-vectorize -fexceptions")
function(rabitq_enable_warnings target)
target_compile_options(${target} PRIVATE
"$<$<COMPILE_LANG_AND_ID:CXX,Clang,GNU>:-Wall;-Wextra>"
)
endfunction()

set(RABITQ_COMMON_SOURCES
src/utils/cpu_features.cpp
Expand Down Expand Up @@ -61,30 +81,41 @@ add_library(rabitq_core STATIC
)

target_include_directories(rabitq_core PUBLIC ${PROJECT_SOURCE_DIR}/include)
target_link_libraries(rabitq_core PUBLIC rabitq_compile_options)
set_target_properties(rabitq_core PROPERTIES POSITION_INDEPENDENT_CODE ON)
target_compile_options(rabitq_core PRIVATE -fopenmp)
set_source_files_properties(${RABITQ_AVX2_SOURCES} PROPERTIES COMPILE_FLAGS "-mavx2 -mfma")
set_source_files_properties(${RABITQ_AVX512_SOURCES} PROPERTIES COMPILE_FLAGS "-mavx512f -mavx512bw -mavx512dq -mfma")
set_source_files_properties(${RABITQ_AVX512_POPCNT_SOURCES} PROPERTIES COMPILE_FLAGS "-mavx512f -mavx512bw -mavx512dq -mavx512vpopcntdq -mfma")
set_source_files_properties(${RABITQ_HNSW_AVX2_SOURCES} PROPERTIES COMPILE_FLAGS "-mavx2 -mfma")
set_source_files_properties(${RABITQ_HNSW_AVX512_CORE_SOURCES} PROPERTIES COMPILE_FLAGS "-mavx512f -mavx512bw -mavx512dq -mavx2 -mfma")
set_source_files_properties(${RABITQ_HNSW_AVX512_POPCNT_SOURCES} PROPERTIES COMPILE_FLAGS "-mavx512f -mavx512bw -mavx512dq -mavx512vpopcntdq -mfma")
rabitq_enable_warnings(rabitq_core)

set_source_files_properties(${RABITQ_AVX2_SOURCES} PROPERTIES
COMPILE_OPTIONS "-mavx2;-mfma"
)
set_source_files_properties(${RABITQ_AVX512_SOURCES} PROPERTIES
COMPILE_OPTIONS "-mavx512f;-mavx512bw;-mavx512dq;-mfma"
)
set_source_files_properties(${RABITQ_AVX512_POPCNT_SOURCES} PROPERTIES
COMPILE_OPTIONS "-mavx512f;-mavx512bw;-mavx512dq;-mavx512vpopcntdq;-mfma"
)
set_source_files_properties(${RABITQ_HNSW_AVX2_SOURCES} PROPERTIES
COMPILE_OPTIONS "-mavx2;-mfma"
)
set_source_files_properties(${RABITQ_HNSW_AVX512_CORE_SOURCES} PROPERTIES
COMPILE_OPTIONS "-mavx512f;-mavx512bw;-mavx512dq;-mavx2;-mfma"
)
set_source_files_properties(${RABITQ_HNSW_AVX512_POPCNT_SOURCES} PROPERTIES
COMPILE_OPTIONS "-mavx512f;-mavx512bw;-mavx512dq;-mavx512vpopcntdq;-mfma"
)

add_library(rabitq_headers INTERFACE)
target_include_directories(rabitq_headers INTERFACE ${PROJECT_SOURCE_DIR}/include)
target_link_libraries(rabitq_headers INTERFACE rabitq_core)

add_subdirectory(sample/cpp)

option(RABITQ_BUILD_PYTHON_BINDINGS "Build Python bindings" OFF)
if(RABITQ_BUILD_SAMPLES)
add_subdirectory(sample/cpp)
endif()

if(RABITQ_BUILD_PYTHON_BINDINGS)
add_subdirectory(python_bindings)
endif()

# Testing
option(RABITQ_BUILD_TESTS "Build tests" OFF)

if(RABITQ_BUILD_TESTS)
enable_testing()
add_subdirectory(tests)
Expand Down
7 changes: 6 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ persistence.

#### Requirements

- CMake 3.10 or newer
- CMake 3.15 or newer
- a C++17 compiler with OpenMP support
- an x86-64 CPU supported by the selected kernels: most paths accept either
AVX2 with FMA or AVX-512F/BW/DQ with FMA
Expand All @@ -111,6 +111,11 @@ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
cmake --build build --parallel
```

Release builds enable native CPU tuning by default. To build a binary that can
be moved between AVX2- and AVX-512-capable machines, configure with
`-DRABITQ_ENABLE_NATIVE_OPTIMIZATION=OFF`; the ISA-specific kernels will still
be selected at runtime.

The index example executables are written to `bin/`. Their source code shows
the complete indexing and querying workflows:

Expand Down
7 changes: 6 additions & 1 deletion docs/docs/quick_start.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ a C++17 API for both indexes and low-level quantization.
AVX2 with FMA or AVX-512F/BW/DQ with FMA
- Python 3.9 or newer for the Python package
- A C++17 compiler with OpenMP support
- CMake 3.15 or newer for Python builds, or CMake 3.10 or newer for C++ builds
- CMake 3.15 or newer

Most SIMD entry points select AVX-512 kernels when AVX-512F, AVX-512BW, and
AVX-512DQ are detected; otherwise they use AVX2 when AVX2 and FMA are
Expand Down Expand Up @@ -92,6 +92,11 @@ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
cmake --build build --parallel
```

Release builds enable native CPU tuning by default. To build a binary that can
be moved between AVX2- and AVX-512-capable machines, configure with
`-DRABITQ_ENABLE_NATIVE_OPTIMIZATION=OFF`; the ISA-specific kernels will still
be selected at runtime.

Example executables are written to `bin/`. Their source demonstrates complete
indexing and querying workflows:

Expand Down
56 changes: 11 additions & 45 deletions python_bindings/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,67 +1,33 @@
cmake_minimum_required(VERSION 3.15)

# 1. Tell pybind11 to use the modern FindPython module.
set(PYBIND11_FINDPYTHON ON)
if(PYTHON_EXECUTABLE AND NOT Python_EXECUTABLE)
set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}")
endif()

# 2. Locate pybind11 matching the pip environment
# 2. Locate Python, NumPy, and pybind11 from the active Python environment.
find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy)
find_package(pybind11 CONFIG REQUIRED)

# 3. Python executable is now reliably in Python_EXECUTABLE (new-style finder)
set(_PY_EXEC "${Python_EXECUTABLE}")

# 4. Grab the NumPy and pybind11 include paths from the active Python environment
execute_process(
COMMAND "${_PY_EXEC}" "-c" "import numpy; print(numpy.get_include())"
OUTPUT_VARIABLE NUMPY_INCLUDE_DIR
OUTPUT_STRIP_TRAILING_WHITESPACE
)
execute_process(
COMMAND "${_PY_EXEC}" "-c" "import pybind11; print(pybind11.get_include())"
OUTPUT_VARIABLE PYBIND11_INCLUDE_DIR
OUTPUT_STRIP_TRAILING_WHITESPACE
)
execute_process(
COMMAND "${_PY_EXEC}" "-c"
"import sysconfig; print(sysconfig.get_path('include'))"
OUTPUT_VARIABLE PYTHON_INCLUDE_DIR_FROM_EXEC
OUTPUT_STRIP_TRAILING_WHITESPACE
)
message(STATUS "Using Python: ${_PY_EXEC}")
message(STATUS "pybind11 include: ${PYBIND11_INCLUDE_DIR}")
message(STATUS "Python include: ${PYTHON_INCLUDE_DIR_FROM_EXEC}")
message(STATUS "NumPy include: ${NUMPY_INCLUDE_DIR}")

# 5. Find OpenMP to ensure parallel batched searches compile correctly
find_package(OpenMP REQUIRED)

# 6. Define the shared pybind11 module
# 3. Define the shared pybind11 module.
pybind11_add_module(_rabitqlib
rabitq_bindings.cpp
hnsw_bindings.cpp
ivf_bindings.cpp
symqg_bindings.cpp
)

# 7. Explicitly add include paths so pip-installed pybind11 takes precedence
# over any system-installed pybind11 (e.g. apt pybind11 2.9.x vs pip 2.13.x)
target_include_directories(_rabitqlib BEFORE PRIVATE
${PYBIND11_INCLUDE_DIR} # pip pybind11 (must beat /usr/include/pybind11)
${PYTHON_INCLUDE_DIR_FROM_EXEC} # Python from the active interpreter
${NUMPY_INCLUDE_DIR}
${CMAKE_CURRENT_SOURCE_DIR} # For bindings_common.hpp
${PROJECT_SOURCE_DIR}/include # For rabitqlib core headers
target_include_directories(_rabitqlib PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}
)
rabitq_enable_warnings(_rabitqlib)

# 8. Link OpenMP flags natively
# 4. Imported targets provide the matching Python, NumPy, and OpenMP settings.
target_link_libraries(_rabitqlib PRIVATE
rabitq_core
OpenMP::OpenMP_CXX
rabitq_headers
Python::NumPy
)

# 9. Map files to a 'rabitqlib' directory
# 5. Map files to a 'rabitqlib' directory
# inside the target wheel, completely bypassing your local layout.
install(TARGETS _rabitqlib DESTINATION rabitqlib)
install(FILES __init__.py DESTINATION rabitqlib)
install(FILES __init__.py DESTINATION rabitqlib)
4 changes: 2 additions & 2 deletions sample/cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin)

add_executable(symqg_indexing symqg_indexing.cpp)
add_executable(symqg_querying symqg_querying.cpp)
Expand All @@ -18,5 +18,5 @@ foreach(RABITQ_SAMPLE_TARGET
hnsw_rabitq_querying
)
target_link_libraries(${RABITQ_SAMPLE_TARGET} PRIVATE rabitq_headers)
target_compile_options(${RABITQ_SAMPLE_TARGET} PRIVATE -march=native)
rabitq_enable_warnings(${RABITQ_SAMPLE_TARGET})
endforeach()
34 changes: 15 additions & 19 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
cmake_minimum_required(VERSION 3.10)

# Fetch Google Test
include(FetchContent)
FetchContent_Declare(
Expand All @@ -10,33 +8,31 @@ FetchContent_Declare(
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
FetchContent_MakeAvailable(googletest)

enable_testing()

# Include directories
include_directories(${PROJECT_SOURCE_DIR}/include)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/common)

# Compiler flags
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra")

# Automatically register all tests under unit and <other folders> folders
file(GLOB_RECURSE UNIT_TESTS ${CMAKE_CURRENT_SOURCE_DIR}/unit/*_test.cpp)
file(GLOB_RECURSE INTEG_TESTS ${CMAKE_CURRENT_SOURCE_DIR}/integration/*_test.cpp)
file(GLOB_RECURSE COMMON_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/common/*.cpp)
file(GLOB_RECURSE UNIT_TESTS CONFIGURE_DEPENDS
${CMAKE_CURRENT_SOURCE_DIR}/unit/*_test.cpp
)
file(GLOB_RECURSE INTEG_TESTS CONFIGURE_DEPENDS
${CMAKE_CURRENT_SOURCE_DIR}/integration/*_test.cpp
)
file(GLOB_RECURSE COMMON_SRCS CONFIGURE_DEPENDS
${CMAKE_CURRENT_SOURCE_DIR}/common/*.cpp
)

# add executables
add_executable(rabitq_tests
${COMMON_SRCS}
${UNIT_TESTS}
${INTEG_TESTS}
)
target_include_directories(rabitq_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/common)
rabitq_enable_warnings(rabitq_tests)

# Link google test and rabitqlib headers
target_link_libraries(rabitq_tests
gtest
gtest_main
rabitq_headers
pthread
PRIVATE
GTest::gtest_main
rabitq_headers
)

# Discover tests for CTest
Expand All @@ -48,4 +44,4 @@ message(STATUS "Discovered test files:")
foreach(TEST_SRC ${UNIT_TESTS} ${INTEG_TESTS})
file(RELATIVE_PATH REL_PATH ${CMAKE_CURRENT_SOURCE_DIR} ${TEST_SRC})
message(STATUS " - ${REL_PATH}")
endforeach()
endforeach()
13 changes: 10 additions & 3 deletions tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ binding tests for RaBitQ Library.

## Prerequisites

- CMake 3.10 or newer
- CMake 3.15 or newer
- A GCC- or Clang-compatible C++17 compiler with OpenMP support
- An x86-64 CPU supported by RaBitQ's AVX2 or AVX-512 runtime dispatch
- Network access during the first configuration so CMake can download
Expand Down Expand Up @@ -35,12 +35,19 @@ ctest --test-dir build --output-on-failure

The combined test executable is also available as `build/tests/rabitq_tests`.

Release builds use native CPU tuning by default. Pass
`-DRABITQ_ENABLE_NATIVE_OPTIMIZATION=OFF` when the resulting test binary must
run on a different AVX2- or AVX-512-capable machine.

### Building without Tests

By default, tests are **not built**. If you want to build only the library:
By default, tests are **not built**, while the C++ samples are. To build only
the library:

```bash
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
cmake -S . -B build \
-DRABITQ_BUILD_SAMPLES=OFF \
-DCMAKE_BUILD_TYPE=Release
cmake --build build --parallel
```

Expand Down
Loading