diff --git a/diskann-benchmark/src/backend/exhaustive/product.rs b/diskann-benchmark/src/backend/exhaustive/product.rs index 43f55a8ff..a5d4d2bd2 100644 --- a/diskann-benchmark/src/backend/exhaustive/product.rs +++ b/diskann-benchmark/src/backend/exhaustive/product.rs @@ -85,10 +85,10 @@ mod imp { 5, ); - let offsets = diskann_providers::model::pq::calculate_chunk_offsets_auto( - data.ncols(), - input.num_pq_chunks.get(), - ); + let dim = std::num::NonZeroUsize::new(data.ncols()) + .ok_or_else(|| anyhow::anyhow!("data has zero columns"))?; + let offsets = + diskann_quantization::views::ChunkOffsets::partition(dim, input.num_pq_chunks)?; let base = { let threadpool = rayon::ThreadPoolBuilder::new() @@ -97,7 +97,7 @@ mod imp { threadpool.install(|| -> anyhow::Result<_> { Ok(parameters.train( data.as_view(), - diskann_quantization::views::ChunkOffsetsView::new(offsets.as_slice())?, + offsets.as_view(), diskann_quantization::Parallelism::Rayon, &diskann_quantization::random::StdRngBuilder::new(input.seed), &diskann_quantization::cancel::DontCancel, @@ -109,7 +109,7 @@ mod imp { data.ncols(), base.flatten().into(), vec![0.0; data.ncols()].into(), - offsets.into(), + offsets.as_slice().into(), )?; let training_time: MicroSeconds = start.elapsed().into(); diff --git a/diskann-providers/src/model/graph/provider/async_/bf_tree/quant_vector_provider.rs b/diskann-providers/src/model/graph/provider/async_/bf_tree/quant_vector_provider.rs index c3c08bb58..edcb593d8 100644 --- a/diskann-providers/src/model/graph/provider/async_/bf_tree/quant_vector_provider.rs +++ b/diskann-providers/src/model/graph/provider/async_/bf_tree/quant_vector_provider.rs @@ -353,7 +353,7 @@ mod tests { let c = provider.query_computer(&[-0.5, -0.5]).unwrap(); let expected: f32 = 1.5 * 1.5 * 2.0; assert_eq!( - c.evaluate_similarity(&provider.get_vector_sync(3).unwrap()), + c.evaluate_similarity(provider.get_vector_sync(3).unwrap().as_slice()), expected ); @@ -362,14 +362,14 @@ mod tests { assert_eq!( d.evaluate_similarity( provider.get_vector_sync(0).unwrap().as_slice(), - provider.get_vector_sync(3).unwrap().as_slice(), + provider.get_vector_sync(3).unwrap().as_slice() ), 2.0 ); let slice: &[f32] = &[-0.5, -0.5]; assert_eq!( - d.evaluate_similarity(slice, &provider.get_vector_sync(3).unwrap()), + d.evaluate_similarity(slice, provider.get_vector_sync(3).unwrap().as_slice()), expected, ); } diff --git a/diskann-providers/src/model/graph/provider/async_/fast_memory_quant_vector_provider.rs b/diskann-providers/src/model/graph/provider/async_/fast_memory_quant_vector_provider.rs index 63aad4566..776e5229f 100644 --- a/diskann-providers/src/model/graph/provider/async_/fast_memory_quant_vector_provider.rs +++ b/diskann-providers/src/model/graph/provider/async_/fast_memory_quant_vector_provider.rs @@ -444,10 +444,7 @@ mod tests { // Query Computer. let c = provider.query_computer(&[-0.5, -0.5]).unwrap(); let expected: f32 = 1.5 * 1.5 * 2.0; - assert_eq!( - c.evaluate_similarity(&provider.get_vector_sync(3)), - expected - ); + assert_eq!(c.evaluate_similarity(provider.get_vector_sync(3)), expected); // Distance Computer. let d = provider.distance_computer(); diff --git a/diskann-providers/src/model/mod.rs b/diskann-providers/src/model/mod.rs index f6ae2be75..61addc05a 100644 --- a/diskann-providers/src/model/mod.rs +++ b/diskann-providers/src/model/mod.rs @@ -11,10 +11,10 @@ pub use configuration::IndexConfiguration; pub mod pq; pub use pq::{ FixedChunkPQTable, GeneratePivotArguments, MAX_PQ_TRAINING_SET_SIZE, NUM_KMEANS_REPS_PQ, - NUM_PQ_CENTROIDS, accum_row_inplace, calculate_chunk_offsets_auto, compute_pq_distance, - compute_pq_distance_for_pq_coordinates, direct_distance_impl, distance, - generate_pq_data_from_pivots_from_membuf, generate_pq_data_from_pivots_from_membuf_batch, - generate_pq_pivots, generate_pq_pivots_from_membuf, + NUM_PQ_CENTROIDS, compute_pq_distance, compute_pq_distance_for_pq_coordinates, + direct_distance_impl, distance, generate_pq_data_from_pivots_from_membuf, + generate_pq_data_from_pivots_from_membuf_batch, generate_pq_pivots, + generate_pq_pivots_from_membuf, }; pub mod statistics; diff --git a/diskann-providers/src/model/pq/distance/dynamic.rs b/diskann-providers/src/model/pq/distance/dynamic.rs index c41955efb..352912846 100644 --- a/diskann-providers/src/model/pq/distance/dynamic.rs +++ b/diskann-providers/src/model/pq/distance/dynamic.rs @@ -101,25 +101,6 @@ where } } -impl PreprocessedDistanceFunction<&Vec, f32> for QueryComputer -where - T: Deref, -{ - fn evaluate_similarity(&self, changing: &Vec) -> f32 { - self.evaluate_similarity(changing.as_slice()) - } -} - -impl PreprocessedDistanceFunction<&&[u8], f32> for QueryComputer -where - T: Deref, -{ - fn evaluate_similarity(&self, changing: &&[u8]) -> f32 { - let changing: &[u8] = changing; - self.evaluate_similarity(changing) - } -} - /// Pre-dispatched distance functions for the `FixedChunkPQTable`. #[derive(Debug)] pub struct VTable { diff --git a/diskann-providers/src/model/pq/distance/test_utils.rs b/diskann-providers/src/model/pq/distance/test_utils.rs index 76c240e1f..d543f6372 100644 --- a/diskann-providers/src/model/pq/distance/test_utils.rs +++ b/diskann-providers/src/model/pq/distance/test_utils.rs @@ -13,7 +13,8 @@ use diskann_vector::{ use rand::{Rng, distr::Distribution}; use rand_distr::{Normal, Uniform}; -use crate::model::{FixedChunkPQTable, pq::calculate_chunk_offsets_auto}; +use crate::model::FixedChunkPQTable; +use diskann_quantization::views::ChunkOffsets; /// We need a way to generate random queries. /// @@ -130,7 +131,12 @@ pub(crate) fn generate_expected_vector( /// * N + 1: The number of PQ Pivots pub(crate) fn seed_pivot_table(config: TableConfig) -> FixedChunkPQTable { // Get the chunk offsets for the selected dimension and bytes. - let offsets = calculate_chunk_offsets_auto(config.dim, config.pq_chunks); + let chunk_offsets = ChunkOffsets::partition( + std::num::NonZeroUsize::new(config.dim).unwrap(), + std::num::NonZeroUsize::new(config.pq_chunks).unwrap(), + ) + .unwrap(); + let offsets = chunk_offsets.as_slice(); // Create the pivot table following the schema described in the docstring. let mut pivots = Vec::::new(); diff --git a/diskann-providers/src/model/pq/mod.rs b/diskann-providers/src/model/pq/mod.rs index 6338e39ec..e3c6f21a0 100644 --- a/diskann-providers/src/model/pq/mod.rs +++ b/diskann-providers/src/model/pq/mod.rs @@ -11,10 +11,9 @@ pub use fixed_chunk_pq_table::{ mod pq_construction; pub use pq_construction::{ MAX_PQ_TRAINING_SET_SIZE, NUM_KMEANS_REPS_PQ, NUM_PQ_CENTROIDS, accum_row_inplace, - calculate_chunk_offsets, calculate_chunk_offsets_auto, generate_pq_data_from_pivots, - generate_pq_data_from_pivots_from_membuf, generate_pq_data_from_pivots_from_membuf_batch, - generate_pq_pivots, generate_pq_pivots_from_membuf, get_chunk_from_training_data, - move_train_data_by_centroid, + generate_pq_data_from_pivots, generate_pq_data_from_pivots_from_membuf, + generate_pq_data_from_pivots_from_membuf_batch, generate_pq_pivots, + generate_pq_pivots_from_membuf, move_train_data_by_centroid, }; /// all metadata of individual sub-component files is written in first 4KB for unified files diff --git a/diskann-providers/src/model/pq/pq_construction.rs b/diskann-providers/src/model/pq/pq_construction.rs index 2862d7e26..e3c29ba02 100644 --- a/diskann-providers/src/model/pq/pq_construction.rs +++ b/diskann-providers/src/model/pq/pq_construction.rs @@ -6,6 +6,7 @@ use std::{ io::{Seek, SeekFrom, Write}, mem::size_of, + num::NonZeroUsize, sync::atomic::AtomicBool, vec, }; @@ -19,6 +20,7 @@ use diskann::{ use diskann_quantization::{ CompressInto, product::{BasicTableView, TransposedTable, train::TrainQuantizer}, + views::{ChunkOffsets, ChunkOffsetsView}, }; use diskann_utils::{ io::Metadata, @@ -94,12 +96,11 @@ where ); } - let mut chunk_offsets: Vec = vec![0; parameters.num_pq_chunks() + 1]; - calculate_chunk_offsets( - parameters.dim(), - parameters.num_pq_chunks(), - &mut chunk_offsets, - ); + let dim = NonZeroUsize::new(parameters.dim()) + .ok_or_else(|| ANNError::log_pq_error("dim must be non-zero"))?; + let num_chunks = NonZeroUsize::new(parameters.num_pq_chunks()) + .ok_or_else(|| ANNError::log_pq_error("num_pq_chunks must be non-zero"))?; + let chunk_offsets = ChunkOffsets::partition(dim, num_chunks).bridge_err()?; let trainer = diskann_quantization::product::train::LightPQTrainingParameters::new( parameters.num_centers(), @@ -111,8 +112,7 @@ where .train( MatrixView::try_from(train_data, parameters.num_train(), parameters.dim()) .bridge_err()?, - diskann_quantization::views::ChunkOffsetsView::new(chunk_offsets.as_slice()) - .bridge_err()?, + chunk_offsets.as_view(), diskann_quantization::Parallelism::Rayon, &random_provider, &diskann_quantization::cancel::DontCancel, @@ -125,7 +125,7 @@ where pq_storage.write_pivot_data( &full_pivot_data, ¢roid, - &chunk_offsets, + chunk_offsets.as_slice(), parameters.num_centers(), parameters.dim(), storage_provider, @@ -202,8 +202,10 @@ pub fn generate_pq_pivots_from_membuf>( } } - // Calculate the chunk offsets - calculate_chunk_offsets(parameters.dim(), parameters.num_pq_chunks(), offsets); + // Calculate the chunk offsets, filling the caller-owned buffer. + let dim = NonZeroUsize::new(parameters.dim()) + .ok_or_else(|| ANNError::log_pq_error("dim must be non-zero"))?; + let chunk_offsets_view = ChunkOffsetsView::partition_into(dim, offsets).bridge_err()?; let trainer = diskann_quantization::product::train::LightPQTrainingParameters::new( parameters.num_centers(), @@ -235,7 +237,7 @@ pub fn generate_pq_pivots_from_membuf>( parameters.dim(), ) .bridge_err()?, - diskann_quantization::views::ChunkOffsetsView::new(offsets).bridge_err()?, + chunk_offsets_view, diskann_quantization::Parallelism::Rayon, &rng_builder, &cancelation, @@ -249,35 +251,6 @@ pub fn generate_pq_pivots_from_membuf>( Ok(()) } -/// Gets all instances of a chunk from the training data for all records in the training data. Each vector in the -/// training dataset is divided into chunks and the PQ algorithm handles each vector chunk individually. This method -/// gets the same chunk from each vector in the training data and creates a new vector out of all of them. -/// -/// # Example -/// See tests for examples -#[inline] -pub fn get_chunk_from_training_data( - train_data: &[f32], - num_train: usize, - raw_vector_dim: usize, - chunk_size: usize, - chunk_offset: usize, -) -> Vec { - let mut result: Vec = vec![0.0; num_train * chunk_size]; - - result - // group empty result data into chunks of chunk_size - .chunks_mut(chunk_size) - .enumerate() - // for each chunk, copy the chunk from the training data into the result vector - .for_each(|(chunk_number, result_chunk)| { - let train_data_start = chunk_number * raw_vector_dim + chunk_offset; - let train_data_end = train_data_start + chunk_size; - result_chunk.copy_from_slice(&train_data[train_data_start..train_data_end]); - }); - result -} - /// Calculates the centroid if needed and moves the train_data to to the centroid /// # Arguments /// * `train_data` Dataset @@ -324,36 +297,7 @@ pub fn move_train_data_by_centroid( } } -/// Calculate the number of chunks for the product quantization algorithm. Returns a vector of offsets where -/// each offset corresponds to a chunk based on the index of the chunk in the vector. -/// -/// # Arguments -/// * `dimensions` Number of dimensions of the input data -/// * `num_pq_chunks` - Number of chunks that will be used in the PQ calculation. Each vector will be split into these -/// number of chunks and each chunk will be compressed down to one byte. -/// * `offsets` - An output vector of offsets, where the size is equal to the number of pq chunks + 1. -#[inline] -pub fn calculate_chunk_offsets(dimensions: usize, num_pq_chunks: usize, offsets: &mut [usize]) { - // Calculate each chunk's offset - // If we have 8 dimension and 3 chunks then offsets would be [0,3,6,8] - let mut chunk_offset: usize = 0; - offsets[0] = chunk_offset; - for chunk_index in 0..num_pq_chunks { - chunk_offset += dimensions / num_pq_chunks; - if chunk_index < (dimensions % num_pq_chunks) { - chunk_offset += 1; - } - offsets[chunk_index + 1] = chunk_offset; - } -} - -pub fn calculate_chunk_offsets_auto(dimensions: usize, num_pq_chunks: usize) -> Vec { - let mut offsets = vec![0; num_pq_chunks + 1]; - calculate_chunk_offsets(dimensions, num_pq_chunks, offsets.as_mut_slice()); - offsets -} - -/// Add the row `y` to every row in `x`. +/// Add `y` to every row of `x`. /// /// # Panics /// @@ -672,6 +616,29 @@ mod pq_test { utils::{ParallelIteratorInPool, create_thread_pool_for_test, read_bin_from}, }; + /// Test helper: Gets all instances of a chunk from the training data for all records + /// in the training data. Each vector in the training dataset is divided into chunks + /// and the PQ algorithm handles each vector chunk individually. This helper gets the + /// same chunk from each vector in the training data and returns it as a flat vector. + fn get_chunk_from_training_data( + train_data: &[f32], + num_train: usize, + raw_vector_dim: usize, + chunk_size: usize, + chunk_offset: usize, + ) -> Vec { + let mut result: Vec = vec![0.0; num_train * chunk_size]; + result + .chunks_mut(chunk_size) + .enumerate() + .for_each(|(chunk_number, result_chunk)| { + let train_data_start = chunk_number * raw_vector_dim + chunk_offset; + let train_data_end = train_data_start + chunk_size; + result_chunk.copy_from_slice(&train_data[train_data_start..train_data_end]); + }); + result + } + #[test] fn test_move_train_data_by_centroid() { let dim = 20; @@ -1077,9 +1044,12 @@ mod pq_test { // Pre-emptively construct an offset view to compare mismatched slices. // We want to check that the difference in the mismatched chunks is small. - let mut offsets = vec![0; num_pq_chunks + 1]; - calculate_chunk_offsets(train_dim, num_pq_chunks, &mut offsets); - let offset_view = diskann_quantization::views::ChunkOffsetsView::new(&offsets).unwrap(); + let chunk_offsets = ChunkOffsets::partition( + NonZeroUsize::new(train_dim).unwrap(), + NonZeroUsize::new(num_pq_chunks).unwrap(), + ) + .unwrap(); + let offset_view = chunk_offsets.as_view(); let full_data = MatrixView::try_from(full_data_vector.as_slice(), num_train, train_dim).unwrap(); let pivot_view = diff --git a/diskann-providers/src/model/pq/views.rs b/diskann-providers/src/model/pq/views.rs index 3329c31d2..17028de3c 100644 --- a/diskann-providers/src/model/pq/views.rs +++ b/diskann-providers/src/model/pq/views.rs @@ -16,6 +16,22 @@ impl From> for ANNError { } } +// Compatibility with ANNError. +impl From> for ANNError { + #[track_caller] + fn from(value: Bridge) -> Self { + ANNError::log_pq_error(value.into_inner()) + } +} + +// Compatibility with ANNError. +impl From> for ANNError { + #[track_caller] + fn from(value: Bridge) -> Self { + ANNError::log_pq_error(value.into_inner()) + } +} + // Compatibility with ANNError. impl From> for ANNError { #[track_caller] diff --git a/diskann-quantization/src/views.rs b/diskann-quantization/src/views.rs index 6ef928345..04c4a0953 100644 --- a/diskann-quantization/src/views.rs +++ b/diskann-quantization/src/views.rs @@ -56,6 +56,24 @@ pub enum ChunkOffsetError { }, } +/// Error returned by [`ChunkOffsets::partition`]. +#[derive(Error, Debug)] +#[error("num_chunks {num_chunks} must not exceed dim {dim}")] +pub struct PartitionError { + pub num_chunks: usize, + pub dim: usize, +} + +/// Error returned by [`ChunkOffsetsView::partition_into`]. +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum PartitionIntoError { + #[error("scratch must have a length of at least 2, found {0}")] + ScratchTooSmall(usize), + #[error(transparent)] + PartitionError(#[from] PartitionError), +} + impl ChunkOffsetsBase where T: DenseData, @@ -205,6 +223,83 @@ impl<'a> From> for &'a [usize] { } } +impl ChunkOffsets { + /// Build a chunk-offset plan that partitions `dim` into `num_chunks` + /// near-equal chunks. The first `dim.get() % num_chunks.get()` chunks are + /// one element larger than the rest. + /// + /// Returns an error if the requested partition is not valid (e.g. + /// `num_chunks.get() > dim.get()`). + pub fn partition(dim: NonZeroUsize, num_chunks: NonZeroUsize) -> Result { + if num_chunks.get() > dim.get() { + return Err(PartitionError { + num_chunks: num_chunks.get(), + dim: dim.get(), + }); + } + let mut offsets = vec![0usize; num_chunks.get() + 1].into_boxed_slice(); + fill_chunk_offsets(dim, &mut offsets); + Ok(Self { dim, offsets }) + } +} + +impl<'a> ChunkOffsetsView<'a> { + /// Fill the caller-owned `scratch` buffer with the partition for `dim` + /// into `scratch.len() - 1` chunks and return a validated view borrowing it. + /// + /// See [`ChunkOffsets::partition`] for the partitioning rule. + /// + /// Returns an error if `scratch.len() < 2` or if the requested partition is not + /// valid (e.g. `scratch.len() - 1 > dim.get()`). + pub fn partition_into( + dim: NonZeroUsize, + scratch: &'a mut [usize], + ) -> Result { + if scratch.len() < 2 { + return Err(PartitionIntoError::ScratchTooSmall(scratch.len())); + } + let num_chunks = scratch.len() - 1; + if num_chunks > dim.get() { + return Err(PartitionError { + num_chunks, + dim: dim.get(), + } + .into()); + } + + fill_chunk_offsets(dim, scratch); + Ok(Self { + dim, + offsets: scratch, + }) + } +} + +/// Internal helper: fill `offsets` with the prefix-sum +/// partitioning of `dimensions` into `num_chunks` near-equal chunks, where +/// `num_chunks = offsets.len() - 1`. +/// +/// The first `dimensions.get() % num_chunks` chunks are one element larger than the +/// rest, so each chunk has size `dimensions.get() / num_chunks` or +/// `dimensions.get() / num_chunks + 1` and the total covers `[0, dimensions.get()]`. +/// +/// # Panics +/// +/// Panics if `offsets.len() <= 1`. +fn fill_chunk_offsets(dimensions: NonZeroUsize, offsets: &mut [usize]) { + let num_chunks = offsets.len() - 1; + let dimensions = dimensions.get(); + let mut chunk_offset: usize = 0; + offsets[0] = chunk_offset; + for chunk_index in 0..num_chunks { + chunk_offset += dimensions / num_chunks; + if chunk_index < (dimensions % num_chunks) { + chunk_offset += 1; + } + offsets[chunk_index + 1] = chunk_offset; + } +} + /////////////// // ChunkView // /////////////// @@ -425,6 +520,76 @@ mod tests { ); } + ////////////////////// + // partition builders // + ////////////////////// + + #[test] + fn partition_happy_path() { + let nz = |x: usize| NonZeroUsize::new(x).unwrap(); + + // Even split: 9 / 3 = 3 each. + let offsets = ChunkOffsets::partition(nz(9), nz(3)).unwrap(); + assert_eq!(offsets.as_slice(), &[0, 3, 6, 9]); + assert_eq!(offsets.dim(), 9); + assert_eq!(offsets.len(), 3); + + // Uneven split: 8 / 3 = 2 r 2 -> first two chunks get an extra element. + let offsets = ChunkOffsets::partition(nz(8), nz(3)).unwrap(); + assert_eq!(offsets.as_slice(), &[0, 3, 6, 8]); + + // Single chunk degenerate case. + let offsets = ChunkOffsets::partition(nz(5), nz(1)).unwrap(); + assert_eq!(offsets.as_slice(), &[0, 5]); + + // dimensions == num_pq_chunks: each chunk is size 1. + let offsets = ChunkOffsets::partition(nz(4), nz(4)).unwrap(); + assert_eq!(offsets.as_slice(), &[0, 1, 2, 3, 4]); + + // The view-into variant matches the owning constructor; num_chunks is + // inferred from `scratch.len() - 1`. + let mut scratch = [0usize; 4]; + let view = ChunkOffsetsView::partition_into(nz(8), &mut scratch).unwrap(); + assert_eq!(view.as_slice(), &[0, 3, 6, 8]); + assert_eq!(view.dim(), 8); + assert_eq!(view.len(), 3); + assert_eq!(scratch.as_slice(), &[0, 3, 6, 8]); + } + + #[test] + fn partition_construction_errors() { + let nz = |x: usize| NonZeroUsize::new(x).unwrap(); + + // num_chunks > dim -> TooManyChunks (caught explicitly before partitioning). + let err = ChunkOffsets::partition(nz(3), nz(5)).unwrap_err(); + assert!( + matches!( + err, + PartitionError { + num_chunks: 5, + dim: 3 + } + ), + "expected TooManyChunks, got {err:?}" + ); + + // Scratch length < 2 -> ScratchTooSmall (cannot infer num_chunks). + let mut too_short = [0usize; 1]; + let err = ChunkOffsetsView::partition_into(nz(8), &mut too_short).unwrap_err(); + assert!(matches!(err, PartitionIntoError::ScratchTooSmall(1))); + + // num_chunks > dim via the view builder too. + let mut scratch = [0usize; 6]; + let err = ChunkOffsetsView::partition_into(nz(3), &mut scratch).unwrap_err(); + assert!(matches!( + err, + PartitionIntoError::PartitionError(PartitionError { + num_chunks: 5, + dim: 3 + }) + )); + } + /////////////// // ChunkView // ///////////////