diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index ccc0911b45..7b805ba35b 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -146,13 +146,14 @@ jobs: matrix: config: - { name: 'all', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=ON -DUSE_ZLIB=ON -DUSE_EXR=ON -DUSE_PNG=ON' } - - { name: 'lite', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF -DOPENVDB_USE_DELAYED_LOADING=OFF' } + - { name: 'lite', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF' } - { name: 'half', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_IMATH_HALF=ON' } - { name: 'simd', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_X86_INSTRSET=7 -DUSE_VCL=ON' } # 7=AVX - { name: 'pygrid', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_PYTHON_WRAP_ALL_GRID_TYPES=ON' } - { name: 'asan', image: '2026', build: 'asan', components: 'core,test', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DNANOVDB_USE_OPENVDB=ON -DOPENVDB_AX_STATIC=OFF -DOPENVDB_CORE_STATIC=OFF -DUSE_BLOSC=OFF' } # We never called blosc_destroy(), so disable blosc to silence these errors - { name: 'ubsan', image: '2026', build: 'ubsan', components: 'core,test', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DCMAKE_CXX_FLAGS="-Wno-deprecated-declarations" ' } - { name: 'c++20', image: '2026', build: 'Release', components: 'core,test', cmake: '-DCMAKE_CXX_STANDARD=20' } + - { name: 'abi14', image: '2026', build: 'Release', components: 'core,test', cmake: '-DOPENVDB_USE_FUTURE_ABI_14=ON -DOPENVDB_ENABLE_ASSERTS=ON' } - { name: 'conf', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON' } fail-fast: false steps: @@ -215,7 +216,6 @@ jobs: --components=\"core,axcore,python,bin,render,test,axbin\" --cargs=\' -DCMAKE_CXX_STANDARD=20 - -DOPENVDB_USE_DELAYED_LOADING=OFF -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install ${{ matrix.config.cmake }} \' @@ -420,7 +420,6 @@ jobs: --components="${{ matrix.config.components }}" --cargs=\' -A x64 -G \"Visual Studio 17 2022\" - -DOPENVDB_USE_DELAYED_LOADING=OFF -DUSE_BLOSC=OFF \ -DUSE_ZLIB=OFF \ -DVCPKG_TARGET_TRIPLET=${VCPKG_DEFAULT_TRIPLET} diff --git a/CMakeLists.txt b/CMakeLists.txt index b9315ffe82..d42ece28d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,7 +84,6 @@ option(OPENVDB_ENABLE_VEC_RELATIONAL_OPERATIONS [=[ Enabled deprecated relational < and > operators on Vector types. This option will be switched to OFF and the feature will be subsequently removed in future releases.]=] ON) -option(OPENVDB_USE_DELAYED_LOADING "Build the core OpenVDB library with delayed-loading." ON) option(OPENVDB_CXX_STRICT "Enable or disable pre-defined compiler warnings" OFF) cmake_dependent_option(OPENVDB_INSTALL_CMAKE_MODULES "Install the provided OpenVDB CMake modules when building the core library" @@ -264,7 +263,6 @@ mark_as_advanced( OPENVDB_BUILD_HOUDINI_ABITESTS OPENVDB_CXX_STRICT OPENVDB_ENABLE_RPATH - OPENVDB_USE_DELAYED_LOADING OPENVDB_FUTURE_DEPRECATION OPENVDB_SIMD SYSTEM_LIBRARY_PATHS diff --git a/ci/install_windows.ps1 b/ci/install_windows.ps1 index 6054cb96bf..a4d44da353 100644 --- a/ci/install_windows.ps1 +++ b/ci/install_windows.ps1 @@ -21,20 +21,52 @@ $vcpkgPackages = @( "nanobind" ) +$maxAttempts = 3 + +# curl's schannel backend reports an unreachable CRL/OCSP responder as a +# certificate verification failure (error 60), which vcpkg then treats as +# permanent. Downgrade a missing revocation answer to a warning; the rest of +# certificate validation still applies. +$env:VCPKG_SSL_REVOKE_BEST_EFFORT = "1" + # Update vcpkg vcpkg update -# Allow the vcpkg command to fail once so we can retry with the latest -try { - vcpkg install $vcpkgPackages -} catch { - Write-Host "vcpkg install failed, retrying with latest ports..." - # Retry the installation with updated ports - Push-Location $env:VCPKG_INSTALLATION_ROOT - git pull - Pop-Location - vcpkg update +$installed = $false + +for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { vcpkg install $vcpkgPackages + + # A failing native command does not raise a terminating error, so the exit + # code has to be inspected explicitly rather than relying on try/catch. + if ($LASTEXITCODE -eq 0) { + $installed = $true + break + } + + if ($attempt -eq $maxAttempts) { + break + } + + # vcpkg fetches port sources directly from upstream hosts and won't retry + # downloads it classifies as permanent failures, so a single flaky TLS + # handshake aborts the whole install. + Write-Host "vcpkg install failed (attempt $attempt of $maxAttempts), retrying..." + Start-Sleep -Seconds 15 + + # Refresh the ports before the last attempt in case the failure is caused + # by a stale port rather than the network. + if ($attempt -eq ($maxAttempts - 1)) { + Write-Host "Retrying with latest ports..." + Push-Location $env:VCPKG_INSTALLATION_ROOT + git pull + Pop-Location + vcpkg update + } +} + +if (-not $installed) { + throw "vcpkg install failed after $maxAttempts attempts" } Write-Host "vcpkg install completed successfully" diff --git a/cmake/FindOpenVDB.cmake b/cmake/FindOpenVDB.cmake index 400afd2544..17a4c2da53 100644 --- a/cmake/FindOpenVDB.cmake +++ b/cmake/FindOpenVDB.cmake @@ -64,8 +64,6 @@ This will define the following variables: True if the OpenVDB Library has been built with log4cplus support ``OpenVDB_USES_IMATH_HALF`` True if the OpenVDB Library has been built with Imath half support -``OpenVDB_USES_DELAYED_LOADING`` - True if the OpenVDB Library has been built with delayed-loading ``OpenVDB_ABI`` Set if this module was able to determine the ABI number the located OpenVDB Library was built against. Unset otherwise. @@ -522,7 +520,6 @@ set(OpenVDB_USES_BLOSC ${USE_BLOSC}) set(OpenVDB_USES_ZLIB ${USE_ZLIB}) set(OpenVDB_USES_LOG4CPLUS ${USE_LOG4CPLUS}) set(OpenVDB_USES_IMATH_HALF ${USE_IMATH_HALF}) -set(OpenVDB_USES_DELAYED_LOADING ${OPENVDB_USE_DELAYED_LOADING}) set(OpenVDB_DEFINITIONS) if(WIN32) @@ -555,7 +552,6 @@ if(_OPENVDB_HAS_NEW_VERSION_HEADER) OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_IMATH_HALF" OpenVDB_USES_IMATH_HALF) OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_BLOSC" OpenVDB_USES_BLOSC) OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_ZLIB" OpenVDB_USES_ZLIB) - OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_DELAYED_LOADING" OpenVDB_USES_DELAYED_LOADING) elseif(NOT OPENVDB_USE_STATIC_LIBS) # Use GetPrerequisites to see which libraries this OpenVDB lib has linked to # which we can query for optional deps. This basically runs ldd/otoll/objdump @@ -602,11 +598,6 @@ elseif(NOT OPENVDB_USE_STATIC_LIBS) if(NOT ${_HAS_DEP} EQUAL -1) set(OpenVDB_USES_IMATH_HALF ON) endif() - - string(FIND ${PREREQUISITE} "boost_iostreams" _HAS_DEP) - if(NOT ${_HAS_DEP} EQUAL -1) - set(OpenVDB_USES_DELAYED_LOADING ON) - endif() endforeach() unset(_OPENVDB_PREREQUISITE_LIST) @@ -628,10 +619,6 @@ if(OpenVDB_USES_IMATH_HALF) find_package(Imath REQUIRED CONFIG) endif() -if(OpenVDB_USES_DELAYED_LOADING) - find_package(Boost REQUIRED COMPONENTS iostreams) -endif() - if(UNIX) find_package(Threads REQUIRED) endif() @@ -643,11 +630,6 @@ endif() set(_OPENVDB_VISIBLE_DEPENDENCIES "") -if(OpenVDB_USES_DELAYED_LOADING) - list(APPEND _OPENVDB_VISIBLE_DEPENDENCIES Boost::iostreams) - list(APPEND OpenVDB_DEFINITIONS OPENVDB_USE_DELAYED_LOADING) -endif() - if(OpenVDB_USES_IMATH_HALF) list(APPEND _OPENVDB_VISIBLE_DEPENDENCIES Imath::Imath) endif() diff --git a/doc/changes.txt b/doc/changes.txt index 3c05074c81..033231bbaa 100644 --- a/doc/changes.txt +++ b/doc/changes.txt @@ -2034,7 +2034,7 @@ Bug fixes: New features: - Added @vdblink{tools::FindActiveValues,FindActiveValues}, which counts the active values in a tree that intersect a given bounding box. -- Added @vdblink{io::DelayedLoadMetadata,DelayedLoadMetadata}, which stores +- Added @c io::DelayedLoadMetadata, which stores mask offsets and compression sizes on write to accelerate delayed load reading. @@ -2933,7 +2933,7 @@ New features: - Added a toggle to the @vdblink::tools::clip() clip@endlink tool to invert the clipping mask. - Custom leaf node implementations may now optimize their file layout - by inheriting from @vdblink::io::MultiPass io::MultiPass@endlink. + by inheriting from @c io::MultiPass. Voxel data for grids with such leaf nodes will be written and read in multiple passes, allowing blocks of related data to be stored contiguously. [Contributed by Double Negative] diff --git a/nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc b/nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc index 2298508a93..bab45dccd6 100644 --- a/nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc +++ b/nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc @@ -257,7 +257,7 @@ int main(int argc, char* argv[]) } if (verbose) std::cout << "Opening OpenVDB file named \"" << inputFile << "\"" << std::endl; openvdb::io::File file(inputFile); - file.open(false); //disable delayed loading + file.open(); if (gridName.empty()) {// convert all grid in the file auto grids = file.getGrids(); std::vector > handles; diff --git a/nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp b/nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp index 8826a3000c..7154e02740 100644 --- a/nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp +++ b/nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp @@ -50,7 +50,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read input VDB file"); openvdb::initialize(); openvdb::io::File inFile(argv[1]); - inFile.open(false); // disable delayed loading + inFile.open(); auto baseGrids = inFile.getGrids(); inFile.close(); auto grid = openvdb::gridPtrCast(baseGrids->at(0)); diff --git a/nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp b/nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp index 6f6000f03c..6e65006934 100644 --- a/nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp +++ b/nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp @@ -44,7 +44,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read input VDB file"); openvdb::initialize(); openvdb::io::File inFile(argv[1]); - inFile.open(false); // disable delayed loading + inFile.open(); auto baseGrids = inFile.getGrids(); inFile.close(); auto grid = openvdb::gridPtrCast(baseGrids->at(0)); diff --git a/nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp b/nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp index 8b6626558b..e8cba39e02 100644 --- a/nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp +++ b/nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp @@ -41,7 +41,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read first input VDB file"); openvdb::initialize(); openvdb::io::File inFile1(argv[1]); - inFile1.open(false); // disable delayed loading + inFile1.open(); auto baseGrids1 = inFile1.getGrids(); inFile1.close(); auto grid1 = openvdb::gridPtrCast(baseGrids1->at(0)); @@ -51,7 +51,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read second input VDB file"); openvdb::initialize(); openvdb::io::File inFile2(argv[2]); - inFile2.open(false); // disable delayed loading + inFile2.open(); auto baseGrids2 = inFile2.getGrids(); inFile2.close(); auto grid2 = openvdb::gridPtrCast(baseGrids2->at(0)); diff --git a/nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc b/nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc index b8ca62c838..7b8f12256e 100644 --- a/nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc +++ b/nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc @@ -37,7 +37,7 @@ void runOpenVDB(nanovdb::GridHandle& handle, int nu openvdb::initialize(); std::string filename = "C:/Users/william/Downloads/dragon.vdb"; openvdb::io::File file(filename); - file.open(false); //disable delayed loading + file.open(); auto srcGrid = file.readGrid("ls_dragon"); std::cout << "Loading OpenVDB grid[" << srcGrid->getName() << "]...\n"; #endif diff --git a/nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp b/nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp index 19795c5dfd..f86980a72b 100644 --- a/nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp +++ b/nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp @@ -81,7 +81,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read input VDB file"); openvdb::initialize(); openvdb::io::File inFile(argv[1]); - inFile.open(false); // disable delayed loading + inFile.open(); auto baseGrids = inFile.getGrids(); inFile.close(); auto grid = openvdb::gridPtrCast(baseGrids->at(0)); diff --git a/nanovdb/nanovdb/unittest/TestOpenVDB.cc b/nanovdb/nanovdb/unittest/TestOpenVDB.cc index fd47a7e017..8ada169df9 100644 --- a/nanovdb/nanovdb/unittest/TestOpenVDB.cc +++ b/nanovdb/nanovdb/unittest/TestOpenVDB.cc @@ -140,7 +140,7 @@ class TestOpenVDB : public ::testing::Test mTimer.start("Reading grid from the file \"" + fileName + "\""); try { openvdb::io::File file(fileName); - file.open(false); //disable delayed loading + file.open(); grid = openvdb::gridPtrCast(file.readGrid(file.beginName().gridName())); } catch(const std::exception& e) { std::cerr << "An exception occurred: \"" << e.what() << "\"" << std::endl; @@ -2860,7 +2860,7 @@ TEST_F(TestOpenVDB, LevelSetFiles) //mTimer.start("\nReading grid from the file \"" + fileName + "\""); try { openvdb::io::File file(fileName); - file.open(false); //disable delayed loading + file.open(); auto srcGrid = openvdb::gridPtrCast(file.readGrid(file.beginName().gridName())); const size_t pos = fileName.find_last_of("/\\") + 1; @@ -2938,7 +2938,7 @@ TEST_F(TestOpenVDB, FogFiles) //mTimer.start("Reading grid from the file \"" + fileName + "\""); try { openvdb::io::File file(fileName); - file.open(false); //disable delayed loading + file.open(); auto srcGrid = openvdb::gridPtrCast(file.readGrid(file.beginName().gridName())); const size_t pos = fileName.find_last_of("/\\") + 1; @@ -3008,7 +3008,7 @@ TEST_F(TestOpenVDB, PointFiles) //mTimer.start("Reading grid from the file \"" + fileName + "\""); try { openvdb::io::File file(fileName); - file.open(false); //disable delayed loading + file.open(); auto srcGrid = openvdb::gridPtrCast(file.readGrid(file.beginName().gridName())); //std::cerr << "Read PointDataGrid named \"" << srcGrid->getName() << "\"" << std::endl; diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 302f890f28..b1aaf5ef3c 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -117,19 +117,6 @@ endif() # Collect and configure lib dependencies -if(OPENVDB_USE_DELAYED_LOADING) - find_package(Boost ${MINIMUM_BOOST_VERSION} REQUIRED COMPONENTS iostreams) - - if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_BOOST_VERSION) - # The X.Y.Z boost version value isn't available until CMake 3.14 - set(FULL_BOOST_VERSION "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}") - if(${FULL_BOOST_VERSION} VERSION_LESS FUTURE_MINIMUM_BOOST_VERSION) - message(DEPRECATION "Support for Boost versions < ${FUTURE_MINIMUM_BOOST_VERSION} " - "is deprecated and will be removed.") - endif() - endif() -endif() - find_package(TBB ${MINIMUM_TBB_VERSION} REQUIRED COMPONENTS tbb) if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_TBB_VERSION) if(${TBB_VERSION} VERSION_LESS FUTURE_MINIMUM_TBB_VERSION) @@ -252,27 +239,6 @@ if(UNIX) list(APPEND OPENVDB_CORE_DEPENDENT_LIBS Threads::Threads) endif() -# Pull in Boost last as houdini's boost (hboost) is fully namespaced and libs -# are renamed too. Boost can be pulled in at any time, but do it last so that, -# if it's in a shared place (like /usr/local) it doesn't accidently pull in -# other headers. - -if(OPENVDB_USE_DELAYED_LOADING) - list(APPEND OPENVDB_CORE_DEPENDENT_LIBS Boost::iostreams) - if(WIN32) - # Boost headers contain #pragma commands on Windows which causes Boost - # libraries to be linked in automatically. Custom boost installations - # may find that these naming conventions don't always match and can - # cause linker errors. This option disables this feature of Boost. Note - # -DBOOST_ALL_NO_LIB can also be provided manually. - if(OPENVDB_DISABLE_BOOST_IMPLICIT_LINKING) - list(APPEND OPENVDB_CORE_DEPENDENT_LIBS - Boost::disable_autolinking # add -DBOOST_ALL_NO_LIB - ) - endif() - endif() -endif() - ########################################################################## ##### Core library configuration @@ -374,13 +340,12 @@ configure_file(version.h.in openvdb/version.h) set(OPENVDB_LIBRARY_SOURCE_FILES Grid.cc io/Archive.cc + io/Codec.cc io/Compression.cc - io/DelayedLoadMetadata.cc io/File.cc io/GridDescriptor.cc io/Queue.cc io/Stream.cc - io/TempFile.cc math/Maps.cc math/Proximity.cc math/QuantizedUnitVec.cc @@ -411,16 +376,28 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES TypeList.h ) +set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES + codecs/BoolCodec.h + codecs/PointDataCodec.h + codecs/PointIndexCodec.h + codecs/ScalarCodec.h + codecs/TopologyCodec.h + codecs/ValueMaskCodec.h +) + +set(OPENVDB_LIBRARY_CODECS_IMPL_INCLUDE_FILES + codecs/impl/ScalarLeafCodec.h +) + set(OPENVDB_LIBRARY_IO_INCLUDE_FILES io/Archive.h + io/Codec.h io/Compression.h - io/DelayedLoadMetadata.h io/File.h io/GridDescriptor.h io/io.h io/Queue.h io/Stream.h - io/TempFile.h ) set(OPENVDB_LIBRARY_MATH_INCLUDE_FILES @@ -461,6 +438,7 @@ set(OPENVDB_LIBRARY_POINTS_INCLUDE_FILES points/PointConversion.h points/PointCount.h points/PointDataGrid.h + points/PointDataIO.h points/PointDelete.h points/PointGroup.h points/PointMask.h @@ -612,6 +590,8 @@ if(USE_EXPLICIT_INSTANTIATION) # inexpensive to generate in this case. set(OPENVDB_LIBRARY_INSTANTIATION_HEADERS ${OPENVDB_LIBRARY_TOOLS_INCLUDE_FILES} + ${OPENVDB_LIBRARY_CODECS_INCLUDE_FILES} + ${OPENVDB_LIBRARY_CODECS_IMPL_INCLUDE_FILES} ) # A new source file is generated by CMake for every header @@ -686,9 +666,6 @@ endif() if(USE_LOG4CPLUS) list(APPEND OPENVDB_CORE_PUBLIC_DEFINES -DOPENVDB_USE_LOG4CPLUS) endif() -if(OPENVDB_USE_DELAYED_LOADING) - list(APPEND OPENVDB_CORE_PUBLIC_DEFINES -DOPENVDB_USE_DELAYED_LOADING) -endif() ########################################################################## @@ -799,6 +776,8 @@ endif() install(FILES ${OPENVDB_LIBRARY_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/openvdb/version.h DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb) install(FILES ${OPENVDB_LIBRARY_IO_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/io) +install(FILES ${OPENVDB_LIBRARY_CODECS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/codecs) +install(FILES ${OPENVDB_LIBRARY_CODECS_IMPL_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/codecs/impl) install(FILES ${OPENVDB_LIBRARY_MATH_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/math) install(FILES ${OPENVDB_LIBRARY_POINTS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/points) install(FILES ${OPENVDB_LIBRARY_POINTS_IMPL_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/points/impl) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index fc566b8387..d32f064c8d 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -9,6 +9,7 @@ #include "Types.h" #include "io/io.h" #include "math/Transform.h" +#include "tree/LeafManager.h" #include "tree/Tree.h" #include "util/Assert.h" #include "util/logging.h" @@ -457,12 +458,12 @@ class OPENVDB_API GridBase: public MetaMap virtual void readBuffers(std::istream&) = 0; /// Read all of this grid's data buffers that intersect the given index-space bounding box. virtual void readBuffers(std::istream&, const CoordBBox&) = 0; - /// @brief Read all of this grid's data buffers that are not yet resident in memory - /// (because delayed loading is in effect). - /// @details If this grid was read from a memory-mapped file, this operation - /// disconnects the grid from the file. - /// @sa io::File::open, io::MappedFile + +#if OPENVDB_ABI_VERSION_NUMBER < 14 + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void readNonresidentBuffers() const = 0; +#endif + /// Write out all data buffers for this grid. virtual void writeBuffers(std::ostream&) const = 0; @@ -941,12 +942,12 @@ class Grid: public GridBase void readBuffers(std::istream&) override; /// Read all of this grid's data buffers that intersect the given index-space bounding box. void readBuffers(std::istream&, const CoordBBox&) override; - /// @brief Read all of this grid's data buffers that are not yet resident in memory - /// (because delayed loading is in effect). - /// @details If this grid was read from a memory-mapped file, this operation - /// disconnects the grid from the file. - /// @sa io::File::open, io::MappedFile - void readNonresidentBuffers() const override; + +#if OPENVDB_ABI_VERSION_NUMBER < 14 + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void readNonresidentBuffers() const override { } +#endif + /// Write out all data buffers for this grid. void writeBuffers(std::ostream&) const override; @@ -1183,14 +1184,34 @@ struct TreeAdapter > //////////////////////////////////////// +namespace points { + +template class PointDataLeafNode; + +/// @brief Type trait that evaluates to true only for @c PointDataLeafNode instantiations. +template +struct IsPointDataLeafNode : std::false_type {}; + +template +struct IsPointDataLeafNode> : std::true_type {}; + +} // namespace points + + /// @brief Metafunction that specifies whether a given leaf node, tree, or grid type -/// requires multiple passes to read and write voxel data -/// @details Multi-pass I/O allows one to optimize the data layout of leaf nodes -/// for certain access patterns during delayed loading. -/// @sa io::MultiPass +/// requires multiple passes to read and write voxel data. +/// @details Multi-pass I/O allows leaf nodes to optimize their serialization layout +/// for delayed-load access patterns. Only @c PointDataLeafNode supports multi-pass I/O. +/// Defining a custom leaf node that inherits @c io::PointDataGridMultiPass is no longer permitted. +/// @sa points::IsPointDataLeafNode template struct HasMultiPassIO { - static const bool value = std::is_base_of::value; + static_assert( + !std::is_base_of::value + || points::IsPointDataLeafNode::value, + "Only PointDataLeafNode may inherit from io::PointDataGridMultiPass; " + "use points::IsPointDataLeafNode to test for multi-pass I/O support."); + static const bool value = points::IsPointDataLeafNode::value; }; // Partial specialization for Tree types @@ -1612,6 +1633,28 @@ inline void Grid::readTopology(std::istream& is) { tree().readTopology(is, saveFloatAsHalf()); + // When called from the legacy (non-codec) TopologyOnly path, the stream + // metadata carries a flag requesting that leaf buffers be allocated and + // filled with the background value (PartialCreate leaves them + // unallocated after readTopology). + if (io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is)) { + if (meta->allocateLeafBuffers()) { + meta->setAllocateLeafBuffers(false); + if constexpr (!std::is_void_v) { + const auto background = tree().root().background(); + tree::LeafManager leafManager(tree()); + leafManager.foreach([&background](auto& leaf, size_t) { + using LeafType = std::decay_t; + if constexpr (!std::is_same_v) { + if (leaf.buffer().empty()) { + leaf.buffer().allocate(); + leaf.buffer().fill(background); + } + } + }); + } + } + } } @@ -1672,14 +1715,6 @@ Grid::readBuffers(std::istream& is, const CoordBBox& bbox) } -template -inline void -Grid::readNonresidentBuffers() const -{ - tree().readNonresidentBuffers(); -} - - template inline void Grid::writeBuffers(std::ostream& os) const diff --git a/openvdb/openvdb/codecs/BoolCodec.h b/openvdb/openvdb/codecs/BoolCodec.h new file mode 100644 index 0000000000..3da1839101 --- /dev/null +++ b/openvdb/openvdb/codecs/BoolCodec.h @@ -0,0 +1,154 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_BOOLCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_BOOLCODEC_HAS_BEEN_INCLUDED + +#include + +#include + +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct ReadBoolBuffersOp +{ + using TreeT = typename GridT::TreeType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + ReadBoolBuffersOp(std::istream& _is, bool _saveFloatAsHalf, + const ValueT& _background, const CoordBBox* _clipBBox) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) + , clipBBox(_clipBBox) { } + + void operator()(RootT& root, size_t) + { + // Clip root-level tiles and prune children that were clipped. + if (clipBBox) { + root.clip(*clipBBox); + } + } + + template + void operator()(NodeT& node, size_t) + { + // Clip internal node tiles and prune children that were clipped. + if (clipBBox) { + node.clip(*clipBBox, background); + } + } + + void operator()(LeafT& leaf, size_t) + { + // Read in the value mask. + leaf.getValueMask().load(is); + // Read in the origin. + Coord origin; + is.read(reinterpret_cast(&origin), sizeof(Coord::ValueType) * 3); + leaf.setOrigin(origin); + + // Read in the mask for the voxel values. + typename LeafT::Buffer::NodeMaskType nodeMask; + nodeMask.load(is); + typename LeafT::Buffer temp(nodeMask); + leaf.swap(temp); + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; + const CoordBBox* clipBBox = nullptr; +}; // struct ReadBoolBuffersOp + +template +struct WriteBoolBuffersOp +{ + using TreeT = typename GridT::TreeType; + using LeafT = typename TreeT::LeafNodeType; + + WriteBoolBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + // Write out the value mask. + leaf.getValueMask().save(os); + + // Write out the origin. + os.write(reinterpret_cast(&leaf.origin()), sizeof(Coord::ValueType) * 3); + + // Write out the voxel values. + leaf.buffer().storage().save(os); + } + + std::ostream& os; + const bool saveFloatAsHalf; +}; // struct WriteBoolBuffersOp + +} // namespace internal + +template +struct BoolCodec final: public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~BoolCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) const final + { + GridT& grid = static_cast(*data.grid); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in BoolCodec"); + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + std::unique_ptr clipIndexBBox; + if (options.clipBBox.isSorted()) { + clipIndexBBox = std::make_unique(grid.constTransform().worldToIndexNodeCentered(options.clipBBox)); + } + + internal::ReadBoolBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), clipIndexBBox.get()); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final + { + const GridT& grid = static_cast(gridBase); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in BoolCodec"); + } + + internal::WriteBoolBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); + } +}; // struct BoolCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_BOOLCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h new file mode 100644 index 0000000000..18484facd1 --- /dev/null +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -0,0 +1,470 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED + +#include + +#include + +#include +#include +#include + +#include "impl/ScalarLeafCodec.h" +#include "TopologyCodec.h" + +#include +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +/// Look up an existing paged stream or create a new one for the given attribute index +template +inline typename PagedStreamPtrT::element_type* getOrCreatePagedStream( + std::map& pagedStreams, Index attributeIndex) +{ + auto it = pagedStreams.find(attributeIndex); + if (it != pagedStreams.end()) return it->second.get(); + using PagedStreamT = typename PagedStreamPtrT::element_type; + auto ptr = std::make_shared(); + auto& stored = (pagedStreams[attributeIndex] = std::move(ptr)); + return stored.get(); +} + +//////////////////////////////////////// +// Read-side functions + +template +inline void readPointDataVoxelSizes(const std::vector& leaves, + std::istream& is, std::unordered_map& voxelBufferSizes) +{ + for (auto* leaf : leaves) { + uint16_t voxelBufferSize; + is.read(reinterpret_cast(&voxelBufferSize), sizeof(uint16_t)); + voxelBufferSizes[leaf->origin()] = voxelBufferSize; + } +} + +template +inline void readPointDataDescriptors(const std::vector& leaves, + std::istream& is) +{ + points::AttributeSet::Descriptor::Ptr sharedDescriptor; + for (auto* leaf : leaves) { + points::AttributeSet::UniquePtr attrSet = leaf->stealAttributeSet(); + if (sharedDescriptor) { + // Reuse shared descriptor from first leaf + attrSet->resetDescriptor(sharedDescriptor, /*allowMismatchingDescriptors=*/true); + } + else { + uint8_t header; + is.read(reinterpret_cast(&header), sizeof(uint8_t)); + attrSet->readDescriptor(is); + if (header & uint8_t(1)) { + // Store descriptor for subsequent leaves + sharedDescriptor = attrSet->descriptorPtr(); + } + // a forwards-compatibility mechanism for future use, + // if a 0x2 bit is set, read and skip over a specific number of bytes + if (header & uint8_t(2)) { + uint64_t bytesToSkip; + is.read(reinterpret_cast(&bytesToSkip), sizeof(uint64_t)); + if (bytesToSkip > uint64_t(0)) { + std::vector tempData(bytesToSkip); + is.read(reinterpret_cast(&tempData[0]), bytesToSkip); + } + } + // this reader is only able to read headers with 0x1 and 0x2 bits set + if (header > uint8_t(3)) { + OPENVDB_THROW(IoError, "Unrecognised header flags in PointDataLeafNode"); + } + } + attrSet->readMetadata(is); + leaf->replaceAttributeSet(attrSet.release(), /*allowMismatchingDescriptors=*/true); + } +} + +template +inline void readPointDataAttributeSizes(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(true); + array->readPagedBuffers(*pagedStream); + } + } +} + +template +inline void readPointDataVoxelData(const std::vector& leaves, + std::istream& is, bool saveFloatAsHalf, + const typename LeafT::ValueType& background, + [[maybe_unused]] const std::unordered_map& voxelBufferSizes, + const typename LeafT::ValueType* storageBackground = nullptr) +{ + using BaseLeaf = typename LeafT::BaseLeaf; + for (auto* leaf : leaves) { + OPENVDB_ASSERT(voxelBufferSizes.find(leaf->origin()) != voxelBufferSizes.end()); + BaseLeaf& baseLeaf = static_cast(*leaf); + readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background, /*skip=*/false, /*clipBBox=*/nullptr, storageBackground); + } +} + +template +inline void readPointDataAttributeData(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(false); + array->readPagedBuffers(*pagedStream); + } + } +} + +template +inline void skipPointDataAttributeSizes(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(true); + array->skipPagedBuffers(*pagedStream); + } + } +} + +template +inline void skipPointDataAttributeData(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(false); + array->skipPagedBuffers(*pagedStream); + } + } +} + +//////////////////////////////////////// +// Write-side functions + +template +inline Index countPointDataPasses(const std::vector& leaves) +{ + Index maxRequiredPasses = 0; + for (const auto* leaf : leaves) { + const Index requiredPasses = leaf->buffers(); + if (requiredPasses > maxRequiredPasses) { + maxRequiredPasses = requiredPasses; + } + } + return maxRequiredPasses; +} + +template +inline void writePointDataVoxelSizes(const std::vector& leaves, + std::ostream& os, bool& matching, + points::AttributeSet::Descriptor::Ptr& sharedDescriptor) +{ + bool descriptorChecked = false; + matching = true; + for (const auto* leaf : leaves) { + io::writeCompressedValuesSize(os, leaf->buffer().data(), LeafT::SIZE); + + // Track descriptor matching + const auto& descriptor = leaf->attributeSet().descriptorPtr(); + if (!descriptorChecked) { + // First leaf - store descriptor + descriptorChecked = true; + sharedDescriptor = descriptor; + } + else if (matching && *sharedDescriptor != *descriptor) { + matching = false; + } + } +} + +template +inline void writePointDataDescriptors(const std::vector& leaves, + std::ostream& os, bool matching, + const points::AttributeSet::Descriptor::Ptr&) +{ + bool firstWrite = true; + for (const auto* leaf : leaves) { + const points::AttributeSet& attributeSet = leaf->attributeSet(); + if (matching) { + // Shared descriptor - only write on first leaf + if (firstWrite) { + firstWrite = false; + uint8_t header(1); + os.write(reinterpret_cast(&header), sizeof(uint8_t)); + attributeSet.writeDescriptor(os, /*transient=*/false); + } + } + else { + // Non-shared descriptor - write on every leaf + uint8_t header(0); + os.write(reinterpret_cast(&header), sizeof(uint8_t)); + attributeSet.writeDescriptor(os, /*transient=*/false); + } + attributeSet.writeMetadata(os, /*transient=*/false, /*paged=*/true); + } +} + +template +inline void writePointDataAttributeSizes(const std::vector& leaves, + std::ostream& os, Index attributeIndex) +{ + std::map pagedStreams; + for (const auto* leaf : leaves) { + const points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + leaf->attributeSet().getConst(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setOutputStream(os); + pagedStream->setSizeOnly(true); + array->writePagedBuffers(*pagedStream, /*outputTransient*/false); + } + } + // Flush paged streams to write any remaining buffered page headers + for (auto& pair : pagedStreams) { + pair.second->flush(); + } +} + +template +inline void writePointDataVoxelData(const std::vector& leaves, + std::ostream& os, bool saveFloatAsHalf) +{ + using BaseLeaf = typename LeafT::BaseLeaf; + for (const auto* leaf : leaves) { + const BaseLeaf& baseLeaf = static_cast(*leaf); + writeScalarLeafBuffers(baseLeaf, os, saveFloatAsHalf); + } +} + +template +inline void writePointDataAttributeData(const std::vector& leaves, + std::ostream& os, Index attributeIndex) +{ + std::map pagedStreams; + for (const auto* leaf : leaves) { + const points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + leaf->attributeSet().getConst(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setOutputStream(os); + pagedStream->setSizeOnly(false); + array->writePagedBuffers(*pagedStream, /*outputTransient*/false); + } + } + // Flush paged streams to write any remaining buffered page data + for (auto& pair : pagedStreams) { + pair.second->flush(); + } +} + +} // namespace internal + +/// Per-grid-type codec-specific options for PointDataCodec +/// Contains point attribute filtering options +struct OPENVDB_API PointDataCodecTypeData : public io::ReadTypedOptions +{ + // Point Attribute Options - which attributes to read + std::vector pointAttributeNames; +}; // struct PointDataCodecTypeData + +template +struct PointDataCodec final: public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~PointDataCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) const final + { + OPENVDB_ASSERT(dynamic_cast(data.grid.get())); + + GridT& grid = static_cast(*data.grid); + + std::vector pointAttributeNames; + + // Look up point-specific options if provided + auto it = options.typeData.find(name()); + if (it != options.typeData.end()) { + auto& pointTypeData = io::ReadTypedOptions::cast(it->second); + pointAttributeNames = pointTypeData.pointAttributeNames; + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + uint16_t numPasses = 1; + is.read(reinterpret_cast(&numPasses), sizeof(uint16_t)); + // The pass layout is: voxel sizes (1) + descriptors (1) + attribute + // sizes (N) + voxel data (1) + attribute data (N) = 2N + 4 passes. + // A leafless grid stores numPasses == 0, and malformed files may store + // numPasses < 4; guard against unsigned underflow in either case. + const Index attributes = numPasses >= 4 ? Index(numPasses - 4) / 2 : 0; + + using LeafT = typename GridT::TreeType::LeafNodeType; + std::vector leaves; + tree.getNodes(leaves); + + // Pass 0: read voxel data sizes + std::unordered_map voxelBufferSizes; + internal::readPointDataVoxelSizes(leaves, is, voxelBufferSizes); + + // Pass 1: read descriptor and attribute metadata + internal::readPointDataDescriptors(leaves, is); + + // Build set of attribute indices to skip based on pointAttributeNames. + // An empty pointAttributeNames means no filtering (read all attributes). + std::set skipIndices; + if (!pointAttributeNames.empty() && !leaves.empty()) { + // Attribute filtering requires homogeneous descriptors across all + // leaves because skip decisions are made per-index across all leaves. + const auto* firstDesc = &leaves[0]->attributeSet().descriptor(); + for (size_t i = 1; i < leaves.size(); ++i) { + if (&leaves[i]->attributeSet().descriptor() != firstDesc) { + OPENVDB_THROW(IoError, + "Attribute filtering is not supported for PointDataGrids " + "with heterogeneous descriptors"); + } + } + const auto& nameMap = firstDesc->map(); + const std::set wantedNames( + pointAttributeNames.begin(), + pointAttributeNames.end()); + for (const auto& namePos : nameMap) { + if (wantedNames.find(namePos.first) == wantedNames.end()) { + skipIndices.insert(static_cast(namePos.second)); + } + } + } + + // Passes 2..N+1: read attribute buffer sizes + std::map pagedStreams; + for (Index i = 0; i < attributes; ++i) { + if (skipIndices.count(i)) { + internal::skipPointDataAttributeSizes(leaves, is, i, pagedStreams); + } else { + internal::readPointDataAttributeSizes(leaves, is, i, pagedStreams); + } + } + + // Pass N+2: read voxel data + using ValueT = typename GridT::TreeType::ValueType; + auto& topoData = static_cast&>(data); + internal::readPointDataVoxelData(leaves, is, saveFloatAsHalf, + tree.background(), voxelBufferSizes, &topoData.storageBackground); + + // Passes N+3..2N+2: read attribute data buffers + for (Index i = 0; i < attributes; ++i) { + if (skipIndices.count(i)) { + internal::skipPointDataAttributeData(leaves, is, i, pagedStreams); + } else { + internal::readPointDataAttributeData(leaves, is, i, pagedStreams); + } + } + + // Drop skipped attributes from each leaf's AttributeSet + if (!skipIndices.empty() && !leaves.empty()) { + const std::vector dropPositions(skipIndices.begin(), skipIndices.end()); + auto filteredDescriptor = + leaves[0]->attributeSet().descriptorPtr()->duplicateDrop(dropPositions); + for (auto* leaf : leaves) { + leaf->dropAttributes(dropPositions, + leaf->attributeSet().descriptor(), filteredDescriptor); + } + } + + // PointDataGrid uses multiple passes, so clip after reading + // the buffers if bbox is not infinite. + if (options.clipBBox.isSorted()) { + CoordBBox indexBBox = + grid.constTransform().worldToIndexNodeCentered(options.clipBBox); + grid.tree().root().clip(indexBBox); + } + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final + { + const GridT& grid = static_cast(gridBase); + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + using LeafT = typename GridT::TreeType::LeafNodeType; + std::vector leaves; + grid.tree().getNodes(leaves); + + // Determine how many leaf buffer passes are required for this grid + uint16_t numPasses = + static_cast(internal::countPointDataPasses(leaves)); + os.write(reinterpret_cast(&numPasses), sizeof(uint16_t)); + + // See readBuffers(): a leafless grid yields numPasses == 0, so guard + // against unsigned underflow rather than computing (numPasses - 4) / 2. + const Index attributes = numPasses >= 4 ? Index(numPasses - 4) / 2 : 0; + + // Pass 0: write voxel data sizes + descriptor tracking + bool matching = true; + points::AttributeSet::Descriptor::Ptr sharedDescriptor; + internal::writePointDataVoxelSizes(leaves, os, matching, sharedDescriptor); + + // Pass 1: write descriptor and attribute metadata + internal::writePointDataDescriptors(leaves, os, matching, sharedDescriptor); + + // Passes 2..N+1: write attribute buffer sizes (page headers) + for (Index i = 0; i < attributes; ++i) { + internal::writePointDataAttributeSizes(leaves, os, i); + } + + // Pass N+2: write voxel data + internal::writePointDataVoxelData(leaves, os, saveFloatAsHalf); + + // Passes N+3..2N+2: write attribute data buffers (page data) + for (Index i = 0; i < attributes; ++i) { + internal::writePointDataAttributeData(leaves, os, i); + } + } +}; // struct PointDataCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h new file mode 100644 index 0000000000..5b4d6b62cf --- /dev/null +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -0,0 +1,163 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED + +#include + +#include + +#include "impl/ScalarLeafCodec.h" +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct ReadPointIndexBuffersOp +{ + using TreeT = typename GridT::TreeType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + ReadPointIndexBuffersOp(std::istream& _is, bool _saveFloatAsHalf, + const ValueT& _background, const ValueT* _storageBackground = nullptr) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) + , storageBackground(_storageBackground) { } + + template + void operator()(NodeT&, size_t) { } + + void operator()(LeafT& leaf, size_t) + { + using BaseLeaf = typename LeafT::BaseLeaf; + + // Read the value mask and voxel data via base class + BaseLeaf& baseLeaf = static_cast(leaf); + readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background, /*skip=*/false, /*clipBBox=*/nullptr, storageBackground); + + // Read the number of indices. + Index64 numIndices = Index64(0); + is.read(reinterpret_cast(&numIndices), sizeof(Index64)); + + // Read the indices data. + leaf.indices().resize(size_t(numIndices)); + is.read(reinterpret_cast(leaf.indices().data()), numIndices * sizeof(ValueT)); + + // Reserved for future use. + Index64 auxDataBytes = Index64(0); + is.read(reinterpret_cast(&auxDataBytes), sizeof(Index64)); + if (auxDataBytes > 0) { + // For now, read and discard any auxiliary data. + std::unique_ptr auxData{new char[auxDataBytes]}; + is.read(auxData.get(), auxDataBytes); + } + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; + const ValueT* storageBackground = nullptr; +}; // struct ReadPointIndexBuffersOp + +template +struct WritePointIndexBuffersOp +{ + using TreeT = typename GridT::TreeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + WritePointIndexBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + using BaseLeaf = typename LeafT::BaseLeaf; + + // Write out the value mask and voxel values via base class + const BaseLeaf& baseLeaf = static_cast(leaf); + writeScalarLeafBuffers(baseLeaf, os, saveFloatAsHalf); + + // Write the number of indices. + Index64 numIndices = Index64(leaf.indices().size()); + os.write(reinterpret_cast(&numIndices), sizeof(Index64)); + + // Write the indices data. + os.write(reinterpret_cast(leaf.indices().data()), numIndices * sizeof(ValueT)); + + // Reserved for future use. + const Index64 auxDataBytes = Index64(0); + os.write(reinterpret_cast(&auxDataBytes), sizeof(Index64)); + } + + std::ostream& os; + const bool saveFloatAsHalf; +}; // struct WritePointIndexBuffersOp + +} // namespace internal + +template +struct PointIndexCodec final: public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~PointIndexCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) const final + { + OPENVDB_ASSERT(dynamic_cast(data.grid.get())); + + GridT& grid = static_cast(*data.grid); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in PointIndexCodec"); + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + if (options.clipBBox.isSorted()) { + diagnostics.addWarning(grid.getName(), "bounding box clipping is not supported for PointIndexGrids"); + } + + using ValueT = typename GridT::TreeType::ValueType; + auto& topoData = static_cast&>(data); + internal::ReadPointIndexBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), &topoData.storageBackground); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final + { + const GridT& grid = static_cast(gridBase); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in PointIndexCodec"); + } + + internal::WritePointIndexBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); + } +}; // struct PointIndexCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h new file mode 100644 index 0000000000..e181f54df7 --- /dev/null +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -0,0 +1,194 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_SCALARCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_SCALARCODEC_HAS_BEEN_INCLUDED + +#include +#include +#include +#include +#include + +#include "impl/ScalarLeafCodec.h" +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct WriteBuffersOp +{ + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + WriteBuffersOp(std::ostream& _os, bool _saveFloatAsHalf, const ValueT& _background) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + // Pass the background explicitly so leaf compression does not depend on + // the stream's background pointer (which the codec path no longer sets). + writeScalarLeafBuffers(leaf, os, saveFloatAsHalf, &background); + } + + std::ostream& os; + const bool saveFloatAsHalf; + const ValueT& background; +}; // struct WriteBuffersOp + + +template +struct ReadBuffersOp +{ + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + using StorageLeafT = typename StorageTreeT::LeafNodeType; + using StorageValueT = typename StorageTreeT::ValueType; + + ReadBuffersOp(std::istream& _is, bool _saveFloatAsHalf, const ValueT& _background, + const CoordBBox* _clipBBox, const StorageValueT* _storageBackground = nullptr) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) + , clipBBox(_clipBBox) + , storageBackground(_storageBackground) { } + + void operator()(RootT& root, size_t) + { + if (clipBBox) { + root.clip(*clipBBox); + } + } + + template + void operator()(NodeT& node, size_t) + { + if (clipBBox) { + node.clip(*clipBBox, background); + } + } + + void operator()(LeafT& leaf, size_t) + { + readScalarLeafBuffers(leaf, is, saveFloatAsHalf, background, /*skip=*/false, clipBBox, storageBackground); + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; + const CoordBBox* clipBBox = nullptr; + const StorageValueT* storageBackground = nullptr; +}; // struct ReadBuffersOp + + +// Free-standing function for both standard and conversion codec cases +// Uses StorageGridT = GridT by default, but allows different storage type for conversions +template +void scalarCodecReadBuffers(GridT& grid, std::istream& is, const io::ReadOptions& options, + const typename StorageGridT::TreeType::ValueType* storageBackground) +{ + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ScalarCodec"); + } + + using TreeT = typename GridT::TreeType; + using StorageTreeT = typename StorageGridT::TreeType; + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + std::unique_ptr clipIndexBBox; + if (options.clipBBox.isSorted()) { + clipIndexBBox = std::make_unique(grid.constTransform().worldToIndexNodeCentered(options.clipBBox)); + } + + // Works for both standard (TreeT == StorageTreeT) and conversion cases + ReadBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), + clipIndexBBox.get(), storageBackground); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); +} + +// Free-standing function for write case (no StorageGridT needed) +template +void scalarCodecWriteBuffers(const GridT& grid, std::ostream& os) +{ + using TreeType = typename GridT::TreeType; + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ScalarCodec"); + } + + WriteBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf(), grid.tree().background()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); +} + +} // namespace internal + +template +struct ScalarCodec final: public TopologyCodec +{ + static_assert(GridT::TreeType::RootNodeType::template SameConfiguration< + typename StorageGridT::TreeType::RootNodeType>::value, + "GridT and StorageGridT must have the same configuration"); + + using Ptr = std::unique_ptr>; + + ~ScalarCodec() noexcept = default; + + static inline std::string name() + { + if constexpr (std::is_same_v) { + return GridT::gridType(); + } else { + std::string buildType = typeNameAsString(); + return StorageGridT::gridType() + "_to_" + buildType; + } + } + + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) const final + { + using StorageValueT = typename StorageGridT::TreeType::ValueType; + GridT& grid = static_cast(*data.grid); + + // readTopology() has already set the tiles and leaf buffers to the + // background, so there is nothing left to read. + if (options.readMode == io::ReadMode::TopologyOnly) return; + + auto& topoData = static_cast&>(data); + internal::scalarCodecReadBuffers(grid, is, options, &topoData.storageBackground); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final + { + // Note: the write body must live inside the negated if constexpr branch + // so it is not instantiated for read-only codecs. A bare + // `if constexpr (Mode == ReadOnly) return;` would still instantiate the + // code that follows, which fails to compile for the scalar-to-mask/bool + // convert codecs (their leaf buffers expose WordType*, not ValueType*). + if constexpr (Mode != io::CodecMode::ReadOnly) { + const GridT& grid = static_cast(gridBase); + internal::scalarCodecWriteBuffers(grid, os); + } + } +}; // struct ScalarCodec + + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_SCALARCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h new file mode 100644 index 0000000000..47f07b7ba0 --- /dev/null +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -0,0 +1,399 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_TOPOLOGYCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_TOPOLOGYCODEC_HAS_BEEN_INCLUDED + +#include +#include +#include +#include +#include +#include +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { + +/// Per-read-operation state shared between readTopology() and readBuffers(). +/// Stores the storage-typed background so readBuffers() can pass it directly +/// to readCompressedValues(), bypassing the stream background ptr entirely. +template +struct TopologyCodecData : public io::CodecData +{ + StorageValueT storageBackground{}; +}; // struct TopologyCodecData + +namespace internal { + +template +struct WriteTopologyOp +{ + using ValueT = typename TreeT::ValueType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + + WriteTopologyOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + void operator()(const RootT& root, size_t) + { + int32_t bufferCount = 1; + os.write(reinterpret_cast(&bufferCount), sizeof(int32_t)); + + background = &root.background(); + if (!saveFloatAsHalf) { + os.write(reinterpret_cast(background), sizeof(ValueT)); + } else { + ValueT truncatedVal = io::truncateRealToHalf(*background); + os.write(reinterpret_cast(&truncatedVal), sizeof(ValueT)); + } + + const Index numTiles = root.tileCount(), numChildren = root.childCount(); + os.write(reinterpret_cast(&numTiles), sizeof(Index)); + os.write(reinterpret_cast(&numChildren), sizeof(Index)); + + // Write tiles. + for (auto iter = root.cbeginValueAll(); iter; ++iter) { + const auto& ijk = iter.getCoord(); + os.write(reinterpret_cast(ijk.asPointer()), 3 * sizeof(Int32)); + ValueT value; + bool active = root.getTileValueUnsafe(ijk, value); + os.write(reinterpret_cast(&value), sizeof(ValueT)); + os.write(reinterpret_cast(&active), sizeof(bool)); + } + + rootChildLevel = RootT::LEVEL-1; + } + + template + void operator()(const NodeT& node, size_t) + { + // Write origin for RootNode children only + if (rootChildLevel == NodeT::LEVEL) { + const auto& ijk = node.origin(); + os.write(reinterpret_cast(ijk.asPointer()), 3 * sizeof(Int32)); + } + + const auto& childMask = node.getChildMask(); + const auto& valueMask = node.getValueMask(); + + childMask.save(os); + valueMask.save(os); + + { + // Copy all of this node's values into an array. + std::unique_ptr valuePtr(new ValueT[NodeT::NUM_VALUES]); + ValueT* values = valuePtr.get(); + const ValueT zero = zeroVal(); + for (Index i = 0; i < NodeT::NUM_VALUES; ++i) { + values[i] = (node.isChildMaskOff(i) ? node.getValueUnsafe(i) : zero); + } + // Compress (optionally) and write out the contents of the array. + io::writeCompressedValues(os, values, NodeT::NUM_VALUES, valueMask, childMask, saveFloatAsHalf, background); + } + } + + void operator()(const LeafT& leaf, size_t) + { + leaf.getValueMask().save(os); + } + + std::ostream& os; + const bool saveFloatAsHalf; + const ValueT* background = nullptr; + Index rootChildLevel = std::numeric_limits::max(); +}; // struct WriteTopologyOp + + +template +struct ReadTopologyOp +{ + using ValueT = typename TreeT::ValueType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using StorageValueT = typename StorageTreeT::ValueType; + + ReadTopologyOp(std::istream& _is, bool _saveFloatAsHalf, io::ReadDiagnostics& _diagnostics, + const std::string& _gridName) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , diagnostics(_diagnostics) + , gridName(_gridName) { } + + void operator()(RootT& root) + { + using ChildT = typename RootT::ChildNodeType; + + int32_t bufferCount; + is.read(reinterpret_cast(&bufferCount), sizeof(int32_t)); + if (bufferCount != 1) { + diagnostics.addWarning(gridName, "multi-buffer trees are no longer supported"); + } + + // Delete the existing tree. + root.clear(); + + // Read a RootNode that was stored in the current format. + + is.read(reinterpret_cast(&storageBackground), sizeof(StorageValueT)); + background = static_cast(storageBackground); + + Index numTiles = 0, numChildren = 0; + is.read(reinterpret_cast(&numTiles), sizeof(Index)); + is.read(reinterpret_cast(&numChildren), sizeof(Index)); + + Int32 vec[3]; + StorageValueT value; + bool active; + + // Read tiles. + for (Index n = 0; n < numTiles; ++n) { + is.read(reinterpret_cast(vec), 3 * sizeof(Int32)); + is.read(reinterpret_cast(&value), sizeof(StorageValueT)); + is.read(reinterpret_cast(&active), sizeof(bool)); + Coord origin(vec); + if constexpr (std::is_same_v) { + root.addTile(origin, value, active); + } else { + root.addTile(origin, static_cast(value), active); + } + } + + // Read child nodes. + for (Index n = 0; n < numChildren; ++n) { + is.read(reinterpret_cast(vec), 3 * sizeof(Int32)); + Coord origin(vec); + ChildT* child = new ChildT(PartialCreate(), origin, background); + (*this)(*child); + root.addChild(child); + } + } + + template + void operator()(NodeT& node) + { + using ChildT = typename NodeT::ChildNodeType; + using NodeMaskT = typename NodeT::NodeMaskType; + + NodeMaskT childMask, valueMask; + childMask.load(is); + valueMask.load(is); + node.setValueMaskUnsafe(valueMask); + + const bool oldVersion = + (io::getFormatVersion(is) < OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION); + const Index numValues = (oldVersion ? childMask.countOff() : NodeT::NUM_VALUES); + { + // Read in (and uncompress, if necessary) all of this node's values + // into a contiguous array. + std::unique_ptr valuePtr(new StorageValueT[numValues]); + StorageValueT* values = valuePtr.get(); + io::readCompressedValues(is, values, numValues, valueMask, saveFloatAsHalf, &storageBackground); + + // Copy values from the array into this node's table. + if (oldVersion) { + // The node's member child mask is still empty at this point + // (PartialCreate; setChildUnsafe runs below), so iterate the + // local childMask's off-bits to match the legacy ordering and + // avoid over-reading the countOff-sized values array. + Index n = 0; + for (auto iter = childMask.beginOff(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[n++])); + } + OPENVDB_ASSERT(n == numValues); + } else { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); + } + } + } + + // Read in all child nodes and insert them into the table at their proper locations. + // Register the child before recursing so that node's destructor frees it on a throw. + for (auto iter = childMask.beginOn(); iter; ++iter) { + Coord origin = node.offsetToGlobalCoord(iter.pos()); + auto* child = new ChildT(PartialCreate(), origin, background); + node.setChildUnsafe(iter.pos(), child); + (*this)(*child); + } + } + + void operator()(LeafT& leaf) + { + typename LeafT::NodeMaskType valueMask; + valueMask.load(is); + leaf.setValueMask(valueMask); + } + + std::istream& is; + bool saveFloatAsHalf; + ValueT background; + StorageValueT storageBackground; + io::ReadDiagnostics& diagnostics; + std::string gridName; +}; // struct ReadTopologyOp + +template +struct SetTilesToBackgroundOp +{ + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + explicit SetTilesToBackgroundOp(const ValueT& background) + : mBackground(background) { } + + bool operator()(RootT& root, size_t) const + { + for (auto it = root.beginValueAll(); it; ++it) { + it.setValue(mBackground); + } + return true; + } + + template + bool operator()(NodeT& node, size_t) const + { + for (Index i = 0; i < NodeT::NUM_VALUES; ++i) { + if (node.isChildMaskOff(i)) { + node.setValueOnlyUnsafe(i, mBackground); + } + } + return true; + } + + bool operator()(LeafT&, size_t) const { return true; } + +private: + const ValueT mBackground; +}; // struct SetTilesToBackgroundOp + +template +void setTilesToBackground(TreeT& tree) +{ + const typename TreeT::ValueType& background = tree.root().background(); + SetTilesToBackgroundOp op(background); + tree::DynamicNodeManager nodeManager(tree); + nodeManager.foreachTopDown(op); +} + +// Free-standing function for read case (supports type conversion via StorageGridT). +// codecData receives the storage-typed background value so readBuffers() callers +// can pass it explicitly to readCompressedValues(), avoiding the stream background ptr. +template +void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::ReadOptions& options, + io::ReadDiagnostics& diagnostics, TopologyCodecData& codecData) +{ + io::checkFormatVersion(is); + + GridT& grid = static_cast(gridBase); + grid.tree().clearAllAccessors(); + + internal::ReadTopologyOp readTopologyOp(is, grid.saveFloatAsHalf(), diagnostics, grid.getName()); + readTopologyOp(grid.tree().root()); + + // Restore the (value-typed) background on the root. ReadTopologyOp only reads + // the on-disk background into a local; without this the grid would retain the + // default background from GridT::create(). Pass updateChildNodes=false so the + // already-populated child nodes are left untouched. + grid.tree().root().setBackground(readTopologyOp.background, /*updateChildNodes=*/false); + + // Copy storageBackground out of the stack-local ReadTopologyOp into codecData + // so it stays alive until readBuffers() completes. readBuffers() passes it + // explicitly to readCompressedValues(), so the stream background ptr is never + // used and setGridBackgroundValuePtr() is not needed in the codec path. + codecData.storageBackground = readTopologyOp.storageBackground; + + if (options.readMode == io::ReadMode::TopologyOnly) { + internal::setTilesToBackground(grid.tree()); + // allocate leaf buffers in parallel and fill with the background value; + // ReadTopologyOp uses PartialCreate which leaves buffers unallocated. + const auto background = grid.tree().root().background(); + tree::LeafManager leafManager(grid.tree()); + leafManager.foreach([&background](auto& leaf, size_t) { + using LeafType = std::decay_t; + if constexpr (!std::is_same_v) { + if (leaf.buffer().empty()) { + leaf.buffer().allocate(); + leaf.buffer().fill(background); + } + } + }); + return; + } +} + +// Free-standing function for write case (no StorageGridT needed) +template +void topologyCodecWriteTopology(const GridBase& gridBase, std::ostream& os) +{ + const GridT& grid = static_cast(gridBase); + using TreeType = typename GridT::TreeType; + + internal::WriteTopologyOp writeTopologyOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeTopologyOp); +} + +} // namespace internal + +template +struct TopologyCodec : public io::Codec +{ + using StorageValueT = typename StorageGridT::TreeType::ValueType; + using Ptr = std::unique_ptr>; + + ~TopologyCodec() noexcept = default; + + io::CodecData::Ptr createData() override + { + auto data = std::make_unique>(); + data->grid = GridT::create(); + return data; + } + + void readTopology(std::istream& is, io::CodecData& data, const io::ReadOptions& options, + io::ReadDiagnostics& diagnostics) const final + { + // Warn when a conversion readMode was requested but this codec is a + // non-conversion instance (GridT == StorageGridT), meaning no conversion + // codec was registered for this grid type and we are falling back to the + // original type. + if constexpr (std::is_same_v) { + if (options.readMode == io::ReadMode::Half || + options.readMode == io::ReadMode::Bool || + options.readMode == io::ReadMode::Mask) + { + const std::string modeStr = + options.readMode == io::ReadMode::Half ? "Half" : + options.readMode == io::ReadMode::Bool ? "Bool" : "Mask"; + diagnostics.addWarning(data.grid->getName(), + "ReadMode::" + modeStr + " conversion is not supported for grid type '" + + GridT::gridType() + "'; reading as original type"); + } + } + auto& topoData = static_cast&>(data); + internal::topologyCodecReadTopology(*data.grid, is, options, diagnostics, topoData); + } + + void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final + { + // Disable implementation when read only. The body must live inside the + // negated if constexpr branch so it is not instantiated for read-only + // codecs; a bare `if constexpr (...) return;` still instantiates what + // follows. + if constexpr (Mode != io::CodecMode::ReadOnly) { + internal::topologyCodecWriteTopology(gridBase, os); + } + } +}; // struct TopologyCodec + + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_TOPOLOGYCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/codecs/ValueMaskCodec.h b/openvdb/openvdb/codecs/ValueMaskCodec.h new file mode 100644 index 0000000000..d3e3e2a938 --- /dev/null +++ b/openvdb/openvdb/codecs/ValueMaskCodec.h @@ -0,0 +1,146 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_VALUEMASKCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_VALUEMASKCODEC_HAS_BEEN_INCLUDED + +#include + +#include +#include + +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct ReadValueMaskBuffersOp +{ + using TreeT = typename GridT::TreeType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + ReadValueMaskBuffersOp(std::istream& _is, bool _saveFloatAsHalf, + const ValueT& _background, const CoordBBox* _clipBBox) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) + , clipBBox(_clipBBox) { } + + void operator()(RootT& root, size_t) + { + // Clip root-level tiles and prune children that were clipped. + if (clipBBox) { + root.clip(*clipBBox); + } + } + + template + void operator()(NodeT& node, size_t) + { + // Clip internal node tiles and prune children that were clipped. + if (clipBBox) { + node.clip(*clipBBox, background); + } + } + + void operator()(LeafT& leaf, size_t) + { + // Read in the value mask. + leaf.getValueMask().load(is); + // Read in the origin. + Coord origin; + origin.read(is); + leaf.setOrigin(origin); + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; + const CoordBBox* clipBBox = nullptr; +}; // struct ReadValueMaskBuffersOp + +template +struct WriteValueMaskBuffersOp +{ + using TreeT = typename GridT::TreeType; + using LeafT = typename TreeT::LeafNodeType; + + WriteValueMaskBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + // Write out the value mask. + leaf.getValueMask().save(os); + + // Write out the origin. + leaf.origin().write(os); + } + + std::ostream& os; + const bool saveFloatAsHalf; +}; // struct WriteValueMaskBuffersOp + +} // namespace internal + +template +struct ValueMaskCodec final: public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~ValueMaskCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) const final + { + GridT& grid = static_cast(*data.grid); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ValueMaskCodec"); + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + std::unique_ptr clipIndexBBox; + if (options.clipBBox.isSorted()) { + clipIndexBBox = std::make_unique(grid.constTransform().worldToIndexNodeCentered(options.clipBBox)); + } + + internal::ReadValueMaskBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), clipIndexBBox.get()); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final + { + const GridT& grid = static_cast(gridBase); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ValueMaskCodec"); + } + + internal::WriteValueMaskBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); + } +}; // struct ValueMaskCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_VALUEMASKCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h b/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h new file mode 100644 index 0000000000..2288f8a5a2 --- /dev/null +++ b/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h @@ -0,0 +1,125 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_IMPL_SCALARLEAFCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_IMPL_SCALARLEAFCODEC_HAS_BEEN_INCLUDED + +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +void writeScalarLeafBuffers(const LeafT& leaf, std::ostream& os, bool saveFloatAsHalf, + const typename LeafT::ValueType* background = nullptr) +{ + using NodeMaskT = typename LeafT::NodeMaskType; + + // Write out the value mask. + leaf.getValueMask().save(os); + + leaf.buffer().data(); // load values + + io::writeCompressedValues(os, leaf.buffer().data(), LeafT::SIZE, + leaf.getValueMask(), /*childMask=*/NodeMaskT(), saveFloatAsHalf, background); +} + + +template +void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, + const typename LeafT::ValueType& background, bool skip = false, + const math::CoordBBox* clipBBox = nullptr, + const typename StorageLeafT::ValueType* storageBackground = nullptr) +{ + using ValueT = typename LeafT::ValueType; + using NodeMaskT = typename LeafT::NodeMaskType; + using StorageBufferT = typename StorageLeafT::Buffer; + using StorageValueT = typename StorageLeafT::ValueType; + + constexpr Index SIZE = LeafT::SIZE; + + SharedPtr meta = io::getStreamMetadataPtr(is); + const bool seekable = meta && meta->seekable(); + + auto& valueMask = leaf.getValueMask(); + + // Load or seek the value mask + if (seekable) valueMask.seek(is); + else valueMask.load(is); + + // Pre-node-mask-compression format stored the origin and buffer count + // inline after the value mask. + int8_t numBuffers = 1; + if (io::getFormatVersion(is) < OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) { + Coord origin; + is.read(reinterpret_cast(&origin), sizeof(Coord::ValueType) * 3); + leaf.setOrigin(origin); + + is.read(reinterpret_cast(&numBuffers), sizeof(int8_t)); + + if (numBuffers > 1) { + OPENVDB_THROW(IoError, "Old file format 221 (FLOAT_FRUSTUM_BBOX) with multiple buffers is not supported"); + } + } + + if (skip) { + if (seekable) { + io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf, storageBackground); + } else { + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); + } + // Clear the value mask so that the skipped leaf has no active + // voxels. Without this, the leaf retains its on-disk active + // topology even though no data was read into its buffer. + valueMask.setOff(); + return; + } + + if constexpr (std::is_same_v) { + // ValueMask leaf: value == active state, already captured in the value mask above. + // Seek/consume past the storage buffer without populating any separate leaf buffer. + if (seekable) { + io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf, storageBackground); + } else { + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); + } + } else if constexpr (std::is_same_v) { + // Bool leaf: must read storage values regardless of seekability, then convert. + if constexpr (std::is_same_v) { + io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); + } else { + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); + for (Index i = 0; i < SIZE; ++i) { + leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); + } + } + } else { + leaf.buffer().allocate(); + if constexpr (std::is_same_v) { + io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); + } else { + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); + for (Index i = 0; i < SIZE; ++i) { + leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); + } + } + } + + if (clipBBox) { + leaf.clip(*clipBBox, background); + } +} + +} // namespace internal +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_IMPL_SCALARLEAFCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index b8facd3715..d1d2451943 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -4,7 +4,6 @@ #include "Archive.h" #include "GridDescriptor.h" -#include "DelayedLoadMetadata.h" #include "io.h" #include @@ -13,35 +12,6 @@ #include #include -#ifdef OPENVDB_USE_DELAYED_LOADING -// Boost.Interprocess uses a header-only portion of Boost.DateTime -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunused-macros" -#endif -#define BOOST_DATE_TIME_NO_LIB -#ifdef __clang__ -#pragma clang diagnostic pop -#endif -#include -#include -#include -#include - -#ifdef _WIN32 -#include // open_existing_file(), close_file() -extern "C" __declspec(dllimport) bool __stdcall GetFileTime( - void* fh, void* ctime, void* atime, void* mtime); -// boost::interprocess::detail was renamed to boost::interprocess::ipcdetail in Boost 1.48. -// Ensure that both namespaces exist. -namespace boost { namespace interprocess { namespace detail {} namespace ipcdetail {} } } -#else -#include // for struct stat -#include // for stat() -#include // for unlink() -#endif -#endif // OPENVDB_USE_DELAYED_LOADING - #include #include // for std::find_if() @@ -209,9 +179,10 @@ struct StreamMetadata::Impl uint32_t mPass = 0; MetaMap mGridMetadata; AuxDataMap mAuxData; - bool mDelayedLoadMeta = DelayedLoadMetadata::isRegisteredType(); + bool mDelayedLoadMeta = false; uint64_t mLeaf = 0; uint32_t mTest = 0; // for testing only + bool mAllocateLeafBuffers = false; }; // struct StreamMetadata @@ -273,8 +244,8 @@ bool StreamMetadata::writeGridStats() const { return mImpl->mWriteGr bool StreamMetadata::seekable() const { return mImpl->mSeekable; } bool StreamMetadata::delayedLoadMeta() const { return mImpl->mDelayedLoadMeta; } bool StreamMetadata::countingPasses() const { return mImpl->mCountingPasses; } +bool StreamMetadata::allocateLeafBuffers() const { return mImpl->mAllocateLeafBuffers; } uint32_t StreamMetadata::pass() const { return mImpl->mPass; } -uint64_t StreamMetadata::leaf() const { return mImpl->mLeaf; } MetaMap& StreamMetadata::gridMetadata() { return mImpl->mGridMetadata; } const MetaMap& StreamMetadata::gridMetadata() const { return mImpl->mGridMetadata; } uint32_t StreamMetadata::__test() const { return mImpl->mTest; } @@ -291,8 +262,8 @@ void StreamMetadata::setHalfFloat(bool b) { mImpl->mHalfFloat = b; void StreamMetadata::setWriteGridStats(bool b) { mImpl->mWriteGridStats = b; } void StreamMetadata::setSeekable(bool b) { mImpl->mSeekable = b; } void StreamMetadata::setCountingPasses(bool b) { mImpl->mCountingPasses = b; } +void StreamMetadata::setAllocateLeafBuffers(bool b) { mImpl->mAllocateLeafBuffers = b; } void StreamMetadata::setPass(uint32_t i) { mImpl->mPass = i; } -void StreamMetadata::setLeaf(uint64_t i) { mImpl->mLeaf = i; } void StreamMetadata::__setTest(uint32_t t) { mImpl->mTest = t; } std::string @@ -306,7 +277,6 @@ StreamMetadata::str() const ostr << "compression: " << compressionToString(compression()) << "\n"; ostr << "half_float: " << halfFloat() << "\n"; ostr << "seekable: " << seekable() << "\n"; - ostr << "delayed_load_meta: " << delayedLoadMeta() << "\n"; ostr << "pass: " << pass() << "\n"; ostr << "counting_passes: " << countingPasses() << "\n"; ostr << "write_grid_stats_metadata: " << writeGridStats() << "\n"; @@ -339,73 +309,6 @@ writeAsType(std::ostream& os, const std::any& val) return false; } -struct PopulateDelayedLoadMetadataOp -{ - DelayedLoadMetadata& metadata; - uint32_t compression; - - PopulateDelayedLoadMetadataOp(DelayedLoadMetadata& _metadata, uint32_t _compression) - : metadata(_metadata) - , compression(_compression) { } - - template - void operator()(const GridT& grid) const - { - using TreeT = typename GridT::TreeType; - using ValueT = typename TreeT::ValueType; - using LeafT = typename TreeT::LeafNodeType; - using MaskT = typename LeafT::NodeMaskType; - - const TreeT& tree = grid.constTree(); - const Index64 leafCount = tree.leafCount(); - - // early exit if not leaf nodes - if (leafCount == Index64(0)) return; - - metadata.resizeMask(leafCount); - - if (compression & (COMPRESS_BLOSC | COMPRESS_ZIP)) { - metadata.resizeCompressedSize(leafCount); - } - - const auto background = tree.background(); - const bool saveFloatAsHalf = grid.saveFloatAsHalf(); - - tree::LeafManager leafManager(tree); - - leafManager.foreach( - [&](const LeafT& leaf, size_t idx) { - // set mask value - MaskCompress maskCompressData( - leaf.valueMask(), /*childMask=*/MaskT(), leaf.buffer().data(), background); - metadata.setMask(idx, maskCompressData.metadata); - - if (compression & (COMPRESS_BLOSC | COMPRESS_ZIP)) { - // set compressed size value - size_t sizeBytes(8); - size_t compressedSize = io::writeCompressedValuesSize( - leaf.buffer().data(), LeafT::SIZE, - leaf.valueMask(), maskCompressData.metadata, saveFloatAsHalf, compression); - metadata.setCompressedSize(idx, compressedSize+sizeBytes); - } - } - ); - } -}; - -bool populateDelayedLoadMetadata(DelayedLoadMetadata& metadata, - const GridBase& gridBase, uint32_t compression) -{ - PopulateDelayedLoadMetadataOp op(metadata, compression); - - using AllowedTypes = TypeList< - Int32Grid, Int64Grid, - FloatGrid, DoubleGrid, - Vec3IGrid, Vec3SGrid, Vec3DGrid>; - - return gridBase.apply(op); -} - } // unnamed namespace std::ostream& @@ -443,117 +346,6 @@ operator<<(std::ostream& os, const StreamMetadata::AuxDataMap& auxData) //////////////////////////////////////// -#ifdef OPENVDB_USE_DELAYED_LOADING - - -// Memory-mapping a VDB file permits threaded input (and output, potentially, -// though that might not be practical for compressed files or files containing -// multiple grids). In particular, a memory-mapped file can be loaded lazily, -// meaning that the voxel buffers of the leaf nodes of a grid's tree are not allocated -// until they are actually accessed. When access to its buffer is requested, -// a leaf node allocates memory for the buffer and then streams in (and decompresses) -// its contents from the memory map, starting from a stream offset that was recorded -// at the time the node was constructed. The memory map must persist as long as -// there are unloaded leaf nodes; this is ensured by storing a shared pointer -// to the map in each unloaded node. - -class MappedFile::Impl -{ -public: - Impl(const std::string& filename, bool autoDelete) - : mMap(filename.c_str(), boost::interprocess::read_only) - , mRegion(mMap, boost::interprocess::read_only) - , mAutoDelete(autoDelete) - { - if (mAutoDelete) { -#ifndef _WIN32 - // On Unix systems, unlink the file so that it gets deleted once it is closed. - ::unlink(mMap.get_name()); -#endif - } - } - - ~Impl() - { - std::string filename; - if (const char* s = mMap.get_name()) filename = s; - OPENVDB_LOG_DEBUG_RUNTIME("closing memory-mapped file " << filename); - if (mNotifier) mNotifier(filename); - if (mAutoDelete) { - if (!boost::interprocess::file_mapping::remove(filename.c_str())) { - if (errno != ENOENT) { - // Warn if the file exists but couldn't be removed. - std::string mesg = getErrorString(); - if (!mesg.empty()) mesg = " (" + mesg + ")"; - OPENVDB_LOG_WARN("failed to remove temporary file " << filename << mesg); - } - } - } - } - - boost::interprocess::file_mapping mMap; - boost::interprocess::mapped_region mRegion; - bool mAutoDelete; - Notifier mNotifier; -#if OPENVDB_ABI_VERSION_NUMBER <= 12 - mutable std::atomic mLastWriteTime; -#endif - -private: - Impl(const Impl&); // not copyable - Impl& operator=(const Impl&); // not copyable -}; - - -MappedFile::MappedFile(const std::string& filename, bool autoDelete): - mImpl(new Impl(filename, autoDelete)) -{ -} - - -MappedFile::~MappedFile() -{ -} - - -std::string -MappedFile::filename() const -{ - std::string result; - if (const char* s = mImpl->mMap.get_name()) result = s; - return result; -} - - -SharedPtr -MappedFile::createBuffer() const -{ - return SharedPtr{ - new boost::iostreams::stream_buffer{ - static_cast(mImpl->mRegion.get_address()), mImpl->mRegion.get_size()}}; -} - - -void -MappedFile::setNotifier(const Notifier& notifier) -{ - mImpl->mNotifier = notifier; -} - - -void -MappedFile::clearNotifier() -{ - mImpl->mNotifier = nullptr; -} - - -#endif // OPENVDB_USE_DELAYED_LOADING - - -//////////////////////////////////////// - - std::string getErrorString(int errorNum) { @@ -595,6 +387,35 @@ Archive::copy() const } +void +Archive::enableReadDiagnostics() +{ + mReadDiagnostics.enable(); +} + + +void +Archive::disableReadDiagnostics() +{ + mReadDiagnostics.disable(); + mReadDiagnostics.clear(); +} + + +const ReadDiagnostics& +Archive::readDiagnostics() const +{ + return mReadDiagnostics; +} + + +void +Archive::clearReadDiagnostics() +{ + mReadDiagnostics.clear(); +} + + //////////////////////////////////////// @@ -888,25 +709,6 @@ setGridBackgroundValuePtr(std::ios_base& strm, const void* background) } -#ifdef OPENVDB_USE_DELAYED_LOADING -MappedFile::Ptr -getMappedFilePtr(std::ios_base& strm) -{ - if (const void* ptr = strm.pword(GetSteamState().mappedFile)) { - return *static_cast(ptr); - } - return MappedFile::Ptr(); -} - - -void -setMappedFilePtr(std::ios_base& strm, io::MappedFile::Ptr& mappedFile) -{ - strm.pword(GetSteamState().mappedFile) = &mappedFile; -} -#endif // OPENVDB_USE_DELAYED_LOADING - - StreamMetadata::Ptr getStreamMetadataPtr(std::ios_base& strm) { @@ -1095,6 +897,28 @@ Archive::readGridCount(std::istream& is) //////////////////////////////////////// +io::Codec* +Archive::findCodec(const std::string& gridType, const io::ReadOptions& options) +{ + // if readMode is Half, then search for a codec that converts + // from the storage grid type to the grid type + if (options.readMode == ReadMode::Half) { + if (auto* codec = io::CodecRegistry::get(gridType + "_to_half")) return codec; + } else if (options.readMode == ReadMode::Bool) { + if (auto* codec = io::CodecRegistry::get(gridType + "_to_bool")) return codec; + } else if (options.readMode == ReadMode::Mask) { + if (auto* codec = io::CodecRegistry::get(gridType + "_to_mask")) return codec; + } + + // Determine the I/O codec to use to read this grid (also the fallback when + // no conversion codec is registered for a Half/Bool/Mask readMode). + return io::CodecRegistry::get(gridType); +} + + +//////////////////////////////////////// + + void Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) const { @@ -1126,35 +950,12 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co //////////////////////////////////////// -//static -bool -Archive::isDelayedLoadingEnabled() -{ -#ifdef OPENVDB_USE_DELAYED_LOADING - return (nullptr == std::getenv("OPENVDB_DISABLE_DELAYED_LOAD")); -#else - return false; -#endif -} - - -namespace { - -struct NoBBox {}; - -template -void -doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const BoxType& bbox) +GridBase::Ptr +Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOptions& readOptions, ReadDiagnostics& diagnostics) { - struct Local { - static void readBuffers(GridBase& g, std::istream& istrm, NoBBox) { g.readBuffers(istrm); } - static void readBuffers(GridBase& g, std::istream& istrm, const CoordBBox& indexBBox) { - g.readBuffers(istrm, indexBBox); - } - static void readBuffers(GridBase& g, std::istream& istrm, const BBoxd& worldBBox) { - g.readBuffers(istrm, g.constTransform().worldToIndexNodeCentered(worldBBox)); - } - }; + // Read the compression settings for this grid and tag the stream with them + // so that downstream functions can reference them. + readGridCompression(is); // Restore the file-level stream metadata on exit. struct OnExit { @@ -1165,6 +966,27 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const }; OnExit restore(is); + // Find the codec for the grid type and options. + io::Codec* codec = findCodec(gd.gridType(), readOptions); + + GridBase::Ptr grid; + io::CodecData::Ptr codecData; + + // Create the grid. + if (codec) { + codecData = codec->createData(); + if (codecData->grid) grid = codecData->grid; + } else { + if (!GridBase::isRegistered(gd.gridType())) { + OPENVDB_THROW(KeyError, "Cannot read grid " + << GridDescriptor::nameAsString(gd.uniqueName()) + << ": grid type " << gd.gridType() << " is not registered"); + } + + grid = GridBase::createGrid(gd.gridType()); + } + grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); + // Stream metadata varies per grid, and it needs to persist // in case delayed load is in effect. io::StreamMetadata::Ptr streamMetadata; @@ -1182,75 +1004,71 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const grid->readMeta(is); - // Add a description of the compression settings to the grid as metadata. - /// @todo Would this be useful? - //const uint32_t c = getDataCompression(is); - //grid->insertMeta(GridBase::META_FILE_COMPRESSION, - // StringMetadata(compressionToString(c))); - - const VersionId version = getLibraryVersion(is); - if (version.first < 6 || (version.first == 6 && version.second <= 1)) { - // If delay load metadata exists, but the file format version does not support - // delay load metadata, this likely means the original grid was read and then - // written using a prior version of OpenVDB and ABI>=5 where unknown metadata - // can be blindly copied. This means that it is possible for the metadata to - // no longer be in sync with the grid, so we remove it to ensure correctness. - - if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { - grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } + // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists + if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { + grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); } streamMetadata->gridMetadata() = static_cast(*grid); const GridClass gridClass = grid->getGridClass(); io::setGridClass(is, gridClass); - // reset leaf value to zero - streamMetadata->setLeaf(0); - - // drop DelayedLoadMetadata from the grid as it is only useful for IO - // a stream metadata non-zero value disables this behaviour for testing - - if (streamMetadata->__test() == uint32_t(0)) { - if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { - grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); + grid->readTransform(is); + const bool readTopology = readOptions.readMode != io::ReadMode::MetadataOnly && !gd.isInstance(); + const bool readBuffers = readTopology && readOptions.readMode != io::ReadMode::TopologyOnly; + if (readTopology) { + // read topology + if (codec) { + codec->readTopology(is, *codecData, readOptions, diagnostics); + } else { + io::StreamMetadata::Ptr allocateLeafBuffersMeta; + if (readOptions.readMode == io::ReadMode::TopologyOnly) { + // Signal Grid::readTopology to allocate leaf buffers and + // fill them with the background value. + allocateLeafBuffersMeta = io::getStreamMetadataPtr(is); + if (allocateLeafBuffersMeta) { + allocateLeafBuffersMeta->setAllocateLeafBuffers(true); + } + } + try { + grid->readTopology(is); + } catch (...) { + // Grid::readTopology() clears the flag on success, but if + // it throws the flag must not leak into subsequent grid reads. + if (allocateLeafBuffersMeta) { + allocateLeafBuffersMeta->setAllocateLeafBuffers(false); + } + throw; + } } } - - grid->readTransform(is); - if (!gd.isInstance()) { - grid->readTopology(is); - Local::readBuffers(*grid, is, bbox); + if (readBuffers) { + // read buffers + if (codec) { + OPENVDB_ASSERT(gd.getEndPos() >= gd.getGridPos()); + const Index64 size = static_cast(gd.getEndPos() - gd.getGridPos()); + codec->readBuffers(is, size, *codecData, readOptions, diagnostics); + } else { + const auto& worldBBox = readOptions.clipBBox; + const bool clip = worldBBox.isSorted(); + if (clip) { + const auto indexBBox = grid->constTransform().worldToIndexNodeCentered(worldBBox); + grid->readBuffers(is, indexBBox); + } else { + grid->readBuffers(is); + } + } } -} -} // unnamed namespace - - -void -Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is) -{ - // Read the compression settings for this grid and tag the stream with them - // so that downstream functions can reference them. - readGridCompression(is); - - doReadGrid(grid, gd, is, NoBBox()); + return grid; } -void -Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, - std::istream& is, const BBoxd& worldBBox) -{ - readGridCompression(is); - doReadGrid(grid, gd, is, worldBBox); -} -void -Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, - std::istream& is, const CoordBBox& indexBBox) +GridBase::Ptr +Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOptions& readOptions) { - readGridCompression(is); - doReadGrid(grid, gd, is, indexBBox); + ReadDiagnostics nullDiagnostics; + return readGrid(gd, is, readOptions, nullDiagnostics); } @@ -1259,15 +1077,15 @@ Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, void Archive::write(std::ostream& os, const GridPtrVec& grids, bool seekable, - const MetaMap& metadata) const + const MetaMap& metadata, const io::WriteOptions& writeOptions) const { - this->write(os, GridCPtrVec(grids.begin(), grids.end()), seekable, metadata); + this->write(os, GridCPtrVec(grids.begin(), grids.end()), seekable, metadata, writeOptions); } void Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, - const MetaMap& metadata) const + const MetaMap& metadata, const io::WriteOptions& writeOptions) const { // Set stream flags so that downstream functions can reference them. io::StreamMetadata::Ptr streamMetadata = io::getStreamMetadataPtr(os); @@ -1339,7 +1157,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, // Get the name of the other grid. gd.setInstanceParentName(mapIter->second.uniqueName()); // Write out this grid's descriptor and metadata, but not its tree. - writeGridInstance(gd, grid, os, seekable); + writeGridInstance(gd, grid, os, seekable, writeOptions); OPENVDB_LOG_DEBUG_RUNTIME("io::Archive::write(): " << GridDescriptor::nameAsString(gd.uniqueName()) @@ -1348,7 +1166,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, << GridDescriptor::nameAsString(gd.instanceParentName())); } else { // Write out the grid descriptor and its associated grid. - writeGrid(gd, grid, os, seekable); + writeGrid(gd, grid, os, seekable, writeOptions); // Record the grid's tree pointer so that the tree doesn't get written // more than once. treeMap[treePtr] = gd; @@ -1364,7 +1182,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, void Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, - std::ostream& os, bool seekable) const + std::ostream& os, bool seekable, const io::WriteOptions& writeOptions) const { // Restore file-level stream metadata on exit. struct OnExit { @@ -1375,6 +1193,9 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, }; OnExit restore(os); + // Find the codec for the grid type and options. + io::Codec* codec = findCodec(gd.gridType()); + // Stream metadata varies per grid, so make a copy of the file-level stream metadata. io::StreamMetadata::Ptr streamMetadata; if (io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(os)) { @@ -1403,35 +1224,36 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, // Save the compression settings for this grid. setGridCompression(os, *grid); - // copy grid and add delay load metadata - const auto copyOfGrid = grid->copyGrid(); // shallow copy - const auto nonConstCopyOfGrid = ConstPtrCast(copyOfGrid); - nonConstCopyOfGrid->insertMeta(GridBase::META_FILE_DELAYED_LOAD, - DelayedLoadMetadata()); - DelayedLoadMetadata::Ptr delayLoadMeta = - nonConstCopyOfGrid->getMetadata(GridBase::META_FILE_DELAYED_LOAD); - if (!populateDelayedLoadMetadata(*delayLoadMeta, *grid, compression())) { - nonConstCopyOfGrid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } - // Save the grid's metadata and transform. if (getWriteGridStatsMetadata(os)) { // Compute and add grid statistics metadata. + const auto copyOfGrid = grid->copyGrid(); // shallow copy + const auto nonConstCopyOfGrid = ConstPtrCast(copyOfGrid); nonConstCopyOfGrid->addStatsMetadata(); nonConstCopyOfGrid->insertMeta(GridBase::META_FILE_COMPRESSION, StringMetadata(compressionToString(getDataCompression(os)))); + copyOfGrid->writeMeta(os); + } else { + grid->writeMeta(os); } - copyOfGrid->writeMeta(os); grid->writeTransform(os); // Save the grid's structure. - grid->writeTopology(os); + if (codec) { + codec->writeTopology(os, *grid, writeOptions); + } else { + grid->writeTopology(os); + } // Now we know the grid block storage position. if (seekable) gd.setBlockPos(os.tellp()); // Save out the data blocks of the grid. - grid->writeBuffers(os); + if (codec) { + codec->writeBuffers(os, *grid, writeOptions); + } else { + grid->writeBuffers(os); + } // Now we know the end position of this grid. if (seekable) gd.setEndPos(os.tellp()); @@ -1450,7 +1272,7 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, void Archive::writeGridInstance(GridDescriptor& gd, GridBase::ConstPtr grid, - std::ostream& os, bool seekable) const + std::ostream& os, bool seekable, const io::WriteOptions&) const { // Write out the Descriptor's header information (grid name, type // and instance parent name). diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index 4e4e680273..5d15b9cdc2 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -5,11 +5,14 @@ #define OPENVDB_IO_ARCHIVE_HAS_BEEN_INCLUDED #include -#include "Compression.h" // for COMPRESS_ZIP, etc. #include #include #include #include // for VersionId + +#include "Codec.h" +#include "Compression.h" // for COMPRESS_ZIP, etc. + #include #include #include @@ -89,14 +92,23 @@ class OPENVDB_API Archive void setGridStatsMetadataEnabled(bool b) { mEnableGridStats = b; } /// @brief Write the grids in the given container to this archive's output stream. - virtual void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const {} + virtual void write(const GridCPtrVec&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const {} + + /// @brief Return @c false (delayed loading has been removed). + static bool isDelayedLoadingEnabled() { return false; } - /// @brief Return @c true if delayed loading is enabled. - /// @details If enabled, delayed loading can be disabled for individual files, - /// but not vice-versa. - /// @note Define the environment variable @c OPENVDB_DISABLE_DELAYED_LOAD - /// to disable delayed loading unconditionally. - static bool isDelayedLoadingEnabled(); + /// @brief Enable collection of read diagnostics (warnings, etc.) during I/O operations. + void enableReadDiagnostics(); + + /// @brief Disable collection of read diagnostics. + void disableReadDiagnostics(); + + /// @brief Return a const reference to the diagnostics collector. + const ReadDiagnostics& readDiagnostics() const; + + /// @brief Clear the diagnostics list while keeping collection active. + void clearReadDiagnostics(); protected: /// @brief Return @c true if the input stream contains grid offsets @@ -131,14 +143,15 @@ class OPENVDB_API Archive /// Read in and return the number of grids on the input stream. static int32_t readGridCount(std::istream&); - /// Populate the given grid from the input stream. - static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&); - /// @brief Populate the given grid from the input stream, but only where it - /// intersects the given world-space bounding box. - static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&, const BBoxd&); - /// @brief Populate the given grid from the input stream, but only where it - /// intersects the given index-space bounding box. - static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&, const CoordBBox&); + /// @brief Find the codec for the given grid type and options. + static io::Codec* findCodec(const std::string& gridType, const io::ReadOptions& options = io::ReadOptions{}); + + /// @brief Read in and create the grid represented by the given grid descriptor using the + /// given input stream, using the provided options if given. + static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, + const io::ReadOptions& readOptions, ReadDiagnostics& diagnostics); + static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, + const io::ReadOptions& readOptions = io::ReadOptions{}); using NamedGridMap = std::map; @@ -149,13 +162,16 @@ class OPENVDB_API Archive /// Write the given grid descriptor and grid to an output stream /// and update the GridDescriptor offsets. /// @param seekable if true, the output stream supports seek operations - void writeGrid(GridDescriptor&, GridBase::ConstPtr, std::ostream&, bool seekable) const; + /// @param writeOptions options controlling how grid data is written + void writeGrid(GridDescriptor&, GridBase::ConstPtr, std::ostream&, bool seekable, + const io::WriteOptions& writeOptions = io::WriteOptions{}) const; /// Write the given grid descriptor and grid metadata to an output stream /// and update the GridDescriptor offsets, but don't write the grid's tree, /// since it is shared with another grid. /// @param seekable if true, the output stream supports seek operations + /// @param writeOptions options controlling how grid data is written void writeGridInstance(GridDescriptor&, GridBase::ConstPtr, - std::ostream&, bool seekable) const; + std::ostream&, bool seekable, const io::WriteOptions& writeOptions = io::WriteOptions{}) const; /// @brief Read the magic number, version numbers, UUID, etc. from the given input stream. /// @return @c true if the input UUID differs from the previously-read UUID. @@ -167,10 +183,15 @@ class OPENVDB_API Archive //@{ /// Write the given grids to an output stream. - void write(std::ostream&, const GridPtrVec&, bool seekable, const MetaMap& = MetaMap()) const; - void write(std::ostream&, const GridCPtrVec&, bool seekable, const MetaMap& = MetaMap()) const; + void write(std::ostream&, const GridPtrVec&, bool seekable, const MetaMap&, + const io::WriteOptions& writeOptions = io::WriteOptions{}) const; + void write(std::ostream&, const GridCPtrVec&, bool seekable, const MetaMap&, + const io::WriteOptions& writeOptions = io::WriteOptions{}) const; //@} + /// Diagnostics collector for read operations (always valid; enabled/disabled via flag) + mutable ReadDiagnostics mReadDiagnostics; + private: friend class ::TestFile; diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc new file mode 100644 index 0000000000..e2a3213814 --- /dev/null +++ b/openvdb/openvdb/io/Codec.cc @@ -0,0 +1,135 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#include "Codec.h" + +#include +#include +#include +#include +#include + +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace io { + +namespace { + +using CodecFactoryMap = std::map; +using CodecFactoryMapCIter = CodecFactoryMap::const_iterator; + +struct LockedCodecRegistry { + LockedCodecRegistry() {} + ~LockedCodecRegistry() {} + std::mutex mMutex; + CodecFactoryMap mMap; +}; + +// Global function for accessing the registry +static LockedCodecRegistry* +getCodecRegistry() +{ + static LockedCodecRegistry registry; + return ®istry; +} + +} // unnamed namespace + + +namespace internal { + +template +struct RegisterCodec { inline void operator()() { CodecRegistry::registerCodec>(); } }; + +template +struct RegisterConvertCodec { + inline void operator()() + { + CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); + } +}; // struct RegisterConvertCodec + +void initialize() +{ + CodecRegistry::clear(); + NumericGridTypes::foreach(); + Vec3GridTypes::foreach(); + + CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); + + // register the plugin that converts from scalar to mask/bool + NumericGridTypes::foreach(); + + // register the plugin that converts from float to half + CodecRegistry::registerCodec>(); +} + +void uninitialize() +{ + CodecRegistry::clear(); +} + +} // namespace internal + + +//////////////////////////////////////// + + +bool +CodecRegistry::isRegistered(const std::string& name) +{ + LockedCodecRegistry* registry = getCodecRegistry(); + std::lock_guard lock(registry->mMutex); + + return (registry->mMap.find(name) != registry->mMap.end()); +} + + +void +CodecRegistry::registerCodecByName(const std::string& name, Codec::Ptr&& codec) +{ + LockedCodecRegistry* registry = getCodecRegistry(); + std::lock_guard lock(registry->mMutex); + + if (registry->mMap.find(name) != registry->mMap.end()) { + OPENVDB_THROW(KeyError, + "Cannot register codec " << name << ". Codec is already registered"); + } + + registry->mMap[name] = std::move(codec); +} + + +Codec* +CodecRegistry::get(const std::string& name) +{ + LockedCodecRegistry* registry = getCodecRegistry(); + std::lock_guard lock(registry->mMutex); + + CodecFactoryMapCIter iter = registry->mMap.find(name); + + return (iter != registry->mMap.end()) ? iter->second.get() : nullptr; +} + + +void +CodecRegistry::clear() +{ + LockedCodecRegistry* registry = getCodecRegistry(); + std::lock_guard lock(registry->mMutex); + + registry->mMap.clear(); +} + + +} // namespace io +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h new file mode 100644 index 0000000000..473102b57a --- /dev/null +++ b/openvdb/openvdb/io/Codec.h @@ -0,0 +1,483 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED + +#include +#include +#include +#include +#include +#include + +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { + +// Forward declaration +class GridBase; + +namespace io { + +namespace internal { + +// Global registration of codecs. These initialization functions are NOT intended +// to be called directly by the user. +// While they are thread-safe, they lack the early-exit logic of openvdb::initialize() - +// this can result in exceptions being thrown when attempting to register a codec that is +// already registered. +// They are also used extensively in unit tests. + +OPENVDB_API void initialize(); +OPENVDB_API void uninitialize(); + +} // namespace internal + +//////////////////////////////////////// + +/// @brief Controls which operations a codec exposes. +/// +/// Passed to codec registration to restrict how the codec may be used at +/// runtime. A @c ReadOnly codec can still be used to read existing files but +/// will not be selected as the write codec for new grids. +enum class CodecMode { + /// Both @c readTopology()/@c readBuffers() and + /// @c writeTopology()/@c writeBuffers() are enabled. This is the default. + ReadWrite, + /// Only @c readTopology() and @c readBuffers() are enabled; the codec + /// will not be offered as a write target. + ReadOnly +}; + +/// @brief Controls how grid data is read. +/// +/// Passed via @c ReadOptions::readMode to influence which portions of a grid +/// are deserialized and whether any on-the-fly conversion is performed during read. +/// Codecs are expected to honour this setting in their @c readTopology() and +/// @c readBuffers() implementations. +enum class ReadMode { + /// Deserialize both topology and all value buffers. This is the default + /// behaviour and produces a fully populated grid identical to the one + /// that was written. + Original, + /// Deserialize topology and value buffers, converting each voxel value + /// to half-precision floating point on the fly. Codecs that do not + /// support native half conversion should fall back to @c Original and + /// record a @c ReadDiagnostic warning. + Half, + /// Deserialize topology and value buffers, converting each voxel value + /// to @c bool (non-zero to @c true, zero to @c false). Produces a + /// @c BoolGrid whose active set matches the source grid's active set. + Bool, + /// Deserialize topology only and promote the result to a mask grid, + /// discarding all value data. Equivalent to reading @c TopologyOnly and + /// then constructing a @c MaskGrid from the active-voxel set, but may + /// be performed more efficiently inside the codec. + Mask, + /// Deserialize topology only; value buffers are skipped. The resulting + /// grid has a valid tree structure (active/inactive state, node + /// hierarchy) and all leaf buffers are allocated and filled with the + /// grid's background value. Useful when only the active-voxel mask is + /// needed and avoiding the cost of reading large value buffers is + /// desirable. + TopologyOnly, + /// Deserialize grid metadata and transform only; no topology, no value + /// buffers. The codec is still used to construct the correct grid type, + /// but its @c readTopology()/@c readBuffers() are not called. + MetadataOnly +}; + +/// @brief Base class for per-grid-type, codec-specific read options. +/// +/// Codecs that require type-specific read configuration should derive from +/// this class and store an instance in @c ReadOptions::typeData, keyed by +/// the grid type string (e.g. @c "Vec3SGrid"). +/// +/// @par Example +/// @code +/// struct MyCodecReadOptions : public ReadTypedOptions { +/// bool someFlag = false; +/// }; +/// +/// ReadOptions options; +/// auto data = std::make_shared(); +/// data->someFlag = true; +/// options.typeData["Vec3SGrid"] = data; +/// @endcode +struct OPENVDB_API ReadTypedOptions +{ + using Ptr = std::shared_ptr; + + virtual ~ReadTypedOptions() = default; + + /// @brief Downcast a @c ReadTypedOptions::Ptr to a concrete derived type. + /// + /// Performs a @c static_cast to @c T after verifying via @c dynamic_cast + /// (when @c OPENVDB_ASSERT is enabled) that @p data actually holds an + /// instance of @c T. Calling this with a mismatched type is undefined + /// behaviour in release builds. + /// + /// @tparam T The concrete @c ReadTypedOptions subclass to cast to. + /// @param data A shared pointer to the base @c ReadTypedOptions object. + /// @return A reference to the underlying @c T instance. + template + static T& cast(const Ptr& data) { + OPENVDB_ASSERT(dynamic_cast(data.get()) != nullptr); + return *static_cast(data.get()); + } +}; // struct ReadTypedOptions + +/// @brief Global read configuration passed to every codec during deserialization. +/// +/// An instance of this struct is threaded through all codec @c readTopology() and +/// @c readBuffers() calls so that site-wide policy (clipping, read mode) and any +/// type-specific overrides are available in a single place. +/// +/// @par Clipping +/// When @c clipBBox is non-empty (i.e. @c !clipBBox.empty()), codecs should +/// restrict the voxel data they load to the region that intersects that world-space +/// bounding box. An empty bbox (the default) means no clipping is applied. +/// +/// @par Read mode +/// @c readMode controls the granularity and on-the-fly conversion applied +/// during deserialization. The default, @c ReadMode::Original, loads both +/// topology and value buffers without modification. +/// @c ReadMode::Half, @c ReadMode::Bool, and @c ReadMode::Mask request +/// in-place type conversion as data is read. +/// @c ReadMode::TopologyOnly skips value buffers entirely, which can be +/// significantly faster when only the active-voxel mask is needed. +/// @c ReadMode::MetadataOnly skips both topology and value buffers, +/// reading only grid metadata and transform. +/// +/// @par Per-type options +/// @c typeData allows callers to attach codec-specific configuration for +/// individual grid types. Entries are keyed by the grid type string +/// (e.g. @c "Vec3SGrid") and hold a @c ReadTypedOptions-derived object. +/// Codecs retrieve their entry via @c ReadTypedOptions::cast(). +struct OPENVDB_API ReadOptions +{ + /// World-space bounding box used to spatially clip the read. + /// An empty bbox (the default) disables clipping. + BBoxd clipBBox = BBoxd(); + + /// Controls which portions of each grid are deserialized. + /// Defaults to @c ReadMode::Original (full topology + value buffers). + ReadMode readMode = ReadMode::Original; + + /// Optional per-grid-type codec configuration, keyed by grid type string. + /// Values are @c ReadTypedOptions subclass instances; use + /// @c ReadTypedOptions::cast() to retrieve the concrete type. + std::unordered_map typeData; +}; // struct ReadOptions + +/// @brief Global write configuration passed to every codec during serialization. +/// +/// Currently carries no fields, but is provided for forward compatibility: +/// future write-time options (e.g. compression hints, metadata policies) can +/// be added here without changing the codec interface. +struct OPENVDB_API WriteOptions +{ +}; // struct WriteOptions + +/// @brief Severity level for a read diagnostic. +/// @note Currently only @c Warning is defined. +enum class DiagnosticSeverity { Warning }; + +/// @brief A single read diagnostic message. +/// @note @c context is typically the grid name or another identifier that +/// locates the source of the message. +struct ReadDiagnostic { + DiagnosticSeverity severity; + std::string context; + std::string message; +}; + +/// @brief Thread-safe collection of read diagnostics accumulated during a codec read. +/// +/// @details Diagnostics report situations where a requested read option could +/// not be honoured by the codec — for example, clipping that was not applied +/// natively and had to fall back to a post-process, or an option that the codec +/// ignores entirely. The object must be explicitly enabled before any messages +/// are recorded; it is disabled by default. +struct OPENVDB_API ReadDiagnostics { + ReadDiagnostics() = default; + ReadDiagnostics(const ReadDiagnostics& other) { + std::lock_guard lock(other.mMutex); + mEnabled = other.mEnabled; + mDiagnostics = other.mDiagnostics; + } + ReadDiagnostics& operator=(const ReadDiagnostics& other) { + if (this != &other) { + std::scoped_lock lock(mMutex, other.mMutex); + mEnabled = other.mEnabled; + mDiagnostics = other.mDiagnostics; + } + return *this; + } + /// @brief Append a warning diagnostic. No-op if the object is disabled. + /// @note This method is thread-safe. + void addWarning(const std::string& context, const std::string& message) { + if (!mEnabled) return; + std::lock_guard lock(mMutex); + mDiagnostics.push_back({DiagnosticSeverity::Warning, context, message}); + } + /// @brief Remove all recorded diagnostics. + /// @note This method is thread-safe. + void clear() { + std::lock_guard lock(mMutex); + mDiagnostics.clear(); + } + /// @brief Enable diagnostic recording. + void enable() { mEnabled = true; } + /// @brief Disable diagnostic recording. + void disable() { mEnabled = false; } + /// @brief Return @c true if diagnostic recording is enabled. + bool enabled() const { return mEnabled; } + /// @brief Return the list of recorded diagnostics. + const std::vector& diagnostics() const { return mDiagnostics; } + /// @brief Return @c true if no diagnostics have been recorded. + bool empty() const { return mDiagnostics.empty(); } +private: + bool mEnabled = false; + std::vector mDiagnostics; + mutable std::mutex mMutex; +}; // struct ReadDiagnostics + +/// @brief Mutable per-operation state created by @c Codec::createData() and +/// passed into every read and write call on a given codec. +/// +/// @details The main purpose of this class is to provide a mechanism for passing +/// additional state between the different methods in the codec. @c ReadOptions is +/// immutable and cannot carry per-operation state; the codec itself is stateless +/// by design (to allow sharing across threads) and therefore also cannot be used +/// for this purpose. +/// +/// The base class provides the single field that every codec requires: +/// a shared pointer to the @c GridBase being populated (on read) or inspected +/// (on write). Derived codecs that need additional transient state should +/// subclass @c CodecData and add those fields. +/// +/// A @c CodecData instance is created once per grid per read or write operation +/// by @c Codec::createData(), which also allocates the concrete @c GridBase +/// subclass and stores it in @c grid. +/// +/// @par Deriving from CodecData +/// @code +/// struct MyCodecData : public io::CodecData { +/// // Additional per-operation state goes here +/// SomeCache intermediateBuffer; +/// }; +/// @endcode +/// +/// Inside @c Codec::createData(), the derived type is allocated and returned: +/// @code +/// io::CodecData::Ptr createData() override { +/// auto data = std::make_unique(); +/// data->grid = MyGridType::create(); +/// return data; +/// } +/// @endcode +/// +/// In the read and write methods, downcast the reference back to the concrete +/// type to access the extra fields: +/// @code +/// void readBuffers(std::istream& is, int64_t size, io::CodecData& data, ...) const override { +/// auto& myData = static_cast(data); +/// MyGridType& grid = static_cast(*myData.grid); +/// // Use myData.intermediateBuffer, grid, etc. +/// } +/// @endcode +struct OPENVDB_API CodecData +{ + using Ptr = std::unique_ptr; + + virtual ~CodecData() = default; + + /// The grid being populated on read, or being serialized on write. + SharedPtr grid; +}; // struct CodecData + +/// @brief Abstract base class for grid I/O codecs. +/// +/// @details A codec encapsulates the serialization and deserialization logic +/// for a specific grid type (or family of grid types). The I/O subsystem +/// selects a codec by name at runtime, calls @c createData() to allocate +/// per-operation state, and then dispatches to the appropriate read or write +/// methods. +/// +/// Codecs must be stateless: all mutable per-operation state is stored in a +/// @c CodecData object (see above) that is created fresh for each grid read +/// or write. This ensures that a single registered codec instance can safely +/// handle concurrent I/O operations. +/// +/// @par Implementing a new codec +/// +/// 1. **Declare the codec struct**, deriving publicly from @c Codec (or from a +/// convenience intermediate that already handles topology, if one exists for +/// the grid family in question): +/// @code +/// struct MyCodec : public io::Codec { ... }; +/// @endcode +/// +/// 2. **Provide a static @c name() method** that returns the unique string +/// identifier under which the codec will be registered. Any globally +/// unique ASCII string is acceptable: +/// @code +/// static std::string name() { return "mycodec"; } +/// @endcode +/// +/// 3. **Override @c createData()** (the only pure-virtual method). Allocate a +/// @c CodecData (or a derived subclass if extra per-operation state is +/// needed), create the appropriate @c GridBase subclass, assign it to +/// @c CodecData::grid, and return the object: +/// @code +/// io::CodecData::Ptr createData() override { +/// auto data = std::make_unique(); +/// data->grid = MyGridType::create(); +/// return data; +/// } +/// @endcode +/// +/// 4. **Override the read and/or write methods** as required. All four +/// methods have default no-op implementations, so override only those that +/// the codec actually uses. The I/O layer separates topology (the tree +/// structure and active/inactive voxel mask) from value buffers (the actual +/// voxel data), allowing callers to request topology-only reads via +/// @c ReadOptions::readMode. Implementations should honour that setting: +/// @code +/// void readTopology(std::istream& is, io::CodecData& data, +/// const io::ReadOptions& options, +/// io::ReadDiagnostics& diagnostics) const override +/// { +/// MyGridType& grid = static_cast(*data.grid); +/// // Deserialize the tree structure into grid... +/// } +/// +/// void readBuffers(std::istream& is, int64_t size, CodecData& data, +/// const io::ReadOptions& options, +/// io::ReadDiagnostics& diagnostics) const override +/// { +/// if (options.readMode == io::ReadMode::TopologyOnly) return; +/// MyGridType& grid = static_cast(*data.grid); +/// // Deserialize voxel values into grid... +/// } +/// @endcode +/// If a requested option (e.g. spatial clipping) cannot be honoured natively, +/// record a warning via @c diagnostics.addWarning() rather than silently ignoring it. +/// +/// 5. **Register the codec** once at start-up, typically from the library's +/// @c initialize() function, using @c CodecRegistry::registerCodec(): +/// @code +/// io::CodecRegistry::registerCodec(); +/// @endcode +/// +/// @note The codec struct itself is never copied; the registry takes ownership +/// of a single heap-allocated instance via @c Codec::Ptr +/// (@c std::unique_ptr). +struct OPENVDB_API Codec +{ + using Ptr = std::unique_ptr; + + virtual ~Codec() = default; + + /// @brief Allocate per-operation codec state, including the target grid. + /// + /// This is the only pure-virtual method. Implementations must create a + /// concrete @c GridBase subclass appropriate for the codec, store it in + /// @c CodecData::grid, and return the @c CodecData (or a derived subclass + /// carrying additional state). The returned object is passed by reference + /// to every subsequent read or write call for the same grid. + /// + /// @return A fully initialized @c CodecData whose @c grid field is non-null. + virtual CodecData::Ptr createData() = 0; + + /// @brief Deserialize the grid topology (tree structure and active-voxel + /// mask) from @a is into the grid held by @a data. + virtual void readTopology(std::istream& /*is*/, CodecData& /*data*/, + const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) const { } + + /// @brief Deserialize all voxel-value buffers from @a is into the grid + /// held by @a data. + /// + /// The default implementation is a no-op. Override to populate leaf-node + /// value buffers after topology has been established. When + /// @c options.readMode is @c ReadMode::TopologyOnly this method will not + /// be called by the I/O layer, so implementations may also guard against + /// that mode internally for safety. If @c options.clipBBox is non-empty, + /// restrict the loaded data to the region that intersects it; if the codec + /// cannot honour clipping natively, fall back to a post-process and record + /// a warning via @a diagnostics. The @c size argument is the number of bytes + /// occupied by the readBuffers data section, measured from the stream + /// position at which this method is called, or -1 when the size is not known + /// (a non-seekable stream, or a stream without grid offsets). + virtual void readBuffers(std::istream& /*is*/, int64_t /*size*/, CodecData& /*data*/, + const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) const { } + + /// @brief Serialize the grid topology (tree structure and active-voxel + /// mask) from @a grid to @a os. + /// + /// The default implementation is a no-op. Override when the codec stores + /// topology as a distinct section that precedes the value buffers in the + /// stream. + virtual void writeTopology(std::ostream& /*os*/, const GridBase& /*grid*/, + const WriteOptions& /*options*/) const { } + + /// @brief Serialize all voxel-value buffers from @a grid to @a os. + /// + /// The default implementation is a no-op. Override to write the leaf-node + /// value buffers that follow the topology section. + virtual void writeBuffers(std::ostream& /*os*/, const GridBase& /*grid*/, + const WriteOptions& /*options*/) const { } +}; // struct Codec + +/// @brief A thread-safe, process-global registry that maps codec names to +/// @c Codec instances. +/// +/// Codecs are identified by a unique string name (typically provided by @c Codec::name()) +/// and stored as @c Codec::Ptr (i.e. @c std::unique_ptr). +/// +/// @par Typical usage +/// @code +/// // Register a custom codec once at start-up (e.g. from initialize()): +/// CodecRegistry::registerCodec(); +/// @endcode +/// +/// @note Attempting to register a codec whose name is already present throws +/// @c openvdb::KeyError. Call @c isRegistered() first when the caller +/// cannot guarantee uniqueness. +struct OPENVDB_API CodecRegistry +{ + /// Return @c true if a codec with the given @a name has been registered. + static bool isRegistered(const std::string& name); + + /// Register a codec under the explicit string @a name, transferring + /// ownership of @a codec to the registry. + /// @throw KeyError if @a name is already registered. + static void registerCodecByName(const std::string& name, Codec::Ptr&& codec); + + /// Convenience wrapper that registers @c CodecT using the name returned + /// by @c CodecT::name(), constructing the instance internally. + /// @throw KeyError if the codec is already registered. + template + static void registerCodec() + { + registerCodecByName(CodecT::name(), std::make_unique()); + } + + /// Return a raw (non-owning) pointer to the codec registered under + /// @a name, or @c nullptr if no such codec exists. + /// The returned pointer remains valid for the lifetime of the registry. + static Codec* get(const std::string& name); + + /// Deregister all codecs and reset the registry to an empty state. + static void clear(); +}; // struct CodecRegistry + +} // namespace io +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Compression.h b/openvdb/openvdb/io/Compression.h index fee97f23fc..a81b1ef1e5 100644 --- a/openvdb/openvdb/io/Compression.h +++ b/openvdb/openvdb/io/Compression.h @@ -9,7 +9,6 @@ #include // for negative() #include #include "io.h" // for getDataCompression(), etc. -#include "DelayedLoadMetadata.h" #include #include #include @@ -236,28 +235,20 @@ OPENVDB_API void bloscFromStream(std::istream&, char* data, size_t numBytes); /// @param count the number of elements to read in /// @param compression whether and how the data is compressed (either COMPRESS_NONE, /// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC) -/// @param metadata optional pointer to a DelayedLoadMetadata object that stores -/// the size of the compressed buffer -/// @param metadataOffset offset into DelayedLoadMetadata, ignored if pointer is null /// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled /// without Blosc support. /// @details This default implementation is instantiated only for types /// whose size can be determined by the sizeof() operator. template inline void -readData(std::istream& is, T* data, Index count, uint32_t compression, - DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) +readData(std::istream& is, T* data, Index count, uint32_t compression) { const bool seek = data == nullptr; if (seek) { OPENVDB_ASSERT(!getStreamMetadataPtr(is) || getStreamMetadataPtr(is)->seekable()); } - const bool hasCompression = compression & (COMPRESS_BLOSC | COMPRESS_ZIP); - if (metadata && seek && hasCompression) { - size_t compressedSize = metadata->getCompressedSize(metadataOffset); - is.seekg(compressedSize, std::ios_base::cur); - } else if (compression & COMPRESS_BLOSC) { + if (compression & COMPRESS_BLOSC) { bloscFromStream(is, reinterpret_cast(data), sizeof(T) * count); } else if (compression & COMPRESS_ZIP) { unzipFromStream(is, reinterpret_cast(data), sizeof(T) * count); @@ -271,8 +262,7 @@ readData(std::istream& is, T* data, Index count, uint32_t compression, /// Specialization for std::string input template<> inline void -readData(std::istream& is, std::string* data, Index count, uint32_t /*compression*/, - DelayedLoadMetadata* /*metadata*/, size_t /*metadataOffset*/) +readData(std::istream& is, std::string* data, Index count, uint32_t /*compression*/) { for (Index i = 0; i < count; ++i) { size_t len = 0; @@ -294,25 +284,22 @@ template struct HalfReader; /// Partial specialization for non-floating-point types (no half to float promotion) template struct HalfReader { - static inline void read(std::istream& is, T* data, Index count, uint32_t compression, - DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) { - readData(is, data, count, compression, metadata, metadataOffset); + static inline void read(std::istream& is, T* data, Index count, uint32_t compression) { + readData(is, data, count, compression); } }; /// Partial specialization for floating-point types template struct HalfReader { using HalfT = typename RealToHalf::HalfT; - static inline void read(std::istream& is, T* data, Index count, uint32_t compression, - DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) { + static inline void read(std::istream& is, T* data, Index count, uint32_t compression) { if (count < 1) return; if (data == nullptr) { // seek mode - pass through null pointer - readData(is, nullptr, count, compression, metadata, metadataOffset); + readData(is, nullptr, count, compression); } else { std::vector halfData(count); // temp buffer into which to read half float values - readData(is, reinterpret_cast(&halfData[0]), count, compression, - metadata, metadataOffset); + readData(is, reinterpret_cast(&halfData[0]), count, compression); // Copy half float values from the temporary buffer to the full float output array. std::copy(halfData.begin(), halfData.end(), data); } @@ -461,10 +448,11 @@ struct HalfWriter { /// which positions in the buffer correspond to active values /// @param fromHalf if true, read 16-bit half floats from the input stream /// and convert them to full floats +/// @param background optional background value used when mask compressed template inline void readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, - const MaskT& valueMask, bool fromHalf) + const MaskT& valueMask, bool fromHalf, const ValueT* background = nullptr) { checkFormatVersion(is); @@ -476,16 +464,6 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, const bool seek = (destBuf == nullptr); OPENVDB_ASSERT(!seek || (!meta || meta->seekable())); - // Get delayed load metadata if it exists - - DelayedLoadMetadata::Ptr delayLoadMeta; - uint64_t leafIndex(0); - if (seek && meta && meta->delayedLoadMeta()) { - delayLoadMeta = - meta->gridMetadata().getMetadata("file_delayed_load"); - leafIndex = meta->leaf(); - } - int8_t metadata = NO_MASK_AND_ALL_VALS; if (getFormatVersion(is) >= OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) { @@ -493,21 +471,20 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, // (selection mask and/or inactive value(s)) is saved. if (seek && !maskCompressed) { is.seekg(/*bytes=*/1, std::ios_base::cur); - } else if (seek && delayLoadMeta) { - metadata = delayLoadMeta->getMask(leafIndex); - is.seekg(/*bytes=*/1, std::ios_base::cur); } else { is.read(reinterpret_cast(&metadata), /*bytes=*/1); } } - ValueT background = zeroVal(); - if (const void* bgPtr = getGridBackgroundValuePtr(is)) { - background = *static_cast(bgPtr); + ValueT bgValue = zeroVal(); + if (background) { + bgValue = *background; + } else if (const void* bgPtr = getGridBackgroundValuePtr(is)) { + bgValue = *static_cast(bgPtr); } - ValueT inactiveVal1 = background; + ValueT inactiveVal1 = bgValue; ValueT inactiveVal0 = - ((metadata == NO_MASK_OR_INACTIVE_VALS) ? background : math::negative(background)); + ((metadata == NO_MASK_OR_INACTIVE_VALS) ? bgValue : math::negative(bgValue)); if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL || metadata == MASK_AND_ONE_INACTIVE_VAL || @@ -562,10 +539,10 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, // Read in the buffer. if (fromHalf) { HalfReader::isReal, ValueT>::read( - is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex); + is, (seek ? nullptr : tempBuf), tempCount, compression); } else { readData( - is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex); + is, (seek ? nullptr : tempBuf), tempCount, compression); } // If mask compression is enabled and the number of active values read into @@ -643,10 +620,12 @@ writeCompressedValuesSize(ValueT* srcBuf, Index srcCount, /// @param childMask a bitmask (typically, a node's child mask) indicating /// which positions in the buffer correspond to child node pointers /// @param toHalf if true, convert floating-point values to 16-bit half floats +/// @param background optional background value used when mask compressed template inline void writeCompressedValues(std::ostream& os, const ValueT* srcBuf, Index srcCount, - const MaskT& valueMask, const MaskT& childMask, bool toHalf) + const MaskT& valueMask, const MaskT& childMask, bool toHalf, + const ValueT* background = nullptr) { // Get the stream's compression settings. const uint32_t compress = getDataCompression(os); @@ -668,12 +647,14 @@ writeCompressedValues(std::ostream& os, const ValueT* srcBuf, Index srcCount, // an inside/outside bitmask. const ValueT zero = zeroVal(); - ValueT background = zero; - if (const void* bgPtr = getGridBackgroundValuePtr(os)) { - background = *static_cast(bgPtr); + ValueT bgValue = zero; + if (background) { + bgValue = *background; + } else if (const void* bgPtr = getGridBackgroundValuePtr(os)) { + bgValue = *static_cast(bgPtr); } - MaskCompress maskCompressData(valueMask, childMask, srcBuf, background); + MaskCompress maskCompressData(valueMask, childMask, srcBuf, bgValue); metadata = maskCompressData.metadata; os.write(reinterpret_cast(&metadata), /*bytes=*/1); diff --git a/openvdb/openvdb/io/DelayedLoadMetadata.cc b/openvdb/openvdb/io/DelayedLoadMetadata.cc deleted file mode 100644 index 1f958b0a4c..0000000000 --- a/openvdb/openvdb/io/DelayedLoadMetadata.cc +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -#include "DelayedLoadMetadata.h" - -#include -#include - -#ifdef OPENVDB_USE_BLOSC -#include - -namespace { - -inline size_t padMask(size_t bytes) -{ - return size_t(std::ceil(static_cast(bytes+1) / - sizeof(openvdb::io::DelayedLoadMetadata::MaskType))); -} - -inline size_t padCompressedSize(size_t bytes) -{ - return size_t(std::ceil(static_cast(bytes+1) / - sizeof(openvdb::io::DelayedLoadMetadata::CompressedSizeType))); -} - -} // namespace - -#endif - -namespace openvdb { -OPENVDB_USE_VERSION_NAMESPACE -namespace OPENVDB_VERSION_NAME { -namespace io { - -DelayedLoadMetadata::DelayedLoadMetadata(const DelayedLoadMetadata& other) - : Metadata() - , mMask(other.mMask) - , mCompressedSize(other.mCompressedSize) -{ -} - -Name DelayedLoadMetadata::typeName() const -{ - return DelayedLoadMetadata::staticTypeName(); -} - -Metadata::Ptr DelayedLoadMetadata::copy() const -{ - Metadata::Ptr metadata(new DelayedLoadMetadata()); - metadata->copy(*this); - return metadata; -} - -void DelayedLoadMetadata::copy(const Metadata& other) -{ - const DelayedLoadMetadata* t = dynamic_cast(&other); - if (t == nullptr) OPENVDB_THROW(TypeError, "Incompatible type during copy"); - mMask = t->mMask; - mCompressedSize = t->mCompressedSize; -} - -std::string DelayedLoadMetadata::str() const -{ - return ""; -} - -bool DelayedLoadMetadata::asBool() const -{ - return false; -} - -Index32 DelayedLoadMetadata::size() const -{ - if (mMask.empty() && mCompressedSize.empty()) return Index32(0); - - // count - size_t size = sizeof(Index32); - - { // mask - size += sizeof(Index32); - size_t compressedSize = compression::bloscCompressedSize( - reinterpret_cast(mMask.data()), mMask.size()*sizeof(MaskType)); - - if (compressedSize > 0) size += compressedSize; - else size += mMask.size()*sizeof(MaskType); - } - { // compressed size - size += sizeof(Index32); - if (!mCompressedSize.empty()) { - size_t compressedSize = compression::bloscCompressedSize( - reinterpret_cast(mCompressedSize.data()), mCompressedSize.size()*sizeof(CompressedSizeType)); - - if (compressedSize > 0) size += compressedSize; - else size += mCompressedSize.size()*sizeof(CompressedSizeType); - } - } - - return static_cast(size); -} - -void DelayedLoadMetadata::clear() -{ - mMask.clear(); - mCompressedSize.clear(); -} - -bool DelayedLoadMetadata::empty() const -{ - return mMask.empty() && mCompressedSize.empty(); -} - -void DelayedLoadMetadata::resizeMask(size_t size) -{ - mMask.resize(size); -} - -void DelayedLoadMetadata::resizeCompressedSize(size_t size) -{ - mCompressedSize.resize(size); -} - -DelayedLoadMetadata::MaskType DelayedLoadMetadata::getMask(size_t index) const -{ - OPENVDB_ASSERT(DelayedLoadMetadata::isRegisteredType()); - OPENVDB_ASSERT(index < mMask.size()); - return mMask[index]; -} - -void DelayedLoadMetadata::setMask(size_t index, const MaskType& value) -{ - OPENVDB_ASSERT(index < mMask.size()); - mMask[index] = value; -} - -DelayedLoadMetadata::CompressedSizeType DelayedLoadMetadata::getCompressedSize(size_t index) const -{ - OPENVDB_ASSERT(DelayedLoadMetadata::isRegisteredType()); - OPENVDB_ASSERT(index < mCompressedSize.size()); - return mCompressedSize[index]; -} - -void DelayedLoadMetadata::setCompressedSize(size_t index, const CompressedSizeType& value) -{ - OPENVDB_ASSERT(index < mCompressedSize.size()); - mCompressedSize[index] = value; -} - -void DelayedLoadMetadata::readValue(std::istream& is, Index32 numBytes) -{ - if (numBytes == 0) return; - - // initial header size - size_t total = sizeof(Index32); - - Index32 count = 0; - is.read(reinterpret_cast(&count), sizeof(Index32)); - total += sizeof(Index32); - - Index32 bytes = 0; - is.read(reinterpret_cast(&bytes), sizeof(Index32)); - total += sizeof(Index32); - - if (bytes > Index32(0)) { - std::unique_ptr compressedBuffer(new char[bytes]); - is.read(reinterpret_cast(compressedBuffer.get()), bytes); - - total += bytes; - -#ifdef OPENVDB_USE_BLOSC - // pad to include BLOSC_MAX_OVERHEAD - size_t uncompressedBytes = openvdb::compression::bloscUncompressedSize(compressedBuffer.get()); - const size_t paddedCount = padMask(uncompressedBytes + BLOSC_MAX_OVERHEAD); - - mMask.reserve(paddedCount); - mMask.resize(count); - - // resize should never modify capacity for smaller vector sizes - OPENVDB_ASSERT(mMask.capacity() >= paddedCount); - - compression::bloscDecompress(reinterpret_cast(mMask.data()), count*sizeof(MaskType), mMask.capacity()*sizeof(MaskType), compressedBuffer.get()); -#endif - } else { - mMask.resize(count); - is.read(reinterpret_cast(mMask.data()), count*sizeof(MaskType)); - total += count*sizeof(MaskType); - } - - is.read(reinterpret_cast(&bytes), sizeof(Index32)); - - if (bytes != std::numeric_limits::max()) { - if (bytes > Index32(0)) { - std::unique_ptr compressedBuffer(new char[bytes]); - is.read(reinterpret_cast(compressedBuffer.get()), bytes); - - total += size_t(bytes); - -#ifdef OPENVDB_USE_BLOSC - // pad to include BLOSC_MAX_OVERHEAD - size_t uncompressedBytes = openvdb::compression::bloscUncompressedSize(compressedBuffer.get()); - const size_t paddedCount = padCompressedSize(uncompressedBytes + BLOSC_MAX_OVERHEAD); - - mCompressedSize.reserve(paddedCount); - mCompressedSize.resize(count); - - // resize should never modify capacity for smaller vector sizes - OPENVDB_ASSERT(mCompressedSize.capacity() >= paddedCount); - - compression::bloscDecompress(reinterpret_cast(mCompressedSize.data()), count*sizeof(CompressedSizeType), mCompressedSize.capacity()*sizeof(CompressedSizeType), compressedBuffer.get()); -#endif - } else { - mCompressedSize.resize(count); - is.read(reinterpret_cast(mCompressedSize.data()), count*sizeof(CompressedSizeType)); - total += count*sizeof(CompressedSizeType); - } - } - - Index32 totalBytes = static_cast(total); - - if (totalBytes < numBytes) { - // Read and discard any unknown bytes at the end of the metadata for forwards-compatibility - // (without seeking, because the stream might not be seekable). - const size_t BUFFER_SIZE = 1024; - std::vector buffer(BUFFER_SIZE); - for (Index32 bytesRemaining = numBytes - totalBytes; bytesRemaining > 0; ) { - const Index32 bytesToSkip = std::min(bytesRemaining, BUFFER_SIZE); - is.read(&buffer[0], bytesToSkip); - bytesRemaining -= bytesToSkip; - } - } -} - -void DelayedLoadMetadata::writeValue(std::ostream& os) const -{ - // metadata has a limit of 2^32 bytes - OPENVDB_ASSERT(mMask.size() < std::numeric_limits::max()); - OPENVDB_ASSERT(mCompressedSize.size() < std::numeric_limits::max()); - - if (mMask.empty() && mCompressedSize.empty()) return; - - OPENVDB_ASSERT(mCompressedSize.empty() || (mMask.size() == mCompressedSize.size())); - - Index32 count = static_cast(mMask.size()); - os.write(reinterpret_cast(&count), sizeof(Index32)); - - const Index32 zeroSize(0); - const Index32 maxSize(std::numeric_limits::max()); - - { // mask buffer - size_t compressedBytes(0); - std::unique_ptr compressedBuffer; - if (compression::bloscCanCompress()) { - compressedBuffer = compression::bloscCompress( - reinterpret_cast(mMask.data()), - mMask.size()*sizeof(MaskType), compressedBytes, /*resize=*/false); - } - - if (compressedBuffer) { - OPENVDB_ASSERT(compressedBytes < std::numeric_limits::max()); - Index32 bytes(static_cast(compressedBytes)); - os.write(reinterpret_cast(&bytes), sizeof(Index32)); - os.write(reinterpret_cast(compressedBuffer.get()), compressedBytes); - } - else { - os.write(reinterpret_cast(&zeroSize), sizeof(Index32)); - os.write(reinterpret_cast(mMask.data()), - mMask.size()*sizeof(MaskType)); - } - } - - // compressed size buffer - - if (mCompressedSize.empty()) { - // write out maximum Index32 value to denote no compressed sizes stored - os.write(reinterpret_cast(&maxSize), sizeof(Index32)); - } else { - size_t compressedBytes(0); - std::unique_ptr compressedBuffer; - if (compression::bloscCanCompress()) { - compressedBuffer = compression::bloscCompress( - reinterpret_cast(mCompressedSize.data()), - mCompressedSize.size()*sizeof(CompressedSizeType), compressedBytes, /*resize=*/false); - } - - if (compressedBuffer) { - OPENVDB_ASSERT(compressedBytes < std::numeric_limits::max()); - Index32 bytes(static_cast(compressedBytes)); - os.write(reinterpret_cast(&bytes), sizeof(Index32)); - os.write(reinterpret_cast(compressedBuffer.get()), compressedBytes); - } - else { - os.write(reinterpret_cast(&zeroSize), sizeof(Index32)); - os.write(reinterpret_cast(mCompressedSize.data()), - mCompressedSize.size()*sizeof(CompressedSizeType)); - } - } -} - -} // namespace io -} // namespace OPENVDB_VERSION_NAME -} // namespace openvdb diff --git a/openvdb/openvdb/io/DelayedLoadMetadata.h b/openvdb/openvdb/io/DelayedLoadMetadata.h deleted file mode 100644 index 8ba9d6f519..0000000000 --- a/openvdb/openvdb/io/DelayedLoadMetadata.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OPENVDB_DELAYED_LOAD_METADATA_HAS_BEEN_INCLUDED -#define OPENVDB_DELAYED_LOAD_METADATA_HAS_BEEN_INCLUDED - -#include -#include -#include -#include -#include - - -namespace openvdb { -OPENVDB_USE_VERSION_NAMESPACE -namespace OPENVDB_VERSION_NAME { -namespace io { - -/// @brief Store a buffer of data that can be optionally used -/// during reading for faster delayed-load I/O performance -class OPENVDB_API DelayedLoadMetadata: public Metadata -{ -public: - using Ptr = SharedPtr; - using ConstPtr = SharedPtr; - using MaskType = int8_t; - using CompressedSizeType = int64_t; - - DelayedLoadMetadata() = default; - DelayedLoadMetadata(const DelayedLoadMetadata& other); - ~DelayedLoadMetadata() override = default; - - Name typeName() const override; - Metadata::Ptr copy() const override; - void copy(const Metadata&) override; - std::string str() const override; - bool asBool() const override; - Index32 size() const override; - - static Name staticTypeName() { return "__delayedload"; } - - static Metadata::Ptr createMetadata() - { - Metadata::Ptr ret(new DelayedLoadMetadata); - return ret; - } - - static void registerType() - { - Metadata::registerType(DelayedLoadMetadata::staticTypeName(), - DelayedLoadMetadata::createMetadata); - } - - static void unregisterType() - { - Metadata::unregisterType(DelayedLoadMetadata::staticTypeName()); - } - - static bool isRegisteredType() - { - return Metadata::isRegisteredType(DelayedLoadMetadata::staticTypeName()); - } - - /// @brief Delete the contents of the mask and compressed size arrays - void clear(); - /// @brief Return @c true if both arrays are empty - bool empty() const; - - /// @brief Resize the mask array - void resizeMask(size_t size); - /// @brief Resize the compressed size array - void resizeCompressedSize(size_t size); - - /// @brief Return the mask value for a specific index - /// @note throws if index is out-of-range or DelayedLoadMask not registered - MaskType getMask(size_t index) const; - /// @brief Set the mask value for a specific index - /// @note throws if index is out-of-range - void setMask(size_t index, const MaskType& value); - - /// @brief Return the compressed size value for a specific index - /// @note throws if index is out-of-range or DelayedLoadMask not registered - CompressedSizeType getCompressedSize(size_t index) const; - /// @brief Set the compressed size value for a specific index - /// @note throws if index is out-of-range - void setCompressedSize(size_t index, const CompressedSizeType& value); - -protected: - void readValue(std::istream&, Index32 numBytes) override; - void writeValue(std::ostream&) const override; - -private: - std::vector mMask; - std::vector mCompressedSize; -}; // class DelayedLoadMetadata - - -} // namespace io -} // namespace OPENVDB_VERSION_NAME -} // namespace openvdb - -#endif // OPENVDB_DELAYED_LOAD_METADATA_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 5c64de3070..b783008986 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -5,20 +5,11 @@ #include "File.h" -#include "TempFile.h" #include #include #include #include -#ifdef OPENVDB_USE_DELAYED_LOADING -#include -#ifndef _WIN32 -#include -#include -#endif -#endif // OPENVDB_USE_DELAYED_LOADING - #include // stat() #include // for getenv(), strtoul() @@ -34,106 +25,24 @@ OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { namespace io { -// Implementation details of the File class -struct File::Impl -{ - enum { DEFAULT_COPY_MAX_BYTES = 500000000 }; // 500 MB - - struct NoBBox {}; - - // Common implementation of the various File::readGrid() overloads, - // with and without bounding box clipping - template - static GridBase::Ptr readGrid(const File& file, const GridDescriptor& gd, const BoxType& bbox) - { - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(file.inputHasGridOffsets()); - - GridBase::Ptr grid = file.createGrid(gd); - gd.seekToGrid(file.inputStream()); - unarchive(file, grid, gd, bbox); - return grid; - } - - static void unarchive(const File& file, GridBase::Ptr& grid, - const GridDescriptor& gd, NoBBox) - { - file.Archive::readGrid(grid, gd, file.inputStream()); - } - - static void unarchive(const File& file, GridBase::Ptr& grid, - const GridDescriptor& gd, const CoordBBox& indexBBox) - { - file.Archive::readGrid(grid, gd, file.inputStream(), indexBBox); - } - - static void unarchive(const File& file, GridBase::Ptr& grid, - const GridDescriptor& gd, const BBoxd& worldBBox) - { - file.Archive::readGrid(grid, gd, file.inputStream(), worldBBox); - } - - static Index64 getDefaultCopyMaxBytes() - { - Index64 result = DEFAULT_COPY_MAX_BYTES; - if (const char* s = std::getenv("OPENVDB_DELAYED_LOAD_COPY_MAX_BYTES")) { - char* endptr = nullptr; - result = std::strtoul(s, &endptr, /*base=*/10); - } - return result; - } - - std::string mFilename; - // The file-level metadata - MetaMap::Ptr mMeta; - // The file stream that is open for reading - std::unique_ptr mInStream; - // File-level stream metadata (file format, compression, etc.) - StreamMetadata::Ptr mStreamMetadata; - // Flag indicating if we have read in the global information (header, - // metadata, and grid descriptors) for this VDB file - bool mIsOpen; - // Grid descriptors for all grids stored in the file, indexed by grid name - NameMap mGridDescriptors; - // All grids, indexed by unique name (used only when mHasGridOffsets is false) - Archive::NamedGridMap mNamedGrids; - // All grids stored in the file (used only when mHasGridOffsets is false) - GridPtrVecPtr mGrids; -#ifdef OPENVDB_USE_DELAYED_LOADING - // The memory-mapped file - MappedFile::Ptr mFileMapping; - // The buffer for the input stream, if it is a memory-mapped file - SharedPtr mStreamBuf; - // File size limit for copying during delayed loading - Index64 mCopyMaxBytes; -#endif -}; // class File::Impl - - -//////////////////////////////////////// - -File::File(const std::string& filename): mImpl(new Impl) +File::File(const std::string& filename) + : Archive() + , mFilename(filename) { - mImpl->mFilename = filename; - mImpl->mIsOpen = false; -#ifdef OPENVDB_USE_DELAYED_LOADING - mImpl->mCopyMaxBytes = Impl::getDefaultCopyMaxBytes(); -#endif setInputHasGridOffsets(true); } -File::~File() -{ -} - - File::File(const File& other) : Archive(other) - , mImpl(new Impl) + , mFilename(other.mFilename) + , mMeta(other.mMeta) + , mIsOpen(false) + , mGridDescriptors(other.mGridDescriptors) + , mNamedGrids(other.mNamedGrids) + , mGrids(other.mGrids) { - *this = other; } @@ -142,16 +51,12 @@ File::operator=(const File& other) { if (&other != this) { Archive::operator=(other); - const Impl& otherImpl = *other.mImpl; - mImpl->mFilename = otherImpl.mFilename; - mImpl->mMeta = otherImpl.mMeta; - mImpl->mIsOpen = false; // don't want two file objects reading from the same stream -#ifdef OPENVDB_USE_DELAYED_LOADING - mImpl->mCopyMaxBytes = otherImpl.mCopyMaxBytes; -#endif - mImpl->mGridDescriptors = otherImpl.mGridDescriptors; - mImpl->mNamedGrids = otherImpl.mNamedGrids; - mImpl->mGrids = otherImpl.mGrids; + mFilename = other.mFilename; + mMeta = other.mMeta; + mIsOpen = false; // don't want two file objects reading from the same stream + mGridDescriptors = other.mGridDescriptors; + mNamedGrids = other.mNamedGrids; + mGrids = other.mGrids; } return *this; } @@ -170,43 +75,43 @@ File::copy() const const std::string& File::filename() const { - return mImpl->mFilename; + return mFilename; } MetaMap::Ptr File::fileMetadata() { - return mImpl->mMeta; + return mMeta; } MetaMap::ConstPtr File::fileMetadata() const { - return mImpl->mMeta; + return mMeta; } const File::NameMap& File::gridDescriptors() const { - return mImpl->mGridDescriptors; + return mGridDescriptors; } File::NameMap& File::gridDescriptors() { - return mImpl->mGridDescriptors; + return mGridDescriptors; } std::istream& File::inputStream() const { - if (!mImpl->mInStream) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mInStream) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } - return *mImpl->mInStream; + return *mInStream; } @@ -222,11 +127,11 @@ File::getSize() const Index64 result = std::numeric_limits::max(); - std::string mesg = "could not get size of file " + filename(); + std::string mesg = "could not get size of file " + mFilename; #ifdef _WIN32 // Get the file size by seeking to the end of the file. - std::ifstream fstrm(filename()); + std::ifstream fstrm(mFilename); if (fstrm) { fstrm.seekg(0, fstrm.end); result = static_cast(fstrm.tellg()); @@ -236,7 +141,7 @@ File::getSize() const #else // Get the file size using the stat() system call. struct stat info; - if (0 != ::stat(filename().c_str(), &info)) { + if (0 != ::stat(mFilename.c_str(), &info)) { std::string s = getErrorString(); if (!s.empty()) mesg += " (" + s + ")"; OPENVDB_THROW(IoError, mesg); @@ -252,96 +157,31 @@ File::getSize() const } -#ifdef OPENVDB_USE_DELAYED_LOADING -Index64 -File::copyMaxBytes() const -{ - return mImpl->mCopyMaxBytes; -} - - -void -File::setCopyMaxBytes(Index64 bytes) -{ - mImpl->mCopyMaxBytes = bytes; -} -#endif - - //////////////////////////////////////// bool File::isOpen() const { - return mImpl->mIsOpen; + return mIsOpen; } bool -#ifdef OPENVDB_USE_DELAYED_LOADING -File::open(bool delayLoad, const MappedFile::Notifier& notifier) -#else -File::open(bool /*delayLoad = true*/) -#endif // OPENVDB_USE_DELAYED_LOADING +File::open() { - if (isOpen()) { - OPENVDB_THROW(IoError, filename() << " is already open"); + if (mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is already open"); } - mImpl->mInStream.reset(); + mInStream.reset(); - // Open the file. + // Open the file using standard I/O (delayed loading has been removed) std::unique_ptr newStream; - SharedPtr newStreamBuf; -#ifdef OPENVDB_USE_DELAYED_LOADING - MappedFile::Ptr newFileMapping; - if (!delayLoad || !Archive::isDelayedLoadingEnabled()) { -#endif - newStream.reset(new std::ifstream( - filename().c_str(), std::ios_base::in | std::ios_base::binary)); -#ifdef OPENVDB_USE_DELAYED_LOADING - } else { - bool isTempFile = false; - std::string fname = filename(); - if (getSize() < copyMaxBytes()) { - // If the file is not too large, make a temporary private copy of it - // and open the copy instead. The original file can then be modified - // or removed without affecting delayed load. - try { - TempFile tempFile; - std::ifstream fstrm(filename().c_str(), - std::ios_base::in | std::ios_base::binary); - boost::iostreams::copy(fstrm, tempFile); - fname = tempFile.filename(); - isTempFile = true; - } catch (std::exception& e) { - std::string mesg; - if (e.what()) mesg = std::string(" (") + e.what() + ")"; - OPENVDB_LOG_WARN("failed to create a temporary copy of " << filename() - << " for delayed loading" << mesg - << "; will read directly from " << filename() << " instead"); - } - } - - // While the file is open, its mapping, stream buffer and stream - // must all be maintained. Once the file is closed, the buffer and - // the stream can be discarded, but the mapping needs to persist - // if any grids were lazily loaded. - try { - newFileMapping.reset(new MappedFile(fname, /*autoDelete=*/isTempFile)); - newStreamBuf = newFileMapping->createBuffer(); - newStream.reset(new std::istream(newStreamBuf.get())); - } catch (std::exception& e) { - std::ostringstream ostr; - ostr << "could not open file " << filename(); - if (e.what() != nullptr) ostr << " (" << e.what() << ")"; - OPENVDB_THROW(IoError, ostr.str()); - } - } -#endif // OPENVDB_USE_DELAYED_LOADING + newStream.reset(new std::ifstream( + mFilename.c_str(), std::ios_base::in | std::ios_base::binary)); if (newStream->fail()) { - OPENVDB_THROW(IoError, "could not open file " << filename()); + OPENVDB_THROW(IoError, "could not open file " << mFilename); } // Read in the file header. @@ -351,63 +191,67 @@ File::open(bool /*delayLoad = true*/) } catch (IoError& e) { if (e.what() && std::string("not a VDB file") == e.what()) { // Rethrow, adding the filename. - OPENVDB_THROW(IoError, filename() << " is not a VDB file"); + OPENVDB_THROW(IoError, mFilename << " is not a VDB file"); } throw; } -#ifdef OPENVDB_USE_DELAYED_LOADING - mImpl->mFileMapping = newFileMapping; - if (mImpl->mFileMapping) mImpl->mFileMapping->setNotifier(notifier); - mImpl->mStreamBuf = newStreamBuf; -#endif - mImpl->mInStream.swap(newStream); + mInStream.swap(newStream); // Tag the input stream with the file format and library version numbers // and other metadata. - mImpl->mStreamMetadata.reset(new StreamMetadata); - mImpl->mStreamMetadata->setSeekable(true); - io::setStreamMetadataPtr(inputStream(), mImpl->mStreamMetadata, /*transfer=*/false); + mStreamMetadata.reset(new StreamMetadata); + mStreamMetadata->setSeekable(true); + io::setStreamMetadataPtr(inputStream(), mStreamMetadata, /*transfer=*/false); Archive::setFormatVersion(inputStream()); Archive::setLibraryVersion(inputStream()); Archive::setDataCompression(inputStream()); -#ifdef OPENVDB_USE_DELAYED_LOADING - io::setMappedFilePtr(inputStream(), mImpl->mFileMapping); -#endif // Read in the VDB metadata. - mImpl->mMeta = MetaMap::Ptr(new MetaMap); - mImpl->mMeta->readMeta(inputStream()); + mMeta = MetaMap::Ptr(new MetaMap); + mMeta->readMeta(inputStream()); if (!inputHasGridOffsets()) { - OPENVDB_LOG_DEBUG_RUNTIME("file " << filename() << " does not support partial reading"); + OPENVDB_LOG_DEBUG_RUNTIME("file " << mFilename << " does not support partial reading"); - mImpl->mGrids.reset(new GridPtrVec); - mImpl->mNamedGrids.clear(); + mGrids.reset(new GridPtrVec); + mNamedGrids.clear(); // Stream in the entire contents of the file and append all grids to mGrids. const int32_t gridCount = readGridCount(inputStream()); for (int32_t i = 0; i < gridCount; ++i) { GridDescriptor gd; - gd.read(inputStream()); + gd.readHeader(inputStream()); + gd.readStreamPos(inputStream()); - GridBase::Ptr grid = createGrid(gd); - Archive::readGrid(grid, gd, inputStream()); + GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), io::ReadOptions{}); - gridDescriptors().insert(std::make_pair(gd.gridName(), gd)); - mImpl->mGrids->push_back(grid); - mImpl->mNamedGrids[gd.uniqueName()] = grid; + mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); + mGrids->push_back(grid); + mNamedGrids[gd.uniqueName()] = grid; } // Connect instances (grids that share trees with other grids). - for (NameMapCIter it = gridDescriptors().begin(); it != gridDescriptors().end(); ++it) { - Archive::connectInstance(it->second, mImpl->mNamedGrids); + for (NameMapCIter it = mGridDescriptors.begin(); it != mGridDescriptors.end(); ++it) { + Archive::connectInstance(it->second, mNamedGrids); } } else { - // Read in just the grid descriptors. - readGridDescriptors(inputStream()); + mGridDescriptors.clear(); + + for (int32_t i = 0, N = readGridCount(inputStream()); i < N; ++i) { + // Read the grid descriptor. + GridDescriptor gd; + gd.readHeader(inputStream()); + gd.readStreamPos(inputStream()); + + // Add the descriptor to the dictionary. + mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); + + // Skip forward to the next descriptor. + gd.seekToEnd(inputStream()); + } } - mImpl->mIsOpen = true; + mIsOpen = true; return newFile; // true if file is not identical to opened file } @@ -416,18 +260,14 @@ void File::close() { // Reset all data. - mImpl->mMeta.reset(); - mImpl->mGridDescriptors.clear(); - mImpl->mGrids.reset(); - mImpl->mNamedGrids.clear(); - mImpl->mInStream.reset(); - mImpl->mStreamMetadata.reset(); -#ifdef OPENVDB_USE_DELAYED_LOADING - mImpl->mStreamBuf.reset(); - mImpl->mFileMapping.reset(); -#endif - - mImpl->mIsOpen = false; + mMeta.reset(); + mGridDescriptors.clear(); + mGrids.reset(); + mNamedGrids.clear(); + mInStream.reset(); + mStreamMetadata.reset(); + + mIsOpen = false; setInputHasGridOffsets(true); } @@ -438,52 +278,54 @@ File::close() bool File::hasGrid(const Name& name) const { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } - return (findDescriptor(name) != gridDescriptors().end()); + return (findDescriptor(name) != mGridDescriptors.end()); } MetaMap::Ptr File::getMetadata() const { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } // Return a deep copy of the file-level metadata, which was read // when the file was opened. - return MetaMap::Ptr(new MetaMap(*mImpl->mMeta)); + return MetaMap::Ptr(new MetaMap(*mMeta)); } GridPtrVecPtr -File::getGrids() const +File::getGrids(const io::ReadOptions& readOptions) const { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } GridPtrVecPtr ret; if (!inputHasGridOffsets()) { // If the input file doesn't have grid offsets, then all of the grids // have already been streamed in and stored in mGrids. - ret = mImpl->mGrids; + ret = mGrids; } else { ret.reset(new GridPtrVec); Archive::NamedGridMap namedGrids; // Read all grids represented by the GridDescriptors. - for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) { + for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { const GridDescriptor& gd = i->second; - GridBase::Ptr grid = readGrid(gd); + // Seek to the grid in the file. + gd.seekToGrid(inputStream()); + GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), readOptions, mReadDiagnostics); ret->push_back(grid); namedGrids[gd.uniqueName()] = grid; } // Connect instances (grids that share trees with other grids). - for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) { + for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { Archive::connectInstance(i->second, namedGrids); } } @@ -503,11 +345,11 @@ File::retrieveCachedGrid(const Name& name) const // Search by unique name. Archive::NamedGridMap::const_iterator it = - mImpl->mNamedGrids.find(GridDescriptor::stringAsUniqueName(name)); + mNamedGrids.find(GridDescriptor::stringAsUniqueName(name)); // If not found, search by grid name. - if (it == mImpl->mNamedGrids.end()) it = mImpl->mNamedGrids.find(name); - if (it == mImpl->mNamedGrids.end()) { - OPENVDB_THROW(KeyError, filename() << " has no grid named \"" << name << "\""); + if (it == mNamedGrids.end()) it = mNamedGrids.find(name); + if (it == mNamedGrids.end()) { + OPENVDB_THROW(KeyError, mFilename << " has no grid named \"" << name << "\""); } return it->second; } @@ -519,8 +361,8 @@ File::retrieveCachedGrid(const Name& name) const GridPtrVecPtr File::readAllGridMetadata() { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } if (fileVersion() < OPENVDB_FILE_VERSION_FLOAT_FRUSTUM_BBOX) { @@ -533,18 +375,22 @@ File::readAllGridMetadata() if (!inputHasGridOffsets()) { // If the input file doesn't have grid offsets, then all of the grids // have already been streamed in and stored in mGrids. - for (size_t i = 0, N = mImpl->mGrids->size(); i < N; ++i) { + for (size_t i = 0, N = mGrids->size(); i < N; ++i) { // Return copies of the grids, but with empty trees. - ret->push_back((*mImpl->mGrids)[i]->copyGridWithNewTree()); + ret->push_back((*mGrids)[i]->copyGridWithNewTree()); } } else { // Read just the metadata and transforms for all grids. - for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) { + for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { const GridDescriptor& gd = i->second; - GridBase::ConstPtr grid = readGridPartial(gd, /*readTopology=*/false); + // Seek to the grid in the file. + gd.seekToGrid(inputStream()); + io::ReadOptions readOptions; + readOptions.readMode = io::ReadMode::MetadataOnly; + GridBase::ConstPtr grid = Archive::readGrid(gd, inputStream(), readOptions); // Return copies of the grids, but with empty trees. // (As of 0.98.0, at least, it would suffice to just const cast - // the grid pointers returned by readGridPartial(), but shallow + // the grid pointers returned by readGrid(partial=true), but shallow // copying the grids helps to ensure future compatibility.) ret->push_back(grid->copyGridWithNewTree()); } @@ -556,8 +402,8 @@ File::readAllGridMetadata() GridBase::Ptr File::readGridMetadata(const Name& name) { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading."); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading."); } if (fileVersion() < OPENVDB_FILE_VERSION_FLOAT_FRUSTUM_BBOX) { @@ -572,13 +418,16 @@ File::readGridMetadata(const Name& name) ret = readGrid(name); } else { NameMapCIter it = findDescriptor(name); - if (it == gridDescriptors().end()) { - OPENVDB_THROW(KeyError, filename() << " has no grid named \"" << name << "\""); + if (it == mGridDescriptors.end()) { + OPENVDB_THROW(KeyError, mFilename << " has no grid named \"" << name << "\""); } // Seek to and read in the grid from the file. const GridDescriptor& gd = it->second; - ret = readGridPartial(gd, /*readTopology=*/false); + gd.seekToGrid(inputStream()); + io::ReadOptions readOptions; + readOptions.readMode = io::ReadMode::MetadataOnly; + ret = Archive::readGrid(gd, inputStream(), readOptions); } return ret->copyGridWithNewTree(); } @@ -587,34 +436,29 @@ File::readGridMetadata(const Name& name) //////////////////////////////////////// -GridBase::Ptr -File::readGrid(const Name& name) -{ - return readGridByName(name, BBoxd()); -} - - GridBase::Ptr File::readGrid(const Name& name, const BBoxd& bbox) { - return readGridByName(name, bbox); + io::ReadOptions readOptions; + readOptions.clipBBox = bbox; + return readGrid(name, readOptions); } GridBase::Ptr -File::readGridByName(const Name& name, const BBoxd& bbox) +File::readGrid(const Name& name, const io::ReadOptions& readOptions) { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading."); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading."); } - const bool clip = bbox.isSorted(); - // If a grid with the given name was already read and cached // (along with the entire contents of the file, because the file // doesn't support random access), retrieve and return it. GridBase::Ptr grid = retrieveCachedGrid(name); if (grid) { + const auto& bbox = readOptions.clipBBox; + const bool clip = bbox.isSorted(); if (clip) { grid = grid->deepCopyGrid(); grid->clipGrid(bbox); @@ -623,33 +467,49 @@ File::readGridByName(const Name& name, const BBoxd& bbox) } NameMapCIter it = findDescriptor(name); - if (it == gridDescriptors().end()) { - OPENVDB_THROW(KeyError, filename() << " has no grid named \"" << name << "\""); + if (it == mGridDescriptors.end()) { + OPENVDB_THROW(KeyError, mFilename << " has no grid named \"" << name << "\""); } // Seek to and read in the grid from the file. const GridDescriptor& gd = it->second; - grid = (clip ? readGrid(gd, bbox) : readGrid(gd)); + // This method should not be called for files that don't contain grid offsets. + OPENVDB_ASSERT(inputHasGridOffsets()); + // Seek to the grid in the file. + gd.seekToGrid(inputStream()); + grid = Archive::readGrid(gd, inputStream(), readOptions, mReadDiagnostics); if (gd.isInstance()) { /// @todo Refactor to share code with Archive::connectInstance()? NameMapCIter parentIt = findDescriptor(GridDescriptor::nameAsString(gd.instanceParentName())); - if (parentIt == gridDescriptors().end()) { + if (parentIt == mGridDescriptors.end()) { OPENVDB_THROW(KeyError, "missing instance parent \"" << GridDescriptor::nameAsString(gd.instanceParentName()) << "\" for grid " << GridDescriptor::nameAsString(gd.uniqueName()) - << " in file " << filename()); + << " in file " << mFilename); } + // Read the parent without clipping. Archive::readGrid() converts the + // world-space clip region into index space using the grid's own + // transform, but an instance has its own transform that may differ + // from the parent's. Instead, read the full parent tree and clip the + // assembled instance below using the instance's transform, so that the + // retained region matches the requested world-space bbox. + io::ReadOptions parentOptions = readOptions; + parentOptions.clipBBox = BBoxd(); + GridBase::Ptr parent; - if (clip) { - const CoordBBox indexBBox = grid->constTransform().worldToIndexNodeCentered(bbox); - parent = readGrid(parentIt->second, indexBBox); - } else { - parent = readGrid(parentIt->second); + OPENVDB_ASSERT(inputHasGridOffsets()); + parentIt->second.seekToGrid(inputStream()); + parent = Archive::readGrid(parentIt->second, inputStream(), parentOptions, mReadDiagnostics); + if (parent) { + grid->setTree(parent->baseTreePtr()); + const auto& clipBBox = readOptions.clipBBox; + if (clipBBox.isSorted()) { + grid->clipGrid(clipBBox); + } } - if (parent) grid->setTree(parent->baseTreePtr()); } return grid; } @@ -659,53 +519,29 @@ File::readGridByName(const Name& name, const BBoxd& bbox) void -File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta) const +File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta, const io::WriteOptions& writeOptions) const { - if (isOpen()) { + if (mIsOpen) { OPENVDB_THROW(IoError, - filename() << " cannot be written because it is open for reading"); + mFilename << " cannot be written because it is open for reading"); } // Create a file stream and write it out. std::ofstream file; - file.open(filename().c_str(), + file.open(mFilename.c_str(), std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); if (file.fail()) { - OPENVDB_THROW(IoError, "could not open " << filename() << " for writing"); + OPENVDB_THROW(IoError, "could not open " << mFilename << " for writing"); } // Write out the vdb. - Archive::write(file, grids, /*seekable=*/true, meta); + Archive::write(file, grids, /*seekable=*/true, meta, writeOptions); file.close(); } -//////////////////////////////////////// - - -void -File::readGridDescriptors(std::istream& is) -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - gridDescriptors().clear(); - - for (int32_t i = 0, N = readGridCount(is); i < N; ++i) { - // Read the grid descriptor. - GridDescriptor gd; - gd.read(is); - - // Add the descriptor to the dictionary. - gridDescriptors().insert(std::make_pair(gd.gridName(), gd)); - - // Skip forward to the next descriptor. - gd.seekToEnd(is); - } -} - //////////////////////////////////////// @@ -716,20 +552,20 @@ File::findDescriptor(const Name& name) const const Name uniqueName = GridDescriptor::stringAsUniqueName(name); // Find all descriptors with the given grid name. - std::pair range = gridDescriptors().equal_range(name); + std::pair range = mGridDescriptors.equal_range(name); if (range.first == range.second) { // If no descriptors were found with the given grid name, the name might have // a suffix ("name[N]"). In that case, remove the "[N]" suffix and search again. - range = gridDescriptors().equal_range(GridDescriptor::stripSuffix(uniqueName)); + range = mGridDescriptors.equal_range(GridDescriptor::stripSuffix(uniqueName)); } const size_t count = size_t(std::distance(range.first, range.second)); if (count > 1 && name == uniqueName) { - OPENVDB_LOG_WARN(filename() << " has more than one grid named \"" << name << "\""); + OPENVDB_LOG_WARN(mFilename << " has more than one grid named \"" << name << "\""); } - NameMapCIter ret = gridDescriptors().end(); + NameMapCIter ret = mGridDescriptors.end(); if (count > 0) { if (name == uniqueName) { @@ -755,109 +591,23 @@ File::findDescriptor(const Name& name) const //////////////////////////////////////// -GridBase::Ptr -File::createGrid(const GridDescriptor& gd) const -{ - // Create the grid. - if (!GridBase::isRegistered(gd.gridType())) { - OPENVDB_THROW(KeyError, "Cannot read grid " - << GridDescriptor::nameAsString(gd.uniqueName()) - << " from " << filename() << ": grid type " - << gd.gridType() << " is not registered"); - } - - GridBase::Ptr grid = GridBase::createGrid(gd.gridType()); - if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); - - return grid; -} - - -GridBase::ConstPtr -File::readGridPartial(const GridDescriptor& gd, bool readTopology) const -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - GridBase::Ptr grid = createGrid(gd); - - // Seek to grid. - gd.seekToGrid(inputStream()); - - // Read the grid partially. - readGridPartial(grid, inputStream(), gd.isInstance(), readTopology); - - // Promote to a const grid. - GridBase::ConstPtr constGrid = grid; - - return constGrid; -} - - -GridBase::Ptr -File::readGrid(const GridDescriptor& gd) const -{ - return Impl::readGrid(*this, gd, Impl::NoBBox()); -} - - -GridBase::Ptr -File::readGrid(const GridDescriptor& gd, const BBoxd& bbox) const -{ - return Impl::readGrid(*this, gd, bbox); -} - - -GridBase::Ptr -File::readGrid(const GridDescriptor& gd, const CoordBBox& bbox) const -{ - return Impl::readGrid(*this, gd, bbox); -} - - -void -File::readGridPartial(GridBase::Ptr grid, std::istream& is, - bool isInstance, bool readTopology) const -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - // This code needs to stay in sync with io::Archive::readGrid(), in terms of - // the order of operations. - readGridCompression(is); - grid->readMeta(is); - - // drop DelayedLoadMetadata from the grid as it is only useful for IO - if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { - grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } - - grid->readTransform(is); - if (!isInstance && readTopology) { - grid->readTopology(is); - } -} - - -//////////////////////////////////////// - - File::NameIterator File::beginName() const { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } - return File::NameIterator(gridDescriptors().begin()); + return File::NameIterator(mGridDescriptors.begin()); } File::NameIterator File::endName() const { - return File::NameIterator(gridDescriptors().end()); + return File::NameIterator(mGridDescriptors.end()); } + } // namespace io } // namespace OPENVDB_VERSION_NAME } // namespace openvdb diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index 41dd6f9694..920b86a50f 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -34,7 +34,7 @@ class OPENVDB_API File: public Archive using NameMapCIter = NameMap::const_iterator; explicit File(const std::string& filename); - ~File() override; + ~File() override { } /// @brief Copy constructor /// @details The copy will be closed and will not reference the same @@ -54,20 +54,16 @@ class OPENVDB_API File: public Archive /// @details The file does not necessarily exist on disk yet. const std::string& filename() const; -#ifdef OPENVDB_USE_DELAYED_LOADING /// @brief Open the file, read the file header and the file-level metadata, /// and populate the grid descriptors, but do not load any grids into memory. - /// @details If @a delayLoad is true, map the file into memory and enable delayed loading - /// of grids, and if a notifier is provided, call it when the file gets unmapped. - /// @note Define the environment variable @c OPENVDB_DISABLE_DELAYED_LOAD to disable - /// delayed loading unconditionally. /// @throw IoError if the file is not a valid VDB file. /// @return @c true if the file's UUID has changed since it was last read. - /// @see setCopyMaxBytes - bool open(bool delayLoad = true, const MappedFile::Notifier& = MappedFile::Notifier()); -#else - bool open(bool /*delayLoad*/ = false); -#endif + bool open(); + + OPENVDB_DEPRECATED_MESSAGE("Use File::open() instead.This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool open(bool /*delayLoad*/) { return open(); } + OPENVDB_DEPRECATED_MESSAGE("Use File::open() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool open(bool /*delayLoad*/, const MappedFile::Notifier& /*notifier*/) { return open(); } /// Return @c true if the file has been opened for reading. bool isOpen() const; @@ -79,23 +75,10 @@ class OPENVDB_API File: public Archive /// @throw IoError if the file size cannot be determined. Index64 getSize() const; -#ifdef OPENVDB_USE_DELAYED_LOADING - /// @brief Return the size in bytes above which this file will not be - /// automatically copied during delayed loading. - Index64 copyMaxBytes() const; - /// @brief If this file is opened with delayed loading enabled, make a private copy - /// of the file if its size in bytes is less than the specified value. - /// @details Making a private copy ensures that the file can't change on disk - /// before it has been fully read. - /// @warning If the file is larger than this size, it is the user's responsibility - /// to ensure that it does not change on disk before it has been fully read. - /// Undefined behavior and/or a crash might result otherwise. - /// @note Copying is enabled by default, but it can be disabled for individual files - /// by setting the maximum size to zero bytes. A default size limit can be specified - /// by setting the environment variable @c OPENVDB_DELAYED_LOAD_COPY_MAX_BYTES - /// to the desired number of bytes. - void setCopyMaxBytes(Index64 bytes); -#endif + OPENVDB_DEPRECATED_MESSAGE("Always returns 0. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 copyMaxBytes() const { return 0; } + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void setCopyMaxBytes(Index64 /*bytes*/) { } /// Return @c true if a grid of the given name exists in this file. bool hasGrid(const Name&) const; @@ -104,7 +87,7 @@ class OPENVDB_API File: public Archive MetaMap::Ptr getMetadata() const; /// Read the entire contents of the file and return a list of grid pointers. - GridPtrVecPtr getGrids() const; + GridPtrVecPtr getGrids(const io::ReadOptions& readOptions = io::ReadOptions{}) const; /// @brief Read just the grid metadata and transforms from the file and return a list /// of pointers to grids that are empty except for their metadata and transforms. @@ -117,8 +100,8 @@ class OPENVDB_API File: public Archive /// @throw KeyError if no grid with the given name exists in this file. GridBase::Ptr readGridMetadata(const Name&); - /// Read an entire grid, including all of its data blocks. - GridBase::Ptr readGrid(const Name&); + /// Read an entire grid, including all of its data blocks, using the provided options if given. + GridBase::Ptr readGrid(const Name&, const io::ReadOptions& readOptions = io::ReadOptions{}); /// @brief Read a grid, including its data blocks, but only where it /// intersects the given world-space bounding box. GridBase::Ptr readGrid(const Name&, const BBoxd&); @@ -127,12 +110,14 @@ class OPENVDB_API File: public Archive /// @brief Write the grids in the given container to the file whose name /// was given in the constructor. - void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const override; + void write(const GridCPtrVec&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const override; /// @brief Write the grids in the given container to the file whose name /// was given in the constructor. template - void write(const GridPtrContainerT&, const MetaMap& = MetaMap()) const; + void write(const GridPtrContainerT&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const; /// A const iterator that iterates over all names in the file. This is only /// valid once the file has been opened. @@ -163,42 +148,16 @@ class OPENVDB_API File: public Archive NameIterator endName() const; private: - /// Read in all grid descriptors that are stored in the given stream. - void readGridDescriptors(std::istream&); - /// @brief Return an iterator to the descriptor for the grid with the given name. /// If the name is non-unique, return an iterator to the first matching descriptor. NameMapCIter findDescriptor(const Name&) const; - /// Return a newly created, empty grid of the type specified by the given grid descriptor. - GridBase::Ptr createGrid(const GridDescriptor&) const; - - /// @brief Read a grid, including its data blocks, but only where it - /// intersects the given world-space bounding box. - GridBase::Ptr readGridByName(const Name&, const BBoxd&); - - /// Read in and return the partially-populated grid specified by the given grid descriptor. - GridBase::ConstPtr readGridPartial(const GridDescriptor&, bool readTopology) const; - - /// Read in and return the grid specified by the given grid descriptor. - GridBase::Ptr readGrid(const GridDescriptor&) const; - /// Read in and return the region of the grid specified by the given grid descriptor - /// that intersects the given world-space bounding box. - GridBase::Ptr readGrid(const GridDescriptor&, const BBoxd&) const; - /// Read in and return the region of the grid specified by the given grid descriptor - /// that intersects the given index-space bounding box. - GridBase::Ptr readGrid(const GridDescriptor&, const CoordBBox&) const; - - /// @brief Partially populate the given grid by reading its metadata and transform and, - /// if the grid is not an instance, its tree structure, but not the tree's leaf nodes. - void readGridPartial(GridBase::Ptr, std::istream&, bool isInstance, bool readTopology) const; - /// @brief Retrieve a grid from @c mNamedGrids. Return a null pointer /// if @c mNamedGrids was not populated (because this file is random-access). /// @throw KeyError if no grid with the given name exists in this file. GridBase::Ptr retrieveCachedGrid(const Name&) const; - void writeGrids(const GridCPtrVec&, const MetaMap&) const; + void writeGrids(const GridCPtrVec&, const MetaMap&, const io::WriteOptions&) const; MetaMap::Ptr fileMetadata(); MetaMap::ConstPtr fileMetadata() const; @@ -211,8 +170,22 @@ class OPENVDB_API File: public Archive friend class ::TestFile; friend class ::TestStream; - struct Impl; - std::unique_ptr mImpl; + std::string mFilename; + // The file-level metadata + MetaMap::Ptr mMeta; + // The file stream that is open for reading + std::unique_ptr mInStream; + // File-level stream metadata (file format, compression, etc.) + StreamMetadata::Ptr mStreamMetadata; + // Flag indicating if we have read in the global information (header, + // metadata, and grid descriptors) for this VDB file + bool mIsOpen = false; + // Grid descriptors for all grids stored in the file, indexed by grid name + NameMap mGridDescriptors; + // All grids, indexed by unique name (used only when mHasGridOffsets is false) + Archive::NamedGridMap mNamedGrids; + // All grids stored in the file (used only when mHasGridOffsets is false) + GridPtrVecPtr mGrids; }; @@ -220,19 +193,21 @@ class OPENVDB_API File: public Archive inline void -File::write(const GridCPtrVec& grids, const MetaMap& meta) const +File::write(const GridCPtrVec& grids, const MetaMap& meta, + const io::WriteOptions& writeOptions) const { - this->writeGrids(grids, meta); + this->writeGrids(grids, meta, writeOptions); } template inline void -File::write(const GridPtrContainerT& container, const MetaMap& meta) const +File::write(const GridPtrContainerT& container, const MetaMap& meta, + const io::WriteOptions& writeOptions) const { GridCPtrVec grids; std::copy(container.begin(), container.end(), std::back_inserter(grids)); - this->writeGrids(grids, meta); + this->writeGrids(grids, meta, writeOptions); } } // namespace io diff --git a/openvdb/openvdb/io/GridDescriptor.cc b/openvdb/openvdb/io/GridDescriptor.cc index 961f622008..9a59d22d1c 100644 --- a/openvdb/openvdb/io/GridDescriptor.cc +++ b/openvdb/openvdb/io/GridDescriptor.cc @@ -69,8 +69,8 @@ GridDescriptor::writeStreamPos(std::ostream &os) const os.write(reinterpret_cast(&mEndPos), sizeof(int64_t)); } -GridBase::Ptr -GridDescriptor::read(std::istream &is) +void +GridDescriptor::readHeader(std::istream &is) { checkFormatVersion(is); @@ -86,21 +86,28 @@ GridDescriptor::read(std::istream &is) } mInstanceParentName = readString(is); +} + +void +GridDescriptor::readStreamPos(std::istream &is) +{ + is.read(reinterpret_cast(&mGridPos), sizeof(int64_t)); + is.read(reinterpret_cast(&mBlockPos), sizeof(int64_t)); + is.read(reinterpret_cast(&mEndPos), sizeof(int64_t)); +} + +GridBase::Ptr +GridDescriptor::read(std::istream &is) +{ + readHeader(is); + readStreamPos(is); - // Create the grid of the type if it has been registered. if (!GridBase::isRegistered(mGridType)) { OPENVDB_THROW(LookupError, "Cannot read grid." << " Grid type " << mGridType << " is not registered."); } - // else GridBase::Ptr grid = GridBase::createGrid(mGridType); if (grid) grid->setSaveFloatAsHalf(mSaveFloatAsHalf); - - // Read in the offsets. - is.read(reinterpret_cast(&mGridPos), sizeof(int64_t)); - is.read(reinterpret_cast(&mBlockPos), sizeof(int64_t)); - is.read(reinterpret_cast(&mEndPos), sizeof(int64_t)); - return grid; } diff --git a/openvdb/openvdb/io/GridDescriptor.h b/openvdb/openvdb/io/GridDescriptor.h index 9ef03262e2..1aaf5f10a4 100644 --- a/openvdb/openvdb/io/GridDescriptor.h +++ b/openvdb/openvdb/io/GridDescriptor.h @@ -61,8 +61,18 @@ class OPENVDB_API GridDescriptor /// written out separately. void writeStreamPos(std::ostream&) const; + /// @brief Read this descriptor's header information (all data except for + /// stream offsets) from the given stream. + void readHeader(std::istream&); + + /// @brief Read stream positions (grid, block, and end offsets) from the + /// given stream. + void readStreamPos(std::istream&); + /// @brief Read a grid descriptor from the given stream. /// @return an empty grid of the type specified by the grid descriptor. + /// @deprecated Use readHeader() followed by readStreamPos() instead. + OPENVDB_DEPRECATED_MESSAGE("Use readHeader() followed by readStreamPos() instead.") GridBase::Ptr read(std::istream&); /// @brief Append the number @a n to the given name (separated by an ASCII diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 662a2f4cd9..87f9f99058 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -5,14 +5,9 @@ #include "File.h" ///< @todo refactor #include "GridDescriptor.h" -#include "TempFile.h" #include #include -#ifdef OPENVDB_USE_DELAYED_LOADING -#include -#endif - #include // for remove() #include // for std::bind() #include @@ -24,140 +19,83 @@ OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { namespace io { -struct Stream::Impl -{ - Impl(): mOutputStream{nullptr} {} - Impl(const Impl& other) { *this = other; } - Impl& operator=(const Impl& other) - { - if (&other != this) { - mMeta = other.mMeta; ///< @todo deep copy? - mGrids = other.mGrids; ///< @todo deep copy? - mOutputStream = other.mOutputStream; - mFile.reset(); - } - return *this; - } - - MetaMap::Ptr mMeta; - GridPtrVecPtr mGrids; - std::ostream* mOutputStream; - std::unique_ptr mFile; -}; - - -//////////////////////////////////////// - - -#ifdef OPENVDB_USE_DELAYED_LOADING -namespace { - -/// @todo Use MappedFile auto-deletion instead. -void -removeTempFile(const std::string expectedFilename, const std::string& filename) +Stream::Stream(std::istream& is) + : Stream(is, io::ReadOptions{}) { - if (filename == expectedFilename) { - if (0 != std::remove(filename.c_str())) { - std::string mesg = getErrorString(); - if (!mesg.empty()) mesg = " (" + mesg + ")"; - OPENVDB_LOG_WARN("failed to remove temporary file " << filename << mesg); - } - } -} - } -#endif // OPENVDB_USE_DELAYED_LOADING - -Stream::Stream(std::istream& is, bool delayLoad): mImpl(new Impl) +Stream::Stream(std::istream& is, const io::ReadOptions& readOptions) { - if (!is) return; - - (void) delayLoad; - -#ifdef OPENVDB_USE_DELAYED_LOADING - if (delayLoad && Archive::isDelayedLoadingEnabled()) { - // Copy the contents of the stream to a temporary private file - // and open the file instead. - std::unique_ptr tempFile; - try { - tempFile.reset(new TempFile); - } catch (std::exception& e) { - std::string mesg; - if (e.what()) mesg = std::string(" (") + e.what() + ")"; - OPENVDB_LOG_WARN("failed to create a temporary file for delayed loading" << mesg - << "; will read directly from the input stream instead"); - } - if (tempFile) { - boost::iostreams::copy(is, *tempFile); - const std::string& filename = tempFile->filename(); - mImpl->mFile.reset(new File(filename)); - mImpl->mFile->setCopyMaxBytes(0); // don't make a copy of the temporary file - /// @todo Need to pass auto-deletion flag to MappedFile. - mImpl->mFile->open(delayLoad, - std::bind(&removeTempFile, filename, std::placeholders::_1)); - } + // Read modes that stop before consuming all of a grid's data are not + // supported, because a stream is read sequentially - the position after + // a partial read is still inside the previous grid's data, so the next + // grid header would be read from the wrong offset. + // TODO: Skip over the unread bytes of each grid instead of disallowing + // these read modes. This is best implemented alongside the extension that + // adds support for byte skipping in non-seekable streams. + if (readOptions.readMode == io::ReadMode::MetadataOnly || + readOptions.readMode == io::ReadMode::TopologyOnly) { + OPENVDB_THROW(ValueError, "io::ReadMode::" + << (readOptions.readMode == io::ReadMode::MetadataOnly + ? "MetadataOnly" : "TopologyOnly") + << " is not supported when reading from a stream"); } -#endif // OPENVDB_USE_DELAYED_LOADING - - if (!mImpl->mFile) { - readHeader(is); - - // Tag the input stream with the library and file format version numbers - // and the compression options specified in the header. - StreamMetadata::Ptr streamMetadata(new StreamMetadata); - io::setStreamMetadataPtr(is, streamMetadata, /*transfer=*/false); - io::setVersion(is, libraryVersion(), fileVersion()); - io::setDataCompression(is, compression()); - - // Read in the VDB metadata. - mImpl->mMeta.reset(new MetaMap); - mImpl->mMeta->readMeta(is); - - // Read in the number of grids. - const int32_t gridCount = readGridCount(is); - - // Read in all grids and insert them into mGrids. - mImpl->mGrids.reset(new GridPtrVec); - std::vector descriptors; - descriptors.reserve(gridCount); - Archive::NamedGridMap namedGrids; - for (int32_t i = 0; i < gridCount; ++i) { - GridDescriptor gd; - gd.read(is); - descriptors.push_back(gd); - GridBase::Ptr grid = readGrid(gd, is); - mImpl->mGrids->push_back(grid); - namedGrids[gd.uniqueName()] = grid; - } - - // Connect instances (grids that share trees with other grids). - for (size_t i = 0, N = descriptors.size(); i < N; ++i) { - Archive::connectInstance(descriptors[i], namedGrids); - } - } -} - -Stream::Stream(): mImpl(new Impl) -{ -} + if (!is) return; + // Delayed loading has been removed - always read directly from the stream + readHeader(is); + + // Tag the input stream with the library and file format version numbers + // and the compression options specified in the header. + StreamMetadata::Ptr streamMetadata(new StreamMetadata); + io::setStreamMetadataPtr(is, streamMetadata, /*transfer=*/false); + io::setVersion(is, libraryVersion(), fileVersion()); + io::setDataCompression(is, compression()); + + // Read in the VDB metadata. + mMeta.reset(new MetaMap); + mMeta->readMeta(is); + + // Read in the number of grids. + const int32_t gridCount = readGridCount(is); + + // Read in all grids and insert them into mGrids. + mGrids.reset(new GridPtrVec); + std::vector descriptors; + descriptors.reserve(gridCount); + Archive::NamedGridMap namedGrids; + for (int32_t i = 0; i < gridCount; ++i) { + GridDescriptor gd; + gd.readHeader(is); + gd.readStreamPos(is); + descriptors.push_back(gd); + GridBase::Ptr grid = Archive::readGrid(gd, is, readOptions); + mGrids->push_back(grid); + namedGrids[gd.uniqueName()] = grid; + } -Stream::Stream(std::ostream& os): mImpl(new Impl) -{ - mImpl->mOutputStream = &os; + // Connect instances (grids that share trees with other grids). + for (size_t i = 0, N = descriptors.size(); i < N; ++i) { + Archive::connectInstance(descriptors[i], namedGrids); + } } -Stream::~Stream() +Stream::Stream(std::ostream& os) + : Archive() + , mOutputStream(&os) { } -Stream::Stream(const Stream& other): Archive(other), mImpl(new Impl(*other.mImpl)) +Stream::Stream(const Stream& other) + : Archive(other) + , mMeta(other.mMeta) + , mGrids(other.mGrids) + , mOutputStream(other.mOutputStream) { } @@ -166,7 +104,10 @@ Stream& Stream::operator=(const Stream& other) { if (&other != this) { - mImpl.reset(new Impl(*other.mImpl)); + Archive::operator=(other); + mMeta = other.mMeta; + mGrids = other.mGrids; + mOutputStream = other.mOutputStream; } return *this; } @@ -182,39 +123,22 @@ Stream::copy() const //////////////////////////////////////// -GridBase::Ptr -Stream::readGrid(const GridDescriptor& gd, std::istream& is) const -{ - GridBase::Ptr grid; - - if (!GridBase::isRegistered(gd.gridType())) { - OPENVDB_THROW(TypeError, "can't read grid \"" - << GridDescriptor::nameAsString(gd.uniqueName()) << - "\" from input stream because grid type " << gd.gridType() << " is unknown"); - } else { - grid = GridBase::createGrid(gd.gridType()); - if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); - - Archive::readGrid(grid, gd, is); - } - return grid; -} - - void -Stream::write(const GridCPtrVec& grids, const MetaMap& metadata) const +Stream::write(const GridCPtrVec& grids, const MetaMap& metadata, + const io::WriteOptions& writeOptions) const { - if (mImpl->mOutputStream == nullptr) { + if (mOutputStream == nullptr) { OPENVDB_THROW(ValueError, "no output stream was specified"); } - this->writeGrids(*mImpl->mOutputStream, grids, metadata); + this->writeGrids(*mOutputStream, grids, metadata, writeOptions); } void -Stream::writeGrids(std::ostream& os, const GridCPtrVec& grids, const MetaMap& metadata) const +Stream::writeGrids(std::ostream& os, const GridCPtrVec& grids, const MetaMap& metadata, + const io::WriteOptions& writeOptions) const { - Archive::write(os, grids, /*seekable=*/false, metadata); + Archive::write(os, grids, /*seekable=*/false, metadata, writeOptions); } @@ -225,12 +149,10 @@ MetaMap::Ptr Stream::getMetadata() const { MetaMap::Ptr result; - if (mImpl->mFile) { - result = mImpl->mFile->getMetadata(); - } else if (mImpl->mMeta) { + if (mMeta) { // Return a deep copy of the file-level metadata // that was read when this object was constructed. - result.reset(new MetaMap(*mImpl->mMeta)); + result.reset(new MetaMap(*mMeta)); } return result; } @@ -239,10 +161,7 @@ Stream::getMetadata() const GridPtrVecPtr Stream::getGrids() { - if (mImpl->mFile) { - return mImpl->mFile->getGrids(); - } - return mImpl->mGrids; + return mGrids; } } // namespace io diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index e2978dc20d..432e5eb9a9 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -22,21 +22,30 @@ class OPENVDB_API Stream: public Archive { public: /// @brief Read grids from an input stream. - /// @details If @a delayLoad is true, map the contents of the input stream - /// into memory and enable delayed loading of grids. - /// @note Define the environment variable @c OPENVDB_DISABLE_DELAYED_LOAD - /// to disable delayed loading unconditionally. - explicit Stream(std::istream&, bool delayLoad = true); + /// @param is The input stream to read from + explicit Stream(std::istream& is); + + /// @brief Read grids from an input stream using the given read options. + /// @param is The input stream to read from + /// @param readOptions Options controlling how grids are read (e.g. attribute + /// skipping for point data grids) + /// @throw ValueError if @a readOptions requests @c io::ReadMode::MetadataOnly + /// or @c io::ReadMode::TopologyOnly. These modes leave a grid's data + /// partially unread, which a sequentially read stream cannot skip over. + Stream(std::istream& is, const io::ReadOptions& readOptions); + + OPENVDB_DEPRECATED_MESSAGE("Use Stream(std::istream&) instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Stream(std::istream& is, bool /*delayLoad*/) : Stream(is) { } /// Construct an archive for stream output. - Stream(); + Stream() = default; /// Construct an archive for output to the given stream. explicit Stream(std::ostream&); Stream(const Stream&); Stream& operator=(const Stream&); - ~Stream() override; + ~Stream() override { } /// @brief Return a copy of this archive. Archive::Ptr copy() const override; @@ -49,24 +58,22 @@ class OPENVDB_API Stream: public Archive /// @brief Write the grids in the given container to this archive's output stream. /// @throw ValueError if this archive was constructed without specifying an output stream. - void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const override; + void write(const GridCPtrVec&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const override; /// @brief Write the grids in the given container to this archive's output stream. /// @throw ValueError if this archive was constructed without specifying an output stream. template - void write(const GridPtrContainerT&, const MetaMap& = MetaMap()) const; + void write(const GridPtrContainerT&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const; private: - /// Create a new grid of the type specified by the given descriptor, - /// then populate the grid from the given input stream. - /// @return the newly created grid. - GridBase::Ptr readGrid(const GridDescriptor&, std::istream&) const; - - void writeGrids(std::ostream&, const GridCPtrVec&, const MetaMap&) const; - + void writeGrids(std::ostream&, const GridCPtrVec&, const MetaMap&, + const io::WriteOptions&) const; - struct Impl; - std::unique_ptr mImpl; + MetaMap::Ptr mMeta; + GridPtrVecPtr mGrids; + std::ostream* mOutputStream = nullptr; }; @@ -75,11 +82,12 @@ class OPENVDB_API Stream: public Archive template inline void -Stream::write(const GridPtrContainerT& container, const MetaMap& metadata) const +Stream::write(const GridPtrContainerT& container, const MetaMap& metadata, + const io::WriteOptions& writeOptions) const { GridCPtrVec grids; std::copy(container.begin(), container.end(), std::back_inserter(grids)); - this->write(grids, metadata); + this->write(grids, metadata, writeOptions); } } // namespace io diff --git a/openvdb/openvdb/io/TempFile.cc b/openvdb/openvdb/io/TempFile.cc deleted file mode 100644 index 290f2e6b9c..0000000000 --- a/openvdb/openvdb/io/TempFile.cc +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -/// @file TempFile.cc - -#ifdef OPENVDB_USE_DELAYED_LOADING - -#include "TempFile.h" - -#include -#ifndef _WIN32 -#include -#include -#include // for std::getenv(), mkstemp() -#include // for mode_t -#include // for mkdir(), umask() -#include // for access() -#else -#include // for std::filebuf -#endif -#include // for std::tmpnam(), L_tmpnam, P_tmpdir -#include -#include -#include -#include - - -namespace openvdb { -OPENVDB_USE_VERSION_NAMESPACE -namespace OPENVDB_VERSION_NAME { -namespace io { - -struct TempFile::TempFileImpl -{ - const std::string& filename() const { return mPath; } - - bool is_open() const { return mBuffer.is_open(); } - - /// @internal boost::filesystem::unique_path(), etc. might be useful here, - /// but as of 9/2014, Houdini ships without the Boost.Filesystem library, - /// which makes it much less convenient to use that library. -#ifndef _WIN32 - TempFileImpl(std::ostream& os): mFileDescr(-1) { this->init(os); } - - void init(std::ostream& os) - { - std::string fn = this->getTempDir() + "/openvdb_temp_XXXXXX"; - std::vector fnbuf(fn.begin(), fn.end()); - fnbuf.push_back(char(0)); - - //const mode_t savedMode = ::umask(~(S_IRUSR | S_IWUSR)); - mFileDescr = ::mkstemp(&fnbuf[0]); - //::umask(savedMode); - if (mFileDescr < 0) { - OPENVDB_THROW(IoError, "failed to generate temporary file"); - } - - mPath.assign(&fnbuf[0]); - - mDevice = DeviceType(mFileDescr, boost::iostreams::never_close_handle); - mBuffer.open(mDevice); - os.rdbuf(&mBuffer); - - if (!os.good()) { - OPENVDB_THROW(IoError, "failed to open temporary file " + mPath); - } - } - - void close() { mBuffer.close(); if (mFileDescr >= 0) ::close(mFileDescr); } - - static std::string getTempDir() - { - if (const char* dir = std::getenv("OPENVDB_TEMP_DIR")) { - if (0 != ::access(dir, F_OK)) { -#ifdef _WIN32 - ::mkdir(dir); -#else - ::mkdir(dir, S_IRUSR | S_IWUSR | S_IXUSR); -#endif - if (0 != ::access(dir, F_OK)) { - OPENVDB_THROW(IoError, - "failed to create OPENVDB_TEMP_DIR (" + std::string(dir) + ")"); - } - } - return dir; - } - if (const char* dir = std::getenv("TMPDIR")) return dir; - return P_tmpdir; - } - - using DeviceType = boost::iostreams::file_descriptor_sink; - using BufferType = boost::iostreams::stream_buffer; - - std::string mPath; - DeviceType mDevice; - BufferType mBuffer; - int mFileDescr; -#else // _WIN32 - // Use only standard library routines; no POSIX. - - TempFileImpl(std::ostream& os) { this->init(os); } - - void init(std::ostream& os) - { - char fnbuf[L_tmpnam]; - const char* filename = std::tmpnam(fnbuf); - if (!filename) { - OPENVDB_THROW(IoError, "failed to generate name for temporary file"); - } - /// @todo This is not safe, since another process could open a file - /// with this name before we do. Unfortunately, there is no safe, - /// portable way to create a temporary file. - mPath = filename; - - const std::ios_base::openmode mode = (std::ios_base::out | std::ios_base::binary); - os.rdbuf(mBuffer.open(mPath.c_str(), mode)); - if (!os.good()) { - OPENVDB_THROW(IoError, "failed to open temporary file " + mPath); - } - } - - void close() { mBuffer.close(); } - - std::string mPath; - std::filebuf mBuffer; -#endif // _WIN32 - -private: - TempFileImpl(const TempFileImpl&); // disable copying - TempFileImpl& operator=(const TempFileImpl&); // disable assignment -}; - - -TempFile::TempFile(): std::ostream(nullptr), mImpl(new TempFileImpl(*this)) {} -TempFile::~TempFile() { this->close(); } -const std::string& TempFile::filename() const { return mImpl->filename(); } -bool TempFile::is_open() const { return mImpl->is_open(); } -void TempFile::close() { mImpl->close(); } - -} // namespace io -} // namespace OPENVDB_VERSION_NAME -} // namespace openvdb - -#endif // OPENVDB_USE_DELAYED_LOADING diff --git a/openvdb/openvdb/io/TempFile.h b/openvdb/openvdb/io/TempFile.h deleted file mode 100644 index cbbdec2250..0000000000 --- a/openvdb/openvdb/io/TempFile.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -/// @file TempFile.h - -#ifdef OPENVDB_USE_DELAYED_LOADING - -#ifndef OPENVDB_IO_TEMPFILE_HAS_BEEN_INCLUDED -#define OPENVDB_IO_TEMPFILE_HAS_BEEN_INCLUDED - -#include -#include -#include - - -namespace openvdb { -OPENVDB_USE_VERSION_NAMESPACE -namespace OPENVDB_VERSION_NAME { -namespace io { - -/// Output stream to a unique temporary file -class OPENVDB_API TempFile: public std::ostream -{ -public: - /// @brief Create and open a unique file. - /// @details On UNIX systems, the file is created in the directory specified by - /// the environment variable @c OPENVDB_TEMP_DIR, if that variable is defined, - /// or else in the directory specified by @c TMPDIR, if that variable is defined. - /// Otherwise (and on non-UNIX systems), the file is created in the system default - /// temporary directory. - TempFile(); - ~TempFile(); - - /// Return the path to the temporary file. - const std::string& filename() const; - - /// Return @c true if the file is open for writing. - bool is_open() const; - - /// Close the file. - void close(); - -private: - struct TempFileImpl; - std::unique_ptr mImpl; -}; - -} // namespace io -} // namespace OPENVDB_VERSION_NAME -} // namespace openvdb - -#endif // OPENVDB_IO_TEMPFILE_HAS_BEEN_INCLUDED - -#endif // OPENVDB_USE_DELAYED_LOADING diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index ff3adb6d26..26e513b7f6 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -73,12 +73,14 @@ class OPENVDB_API StreamMetadata bool countingPasses() const; void setCountingPasses(bool); + /// @brief Return @c true if readTopology() should allocate and zero-fill + /// leaf buffers after loading the tree structure (topology-only read mode). + bool allocateLeafBuffers() const; + void setAllocateLeafBuffers(bool); + uint32_t pass() const; void setPass(uint32_t); - uint64_t leaf() const; - void setLeaf(uint64_t); - //@{ /// @brief Return a (reference to a) copy of the metadata of the grid /// currently being read or written. @@ -119,9 +121,9 @@ std::ostream& operator<<(std::ostream&, const StreamMetadata::AuxDataMap&); //////////////////////////////////////// -/// @brief Leaf nodes that require multi-pass I/O must inherit from this struct. -/// @sa Grid::hasMultiPassIO() -struct MultiPass {}; +/// @brief Tag for multi-pass I/O. Only points::PointDataLeafNode may inherit from this. +/// @sa Grid::hasMultiPassIO(), points::IsPointDataLeafNode +struct PointDataGridMultiPass {}; //////////////////////////////////////// @@ -129,50 +131,37 @@ struct MultiPass {}; class File; -#ifdef OPENVDB_USE_DELAYED_LOADING -/// @brief Handle to control the lifetime of a memory-mapped .vdb file +// This class is deprecated and will be removed. Delayed loading is no longer supported. +// It is retained for API backwards compatibility only, but all methods are no-ops. class OPENVDB_API MappedFile { public: using Ptr = SharedPtr; + using Notifier = std::function; - ~MappedFile(); - MappedFile(const MappedFile&) = delete; // not copyable - MappedFile& operator=(const MappedFile&) = delete; + OPENVDB_DEPRECATED_MESSAGE("This class is deprecated and will be removed. Delayed loading is no longer supported.") + explicit MappedFile(const std::string& /*filename*/, bool /*autoDelete*/ = false) {} - /// Return the filename of the mapped file. - std::string filename() const; + /// @brief Destructor + ~MappedFile() = default; - /// @brief Return a new stream buffer for the mapped file. - /// @details Typical usage is - /// @code - /// openvdb::io::MappedFile::Ptr mappedFile = ...; - /// auto buf = mappedFile->createBuffer(); - /// std::istream istrm{buf.get()}; - /// // Read from istrm... - /// @endcode - /// The buffer must persist as long as the stream is open. - SharedPtr createBuffer() const; + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; - using Notifier = std::function; - /// @brief Register a function that will be called with this file's name - /// when the file is unmapped. - void setNotifier(const Notifier&); - /// Deregister the notifier. - void clearNotifier(); + OPENVDB_DEPRECATED_MESSAGE("Always returns an empty string. This method is deprecated and will be removed. Delayed loading is no longer supported.") + std::string filename() const { return std::string(); } -private: - friend class File; - friend class ::TestMappedFile; + OPENVDB_DEPRECATED_MESSAGE("Always returns a null stream buffer pointer. This method is deprecated and will be removed. Delayed loading is no longer supported.") + SharedPtr createBuffer() const { return SharedPtr(); } - explicit MappedFile(const std::string& filename, bool autoDelete = false); + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void setNotifier(const Notifier&) {} - class Impl; - std::unique_ptr mImpl; + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void clearNotifier() {} }; // class MappedFile -#endif // OPENVDB_USE_DELAYED_LOADING //////////////////////////////////////// @@ -249,15 +238,10 @@ OPENVDB_API bool getWriteGridStatsMetadata(std::ios_base&); /// and store them as grid metadata when writing to the given stream. OPENVDB_API void setWriteGridStatsMetadata(std::ios_base&, bool writeGridStats); -#ifdef OPENVDB_USE_DELAYED_LOADING -/// @brief Return a shared pointer to the memory-mapped file with which the given stream -/// is associated, or a null pointer if the stream is not associated with a memory-mapped file. -OPENVDB_API SharedPtr getMappedFilePtr(std::ios_base&); -/// @brief Associate the given stream with (a shared pointer to) a memory-mapped file. -/// @note The shared pointer object (not just the io::MappedFile object to which it points) -/// must remain valid until the file is closed. -OPENVDB_API void setMappedFilePtr(std::ios_base&, SharedPtr&); -#endif // OPENVDB_USE_DELAYED_LOADING +OPENVDB_DEPRECATED_MESSAGE("Always returns a null pointer. This function is deprecated and will be removed. Delayed loading is no longer supported.") +inline SharedPtr getMappedFilePtr(std::ios_base&) { return SharedPtr(); } +OPENVDB_DEPRECATED_MESSAGE("This function is deprecated and will be removed. Delayed loading is no longer supported.") +inline void setMappedFilePtr(std::ios_base&, SharedPtr&) { } /// @brief Return a shared pointer to an object that stores metadata (file format, /// compression scheme, etc.) for use when reading from or writing to the given stream. diff --git a/openvdb/openvdb/openvdb.cc b/openvdb/openvdb/openvdb.cc index d4691c36a7..ff163071de 100644 --- a/openvdb/openvdb/openvdb.cc +++ b/openvdb/openvdb/openvdb.cc @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "openvdb.h" -#include "io/DelayedLoadMetadata.h" +#include "io/Codec.h" #include "points/PointDataGrid.h" #include "tools/PointIndexGrid.h" #include "util/logging.h" @@ -71,6 +71,8 @@ initialize() logging::initialize(); + io::internal::initialize(); + // Register metadata. Metadata::clearRegistry(); MetaTypes::foreach(); @@ -131,6 +133,7 @@ __pragma(warning(default:1711)) Metadata::clearRegistry(); GridBase::clearRegistry(); math::MapRegistry::clear(); + io::internal::uninitialize(); points::internal::uninitialize(); #ifdef OPENVDB_USE_BLOSC diff --git a/openvdb/openvdb/openvdb.h b/openvdb/openvdb/openvdb.h index a4db4b7337..8ce06d4167 100644 --- a/openvdb/openvdb/openvdb.h +++ b/openvdb/openvdb/openvdb.h @@ -46,9 +46,6 @@ OPENVDB_API void initialize(); OPENVDB_API void uninitialize(); -// foward declare some default types -namespace io { class DelayedLoadMetadata; } - /// Common tree types using BoolTree = tree::Tree4::Type; using DoubleTree = tree::Tree4::Type; @@ -222,8 +219,7 @@ using MetaTypes = TypeList< Vec4SMetadata, Vec4DMetadata, Mat4SMetadata, - Mat4DMetadata, - io::DelayedLoadMetadata>; + Mat4DMetadata>; } // namespace OPENVDB_VERSION_NAME diff --git a/openvdb/openvdb/points/AttributeArray.cc b/openvdb/openvdb/points/AttributeArray.cc index a24fc5a2bd..19e6d14436 100644 --- a/openvdb/openvdb/points/AttributeArray.cc +++ b/openvdb/openvdb/points/AttributeArray.cc @@ -61,7 +61,6 @@ AttributeArray::AttributeArray(const AttributeArray& rhs, const tbb::spin_mutex: : mIsUniform(rhs.mIsUniform) , mFlags(rhs.mFlags) , mUsePagedRead(rhs.mUsePagedRead) - , mOutOfCore(rhs.mOutOfCore.load()) , mPageHandle() { if (mFlags & PARTIALREAD) mCompressedBytes = rhs.mCompressedBytes; @@ -78,7 +77,6 @@ AttributeArray::operator=(const AttributeArray& rhs) mIsUniform = rhs.mIsUniform; mFlags = rhs.mFlags; mUsePagedRead = rhs.mUsePagedRead; - mOutOfCore.store(rhs.mOutOfCore); if (mFlags & PARTIALREAD) mCompressedBytes = rhs.mCompressedBytes; else if (rhs.mPageHandle) mPageHandle = rhs.mPageHandle->copy(); else mPageHandle.reset(); @@ -190,9 +188,6 @@ AttributeArray::setConstantStride(bool state) bool AttributeArray::operator==(const AttributeArray& other) const { - this->loadData(); - other.loadData(); - if (this->mUsePagedRead != other.mUsePagedRead || this->mFlags != other.mFlags) return false; return this->isEqual(other); diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index 9fba9c4c53..26f03b71cc 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -130,7 +130,7 @@ class OPENVDB_API AttributeArray template friend class AttributeHandle; - AttributeArray(): mPageHandle() { mOutOfCore = 0; } + AttributeArray(): mPageHandle() { } virtual ~AttributeArray() { // if this AttributeArray has been partially read, zero the compressed bytes, @@ -189,10 +189,7 @@ class OPENVDB_API AttributeArray /// Return the number of bytes of memory used by this attribute. virtual size_t memUsage() const = 0; - /// Return the number of bytes of memory used by this attribute array once it - /// has been deserialized (this may be different to memUsage() if delay-loading - /// is in use). Note that this method does NOT consider the fact that a - /// uniform attribute could be expanded and only deals with delay-loading. + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual size_t memUsageIfLoaded() const = 0; /// Create a new attribute array of the given (registered) type, length and stride. @@ -311,14 +308,17 @@ class OPENVDB_API AttributeArray /// Read attribute buffers from a paged stream. virtual void readPagedBuffers(compression::PagedInputStream&) = 0; + /// Skip attribute buffers in a paged stream without reading or + /// allocating any data. + void skipPagedBuffers(compression::PagedInputStream&); /// Write attribute buffers to a paged stream. /// @param outputTransient if true, write out transient attributes virtual void writePagedBuffers(compression::PagedOutputStream&, bool outputTransient) const = 0; - /// Ensures all data is in-core + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void loadData() const = 0; - /// Return @c true if all data has been loaded + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual bool isDataLoaded() const = 0; /// Check the compressed bytes and flags. If they are equal, perform a deeper @@ -366,10 +366,12 @@ class OPENVDB_API AttributeArray mutable tbb::spin_mutex mMutex; uint8_t mFlags = 0; uint8_t mUsePagedRead = 0; - std::atomic mOutOfCore; // interpreted as bool +#if OPENVDB_ABI_VERSION_NUMBER < 14 + std::atomic mOutOfCore{0}; // interpreted as bool +#endif /// used for out-of-core, paged reading union { - compression::PageHandle::Ptr mPageHandle; + std::unique_ptr mPageHandle; size_t mCompressedBytes; }; }; // class AttributeArray @@ -611,11 +613,8 @@ class TypedAttributeArray final: public AttributeArray /// Return the number of bytes of memory used by this attribute. size_t memUsage() const override; - /// Return the number of bytes of memory used by this attribute array once it - /// has been deserialized (this may be different to memUsage() if delay-loading - /// is in use). Note that this method does NOT consider the fact that a - /// uniform attribute could be expanded and only deals with delay-loading. - size_t memUsageIfLoaded() const override; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + size_t memUsageIfLoaded() const override { return memUsage(); } /// Return the value at index @a n (assumes in-core) ValueType getUnsafe(Index n) const; @@ -696,14 +695,14 @@ class TypedAttributeArray final: public AttributeArray /// @param outputTransient if true, write out transient attributes void writePagedBuffers(compression::PagedOutputStream& os, bool outputTransient) const override; - /// Return @c true if this buffer's values have not yet been read from disk. - inline bool isOutOfCore() const; + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool isOutOfCore() const { return false; } - /// Ensures all data is in-core - void loadData() const override; + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void loadData() const override { } - /// Return @c true if all data has been loaded - bool isDataLoaded() const override; + OPENVDB_DEPRECATED_MESSAGE("Always returns true. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool isDataLoaded() const override { return true; } /// Return the raw data buffer inline const StorageType* constData() const { return this->data(); } @@ -716,18 +715,13 @@ class TypedAttributeArray final: public AttributeArray inline const StorageType* data() const { OPENVDB_ASSERT(validData()); return mData.get(); } /// Verify that data is not out-of-core or in a partially-read state - inline bool validData() const { return !(isOutOfCore() || (flags() & PARTIALREAD)); } + inline bool validData() const { return !((flags() & PARTIALREAD)); } private: friend class ::TestAttributeArray; TypedAttributeArray(const TypedAttributeArray&, const tbb::spin_mutex::scoped_lock&); - /// Load data from memory-mapped file. - inline void doLoad() const; - /// Load data from memory-mapped file (unsafe as this function is not protected by a mutex). - inline void doLoadUnsafe() const; - /// Toggle out-of-core state inline void setOutOfCore(const bool); @@ -962,8 +956,6 @@ void AttributeArray::doCopyValues(const AttributeArray& sourceArray, const IterT { // ensure both arrays have float-float or integer-integer value types OPENVDB_ASSERT(sourceArray.valueTypeIsFloatingPoint() == this->valueTypeIsFloatingPoint()); - // ensure both arrays have been loaded from disk (if delay-loaded) - OPENVDB_ASSERT(sourceArray.isDataLoaded() && this->isDataLoaded()); // ensure storage size * stride matches on both arrays OPENVDB_ASSERT(this->storageTypeSize()*this->stride() == sourceArray.storageTypeSize()*sourceArray.stride()); @@ -1024,10 +1016,6 @@ void AttributeArray::copyValues(const AttributeArray& sourceArray, const IterT& OPENVDB_THROW(TypeError, "Cannot copy array data due to mis-match in storage type sizes."); } - // ensure both arrays have been loaded from disk - sourceArray.loadData(); - this->loadData(); - // if the target array is uniform, expand it first this->expand(); @@ -1206,8 +1194,6 @@ template size_t TypedAttributeArray::arrayMemUsage() const { - if (this->isOutOfCore()) return 0; - return (mIsUniform ? 1 : this->dataSize()) * sizeof(StorageType); } @@ -1232,11 +1218,6 @@ template void TypedAttributeArray::deallocate() { - // detach from file if delay-loaded - if (this->isOutOfCore()) { - this->setOutOfCore(false); - this->mPageHandle.reset(); - } if (mData) mData.reset(); } @@ -1304,14 +1285,6 @@ TypedAttributeArray::memUsage() const } -template -size_t -TypedAttributeArray::memUsageIfLoaded() const -{ - return sizeof(*this) + (mIsUniform ? 1 : this->dataSize()) * sizeof(StorageType); -} - - template typename TypedAttributeArray::ValueType TypedAttributeArray::getUnsafe(Index n) const @@ -1329,7 +1302,6 @@ typename TypedAttributeArray::ValueType TypedAttributeArray::get(Index n) const { if (n >= this->dataSize()) OPENVDB_THROW(IndexError, "Out-of-range access."); - if (this->isOutOfCore()) this->doLoad(); return this->getUnsafe(n); } @@ -1366,7 +1338,6 @@ void TypedAttributeArray::setUnsafe(Index n, const ValueType& val) { OPENVDB_ASSERT(n < this->dataSize()); - OPENVDB_ASSERT(!this->isOutOfCore()); OPENVDB_ASSERT(!this->isUniform()); // this unsafe method assumes the data is not uniform, however if it is, this redirects the index @@ -1381,7 +1352,6 @@ void TypedAttributeArray::set(Index n, const ValueType& val) { if (n >= this->dataSize()) OPENVDB_THROW(IndexError, "Out-of-range access."); - if (this->isOutOfCore()) this->doLoad(); if (this->isUniform()) this->expand(); this->setUnsafe(n, val); @@ -1486,12 +1456,6 @@ template void TypedAttributeArray::fill(const ValueType& value) { - if (this->isOutOfCore()) { - tbb::spin_mutex::scoped_lock lock(mMutex); - this->deallocate(); - this->allocate(); - } - const Index size = mIsUniform ? 1 : this->dataSize(); for (Index i = 0; i < size; ++i) { Codec::encode(value, this->data()[i]); @@ -1507,54 +1471,6 @@ TypedAttributeArray::fill(AttributeArray* array, const Value } -template -bool -TypedAttributeArray::isOutOfCore() const -{ - return mOutOfCore; -} - - -template -void -TypedAttributeArray::setOutOfCore(const bool b) -{ - mOutOfCore = b; -} - - -template -void -TypedAttributeArray::doLoad() const -{ - if (!(this->isOutOfCore())) return; - - TypedAttributeArray* self = - const_cast*>(this); - - // This lock will be contended at most once, after which this buffer - // will no longer be out-of-core. - tbb::spin_mutex::scoped_lock lock(self->mMutex); - this->doLoadUnsafe(); -} - - -template -void -TypedAttributeArray::loadData() const -{ - this->doLoad(); -} - - -template -bool -TypedAttributeArray::isDataLoaded() const -{ - return !this->isOutOfCore(); -} - - template void TypedAttributeArray::read(std::istream& is) @@ -1624,8 +1540,6 @@ TypedAttributeArray::readBuffers(std::istream& is) OPENVDB_THROW(IoError, "Cannot read paged AttributeArray buffers."); } - tbb::spin_mutex::scoped_lock lock(mMutex); - this->deallocate(); uint8_t bloscCompressed(0); @@ -1663,13 +1577,6 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu return; } -#ifdef OPENVDB_USE_DELAYED_LOADING - // If this array is being read from a memory-mapped file, delay loading of its data - // until the data is actually accessed. - io::MappedFile::Ptr mappedFile = io::getMappedFilePtr(is.getInputStream()); - const bool delayLoad = (mappedFile.get() != nullptr); -#endif - if (is.sizeOnly()) { size_t compressedBytes(mCompressedBytes); @@ -1682,26 +1589,55 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu OPENVDB_ASSERT(mPageHandle); - tbb::spin_mutex::scoped_lock lock(mMutex); - this->deallocate(); -#ifdef OPENVDB_USE_DELAYED_LOADING - this->setOutOfCore(delayLoad); - is.read(mPageHandle, std::streamsize(mPageHandle->size()), delayLoad); -#else is.read(mPageHandle, std::streamsize(mPageHandle->size()), false); -#endif // OPENVDB_USE_DELAYED_LOADING + std::unique_ptr buffer = mPageHandle->read(); + mData.reset(reinterpret_cast(buffer.release())); + mPageHandle.reset(); -#ifdef OPENVDB_USE_DELAYED_LOADING - if (!delayLoad) { -#endif - std::unique_ptr buffer = mPageHandle->read(); - mData.reset(reinterpret_cast(buffer.release())); - mPageHandle.reset(); -#ifdef OPENVDB_USE_DELAYED_LOADING + // clear page state + + mUsePagedRead = 0; +} + + +inline void +AttributeArray::skipPagedBuffers(compression::PagedInputStream& is) +{ + if (!mUsePagedRead) { + if (!is.sizeOnly()) { + // for non-paged data, seek past the raw data in the stream + std::istream& inputStream = is.getInputStream(); + uint8_t bloscCompressed(0); + if (!mIsUniform) inputStream.read(reinterpret_cast(&bloscCompressed), sizeof(uint8_t)); + auto meta = io::getStreamMetadataPtr(inputStream); + if (meta && meta->seekable()) { + inputStream.seekg(mCompressedBytes, std::ios_base::cur); + } else { + std::vector tempData(mCompressedBytes); + inputStream.read(tempData.data(), mCompressedBytes); + } + mCompressedBytes = 0; + mFlags = static_cast(mFlags & ~PARTIALREAD); + } + return; } -#endif + + if (is.sizeOnly()) + { + size_t compressedBytes(mCompressedBytes); + mCompressedBytes = 0; + mFlags = static_cast(mFlags & ~PARTIALREAD); + OPENVDB_ASSERT(!mPageHandle); + mPageHandle = is.createHandle(compressedBytes); + return; + } + + OPENVDB_ASSERT(mPageHandle); + + is.skip(mPageHandle, std::streamsize(mPageHandle->size())); + mPageHandle.reset(); // clear page state @@ -1744,9 +1680,6 @@ TypedAttributeArray::writeMetadata(std::ostream& os, bool ou bool bloscCompression = io::getDataCompression(os) & io::COMPRESS_BLOSC; - // any compressed data needs to be loaded if out-of-core - if (bloscCompression) this->doLoad(); - size_t compressedBytes = 0; if (!strideOfOne) @@ -1795,8 +1728,6 @@ TypedAttributeArray::writeBuffers(std::ostream& os, bool out OPENVDB_THROW(IoError, "Cannot write out a partially-read AttributeArray."); } - this->doLoad(); - if (this->isUniform()) { os.write(reinterpret_cast(this->data()), sizeof(StorageType)); } @@ -1844,37 +1775,10 @@ TypedAttributeArray::writePagedBuffers(compression::PagedOut OPENVDB_THROW(IoError, "Cannot write out a partially-read AttributeArray."); } - this->doLoad(); - os.write(reinterpret_cast(this->data()), this->arrayMemUsage()); } -template -void -TypedAttributeArray::doLoadUnsafe() const -{ - if (!(this->isOutOfCore())) return; - - // this function expects the mutex to already be locked - - auto* self = const_cast*>(this); - - OPENVDB_ASSERT(self->mPageHandle); - OPENVDB_ASSERT(!(self->mFlags & PARTIALREAD)); - - std::unique_ptr buffer = self->mPageHandle->read(); - - self->mData.reset(reinterpret_cast(buffer.release())); - - self->mPageHandle.reset(); - - // clear all write and out-of-core flags - - self->mOutOfCore = false; -} - - template AttributeArray::AccessorBasePtr TypedAttributeArray::getAccessor() const @@ -1901,9 +1805,6 @@ TypedAttributeArray::isEqual(const AttributeArray& other) co this->mIsUniform != otherT->mIsUniform || this->attributeType() != this->attributeType()) return false; - this->doLoad(); - otherT->doLoad(); - const StorageType *target = this->data(), *source = otherT->data(); if (!target && !source) return true; if (!target || !source) return false; @@ -1995,10 +1896,6 @@ AttributeHandle::AttributeHandle(const AttributeArray& arr OPENVDB_THROW(TypeError, "Cannot bind handle due to incompatible type of AttributeArray."); } - // load data if delay-loaded - - mArray->loadData(); - // bind getter and setter methods AttributeArray::AccessorBasePtr accessor = mArray->getAccessor(); diff --git a/openvdb/openvdb/points/AttributeGroup.cc b/openvdb/openvdb/points/AttributeGroup.cc index 4f022bae7d..431a5ac73a 100644 --- a/openvdb/openvdb/points/AttributeGroup.cc +++ b/openvdb/openvdb/points/AttributeGroup.cc @@ -23,10 +23,6 @@ GroupHandle::GroupHandle(const GroupAttributeArray& array, const GroupType& offs , mBitMask(static_cast(1 << offset)) { OPENVDB_ASSERT(isGroup(mArray)); - - // load data if delay-loaded - - mArray.loadData(); } @@ -36,10 +32,6 @@ GroupHandle::GroupHandle(const GroupAttributeArray& array, const GroupType& bitM , mBitMask(bitMask) { OPENVDB_ASSERT(isGroup(mArray)); - - // load data if delay-loaded - - mArray.loadData(); } diff --git a/openvdb/openvdb/points/AttributeSet.cc b/openvdb/openvdb/points/AttributeSet.cc index 1eebe436ff..71cc065151 100644 --- a/openvdb/openvdb/points/AttributeSet.cc +++ b/openvdb/openvdb/points/AttributeSet.cc @@ -140,17 +140,6 @@ AttributeSet::memUsage() const } -size_t -AttributeSet::memUsageIfLoaded() const -{ - size_t bytes = sizeof(*this) + mDescr->memUsage(); - for (const auto& attr : mAttrs) { - bytes += attr->memUsageIfLoaded(); - } - return bytes; -} - - size_t AttributeSet::find(const std::string& name) const { diff --git a/openvdb/openvdb/points/AttributeSet.h b/openvdb/openvdb/points/AttributeSet.h index 3137eb7413..84cbeb3eaa 100644 --- a/openvdb/openvdb/points/AttributeSet.h +++ b/openvdb/openvdb/points/AttributeSet.h @@ -114,10 +114,8 @@ class OPENVDB_API AttributeSet /// Return the number of bytes of memory used by this attribute set. size_t memUsage() const; - /// Return the number of bytes of memory used by this attribute set once it - /// has been deserialized (this may be different to memUsage() if delay-loading - /// is in use). - size_t memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + size_t memUsageIfLoaded() const { return memUsage(); } /// @brief Return the position of the attribute array whose name is @a name, /// or @c INVALID_POS if no match is found. diff --git a/openvdb/openvdb/points/PointConversion.h b/openvdb/openvdb/points/PointConversion.h index dbdfd97b6d..a77df2f267 100644 --- a/openvdb/openvdb/points/PointConversion.h +++ b/openvdb/openvdb/points/PointConversion.h @@ -134,7 +134,6 @@ populateAttribute( PointDataTreeT& tree, /// @param pointOffsets a vector of cumulative point offsets for each leaf /// @param startOffset a value to shift all the point offsets by /// @param filter an index filter -/// @param inCoreOnly true if out-of-core leaf nodes are to be ignored /// template @@ -143,8 +142,17 @@ convertPointDataGridPosition( PositionAttribute& positionAttribute, const PointDataGridT& grid, const std::vector& pointOffsets, const Index64 startOffset, - const FilterT& filter = NullFilter(), - const bool inCoreOnly = false); + const FilterT& filter = NullFilter()); + +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridPosition() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +inline void +convertPointDataGridPosition( PositionAttribute& positionAttribute, + const PointDataGridT& grid, + const std::vector& pointOffsets, + const Index64 startOffset, + const FilterT& filter, + const bool /*inCoreOnly*/); /// @brief Convert the attribute from a PointDataGrid @@ -156,7 +164,6 @@ convertPointDataGridPosition( PositionAttribute& positionAttribute, /// @param arrayIndex the index in the Descriptor of the array to be converted. /// @param stride the stride of the attribute /// @param filter an index filter -/// @param inCoreOnly true if out-of-core leaf nodes are to be ignored template inline void convertPointDataGridAttribute( TypedAttribute& attribute, @@ -165,8 +172,19 @@ convertPointDataGridAttribute( TypedAttribute& attribute, const Index64 startOffset, const unsigned arrayIndex, const Index stride = 1, - const FilterT& filter = NullFilter(), - const bool inCoreOnly = false); + const FilterT& filter = NullFilter()); + +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridAttribute() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +inline void +convertPointDataGridAttribute( TypedAttribute& attribute, + const PointDataTreeT& tree, + const std::vector& pointOffsets, + const Index64 startOffset, + const unsigned arrayIndex, + const Index stride, + const FilterT& filter, + const bool /*inCoreOnly*/); /// @brief Convert the group from a PointDataGrid @@ -177,18 +195,26 @@ convertPointDataGridAttribute( TypedAttribute& attribute, /// @param startOffset a value to shift all the point offsets by /// @param index the group index to be converted. /// @param filter an index filter -/// @param inCoreOnly true if out-of-core leaf nodes are to be ignored /// +template +inline void +convertPointDataGridGroup( Group& group, + const PointDataTreeT& tree, + const std::vector& pointOffsets, + const Index64 startOffset, + const AttributeSet::Descriptor::GroupIndex index, + const FilterT& filter = NullFilter()); template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridGroup() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") inline void convertPointDataGridGroup( Group& group, const PointDataTreeT& tree, const std::vector& pointOffsets, const Index64 startOffset, const AttributeSet::Descriptor::GroupIndex index, - const FilterT& filter = NullFilter(), - const bool inCoreOnly = false); + const FilterT& filter, + const bool /*inCoreOnly*/); // for internal use only - this traits class extracts T::value_type if defined, // otherwise falls back to using Vec3R diff --git a/openvdb/openvdb/points/PointCount.h b/openvdb/openvdb/points/PointCount.h index d3fef689db..32ae569b5a 100644 --- a/openvdb/openvdb/points/PointCount.h +++ b/openvdb/openvdb/points/PointCount.h @@ -28,28 +28,39 @@ namespace points { /// @brief Count the total number of points in a PointDataTree /// @param tree the PointDataTree in which to count the points /// @param filter an optional index filter -/// @param inCoreOnly if true, points in out-of-core leaf nodes are not counted /// @param threaded enable or disable threading (threading is enabled by default) template inline Index64 pointCount( const PointDataTreeT& tree, const FilterT& filter = NullFilter(), - const bool inCoreOnly = false, const bool threaded = true); +template +OPENVDB_DEPRECATED_MESSAGE("Use pointCount() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +inline Index64 pointCount( const PointDataTreeT& tree, + const FilterT& filter, + const bool /*inCoreOnly*/, + const bool threaded); + /// @brief Populate an array of cumulative point offsets per leaf node. /// @param pointOffsets array of offsets to be populated /// @param tree the PointDataTree from which to populate the offsets /// @param filter an optional index filter -/// @param inCoreOnly if true, points in out-of-core leaf nodes are ignored /// @param threaded enable or disable threading (threading is enabled by default) /// @return The final cumulative point offset. template inline Index64 pointOffsets(std::vector& pointOffsets, const PointDataTreeT& tree, const FilterT& filter = NullFilter(), - const bool inCoreOnly = false, const bool threaded = true); +template +OPENVDB_DEPRECATED_MESSAGE("Use pointOffsets() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +inline Index64 pointOffsets(std::vector& pointOffsets, + const PointDataTreeT& tree, + const FilterT& filter, + const bool /*inCoreOnly*/, + const bool threaded); + /// @brief Generate a new grid with voxel values to store the number of points per voxel /// @param grid the PointDataGrid to use to compute the count grid /// @param filter an optional index filter diff --git a/openvdb/openvdb/points/PointDataGrid.h b/openvdb/openvdb/points/PointDataGrid.h index f3f72eab56..35988239af 100644 --- a/openvdb/openvdb/points/PointDataGrid.h +++ b/openvdb/openvdb/points/PointDataGrid.h @@ -31,140 +31,14 @@ #include // std::pair, std::make_pair #include +#include // io::readCompressedValues(), io::writeCompressedValues(), io::writeCompressedValuesSize() + class TestPointDataLeaf; namespace openvdb { OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { -namespace io -{ - -/// @brief openvdb::io::readCompressedValues specialized on PointDataIndex32 arrays to -/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit -template<> -inline void -readCompressedValues( std::istream& is, PointDataIndex32* destBuf, Index destCount, - const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/) -{ - using compression::bloscDecompress; - - const bool seek = destBuf == nullptr; - - const size_t destBytes = destCount*sizeof(PointDataIndex32); - const size_t maximumBytes = std::numeric_limits::max(); - if (destBytes >= maximumBytes) { - OPENVDB_THROW(openvdb::IoError, "Cannot read more than " << - maximumBytes << " bytes in voxel values.") - } - - uint16_t bytes16; - - const io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is); - - if (seek && meta) { - // buffer size temporarily stored in the StreamMetadata pass - // to avoid having to perform an expensive disk read for 2-bytes - bytes16 = static_cast(meta->pass()); - // seek over size of the compressed buffer - is.seekg(sizeof(uint16_t), std::ios_base::cur); - } - else { - // otherwise read from disk - is.read(reinterpret_cast(&bytes16), sizeof(uint16_t)); - } - - if (bytes16 == std::numeric_limits::max()) { - // read or seek uncompressed data - if (seek) { - is.seekg(destBytes, std::ios_base::cur); - } - else { - is.read(reinterpret_cast(destBuf), destBytes); - } - } - else { - // read or seek uncompressed data - if (seek) { - is.seekg(int(bytes16), std::ios_base::cur); - } - else { - // decompress into the destination buffer - std::unique_ptr bloscBuffer(new char[int(bytes16)]); - is.read(bloscBuffer.get(), bytes16); - std::unique_ptr buffer = bloscDecompress( bloscBuffer.get(), - destBytes, - /*resize=*/false); - std::memcpy(destBuf, buffer.get(), destBytes); - } - } -} - -/// @brief openvdb::io::writeCompressedValues specialized on PointDataIndex32 arrays to -/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit -template<> -inline void -writeCompressedValues( std::ostream& os, const PointDataIndex32* srcBuf, Index srcCount, - const util::NodeMask<3>& /*valueMask*/, - const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/) -{ - using compression::bloscCompress; - - const size_t srcBytes = srcCount*sizeof(PointDataIndex32); - const size_t maximumBytes = std::numeric_limits::max(); - if (srcBytes >= maximumBytes) { - OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << - maximumBytes << " bytes in voxel values.") - } - - const char* charBuffer = reinterpret_cast(srcBuf); - - size_t compressedBytes; - std::unique_ptr buffer = bloscCompress( charBuffer, srcBytes, - compressedBytes, /*resize=*/false); - - if (compressedBytes > 0) { - auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - os.write(reinterpret_cast(buffer.get()), compressedBytes); - } - else { - auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - os.write(reinterpret_cast(srcBuf), srcBytes); - } -} - -template -inline void -writeCompressedValuesSize(std::ostream& os, const T* srcBuf, Index srcCount) -{ - using compression::bloscCompressedSize; - - const size_t srcBytes = srcCount*sizeof(T); - const size_t maximumBytes = std::numeric_limits::max(); - if (srcBytes >= maximumBytes) { - OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << - maximumBytes << " bytes in voxel values.") - } - - const char* charBuffer = reinterpret_cast(srcBuf); - - // calculate voxel buffer size after compression - size_t compressedBytes = bloscCompressedSize(charBuffer, srcBytes); - - if (compressedBytes > 0) { - auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - } - else { - auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - } -} - -} // namespace io - // forward declaration namespace tree { @@ -218,26 +92,22 @@ makeDescriptorUnique(PointDataTreeT& tree); /// /// @note Multiple threads cannot safely access the same AttributeArray when using streaming. template +OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") inline void -setStreamingMode(PointDataTreeT& tree, bool on = true); +setStreamingMode(PointDataTreeT& tree, bool on = true) { (void)tree; (void)on; } -/// @brief Sequentially pre-fetch all delayed-load voxel and attribute data from disk in order -/// to accelerate subsequent random access. -/// -/// @param tree the PointDataTree. -/// @param position if enabled, prefetch the position attribute (default is on) -/// @param otherAttributes if enabled, prefetch all other attributes (default is on) template +OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") inline void -prefetch(PointDataTreeT& tree, bool position = true, bool otherAttributes = true); +prefetch(PointDataTreeT&, bool /*position*/ = true, bool /*otherAttributes*/ = true) { } //////////////////////////////////////// template -class PointDataLeafNode : public tree::LeafNode, io::MultiPass { +class PointDataLeafNode : public tree::LeafNode, io::PointDataGridMultiPass { public: using LeafNodeType = PointDataLeafNode; @@ -504,7 +374,8 @@ class PointDataLeafNode : public tree::LeafNode, io::MultiPass { Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } void evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels = true) const; @@ -1522,13 +1393,6 @@ PointDataLeafNode::memUsage() const return BaseLeaf::memUsage() + mAttributeSet->memUsage(); } -template -inline Index64 -PointDataLeafNode::memUsageIfLoaded() const -{ - return BaseLeaf::memUsageIfLoaded() + mAttributeSet->memUsageIfLoaded(); -} - template inline void PointDataLeafNode::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const @@ -1598,63 +1462,6 @@ makeDescriptorUnique(PointDataTreeT& tree) } -template -inline void -setStreamingMode(PointDataTreeT& tree, bool on) -{ - auto leafIter = tree.beginLeaf(); - for (; leafIter; ++leafIter) { - for (size_t i = 0; i < leafIter->attributeSet().size(); i++) { - leafIter->attributeArray(i).setStreaming(on); - } - } -} - - -template -inline void -prefetch(PointDataTreeT& tree, bool position, bool otherAttributes) -{ - // NOTE: the following is intentionally not multi-threaded, as the I/O - // is faster if done in the order in which it is stored in the file - - auto leaf = tree.cbeginLeaf(); - if (!leaf) return; - - const auto& attributeSet = leaf->attributeSet(); - - // pre-fetch leaf data - - for ( ; leaf; ++leaf) { - leaf->buffer().data(); - } - - // pre-fetch position attribute data (position will typically have index 0) - - size_t positionIndex = attributeSet.find("P"); - - if (position && positionIndex != AttributeSet::INVALID_POS) { - for (leaf = tree.cbeginLeaf(); leaf; ++leaf) { - OPENVDB_ASSERT(leaf->hasAttribute(positionIndex)); - leaf->constAttributeArray(positionIndex).loadData(); - } - } - - // pre-fetch other attribute data - - if (otherAttributes) { - const size_t attributes = attributeSet.size(); - for (size_t attributeIndex = 0; attributeIndex < attributes; attributeIndex++) { - if (attributeIndex == positionIndex) continue; - for (leaf = tree.cbeginLeaf(); leaf; ++leaf) { - OPENVDB_ASSERT(leaf->hasAttribute(attributeIndex)); - leaf->constAttributeArray(attributeIndex).loadData(); - } - } - } -} - - namespace internal { /// @brief Global registration of point data-related types diff --git a/openvdb/openvdb/points/PointDataIO.h b/openvdb/openvdb/points/PointDataIO.h new file mode 100644 index 0000000000..110b93972b --- /dev/null +++ b/openvdb/openvdb/points/PointDataIO.h @@ -0,0 +1,150 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED +#define OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED + + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { + + +//////////////////////////////////////// + + +namespace io +{ + +/// @brief openvdb::io::readCompressedValues specialized on PointDataIndex32 arrays to +/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit +template<> +inline void +readCompressedValues( std::istream& is, PointDataIndex32* destBuf, Index destCount, + const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/, + const PointDataIndex32* /*background*/) +{ + using compression::bloscDecompress; + + const bool seek = destBuf == nullptr; + + const size_t destBytes = destCount*sizeof(PointDataIndex32); + const size_t maximumBytes = std::numeric_limits::max(); + if (destBytes >= maximumBytes) { + OPENVDB_THROW(openvdb::IoError, "Cannot read more than " << + maximumBytes << " bytes in voxel values.") + } + + uint16_t bytes16; + + const io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is); + + if (seek && meta) { + // buffer size temporarily stored in the StreamMetadata pass + // to avoid having to perform an expensive disk read for 2-bytes + bytes16 = static_cast(meta->pass()); + // seek over size of the compressed buffer + is.seekg(sizeof(uint16_t), std::ios_base::cur); + } + else { + // otherwise read from disk + is.read(reinterpret_cast(&bytes16), sizeof(uint16_t)); + } + + if (bytes16 == std::numeric_limits::max()) { + // read or seek uncompressed data + if (seek) { + is.seekg(destBytes, std::ios_base::cur); + } + else { + is.read(reinterpret_cast(destBuf), destBytes); + } + } + else { + // read or seek uncompressed data + if (seek) { + is.seekg(int(bytes16), std::ios_base::cur); + } + else { + // decompress into the destination buffer + std::unique_ptr bloscBuffer(new char[int(bytes16)]); + is.read(bloscBuffer.get(), bytes16); + std::unique_ptr buffer = bloscDecompress( bloscBuffer.get(), + destBytes, + /*resize=*/false); + std::memcpy(destBuf, buffer.get(), destBytes); + } + } +} + +/// @brief openvdb::io::writeCompressedValues specialized on PointDataIndex32 arrays to +/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit +template<> +inline void +writeCompressedValues( std::ostream& os, const PointDataIndex32* srcBuf, Index srcCount, + const util::NodeMask<3>& /*valueMask*/, + const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/, + const PointDataIndex32* /*background*/) +{ + using compression::bloscCompress; + + const size_t srcBytes = srcCount*sizeof(PointDataIndex32); + const size_t maximumBytes = std::numeric_limits::max(); + if (srcBytes >= maximumBytes) { + OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << + maximumBytes << " bytes in voxel values.") + } + + const char* charBuffer = reinterpret_cast(srcBuf); + + size_t compressedBytes; + std::unique_ptr buffer = bloscCompress( charBuffer, srcBytes, + compressedBytes, /*resize=*/false); + + if (compressedBytes > 0) { + auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + os.write(reinterpret_cast(buffer.get()), compressedBytes); + } + else { + auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + os.write(reinterpret_cast(srcBuf), srcBytes); + } +} + +template +inline void +writeCompressedValuesSize(std::ostream& os, const T* srcBuf, Index srcCount) +{ + using compression::bloscCompressedSize; + + const size_t srcBytes = srcCount*sizeof(T); + const size_t maximumBytes = std::numeric_limits::max(); + if (srcBytes >= maximumBytes) { + OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << + maximumBytes << " bytes in voxel values.") + } + + const char* charBuffer = reinterpret_cast(srcBuf); + + // calculate voxel buffer size after compression + size_t compressedBytes = bloscCompressedSize(charBuffer, srcBytes); + + if (compressedBytes > 0) { + auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + } + else { + auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + } +} + +} // namespace io + + +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/points/StreamCompression.cc b/openvdb/openvdb/points/StreamCompression.cc index 3f24b1b0e6..fb99554b70 100644 --- a/openvdb/openvdb/points/StreamCompression.cc +++ b/openvdb/openvdb/points/StreamCompression.cc @@ -7,6 +7,7 @@ #include #include #include +#include #ifdef OPENVDB_USE_BLOSC #include #endif @@ -279,13 +280,6 @@ bloscDecompress(const char*, const size_t, const bool) //////////////////////////////////////// -void -Page::load() const -{ - this->doLoad(); -} - - long Page::uncompressedBytes() const { @@ -297,10 +291,6 @@ Page::uncompressedBytes() const const char* Page::buffer(const int index) const { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) this->load(); -#endif - return mData.get() + index; } @@ -336,51 +326,37 @@ Page::readBuffers(std::istream&is, bool delayed) bool isCompressed = mInfo->compressedBytes > 0; -#ifdef OPENVDB_USE_DELAYED_LOADING - io::MappedFile::Ptr mappedFile = io::getMappedFilePtr(is); - - if (delayed && mappedFile) { - SharedPtr meta = io::getStreamMetadataPtr(is); - OPENVDB_ASSERT(meta); + std::unique_ptr buffer(new char[ + (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)]); + is.read(buffer.get(), (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)); - std::streamoff filepos = is.tellg(); - - // seek over the page - is.seekg((isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes), - std::ios_base::cur); - - mInfo->mappedFile = mappedFile; - mInfo->meta = meta; - mInfo->filepos = filepos; - - OPENVDB_ASSERT(mInfo->mappedFile); - } - else { -#endif - std::unique_ptr buffer(new char[ - (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)]); - is.read(buffer.get(), (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)); - - if (mInfo->compressedBytes > 0) { - this->decompress(buffer); - } else { - this->copy(buffer, -static_cast(mInfo->compressedBytes)); - } - mInfo.reset(); -#ifdef OPENVDB_USE_DELAYED_LOADING + if (mInfo->compressedBytes > 0) { + this->decompress(buffer); + } else { + this->copy(buffer, -static_cast(mInfo->compressedBytes)); } -#endif + mInfo.reset(); } -bool -Page::isOutOfCore() const +void +Page::skipBuffers(std::istream& is) { -#ifdef OPENVDB_USE_DELAYED_LOADING - return bool(mInfo); -#else - return false; -#endif + OPENVDB_ASSERT(mInfo); + + bool isCompressed = mInfo->compressedBytes > 0; + std::streamsize bytes = isCompressed ? + mInfo->compressedBytes : -mInfo->compressedBytes; + + auto meta = io::getStreamMetadataPtr(is); + if (meta && meta->seekable()) { + is.seekg(bytes, std::ios_base::cur); + } else { + std::vector tempData(bytes); + is.read(tempData.data(), bytes); + } + + mInfo.reset(); } @@ -406,47 +382,6 @@ Page::decompress(const std::unique_ptr& temp) } -void -Page::doLoad() const -{ -#ifdef OPENVDB_USE_DELAYED_LOADING - if (!this->isOutOfCore()) return; - - Page* self = const_cast(this); - - // This lock will be contended at most once, after which this buffer - // will no longer be out-of-core. - tbb::spin_mutex::scoped_lock lock(self->mMutex); - if (!this->isOutOfCore()) return; - - OPENVDB_ASSERT(self->mInfo); - - int compressedBytes = static_cast(self->mInfo->compressedBytes); - bool compressed = compressedBytes > 0; - if (!compressed) compressedBytes = -compressedBytes; - - OPENVDB_ASSERT(compressedBytes); - - std::unique_ptr temp(new char[compressedBytes]); - - OPENVDB_ASSERT(self->mInfo->mappedFile); - SharedPtr buf = self->mInfo->mappedFile->createBuffer(); - OPENVDB_ASSERT(buf); - - std::istream is(buf.get()); - io::setStreamMetadataPtr(is, self->mInfo->meta, /*transfer=*/true); - is.seekg(self->mInfo->filepos); - - is.read(temp.get(), compressedBytes); - - if (compressed) self->decompress(temp); - else self->copy(temp, compressedBytes); - - self->mInfo.reset(); -#endif -} - - //////////////////////////////////////// @@ -525,6 +460,23 @@ PagedInputStream::read(PageHandle::Ptr& pageHandle, std::streamsize n, bool dela } +void +PagedInputStream::skip(PageHandle::Ptr& pageHandle, std::streamsize n) +{ + OPENVDB_ASSERT(mByteIndex <= mUncompressedBytes); + + Page& page = pageHandle->page(); + + if (mByteIndex == mUncompressedBytes) { + mUncompressedBytes = static_cast(page.uncompressedBytes()); + page.skipBuffers(*mIs); + mByteIndex = 0; + } + + mByteIndex += int(n); +} + + //////////////////////////////////////// diff --git a/openvdb/openvdb/points/StreamCompression.h b/openvdb/openvdb/points/StreamCompression.h index c7dcd4a8e9..2efbcda35c 100644 --- a/openvdb/openvdb/points/StreamCompression.h +++ b/openvdb/openvdb/points/StreamCompression.h @@ -114,10 +114,6 @@ class OPENVDB_API Page private: struct Info { -#ifdef OPENVDB_USE_DELAYED_LOADING - io::MappedFile::Ptr mappedFile; -#endif - SharedPtr meta; std::streamoff filepos; long compressedBytes; long uncompressedBytes; @@ -128,8 +124,8 @@ class OPENVDB_API Page Page() = default; - /// @brief load the Page into memory - void load() const; + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") + void load() const { } /// @brief Uncompressed bytes of the Paged data, available /// when the header has been read. @@ -146,8 +142,12 @@ class OPENVDB_API Page /// pointers will be stored to load the data lazily. void readBuffers(std::istream&, bool delayed); - /// @brief Test if the data is out-of-core - bool isOutOfCore() const; + /// @brief Skip the Page buffers by seeking past the compressed data + /// without reading or decompressing it. + void skipBuffers(std::istream&); + + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool isOutOfCore() const { return false; } private: /// @brief Convenience method to store a copy of the supplied buffer @@ -156,9 +156,6 @@ class OPENVDB_API Page /// @brief Decompress and store the supplied data void decompress(const std::unique_ptr& temp); - /// @brief Thread-safe loading of the data - void doLoad() const; - std::unique_ptr mInfo = std::unique_ptr(new Info); std::unique_ptr mData; tbb::spin_mutex mMutex; @@ -229,6 +226,10 @@ class OPENVDB_API PagedInputStream /// an immediate read of the data. void read(PageHandle::Ptr& pageHandle, std::streamsize n, bool delayed = true); + /// @brief Skip past the page data referenced by @a pageHandle without + /// reading or decompressing it. + void skip(PageHandle::Ptr& pageHandle, std::streamsize n); + private: int mByteIndex = 0; int mUncompressedBytes = 0; diff --git a/openvdb/openvdb/points/impl/PointConversionImpl.h b/openvdb/openvdb/points/impl/PointConversionImpl.h index 22fc09a5a0..bab3771610 100644 --- a/openvdb/openvdb/points/impl/PointConversionImpl.h +++ b/openvdb/openvdb/points/impl/PointConversionImpl.h @@ -133,15 +133,13 @@ struct ConvertPointDataGridPositionOp { const Index64 startOffset, const math::Transform& transform, const size_t index, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) : mAttribute(attribute) , mPointOffsets(pointOffsets) , mStartOffset(startOffset) , mTransform(transform) , mIndex(index) , mFilter(filter) - , mInCoreOnly(inCoreOnly) { // only accept Vec3f as ValueType static_assert(VecTraits::Size == 3 && @@ -169,8 +167,6 @@ struct ConvertPointDataGridPositionOp { OPENVDB_ASSERT(leaf.pos() < mPointOffsets.size()); - if (mInCoreOnly && leaf->buffer().isOutOfCore()) continue; - Index64 offset = mStartOffset; if (leaf.pos() > 0) offset += mPointOffsets[leaf.pos() - 1]; @@ -196,7 +192,6 @@ struct ConvertPointDataGridPositionOp { const math::Transform& mTransform; const size_t mIndex; const FilterT& mFilter; - const bool mInCoreOnly; }; // ConvertPointDataGridPositionOp @@ -215,15 +210,13 @@ struct ConvertPointDataGridAttributeOp { const Index64 startOffset, const size_t index, const Index stride, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) : mAttribute(attribute) , mPointOffsets(pointOffsets) , mStartOffset(startOffset) , mIndex(index) , mStride(stride) - , mFilter(filter) - , mInCoreOnly(inCoreOnly) { } + , mFilter(filter) { } template void convert(IterT& iter, HandleT& targetHandle, @@ -257,8 +250,6 @@ struct ConvertPointDataGridAttributeOp { OPENVDB_ASSERT(leaf.pos() < mPointOffsets.size()); - if (mInCoreOnly && leaf->buffer().isOutOfCore()) continue; - Index64 offset = mStartOffset; if (leaf.pos() > 0) offset += mPointOffsets[leaf.pos() - 1]; @@ -284,7 +275,6 @@ struct ConvertPointDataGridAttributeOp { const size_t mIndex; const Index mStride; const FilterT& mFilter; - const bool mInCoreOnly; }; // ConvertPointDataGridAttributeOp template @@ -299,14 +289,12 @@ struct ConvertPointDataGridGroupOp { const std::vector& pointOffsets, const Index64 startOffset, const AttributeSet::Descriptor::GroupIndex index, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) : mGroup(group) , mPointOffsets(pointOffsets) , mStartOffset(startOffset) , mIndex(index) - , mFilter(filter) - , mInCoreOnly(inCoreOnly) { } + , mFilter(filter) { } template void convert(IterT& iter, const GroupAttributeArray& groupArray, Index64& offset) const @@ -337,8 +325,6 @@ struct ConvertPointDataGridGroupOp { OPENVDB_ASSERT(leaf.pos() < mPointOffsets.size()); - if (mInCoreOnly && leaf->buffer().isOutOfCore()) continue; - Index64 offset = mStartOffset; if (leaf.pos() > 0) offset += mPointOffsets[leaf.pos() - 1]; @@ -365,7 +351,6 @@ struct ConvertPointDataGridGroupOp { const Index64 mStartOffset; const GroupIndex mIndex; const FilterT& mFilter; - const bool mInCoreOnly; }; // ConvertPointDataGridGroupOp template @@ -582,8 +567,7 @@ convertPointDataGridPosition( PositionAttribute& positionAttribute, const PointDataGridT& grid, const std::vector& pointOffsets, const Index64 startOffset, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) { using TreeType = typename PointDataGridT::TreeType; using LeafManagerT = typename tree::LeafManager; @@ -601,12 +585,26 @@ convertPointDataGridPosition( PositionAttribute& positionAttribute, LeafManagerT leafManager(tree); ConvertPointDataGridPositionOp convert( positionAttribute, pointOffsets, startOffset, grid.transform(), positionIndex, - filter, inCoreOnly); + filter); tbb::parallel_for(leafManager.leafRange(), convert); positionAttribute.compact(); } +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridPosition() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +void +convertPointDataGridPosition( PositionAttribute& positionAttribute, + const PointDataGridT& grid, + const std::vector& pointOffsets, + const Index64 startOffset, + const FilterT& filter, + const bool /*inCoreOnly*/) +{ + convertPointDataGridPosition(positionAttribute, grid, pointOffsets, startOffset, filter); +} + + //////////////////////////////////////// @@ -618,8 +616,7 @@ convertPointDataGridAttribute( TypedAttribute& attribute, const Index64 startOffset, const unsigned arrayIndex, const Index stride, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) { using LeafManagerT = typename tree::LeafManager; @@ -633,12 +630,28 @@ convertPointDataGridAttribute( TypedAttribute& attribute, LeafManagerT leafManager(tree); ConvertPointDataGridAttributeOp convert( attribute, pointOffsets, startOffset, arrayIndex, stride, - filter, inCoreOnly); + filter); tbb::parallel_for(leafManager.leafRange(), convert); attribute.compact(); } +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridAttribute() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +void +convertPointDataGridAttribute( TypedAttribute& attribute, + const PointDataTreeT& tree, + const std::vector& pointOffsets, + const Index64 startOffset, + const unsigned arrayIndex, + const Index stride, + const FilterT& filter, + const bool /*inCoreOnly*/) +{ + convertPointDataGridAttribute(attribute, tree, pointOffsets, startOffset, arrayIndex, stride, filter); +} + + //////////////////////////////////////// @@ -649,8 +662,7 @@ convertPointDataGridGroup( Group& group, const std::vector& pointOffsets, const Index64 startOffset, const AttributeSet::Descriptor::GroupIndex index, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) { using LeafManagerT= typename tree::LeafManager; @@ -662,7 +674,7 @@ convertPointDataGridGroup( Group& group, LeafManagerT leafManager(tree); ConvertPointDataGridGroupOp convert( group, pointOffsets, startOffset, index, - filter, inCoreOnly); + filter); tbb::parallel_for(leafManager.leafRange(), convert); // must call this after modifying point groups in parallel @@ -670,6 +682,22 @@ convertPointDataGridGroup( Group& group, group.finalize(); } + +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridGroup() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +void +convertPointDataGridGroup( Group& group, + const PointDataTreeT& tree, + const std::vector& pointOffsets, + const Index64 startOffset, + const AttributeSet::Descriptor::GroupIndex index, + const FilterT& filter, + const bool /*inCoreOnly*/) +{ + convertPointDataGridGroup(group, tree, pointOffsets, startOffset, index, filter); +} + + template inline float computeVoxelSize( const PositionWrapper& positions, diff --git a/openvdb/openvdb/points/impl/PointCountImpl.h b/openvdb/openvdb/points/impl/PointCountImpl.h index 29c998b9c8..9f913a5ce0 100644 --- a/openvdb/openvdb/points/impl/PointCountImpl.h +++ b/openvdb/openvdb/points/impl/PointCountImpl.h @@ -17,16 +17,14 @@ namespace points { template Index64 pointCount(const PointDataTreeT& tree, const FilterT& filter, - const bool inCoreOnly, const bool threaded) { using LeafManagerT = tree::LeafManager; using LeafRangeT = typename LeafManagerT::LeafRange; auto countLambda = - [&filter, &inCoreOnly] (const LeafRangeT& range, Index64 sum) -> Index64 { + [&filter] (const LeafRangeT& range, Index64 sum) -> Index64 { for (const auto& leaf : range) { - if (inCoreOnly && leaf.buffer().isOutOfCore()) continue; auto state = filter.state(leaf); if (state == index::ALL) { sum += leaf.pointCount(); @@ -48,11 +46,21 @@ Index64 pointCount(const PointDataTreeT& tree, } +template +OPENVDB_DEPRECATED_MESSAGE("Use pointCount() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +Index64 pointCount(const PointDataTreeT& tree, + const FilterT& filter, + const bool /*inCoreOnly*/, + const bool threaded) +{ + return pointCount(tree, filter, threaded); +} + + template Index64 pointOffsets( std::vector& pointOffsets, const PointDataTreeT& tree, const FilterT& filter, - const bool inCoreOnly, const bool threaded) { using LeafT = typename PointDataTreeT::LeafNodeType; @@ -67,8 +75,7 @@ Index64 pointOffsets( std::vector& pointOffsets, LeafManagerT leafManager(tree); leafManager.foreach( - [&pointOffsets, &filter, &inCoreOnly](const LeafT& leaf, size_t pos) { - if (inCoreOnly && leaf.buffer().isOutOfCore()) return; + [&pointOffsets, &filter](const LeafT& leaf, size_t pos) { auto state = filter.state(leaf); if (state == index::ALL) { pointOffsets[pos] = leaf.pointCount(); @@ -90,6 +97,18 @@ Index64 pointOffsets( std::vector& pointOffsets, } +template +OPENVDB_DEPRECATED_MESSAGE("Use pointOffsets() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +Index64 pointOffsets( std::vector& pointOffsets, + const PointDataTreeT& tree, + const FilterT& filter, + const bool /*inCoreOnly*/, + const bool threaded) +{ + return pointOffsets(pointOffsets, tree, filter, threaded); +} + + template typename GridT::Ptr pointCountGrid( const PointDataGridT& points, diff --git a/openvdb/openvdb/points/impl/PointMoveImpl.h b/openvdb/openvdb/points/impl/PointMoveImpl.h index 7d86a66d1f..87f2b35a4a 100644 --- a/openvdb/openvdb/points/impl/PointMoveImpl.h +++ b/openvdb/openvdb/points/impl/PointMoveImpl.h @@ -319,7 +319,6 @@ struct GlobalMovePointsOp // extract target array and ensure data is out-of-core and non-uniform auto& targetArray = leaf.attributeArray(mAttributeIndex); - targetArray.loadData(); targetArray.expand(); // perform the copy @@ -344,7 +343,6 @@ struct GlobalMovePointsOp const LeafT& sourceLeaf = mSourceLeafManager.leaf(sourceLeafIndex); const auto& sourceArray = sourceLeaf.constAttributeArray(mAttributeIndex); - sourceArray.loadData(); targetArray.copyValuesUnsafe(sourceArray, copyIterator); @@ -427,12 +425,10 @@ struct LocalMovePointsOp const Index sourceLeafOffset(mSourceIndices[idx]); LeafT& sourceLeaf = mSourceLeafManager.leaf(sourceLeafOffset); const auto& sourceArray = sourceLeaf.constAttributeArray(mAttributeIndex); - sourceArray.loadData(); // extract target array and ensure data is out-of-core and non-uniform auto& targetArray = leaf.attributeArray(mAttributeIndex); - targetArray.loadData(); targetArray.expand(); // perform the copy diff --git a/openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h b/openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h index 3578b44e83..88ee47f078 100644 --- a/openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h +++ b/openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h @@ -948,7 +948,7 @@ class GridToRasterize // generate leaf offsets (if necessary) if (mLeafOffsets.empty()) { openvdb::points::pointOffsets(mLeafOffsets, mGrid->constTree(), resolvedFilter, - /*inCoreOnly=*/false, mSettings.threaded); + mSettings.threaded); } // set streaming arbitrary attribute array flags diff --git a/openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h b/openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h index f2ce326efd..3b2a66e1c9 100644 --- a/openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h +++ b/openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h @@ -310,10 +310,7 @@ struct WeightPosSumsTransfer auto& leaf = this->mManager.leaf(idx); { - // @todo add API to get the array from the group handle. The handle - // calls loadData but not expand. auto& array = leaf.attributeArray(this->mIndices.mEllipsesGroupIndex.first); - array.loadData(); // so we can call setUnsafe/getUnsafe array.expand(); } diff --git a/openvdb/openvdb/tools/Count.h b/openvdb/openvdb/tools/Count.h index 03dc719b86..c1a3a9f9f9 100644 --- a/openvdb/openvdb/tools/Count.h +++ b/openvdb/openvdb/tools/Count.h @@ -60,19 +60,13 @@ Index64 countActiveTiles(const TreeT& tree, bool threaded = true); /// @brief Return the total amount of memory in bytes occupied by this tree. -/// @details This method returns the total in-core memory usage which can be -/// different to the maximum possible memory usage for trees which have not -/// been fully deserialized (via delay-loading). Thus, this is the current -/// true memory consumption. template Index64 memUsage(const TreeT& tree, bool threaded = true); -/// @brief Return the deserialized memory usage of this tree. This is not -/// necessarily equal to the current memory usage (returned by tools::memUsage) -/// if delay-loading is enabled. See File::open. template -Index64 memUsageIfLoaded(const TreeT& tree, bool threaded = true); +OPENVDB_DEPRECATED_MESSAGE("Use tools::memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +Index64 memUsageIfLoaded(const TreeT& tree, bool threaded = true) { return tools::memUsage(tree, threaded); } /// @brief Return the minimum and maximum active values in this tree. @@ -299,9 +293,9 @@ struct MemUsageOp using RootT = typename TreeType::RootNodeType; using LeafT = typename TreeType::LeafNodeType; - MemUsageOp(const bool inCoreOnly) : mInCoreOnly(inCoreOnly) {} - MemUsageOp(const MemUsageOp& other) : mCount(0), mInCoreOnly(other.mInCoreOnly) {} - MemUsageOp(const MemUsageOp& other, tbb::split) : MemUsageOp(other) {} + MemUsageOp() = default; + MemUsageOp(const MemUsageOp& other) = default; + MemUsageOp(const MemUsageOp&, tbb::split) : mCount(0) {} // accumulate size of the root node in bytes bool operator()(const RootT& root, size_t) @@ -323,8 +317,7 @@ struct MemUsageOp // accumulate size of leaf node in bytes bool operator()(const LeafT& leaf, size_t) { - if (mInCoreOnly) mCount += leaf.memUsage(); - else mCount += leaf.memUsageIfLoaded(); + mCount += leaf.memUsage(); return false; } @@ -334,7 +327,6 @@ struct MemUsageOp } openvdb::Index64 mCount{0}; - const bool mInCoreOnly; }; // struct MemUsageOp /// @brief A DynamicNodeManager operator to find the minimum and maximum active values in this tree. @@ -492,25 +484,12 @@ Index64 countActiveTiles(const TreeT& tree, bool threaded) template Index64 memUsage(const TreeT& tree, bool threaded) { - count_internal::MemUsageOp op(true); + count_internal::MemUsageOp op; tree::DynamicNodeManager nodeManager(tree); nodeManager.reduceTopDown(op, threaded); return op.mCount + sizeof(tree); } -template -Index64 memUsageIfLoaded(const TreeT& tree, bool threaded) -{ - /// @note For numeric (non-point) grids this really doesn't need to - /// traverse the tree and could instead be computed from the node counts. - /// We do so anyway as it ties this method into the tree data structure - /// which makes sure that changes to the tree/nodes are reflected/kept in - /// sync here. - count_internal::MemUsageOp op(false); - tree::DynamicNodeManager nodeManager(tree); - nodeManager.reduceTopDown(op, threaded); - return op.mCount + sizeof(tree); -} template math::MinMax minMax(const TreeT& tree, bool threaded) diff --git a/openvdb/openvdb/tools/Merge.h b/openvdb/openvdb/tools/Merge.h index a7afd4d055..54fbf927b4 100644 --- a/openvdb/openvdb/tools/Merge.h +++ b/openvdb/openvdb/tools/Merge.h @@ -581,16 +581,14 @@ struct UnallocatedBuffer static void allocateAndFill(BufferT& buffer, const ValueT& background) { if (buffer.empty()) { - if (!buffer.isOutOfCore()) { - buffer.allocate(); - buffer.fill(background); - } + buffer.allocate(); + buffer.fill(background); } } static bool isPartiallyConstructed(const BufferT& buffer) { - return !buffer.isOutOfCore() && buffer.empty(); + return buffer.empty(); } }; // struct AllocateAndFillBuffer diff --git a/openvdb/openvdb/tools/PointIndexGrid.h b/openvdb/openvdb/tools/PointIndexGrid.h index 3db213ea38..245b93ed5d 100644 --- a/openvdb/openvdb/tools/PointIndexGrid.h +++ b/openvdb/openvdb/tools/PointIndexGrid.h @@ -1491,7 +1491,8 @@ struct PointIndexLeafNode : public tree::LeafNode Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } //////////////////////////////////////// @@ -1798,13 +1799,6 @@ PointIndexLeafNode::memUsage() const return BaseLeaf::memUsage() + Index64((sizeof(T)*mIndices.capacity()) + sizeof(mIndices)); } -template -inline Index64 -PointIndexLeafNode::memUsageIfLoaded() const -{ - return BaseLeaf::memUsageIfLoaded() + Index64((sizeof(T)*mIndices.capacity()) + sizeof(mIndices)); -} - } // namespace tools diff --git a/openvdb/openvdb/tools/VolumeToMesh.h b/openvdb/openvdb/tools/VolumeToMesh.h index 509fc1187b..b3c268072a 100644 --- a/openvdb/openvdb/tools/VolumeToMesh.h +++ b/openvdb/openvdb/tools/VolumeToMesh.h @@ -616,8 +616,8 @@ template<> inline bool isInsideValue(bool value, bool /*isovalue*/) { return value; } -/// @brief Minor wrapper around the Leaf API to avoid atomic access with -/// delayed loading. +/// @brief Minor wrapper around the Leaf API. This was originally added to avoid +// atomic access with delayed loading, however delayed loading is no longer supported. template ::value> struct LeafBufferAccessor diff --git a/openvdb/openvdb/tree/InternalNode.h b/openvdb/openvdb/tree/InternalNode.h index 913521b570..c61deedbfe 100644 --- a/openvdb/openvdb/tree/InternalNode.h +++ b/openvdb/openvdb/tree/InternalNode.h @@ -82,6 +82,10 @@ class InternalNode /// @param active State assigned to all the tiles InternalNode(const Coord& origin, const ValueType& fillValue, bool active = false); + /// @brief Construct a node without allocating child memory. Children are + /// left unallocated and must be populated single-threaded, or with external + /// synchronization. The valid advanced pattern is: create all nodes + /// single-threaded, then allocate leaf buffers in parallel across distinct leaves. InternalNode(PartialCreate, const Coord&, const ValueType& fillValue, bool active = false); /// @brief Deep copy constructor diff --git a/openvdb/openvdb/tree/LeafBuffer.h b/openvdb/openvdb/tree/LeafBuffer.h index a684db8da6..7d1a965144 100644 --- a/openvdb/openvdb/tree/LeafBuffer.h +++ b/openvdb/openvdb/tree/LeafBuffer.h @@ -8,9 +8,11 @@ #include // for io::readCompressedValues(), etc #include #include +#if OPENVDB_ABI_VERSION_NUMBER < 14 #include -#include // for std::swap #include +#endif +#include // for std::swap #include // for offsetof() #include #include @@ -36,58 +38,29 @@ class LeafBuffer using NodeMaskType = util::NodeMask; static const Index SIZE = 1 << 3 * Log2Dim; -#ifdef OPENVDB_USE_DELAYED_LOADING - struct FileInfo - { - FileInfo(): bufpos(0) , maskpos(0) {} - std::streamoff bufpos; - std::streamoff maskpos; - io::MappedFile::Ptr mapping; - SharedPtr meta; - }; -#endif - /// Default constructor - inline LeafBuffer(): mData(new ValueType[SIZE]) - { -#ifdef OPENVDB_USE_DELAYED_LOADING - mOutOfCore = 0; -#endif - } + inline LeafBuffer(): mData(new ValueType[SIZE]) {} /// Construct a buffer populated with the specified value. explicit inline LeafBuffer(const ValueType&); /// Copy constructor inline LeafBuffer(const LeafBuffer&); - /// Construct a buffer but don't allocate memory for the full array of values. - LeafBuffer(PartialCreate, const ValueType&): mData(nullptr) - { -#ifdef OPENVDB_USE_DELAYED_LOADING - mOutOfCore = 0; -#endif - } + /// @brief Construct a buffer without allocating the value array. + /// The buffer is left unallocated; call @c allocate() before use. + /// Populating the buffer (via @c allocate() or @c data()) must be done + /// single-threaded per leaf, or externally synchronized. The valid + /// advanced pattern is: create all nodes single-threaded, then call + /// @c allocate() in parallel across distinct leaves. + LeafBuffer(PartialCreate, const ValueType&): mData(nullptr) {} /// Destructor inline ~LeafBuffer(); - /// Return @c true if this buffer's values have not yet been read from disk. - bool isOutOfCore() const - { -#ifdef OPENVDB_USE_DELAYED_LOADING - return bool(mOutOfCore); -#else - return false; -#endif - } + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool isOutOfCore() const { return false; } /// Return @c true if memory for this buffer has not yet been allocated. - bool empty() const { return !mData || this->isOutOfCore(); } + bool empty() const { return !mData; } /// Allocate memory for this buffer if it has not already been allocated. bool allocate() { if (mData == nullptr) mData = new ValueType[SIZE]; return true; } -#ifdef OPENVDB_USE_DELAYED_LOADING - /// Enable out-of-core in the LeafBuffer. - void enableOutOfCore(SharedPtr& meta, const std::streamoff& bufpos, - io::MappedFile::Ptr& mappedFile, const std::streamoff& maskpos); -#endif - /// Populate this buffer with a constant value. inline void fill(const ValueType&); @@ -113,16 +86,21 @@ class LeafBuffer /// Return the memory footprint of this buffer in bytes. inline Index memUsage() const; - inline Index memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + inline Index memUsageIfLoaded() const { return memUsage(); } /// Return the number of values contained in this buffer. static Index size() { return SIZE; } /// @brief Return a const pointer to the array of voxel values. - /// @details This method guarantees that the buffer is allocated and loaded. + /// @warning The buffer must already be allocated (call @c allocate() first). + /// First-touch allocation via @c data() is not thread-safe; concurrent access + /// to an unallocated buffer is a programming error flagged in debug builds. /// @warning This method should only be used by experts seeking low-level optimizations. const ValueType* data() const; /// @brief Return a pointer to the array of voxel values. - /// @details This method guarantees that the buffer is allocated and loaded. + /// @warning The buffer must already be allocated (call @c allocate() first). + /// First-touch allocation via @c data() is not thread-safe; concurrent access + /// to an unallocated buffer is a programming error flagged in debug builds. /// @warning This method should only be used by experts seeking low-level optimizations. ValueType* data(); @@ -139,37 +117,12 @@ class LeafBuffer bool deallocate(); - inline void setOutOfCore(bool b) - { - (void) b; -#ifdef OPENVDB_USE_DELAYED_LOADING - mOutOfCore = b; + ValueType* mData = nullptr; +#if OPENVDB_ABI_VERSION_NUMBER < 14 + // Deprecated members kept for ABI compatibility + std::atomic mDeprecatedAtomic{0}; + tbb::spin_mutex mDeprecatedSpinMutex; #endif - } - // To facilitate inlining in the common case in which the buffer is in-core, - // the loading logic is split into a separate function, doLoad(). - inline void loadValues() const - { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) this->doLoad(); -#endif - } - inline void doLoad() const; - inline bool detachFromFile(); - - using FlagsType = std::atomic; - -#ifdef OPENVDB_USE_DELAYED_LOADING - union { - ValueType* mData; - FileInfo* mFileInfo; - }; -#else - ValueType* mData; -#endif - FlagsType mOutOfCore; // interpreted as bool; extra bits reserved for future use - tbb::spin_mutex mMutex; // 1 byte - //int8_t mReserved[3]; // padding for alignment friend class ::TestLeaf; // Allow the parent LeafNode to access this buffer's data pointer. @@ -185,9 +138,6 @@ inline LeafBuffer::LeafBuffer(const ValueType& val) : mData(new ValueType[SIZE]) { -#ifdef OPENVDB_USE_DELAYED_LOADING - mOutOfCore = 0; -#endif this->fill(val); } @@ -196,15 +146,7 @@ template inline LeafBuffer::~LeafBuffer() { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) { - this->detachFromFile(); - } else { - this->deallocate(); - } -#else this->deallocate(); -#endif } @@ -212,25 +154,14 @@ template inline LeafBuffer::LeafBuffer(const LeafBuffer& other) : mData(nullptr) -#ifdef OPENVDB_USE_DELAYED_LOADING - , mOutOfCore(other.mOutOfCore.load()) -#endif { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (other.isOutOfCore()) { - mFileInfo = new FileInfo(*other.mFileInfo); - } else { -#endif - if (other.mData != nullptr) { - this->allocate(); - ValueType* target = mData; - const ValueType* source = other.mData; - Index n = SIZE; - while (n--) *target++ = *source++; - } -#ifdef OPENVDB_USE_DELAYED_LOADING + if (other.mData != nullptr) { + this->allocate(); + ValueType* target = mData; + const ValueType* source = other.mData; + Index n = SIZE; + while (n--) *target++ = *source++; } -#endif } @@ -239,7 +170,6 @@ inline void LeafBuffer::setValue(Index i, const ValueType& val) { OPENVDB_ASSERT(i < SIZE); - this->loadValues(); if (mData) mData[i] = val; } @@ -249,52 +179,22 @@ inline LeafBuffer& LeafBuffer::operator=(const LeafBuffer& other) { if (&other != this) { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) { - this->detachFromFile(); - } else { - if (other.isOutOfCore()) this->deallocate(); - } - if (other.isOutOfCore()) { - mOutOfCore.store(other.mOutOfCore.load(std::memory_order_acquire), - std::memory_order_release); - mFileInfo = new FileInfo(*other.mFileInfo); - } else { -#endif - if (other.mData != nullptr) { - this->allocate(); - ValueType* target = mData; - const ValueType* source = other.mData; - Index n = SIZE; - while (n--) *target++ = *source++; - } -#ifdef OPENVDB_USE_DELAYED_LOADING + if (other.mData != nullptr) { + this->allocate(); + ValueType* target = mData; + const ValueType* source = other.mData; + Index n = SIZE; + while (n--) *target++ = *source++; } -#endif } return *this; } -#ifdef OPENVDB_USE_DELAYED_LOADING -template -void LeafBuffer::enableOutOfCore( - SharedPtr& meta, const std::streamoff& bufpos, - io::MappedFile::Ptr& mappedFile, const std::streamoff& maskpos) -{ - this->setOutOfCore(true); - mFileInfo = new FileInfo; - mFileInfo->meta = meta; - mFileInfo->bufpos = bufpos; - mFileInfo->mapping = mappedFile; - mFileInfo->maskpos = maskpos; -} -#endif template inline void LeafBuffer::fill(const ValueType& val) { - this->detachFromFile(); if (mData != nullptr) { ValueType* target = mData; Index n = SIZE; @@ -307,8 +207,6 @@ template inline bool LeafBuffer::operator==(const LeafBuffer& other) const { - this->loadValues(); - other.loadValues(); const ValueType *target = mData, *source = other.mData; if (!target && !source) return true; if (!target || !source) return false; @@ -323,15 +221,6 @@ inline void LeafBuffer::swap(LeafBuffer& other) { std::swap(mData, other.mData); -#ifdef OPENVDB_USE_DELAYED_LOADING - // Two atomics can't be swapped because it would require hardware support: - // https://en.wikipedia.org/wiki/Double_compare-and-swap - // Note that there's a window in which other.mOutOfCore could be written - // between our load from it and our store to it. - auto tmp = other.mOutOfCore.load(std::memory_order_acquire); - tmp = mOutOfCore.exchange(std::move(tmp)); - other.mOutOfCore.store(std::move(tmp), std::memory_order_release); -#endif } @@ -340,24 +229,7 @@ inline Index LeafBuffer::memUsage() const { size_t n = sizeof(*this); -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) n += sizeof(FileInfo); - else { -#endif - if (mData) n += SIZE * sizeof(ValueType); -#ifdef OPENVDB_USE_DELAYED_LOADING - } -#endif - return static_cast(n); -} - - -template -inline Index -LeafBuffer::memUsageIfLoaded() const -{ - size_t n = sizeof(*this); - n += SIZE * sizeof(ValueType); + if (mData) n += SIZE * sizeof(ValueType); return static_cast(n); } @@ -366,14 +238,10 @@ template inline const typename LeafBuffer::ValueType* LeafBuffer::data() const { - this->loadValues(); + OPENVDB_ASSERT(mData != nullptr); if (mData == nullptr) { LeafBuffer* self = const_cast(this); -#ifdef OPENVDB_USE_DELAYED_LOADING - // This lock will be contended at most once. - tbb::spin_mutex::scoped_lock lock(self->mMutex); -#endif - if (mData == nullptr) self->mData = new ValueType[SIZE]; + self->mData = new ValueType[SIZE]; } return mData; } @@ -382,14 +250,8 @@ template inline typename LeafBuffer::ValueType* LeafBuffer::data() { - this->loadValues(); - if (mData == nullptr) { -#ifdef OPENVDB_USE_DELAYED_LOADING - // This lock will be contended at most once. - tbb::spin_mutex::scoped_lock lock(mMutex); -#endif - if (mData == nullptr) mData = new ValueType[SIZE]; - } + OPENVDB_ASSERT(mData != nullptr); + if (mData == nullptr) mData = new ValueType[SIZE]; return mData; } @@ -400,7 +262,6 @@ LeafBuffer::at(Index i) const { static const ValueType sZero = zeroVal(); OPENVDB_ASSERT(i < SIZE); - this->loadValues(); // We can't use the ternary operator here, otherwise Visual C++ returns // a reference to a temporary. if (mData) return mData[i]; else return sZero; @@ -411,11 +272,7 @@ template inline bool LeafBuffer::deallocate() { - if (mData != nullptr) { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) return false; -#endif delete[] mData; mData = nullptr; return true; @@ -424,62 +281,6 @@ LeafBuffer::deallocate() } -template -inline void -LeafBuffer::doLoad() const -{ -#ifdef OPENVDB_USE_DELAYED_LOADING - if (!this->isOutOfCore()) return; - - LeafBuffer* self = const_cast*>(this); - - // This lock will be contended at most once, after which this buffer - // will no longer be out-of-core. - tbb::spin_mutex::scoped_lock lock(self->mMutex); - if (!this->isOutOfCore()) return; - - std::unique_ptr info(self->mFileInfo); - OPENVDB_ASSERT(info.get() != nullptr); - OPENVDB_ASSERT(info->mapping.get() != nullptr); - OPENVDB_ASSERT(info->meta.get() != nullptr); - - /// @todo For now, we have to clear the mData pointer in order for allocate() to take effect. - self->mData = nullptr; - self->allocate(); - - SharedPtr buf = info->mapping->createBuffer(); - std::istream is(buf.get()); - - io::setStreamMetadataPtr(is, info->meta, /*transfer=*/true); - - NodeMaskType mask; - is.seekg(info->maskpos); - mask.load(is); - - is.seekg(info->bufpos); - io::readCompressedValues(is, self->mData, SIZE, mask, io::getHalfFloat(is)); - - self->setOutOfCore(false); -#endif -} - - -template -inline bool -LeafBuffer::detachFromFile() -{ -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) { - delete mFileInfo; - mFileInfo = nullptr; - this->setOutOfCore(false); - return true; - } -#endif - return false; -} - - //////////////////////////////////////// @@ -524,7 +325,8 @@ class LeafBuffer void swap(LeafBuffer& other) { if (&other != this) std::swap(mData, other.mData); } Index memUsage() const { return sizeof(*this); } - Index memUsageIfLoaded() const { return sizeof(*this); } + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index memUsageIfLoaded() const { return memUsage(); } static Index size() { return SIZE; } /// @brief Return a pointer to the C-style array of words encoding the bits. diff --git a/openvdb/openvdb/tree/LeafNode.h b/openvdb/openvdb/tree/LeafNode.h index 11951c1262..21cae3d7a2 100644 --- a/openvdb/openvdb/tree/LeafNode.h +++ b/openvdb/openvdb/tree/LeafNode.h @@ -83,6 +83,9 @@ class LeafNode /// @param value a value with which to fill the buffer /// @param active the active state to which to initialize all voxels /// @details This constructor does not allocate memory for voxel values. + /// Call @c buffer().allocate() before accessing voxel data. The valid + /// advanced pattern is: create all leaves single-threaded, then call + /// @c allocate() in parallel across distinct leaves. LeafNode(PartialCreate, const Coord& coords, const ValueType& value = zeroVal(), @@ -152,13 +155,14 @@ class LeafNode /// Return @c true if this node contains only active voxels. bool isDense() const { return mValueMask.isOn(); } /// Return @c true if memory for this node's buffer has been allocated. - bool isAllocated() const { return !mBuffer.isOutOfCore() && !mBuffer.empty(); } + bool isAllocated() const { return !mBuffer.empty(); } /// Allocate memory for this node's buffer if it has not already been allocated. bool allocate() { return mBuffer.allocate(); } /// Return the memory in bytes occupied by this node. Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } /// Expand the given bounding box so that it includes this leaf node's active voxels. /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all @@ -466,7 +470,6 @@ class LeafNode template void modifyValue(Index offset, const ModifyOp& op) { - mBuffer.loadValues(); if (!mBuffer.empty()) { // in-place modify value ValueType& val = const_cast(mBuffer[offset]); @@ -487,7 +490,6 @@ class LeafNode template void modifyValueAndActiveState(const Coord& xyz, const ModifyOp& op) { - mBuffer.loadValues(); if (!mBuffer.empty()) { const Index offset = this->coordToOffset(xyz); bool state = mValueMask.isOn(offset); @@ -1262,8 +1264,6 @@ template inline void LeafNode::copyToDense(const CoordBBox& bbox, DenseT& dense) const { - mBuffer.loadValues(); - using DenseValueType = typename DenseT::ValueType; const size_t xStride = dense.xStride(), yStride = dense.yStride(), zStride = dense.zStride(); @@ -1375,10 +1375,6 @@ LeafNode::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo SharedPtr meta = io::getStreamMetadataPtr(is); const bool seekable = meta && meta->seekable(); -#ifdef OPENVDB_USE_DELAYED_LOADING - std::streamoff maskpos = is.tellg(); -#endif - if (seekable) { // Seek over the value mask. mValueMask.seek(is); @@ -1401,37 +1397,16 @@ LeafNode::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo // This node lies completely outside the clipping region. skipCompressedValues(seekable, is, fromHalf); mValueMask.setOff(); - mBuffer.setOutOfCore(false); } else { -#ifdef OPENVDB_USE_DELAYED_LOADING - // If this node lies completely inside the clipping region and it is being read - // from a memory-mapped file, delay loading of its buffer until the buffer - // is actually accessed. (If this node requires clipping, its buffer - // must be accessed and therefore must be loaded.) - io::MappedFile::Ptr mappedFile = io::getMappedFilePtr(is); - const bool delayLoad = ((mappedFile.get() != nullptr) && clipBBox.isInside(nodeBBox)); - - if (delayLoad) { - // Save the offset to the value mask (maskpos), because the in-memory copy - // might change before the value buffer gets read. - mBuffer.enableOutOfCore(meta, is.tellg(), mappedFile, maskpos); - // Skip over voxel values. - skipCompressedValues(seekable, is, fromHalf); - } else { -#endif - mBuffer.allocate(); - io::readCompressedValues(is, mBuffer.mData, SIZE, mValueMask, fromHalf); - mBuffer.setOutOfCore(false); - - // Get this tree's background value. - T background = zeroVal(); - if (const void* bgPtr = io::getGridBackgroundValuePtr(is)) { - background = *static_cast(bgPtr); - } - this->clip(clipBBox, background); -#ifdef OPENVDB_USE_DELAYED_LOADING + mBuffer.allocate(); + io::readCompressedValues(is, mBuffer.mData, SIZE, mValueMask, fromHalf); + + // Get this tree's background value. + T background = zeroVal(); + if (const void* bgPtr = io::getGridBackgroundValuePtr(is)) { + background = *static_cast(bgPtr); } -#endif + this->clip(clipBBox, background); } if (numBuffers > 1) { @@ -1447,9 +1422,6 @@ LeafNode::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo } } } - - // increment the leaf number - if (meta) meta->setLeaf(meta->leaf() + 1); } @@ -1460,8 +1432,6 @@ LeafNode::writeBuffers(std::ostream& os, bool toHalf) const // Write out the value mask. mValueMask.save(os); - mBuffer.loadValues(); - io::writeCompressedValues(os, mBuffer.mData, SIZE, mValueMask, /*childMask=*/NodeMaskType(), toHalf); } @@ -1490,16 +1460,6 @@ LeafNode::memUsage() const } -template -inline Index64 -LeafNode::memUsageIfLoaded() const -{ - // Use sizeof(*this) to capture alignment-related padding - // (but note that sizeof(*this) includes sizeof(mBuffer)). - return sizeof(*this) + mBuffer.memUsageIfLoaded() - sizeof(mBuffer); -} - - template inline void LeafNode::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const diff --git a/openvdb/openvdb/tree/LeafNodeBool.h b/openvdb/openvdb/tree/LeafNodeBool.h index 80dd31cb87..8caccdbbc6 100644 --- a/openvdb/openvdb/tree/LeafNodeBool.h +++ b/openvdb/openvdb/tree/LeafNodeBool.h @@ -145,7 +145,8 @@ class LeafNode /// Return the memory in bytes occupied by this node. Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } /// Expand the given bounding box so that it includes this leaf node's active voxels. /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all @@ -898,15 +899,6 @@ LeafNode::memUsage() const } -template -inline Index64 -LeafNode::memUsageIfLoaded() const -{ - // Use sizeof(*this) to capture alignment-related padding - return sizeof(*this); -} - - template inline void LeafNode::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const diff --git a/openvdb/openvdb/tree/LeafNodeMask.h b/openvdb/openvdb/tree/LeafNodeMask.h index 1a5e9a2bfc..41474874bd 100644 --- a/openvdb/openvdb/tree/LeafNodeMask.h +++ b/openvdb/openvdb/tree/LeafNodeMask.h @@ -144,7 +144,8 @@ class LeafNode /// Return the memory in bytes occupied by this node. Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } /// Expand the given bounding box so that it includes this leaf node's active voxels. /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all @@ -884,15 +885,6 @@ LeafNode::memUsage() const } -template -inline Index64 -LeafNode::memUsageIfLoaded() const -{ - // Use sizeof(*this) to capture alignment-related padding - return sizeof(*this); -} - - template inline void LeafNode::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const diff --git a/openvdb/openvdb/tree/Tree.h b/openvdb/openvdb/tree/Tree.h index e0e99881ba..07ed98e5db 100644 --- a/openvdb/openvdb/tree/Tree.h +++ b/openvdb/openvdb/tree/Tree.h @@ -95,13 +95,7 @@ class OPENVDB_API TreeBase virtual void getIndexRange(CoordBBox& bbox) const = 0; - /// @brief Replace with background tiles any nodes whose voxel buffers - /// have not yet been allocated. - /// @details Typically, unallocated nodes are leaf nodes whose voxel buffers - /// are not yet resident in memory because delayed loading is in effect. - /// @sa readNonresidentBuffers, io::File::open virtual void clipUnallocatedNodes() = 0; - /// Return the total number of unallocated leaf nodes residing in this tree. #if OPENVDB_ABI_VERSION_NUMBER >= 12 virtual Index64 unallocatedLeafCount() const = 0; #else @@ -157,22 +151,21 @@ class OPENVDB_API TreeBase /// @brief Read the tree topology from a stream. /// /// This will read the tree structure and tile values, but not voxel data. - virtual void readTopology(std::istream&, bool saveFloatAsHalf = false); + virtual void readTopology(std::istream&, bool saveFloatAsHalf = false) = 0; /// @brief Write the tree topology to a stream. /// /// This will write the tree structure and tile values, but not voxel data. - virtual void writeTopology(std::ostream&, bool saveFloatAsHalf = false) const; + virtual void writeTopology(std::ostream&, bool saveFloatAsHalf = false) const = 0; /// Read all data buffers for this tree. virtual void readBuffers(std::istream&, bool saveFloatAsHalf = false) = 0; /// Read all of this tree's data buffers that intersect the given bounding box. virtual void readBuffers(std::istream&, const CoordBBox&, bool saveFloatAsHalf = false) = 0; - /// @brief Read all of this tree's data buffers that are not yet resident in memory - /// (because delayed loading is in effect). - /// @details If this tree was read from a memory-mapped file, this operation - /// disconnects the tree from the file. - /// @sa clipUnallocatedNodes, io::File::open, io::MappedFile + +#if OPENVDB_ABI_VERSION_NUMBER < 14 + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void readNonresidentBuffers() const = 0; +#endif /// Write out all the data buffers for this tree. virtual void writeBuffers(std::ostream&, bool saveFloatAsHalf = false) const = 0; @@ -339,12 +332,12 @@ class Tree: public TreeBase void readBuffers(std::istream&, bool saveFloatAsHalf = false) override; /// Read all of this tree's data buffers that intersect the given bounding box. void readBuffers(std::istream&, const CoordBBox&, bool saveFloatAsHalf = false) override; - /// @brief Read all of this tree's data buffers that are not yet resident in memory - /// (because delayed loading is in effect). - /// @details If this tree was read from a memory-mapped file, this operation - /// disconnects the tree from the file. - /// @sa clipUnallocatedNodes, io::File::open, io::MappedFile - void readNonresidentBuffers() const override; + +#if OPENVDB_ABI_VERSION_NUMBER < 14 + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void readNonresidentBuffers() const override { } +#endif + /// Write out all data buffers for this tree. void writeBuffers(std::ostream&, bool saveFloatAsHalf = false) const override; @@ -494,9 +487,6 @@ class Tree: public TreeBase void clip(const CoordBBox&); /// @brief Replace with background tiles any nodes whose voxel buffers /// have not yet been allocated. - /// @details Typically, unallocated nodes are leaf nodes whose voxel buffers - /// are not yet resident in memory because delayed loading is in effect. - /// @sa readNonresidentBuffers, io::File::open void clipUnallocatedNodes() override; /// Return the total number of unallocated leaf nodes residing in this tree. @@ -1140,23 +1130,6 @@ struct Tree5 { //////////////////////////////////////// -inline void -TreeBase::readTopology(std::istream& is, bool /*saveFloatAsHalf*/) -{ - int32_t bufferCount; - is.read(reinterpret_cast(&bufferCount), sizeof(int32_t)); - if (bufferCount != 1) OPENVDB_LOG_WARN("multi-buffer trees are no longer supported"); -} - - -inline void -TreeBase::writeTopology(std::ostream& os, bool /*saveFloatAsHalf*/) const -{ - int32_t bufferCount = 1; - os.write(reinterpret_cast(&bufferCount), sizeof(int32_t)); -} - - inline void TreeBase::print(std::ostream& os, int /*verboseLevel*/) const { @@ -1283,7 +1256,9 @@ void Tree::readTopology(std::istream& is, bool saveFloatAsHalf) { this->clearAllAccessors(); - TreeBase::readTopology(is, saveFloatAsHalf); + int32_t bufferCount; + is.read(reinterpret_cast(&bufferCount), sizeof(int32_t)); + if (bufferCount != 1) OPENVDB_LOG_WARN("multi-buffer trees are no longer supported"); mRoot.readTopology(is, saveFloatAsHalf); } @@ -1292,7 +1267,8 @@ template void Tree::writeTopology(std::ostream& os, bool saveFloatAsHalf) const { - TreeBase::writeTopology(os, saveFloatAsHalf); + int32_t bufferCount = 1; + os.write(reinterpret_cast(&bufferCount), sizeof(int32_t)); mRoot.writeTopology(os, saveFloatAsHalf); } @@ -1315,17 +1291,6 @@ Tree::readBuffers(std::istream &is, const CoordBBox& bbox, bool sa } -template -inline void -Tree::readNonresidentBuffers() const -{ - for (LeafCIter it = this->cbeginLeaf(); it; ++it) { - // Retrieving the value of a leaf voxel forces loading of the leaf node's voxel buffer. - it->getValue(Index(0)); - } -} - - template inline void Tree::writeBuffers(std::ostream &os, bool saveFloatAsHalf) const diff --git a/openvdb/openvdb/unittest/CMakeLists.txt b/openvdb/openvdb/unittest/CMakeLists.txt index bf68272704..446d118027 100644 --- a/openvdb/openvdb/unittest/CMakeLists.txt +++ b/openvdb/openvdb/unittest/CMakeLists.txt @@ -82,12 +82,12 @@ else() TestAttributeSet.cc TestBBox.cc TestClip.cc + TestCodec.cc TestConjGradient.cc TestCoord.cc TestCount.cc TestCpt.cc TestCurl.cc - TestDelayedLoadMetadata.cc TestDense.cc TestDenseSparseTools.cc TestDiagnostics.cc @@ -146,6 +146,7 @@ else() TestParticlesToLevelSet.cc TestPointAdvect.cc TestPointAttribute.cc + TestPointCodec.cc TestPointConversion.cc TestPointCount.cc TestPointDataLeaf.cc diff --git a/openvdb/openvdb/unittest/TestAttributeArray.cc b/openvdb/openvdb/unittest/TestAttributeArray.cc index 0b949c3ece..dc360e6fb1 100644 --- a/openvdb/openvdb/unittest/TestAttributeArray.cc +++ b/openvdb/openvdb/unittest/TestAttributeArray.cc @@ -12,31 +12,6 @@ #include -#ifdef OPENVDB_USE_DELAYED_LOADING -#ifdef __clang__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-macros" -#endif -// Boost.Interprocess uses a header-only portion of Boost.DateTime -#define BOOST_DATE_TIME_NO_LIB -#ifdef __clang__ -#pragma GCC diagnostic pop -#endif -#include -#include - -#ifdef _WIN32 -#include // open_existing_file(), close_file() -// boost::interprocess::detail was renamed to boost::interprocess::ipcdetail in Boost 1.48. -// Ensure that both namespaces exist. -namespace boost { namespace interprocess { namespace detail {} namespace ipcdetail {} } } -#include -#else -#include // for struct stat -#include // for stat() -#endif -#endif // OPENVDB_USE_DELAYED_LOADING - #include #include @@ -113,7 +88,6 @@ class TestAttributeArray: public ::testing::Test void testRegistry(); void testAccessorEval(); - void testDelayedLoad(); }; // class TestAttributeArray @@ -1400,801 +1374,6 @@ TEST_F(TestAttributeArray, testStrided) } } -#ifdef OPENVDB_USE_DELAYED_LOADING -void -TestAttributeArray::testDelayedLoad() -{ - using AttributeArrayI = TypedAttributeArray; - using AttributeArrayF = TypedAttributeArray; - - AttributeArrayI::registerType(); - AttributeArrayF::registerType(); - - SharedPtr mappedFile; - - io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata); - - std::string tempDir; - if (const char* dir = std::getenv("TMPDIR")) tempDir = dir; -#ifdef _WIN32 - if (tempDir.empty()) { - char tempDirBuffer[MAX_PATH+1]; - int tempDirLen = GetTempPath(MAX_PATH+1, tempDirBuffer); - EXPECT_TRUE(tempDirLen > 0 && tempDirLen <= MAX_PATH); - tempDir = tempDirBuffer; - } -#else - if (tempDir.empty()) tempDir = P_tmpdir; -#endif - - { // IO - const Index count = 50; - AttributeArrayI attrA(count); - - for (unsigned i = 0; i < unsigned(count); ++i) { - attrA.set(i, int(i)); - } - - AttributeArrayF attrA2(count); - - std::string filename; - - // write out attribute array to a temp file - { - filename = tempDir + "/openvdb_delayed1"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - attrA.writeMetadata(fileout, false, /*paged=*/true); - compression::PagedOutputStream outputStreamSize(fileout); - outputStreamSize.setSizeOnly(true); - attrA.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout); - outputStream.setSizeOnly(false); - attrA.writePagedBuffers(outputStream, false); - outputStream.flush(); - - attrA2.writeMetadata(fileout, false, /*paged=*/true); - compression::PagedOutputStream outputStreamSize2(fileout); - outputStreamSize2.setSizeOnly(true); - attrA2.writePagedBuffers(outputStreamSize2, false); - outputStreamSize2.flush(); - compression::PagedOutputStream outputStream2(fileout); - outputStream2.setSizeOnly(false); - attrA2.writePagedBuffers(outputStream2, false); - outputStream2.flush(); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check manual loading of data - { - AttributeArrayI attrB; - AttributeArrayF attrB2; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(matchingNamePairs(attrA.type(), attrB.type())); - EXPECT_EQ(attrA.size(), attrB.size()); - EXPECT_EQ(attrA.isUniform(), attrB.isUniform()); - EXPECT_EQ(attrA.isTransient(), attrB.isTransient()); - EXPECT_EQ(attrA.isHidden(), attrB.isHidden()); - - AttributeArrayI attrBcopy(attrB); - AttributeArrayI attrBequal = attrB; - - EXPECT_TRUE(attrB.isOutOfCore()); - EXPECT_TRUE(attrBcopy.isOutOfCore()); - EXPECT_TRUE(attrBequal.isOutOfCore()); - - EXPECT_TRUE(!static_cast(attrB).isDataLoaded()); - EXPECT_TRUE(!static_cast(attrBcopy).isDataLoaded()); - EXPECT_TRUE(!static_cast(attrBequal).isDataLoaded()); - - attrB.loadData(); - attrBcopy.loadData(); - attrBequal.loadData(); - - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrBcopy.isOutOfCore()); - EXPECT_TRUE(!attrBequal.isOutOfCore()); - - EXPECT_TRUE(static_cast(attrB).isDataLoaded()); - EXPECT_TRUE(static_cast(attrBcopy).isDataLoaded()); - EXPECT_TRUE(static_cast(attrBequal).isDataLoaded()); - - EXPECT_EQ(attrA.memUsage(), attrB.memUsage()); - EXPECT_EQ(attrA.memUsage(), attrBcopy.memUsage()); - EXPECT_EQ(attrA.memUsage(), attrBequal.memUsage()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - EXPECT_EQ(attrA.get(i), attrBcopy.get(i)); - EXPECT_EQ(attrA.get(i), attrBequal.get(i)); - } - - attrB2.readMetadata(filein); - compression::PagedInputStream inputStream2(filein); - inputStream2.setSizeOnly(true); - attrB2.readPagedBuffers(inputStream2); - inputStream2.setSizeOnly(false); - attrB2.readPagedBuffers(inputStream2); - - EXPECT_TRUE(matchingNamePairs(attrA2.type(), attrB2.type())); - EXPECT_EQ(attrA2.size(), attrB2.size()); - EXPECT_EQ(attrA2.isUniform(), attrB2.isUniform()); - EXPECT_EQ(attrA2.isTransient(), attrB2.isTransient()); - EXPECT_EQ(attrA2.isHidden(), attrB2.isHidden()); - - AttributeArrayF attrB2copy(attrB2); - AttributeArrayF attrB2equal = attrB2; - - EXPECT_TRUE(attrB2.isOutOfCore()); - EXPECT_TRUE(attrB2copy.isOutOfCore()); - EXPECT_TRUE(attrB2equal.isOutOfCore()); - attrB2.loadData(); - attrB2copy.loadData(); - attrB2equal.loadData(); - - EXPECT_TRUE(!attrB2.isOutOfCore()); - EXPECT_TRUE(!attrB2copy.isOutOfCore()); - EXPECT_TRUE(!attrB2equal.isOutOfCore()); - - EXPECT_EQ(attrA2.memUsage(), attrB2.memUsage()); - EXPECT_EQ(attrA2.memUsage(), attrB2copy.memUsage()); - EXPECT_EQ(attrA2.memUsage(), attrB2equal.memUsage()); - - EXPECT_EQ(attrA2.get(0), attrB2.get(0)); - EXPECT_EQ(attrA2.get(0), attrB2copy.get(0)); - EXPECT_EQ(attrA2.get(0), attrB2equal.get(0)); - } - - // read in using delayed load and check fill() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - EXPECT_TRUE(!attrB.isUniform()); - - attrB.fill(5); - - EXPECT_TRUE(!attrB.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(5, attrB.get(i)); - } - } - - // read in using delayed load and check streaming (write handle) - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - EXPECT_TRUE(!attrB.isUniform()); - - attrB.setStreaming(true); - - { - AttributeWriteHandle handle(attrB); - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrB.isUniform()); - } - - EXPECT_TRUE(!attrB.isUniform()); - } - - // read in using delayed load and check streaming (read handle) - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - EXPECT_TRUE(!attrB.isUniform()); - - attrB.setStreaming(true); - - { - AttributeHandle handle(attrB); - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrB.isUniform()); - } - - EXPECT_TRUE(attrB.isUniform()); - } - - // read in using delayed load and check implicit load through get() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - attrB.get(0); - - EXPECT_TRUE(!attrB.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - } - } - - // read in using delayed load and check implicit load through compress() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - } - - // read in using delayed load and check copy and assignment constructors - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - AttributeArrayI attrC(attrB); - AttributeArrayI attrD = attrB; - - EXPECT_TRUE(attrB.isOutOfCore()); - EXPECT_TRUE(attrC.isOutOfCore()); - EXPECT_TRUE(attrD.isOutOfCore()); - - attrB.loadData(); - attrC.loadData(); - attrD.loadData(); - - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrC.isOutOfCore()); - EXPECT_TRUE(!attrD.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - EXPECT_EQ(attrA.get(i), attrC.get(i)); - EXPECT_EQ(attrA.get(i), attrD.get(i)); - } - } - - // read in using delayed load and check implicit load through AttributeHandle - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - AttributeHandle handle(attrB); - - EXPECT_TRUE(!attrB.isOutOfCore()); - } - - // read in using delayed load and check detaching of file (using collapse()) - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - EXPECT_TRUE(!attrB.isUniform()); - - attrB.collapse(); - - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(attrB.isUniform()); - - EXPECT_EQ(0, attrB.get(0)); - } - - // read in and write out using delayed load to check writing out-of-core attributes - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - std::string filename2 = tempDir + "/openvdb_delayed5"; - std::ofstream fileout2(filename2.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout2, streamMetadata); - io::setDataCompression(fileout2, io::COMPRESS_BLOSC); - - attrB.writeMetadata(fileout2, false, /*paged=*/true); - compression::PagedOutputStream outputStreamSize(fileout2); - outputStreamSize.setSizeOnly(true); - attrB.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout2); - outputStream.setSizeOnly(false); - attrB.writePagedBuffers(outputStream, false); - outputStream.flush(); - - fileout2.close(); - - AttributeArrayI attrB2; - - std::ifstream filein2(filename2.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein2, streamMetadata); - io::setMappedFilePtr(filein2, mappedFile); - - attrB2.readMetadata(filein2); - compression::PagedInputStream inputStream2(filein2); - inputStream2.setSizeOnly(true); - attrB2.readPagedBuffers(inputStream2); - inputStream2.setSizeOnly(false); - attrB2.readPagedBuffers(inputStream2); - - EXPECT_TRUE(attrB2.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrB.get(i), attrB2.get(i)); - } - - filein2.close(); - } - - // Clean up temp files. - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - - AttributeArrayI attrUniform(count); - - // write out uniform attribute array to a temp file - { - filename = tempDir + "/openvdb_delayed2"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - attrUniform.writeMetadata(fileout, false, /*paged=*/true); - - compression::PagedOutputStream outputStreamSize(fileout); - outputStreamSize.setSizeOnly(true); - attrUniform.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout); - outputStream.setSizeOnly(false); - attrUniform.writePagedBuffers(outputStream, false); - outputStream.flush(); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check fill() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isUniform()); - - attrB.fill(5); - - EXPECT_TRUE(attrB.isUniform()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(5, attrB.get(i)); - } - } - - AttributeArrayI attrStrided(count, /*stride=*/3); - - EXPECT_EQ(Index(3), attrStrided.stride()); - - // Clean up temp files. - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - - // write out strided attribute array to a temp file - { - filename = tempDir + "/openvdb_delayed3"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - attrStrided.writeMetadata(fileout, false, /*paged=*/true); - - compression::PagedOutputStream outputStreamSize(fileout); - outputStreamSize.setSizeOnly(true); - attrStrided.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout); - outputStream.setSizeOnly(false); - attrStrided.writePagedBuffers(outputStream, false); - outputStream.flush(); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check fill() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_EQ(Index(3), attrB.stride()); - } - - // Clean up temp files. - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - - // write out compressed attribute array to a temp file - { - filename = tempDir + "/openvdb_delayed4"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - attrA.writeMetadata(fileout, false, /*paged=*/true); - - compression::PagedOutputStream outputStreamSize(fileout); - outputStreamSize.setSizeOnly(true); - attrA.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout); - outputStream.setSizeOnly(false); - attrA.writePagedBuffers(outputStream, false); - outputStream.flush(); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check manual loading of data - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - attrB.loadData(); - EXPECT_TRUE(!attrB.isOutOfCore()); - - EXPECT_EQ(attrA.memUsage(), attrB.memUsage()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - } - } - - // read in using delayed load and check partial read state - { - std::unique_ptr attrB(new AttributeArrayI); - - EXPECT_TRUE(!(attrB->flags() & AttributeArray::PARTIALREAD)); - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB->readMetadata(filein); - - // PARTIALREAD flag should now be set - EXPECT_TRUE(attrB->flags() & AttributeArray::PARTIALREAD); - - // copy-construct and assign AttributeArray - AttributeArrayI attrC(*attrB); - EXPECT_TRUE(attrC.flags() & AttributeArray::PARTIALREAD); - AttributeArrayI attrD = *attrB; - EXPECT_TRUE(attrD.flags() & AttributeArray::PARTIALREAD); - - // verify deleting attrB is safe - attrB.reset(); - - // verify data is not valid - EXPECT_TRUE(!attrC.validData()); - - { // attempting to write a partially-read AttributeArray throws - std::string filename = tempDir + "/openvdb_partial1"; - ScopedFile f(filename); - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - EXPECT_THROW(attrC.writeMetadata(fileout, false, /*paged=*/true), IoError); - } - - // continue loading with copy-constructed AttributeArray - - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrC.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrC.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrC.isOutOfCore()); - attrC.loadData(); - EXPECT_TRUE(!attrC.isOutOfCore()); - - // verify data is now valid - EXPECT_TRUE(attrC.validData()); - - EXPECT_EQ(attrA.memUsage(), attrC.memUsage()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrC.get(i)); - } - } - - // read in using delayed load and check implicit load through get() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - attrB.get(0); - - EXPECT_TRUE(!attrB.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - } - } - -#ifdef OPENVDB_USE_BLOSC - // read in using delayed load and check copy and assignment constructors - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - AttributeArrayI attrC(attrB); - AttributeArrayI attrD = attrB; - - EXPECT_TRUE(attrB.isOutOfCore()); - EXPECT_TRUE(attrC.isOutOfCore()); - EXPECT_TRUE(attrD.isOutOfCore()); - - attrB.loadData(); - attrC.loadData(); - attrD.loadData(); - - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrC.isOutOfCore()); - EXPECT_TRUE(!attrD.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - EXPECT_EQ(attrA.get(i), attrC.get(i)); - EXPECT_EQ(attrA.get(i), attrD.get(i)); - } - } - - // read in using delayed load and check implicit load through AttributeHandle - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - AttributeHandle handle(attrB); - - EXPECT_TRUE(!attrB.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), handle.get(i)); - } - } -#endif - - // Clean up temp files. - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - - // write out invalid serialization flags as metadata to a temp file - { - filename = tempDir + "/openvdb_delayed5"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - // write out unknown serialization flags to check forwards-compatibility - - Index64 bytes(0); - uint8_t flags(0); - uint8_t serializationFlags(Int16(0x10)); - Index size(0); - - fileout.write(reinterpret_cast(&bytes), sizeof(Index64)); - fileout.write(reinterpret_cast(&flags), sizeof(uint8_t)); - fileout.write(reinterpret_cast(&serializationFlags), sizeof(uint8_t)); - fileout.write(reinterpret_cast(&size), sizeof(Index)); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check metadata fail due to serialization flags - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - EXPECT_THROW(attrB.readMetadata(filein), openvdb::IoError); - } - - // cleanup temp files - - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - } -} -TEST_F(TestAttributeArray, testDelayedLoad) { testDelayedLoad(); } -#endif - TEST_F(TestAttributeArray, testDefaultValue) { diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc new file mode 100644 index 0000000000..cfa14de91e --- /dev/null +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -0,0 +1,514 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include + +class TestCodec: public ::testing::Test +{ +public: + void SetUp() override { openvdb::initialize(); } + void TearDown() override { openvdb::uninitialize(); } +}; + +struct MockCodec : public openvdb::io::Codec +{ + static std::string name() { return "mock"; } + + openvdb::io::CodecData::Ptr createData() final { return nullptr; } +}; + +TEST_F(TestCodec, testCodecRegistry) +{ + using namespace openvdb::io; + + // Start clean + CodecRegistry::clear(); + + // Test isRegistered on empty registry + EXPECT_FALSE(CodecRegistry::isRegistered("mock")); + + // Test registerCodecByName + EXPECT_NO_THROW( + CodecRegistry::registerCodecByName("mock", std::make_unique()) + ); + + EXPECT_TRUE(CodecRegistry::isRegistered("mock")); + EXPECT_FALSE(CodecRegistry::isRegistered("nonexistent")); + + // Test duplicate registration throws KeyError + EXPECT_THROW( + CodecRegistry::registerCodecByName("mock", std::make_unique()), + openvdb::KeyError + ); + + // Test registerCodec template form also throws on duplicate + EXPECT_THROW( + CodecRegistry::registerCodec(), + openvdb::KeyError + ); + + // Test get + EXPECT_NE(CodecRegistry::get("mock"), nullptr); + EXPECT_EQ(CodecRegistry::get("nonexistent"), nullptr); + + // Test clear + CodecRegistry::clear(); + EXPECT_FALSE(CodecRegistry::isRegistered("mock")); + EXPECT_NO_THROW(CodecRegistry::clear()); // Clear on empty registry + + // Test registerCodec template form on fresh registry + EXPECT_NO_THROW(CodecRegistry::registerCodec()); + EXPECT_TRUE(CodecRegistry::isRegistered("mock")); + + // Test io::initialize and io::uninitialize + CodecRegistry::clear(); + EXPECT_FALSE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType())); + + EXPECT_NO_THROW(internal::initialize()); + EXPECT_TRUE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType())); + + EXPECT_NO_THROW(internal::uninitialize()); + EXPECT_FALSE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType())); +} + + +TEST_F(TestCodec, testInitializeIdempotent) +{ + using namespace openvdb::io; + + // Calling initialize() twice without uninitialize() in between must not throw. + // Previously registerCodecByName() threw KeyError on the duplicate registration. + CodecRegistry::clear(); + EXPECT_NO_THROW(internal::initialize()); + EXPECT_NO_THROW(internal::initialize()); + + // Codecs must still be registered after the second call. + EXPECT_TRUE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType())); + EXPECT_TRUE(CodecRegistry::isRegistered(openvdb::FloatGrid::gridType())); + + internal::uninitialize(); +} + + +TEST_F(TestCodec, testReadDiagnostics) +{ + using namespace openvdb; + using namespace openvdb::io; + + // ReadDiagnostics struct: disabled by default, addWarning is a no-op until enabled + { + ReadDiagnostics diags; + EXPECT_FALSE(diags.enabled()); + diags.addWarning("grid_a", "something went wrong"); + EXPECT_TRUE(diags.diagnostics().empty()); + + diags.enable(); + diags.addWarning("grid_a", "something went wrong"); + ASSERT_EQ(diags.diagnostics().size(), size_t(1)); + EXPECT_EQ(diags.diagnostics()[0].severity, DiagnosticSeverity::Warning); + + diags.clear(); + EXPECT_TRUE(diags.diagnostics().empty()); + } + + CodecRegistry::clear(); + openvdb::io::internal::initialize(); + + // Archive API and getGrids() with diagnostics + + BoolGrid::Ptr srcGrid = BoolGrid::create(false); + srcGrid->setName("bool_grid"); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), true, true); + + const std::string codecPath = "testReadDiagnostics.vdb"; + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + // Disabled by default; enabling produces no warnings on a clean read + { + io::File f(codecPath); + f.open(); + EXPECT_FALSE(f.readDiagnostics().enabled()); + f.enableReadDiagnostics(); + EXPECT_TRUE(f.readDiagnostics().enabled()); + f.readGrid("bool_grid"); + EXPECT_TRUE(f.readDiagnostics().diagnostics().empty()); + f.close(); + } + + // clearReadDiagnostics() resets entries but keeps diagnostics enabled + { + io::File f(codecPath); + f.open(); + f.enableReadDiagnostics(); + GridPtrVecPtr grids = f.getGrids(); + ASSERT_TRUE(grids && !grids->empty()); + f.clearReadDiagnostics(); + EXPECT_TRUE(f.readDiagnostics().enabled()); + EXPECT_TRUE(f.readDiagnostics().diagnostics().empty()); + f.close(); + } + + std::remove(codecPath.c_str()); +} + + +template +void testIOImpl( + const std::string& gridName, + const typename GridT::ValueType& bgValue, + const typename GridT::ValueType& fillValue) +{ + using namespace openvdb; + using namespace openvdb::io; + + typename GridT::Ptr srcGrid = GridT::create(bgValue); + srcGrid->setName(gridName); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), fillValue, true); + + std::stringstream ss("test"); + if (CodecRegistry::isRegistered(GridT::gridType())) { + ss << "_codec"; + } else { + ss << "_tree"; + } + ss << "_" << GridT::gridType() << ".vdb"; + const std::string path = ss.str(); + { + io::File f(path); + f.write(GridPtrVec{srcGrid}); + } + + typename GridT::Ptr readGrid; + { + io::File f(path); + f.open(); + readGrid = gridPtrCast(f.readGrid(gridName)); + f.close(); + } + ASSERT_TRUE(readGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(readGrid->tree())); + { + auto readAcc = readGrid->getConstAccessor(); + for (typename GridT::ValueOnCIter it = srcGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, readAcc.getValue(it.getCoord())); + } + } + + // clip read + const BBoxd clipBBox(Vec3d(0.0), Vec3d(3.5)); + auto srcClipped = tools::clip(*srcGrid, clipBBox); + + typename GridT::Ptr readClipped; + { + io::File f(path); + f.open(); + readClipped = gridPtrCast(f.readGrid(gridName, clipBBox)); + f.close(); + } + ASSERT_TRUE(readClipped); + EXPECT_TRUE(srcClipped->tree().hasSameTopology(readClipped->tree())); + { + auto readAcc = readClipped->getConstAccessor(); + for (typename GridT::ValueOnCIter it = srcClipped->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, readAcc.getValue(it.getCoord())); + } + } + + // topology-only read + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + typename GridT::Ptr readTopo; + { + io::File f(path); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid(gridName, topoOpts)); + readTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(readTopo); + // TopologyOnly: full tree structure is read (topology + active-voxel masks), + // leaf buffers are allocated and zero-filled, values are not read. + EXPECT_EQ(readTopo->tree().leafCount(), srcGrid->tree().leafCount()); + EXPECT_TRUE(readTopo->tree().leafCount() > 0); + EXPECT_EQ(readTopo->activeVoxelCount(), srcGrid->activeVoxelCount()); + // verify leaf buffers are allocated (bool/mask buffers are always present, skip empty() check) + if constexpr (!std::is_same_v) { + for (auto leafIter = readTopo->tree().cbeginLeaf(); leafIter; ++leafIter) { + EXPECT_FALSE(leafIter->buffer().empty()); + } + } + EXPECT_EQ(readTopo->getName(), gridName); + + // Cleanup + std::remove(path.c_str()); +} + +template +void testCodecIOImpl( + const std::string& gridName, + const typename GridT::ValueType& bgValue, + const typename GridT::ValueType& fillValue) +{ + // initialize to register all the codecs + openvdb::io::CodecRegistry::clear(); + openvdb::io::internal::initialize(); + // ensure the codec is registered + ASSERT_TRUE(openvdb::io::CodecRegistry::isRegistered(GridT::gridType())); + // test the io implementation (codec) + testIOImpl(gridName, bgValue, fillValue); + // clear the codec registry (now read/write falls back to Tree I/O) + openvdb::io::CodecRegistry::clear(); + // ensure the codec is not registered + ASSERT_FALSE(openvdb::io::CodecRegistry::isRegistered(GridT::gridType())); + // test the io implementation (tree I/O) + testIOImpl(gridName, bgValue, fillValue); +} + +TEST_F(TestCodec, testFloatCodecIO) { testCodecIOImpl("float_grid", 0.0f, 1.0f); } +TEST_F(TestCodec, testDoubleCodecIO) { testCodecIOImpl("double_grid", 0.0, 1.0); } +TEST_F(TestCodec, testInt32CodecIO) { testCodecIOImpl("int32_grid", 0, 1); } +TEST_F(TestCodec, testInt64CodecIO) { testCodecIOImpl("int64_grid", openvdb::Int64(0), openvdb::Int64(1)); } +TEST_F(TestCodec, testHalfCodecIO) { testCodecIOImpl("half_grid", openvdb::Half(0.0), openvdb::Half(1.5)); } +TEST_F(TestCodec, testVec3ICodecIO) { testCodecIOImpl("vec3i_grid", openvdb::Vec3i(0), openvdb::Vec3i(1, 2, 3)); } +TEST_F(TestCodec, testVec3SCodecIO) { testCodecIOImpl("vec3s_grid", openvdb::Vec3s(0.0f), openvdb::Vec3s(1.0f, 2.0f, 3.0f)); } +TEST_F(TestCodec, testVec3DCodecIO) { testCodecIOImpl("vec3d_grid", openvdb::Vec3d(0.0), openvdb::Vec3d(1.0, 2.0, 3.0)); } +TEST_F(TestCodec, testBoolCodecIO) { testCodecIOImpl("bool_grid", false, true); } +TEST_F(TestCodec, testMaskCodecIO) { testCodecIOImpl("mask_grid", false, true); } + +TEST_F(TestCodec, testFloatToHalfCodecConversion) +{ + using namespace openvdb; + using namespace openvdb::io; + + openvdb::initialize(); + CodecRegistry::clear(); + + // Verify the conversion codec name + const std::string expectedName = FloatGrid::gridType() + "_to_half"; + EXPECT_EQ((codecs::ScalarCodec::name()), + expectedName); + + // Verify the codec is registered after initialize() + io::internal::initialize(); + EXPECT_TRUE(CodecRegistry::isRegistered(expectedName)); + + // Write a FloatGrid with a known fill value (1.5f is exactly representable in half) + const std::string floatPath = "test_float_to_half.vdb"; + const std::string gridName = "float_grid"; + FloatGrid::Ptr srcGrid = FloatGrid::create(0.0f); + srcGrid->setName(gridName); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + { + io::File f(floatPath); + f.write(GridPtrVec{srcGrid}); + } + + // Read back with ReadMode::Half — triggers the float-to-half conversion codec + ReadOptions halfOpts; + halfOpts.readMode = ReadMode::Half; + + GridBase::Ptr base; + { + io::File f(floatPath); + f.open(); + base = f.readGrid(gridName, halfOpts); + f.close(); + } + ASSERT_TRUE(base); + + // The returned grid must be a HalfGrid + EXPECT_TRUE(base->isType()); + HalfGrid::Ptr halfGrid = gridPtrCast(base); + ASSERT_TRUE(halfGrid); + + // Topology must match the source FloatGrid + EXPECT_TRUE(srcGrid->tree().hasSameTopology(halfGrid->tree())); + + // All active voxel values must equal Half(1.5f) + for (HalfGrid::ValueOnCIter it = halfGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, Half(1.5f)); + } + + // Background value must equal Half(0.0f) + EXPECT_EQ(halfGrid->background(), Half(0.0f)); + + // Cleanup + std::remove(floatPath.c_str()); +} + +template +void testConvertCodecImpl() +{ + using namespace openvdb; + using namespace openvdb::io; + + openvdb::io::CodecRegistry::clear(); + openvdb::io::internal::initialize(); + + // Verify the conversion codec name + const std::string expectedName = + SrcGridT::gridType() + "_to_" + typeNameAsString(); + EXPECT_EQ((codecs::ScalarCodec::name()), + expectedName); + + // Verify the codec is registered after initialize() + EXPECT_TRUE(CodecRegistry::isRegistered(expectedName)); + + // Write a SrcGridT with a non-zero fill value + const std::string testPath = "test_" + expectedName + ".vdb"; + const std::string gridName = SrcGridT::gridType(); + typename SrcGridT::Ptr srcGrid = + SrcGridT::create(typename SrcGridT::ValueType(0)); + srcGrid->setName(gridName); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), + typename SrcGridT::ValueType(1), true); + + { + io::File f(testPath); + f.write(GridPtrVec{srcGrid}); + } + + // Read back with the specified ReadMode — triggers the conversion codec + ReadOptions opts; + opts.readMode = mode; + + GridBase::Ptr base; + { + io::File f(testPath); + f.open(); + base = f.readGrid(gridName, opts); + f.close(); + } + ASSERT_TRUE(base); + + // The returned grid must be a DstGridT + EXPECT_TRUE(base->isType()); + typename DstGridT::Ptr dstGrid = gridPtrCast(base); + ASSERT_TRUE(dstGrid); + + // Topology must match the source grid + EXPECT_TRUE(srcGrid->tree().hasSameTopology(dstGrid->tree())); + + // All active voxel values must be true + for (typename DstGridT::ValueOnCIter it = dstGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, typename DstGridT::ValueType(true)); + } + + // Background value must be false + EXPECT_EQ(dstGrid->background(), typename DstGridT::ValueType(false)); + + // Cleanup + std::remove(testPath.c_str()); +} + +TEST_F(TestCodec, testNumericToBoolCodecConversion) +{ + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); +} + +TEST_F(TestCodec, testNumericToMaskCodecConversion) +{ + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); +} + +// Regression test for the dangling storageBackground pointer bug. +// +// ReadTopologyOp stores storageBackground on its stack frame and registers +// &storageBackground with the stream. Before the fix, topologyCodecReadTopology +// returned and destroyed ReadTopologyOp before readBuffers() ran; readCompressedValues +// then dereferenced the dead pointer to reconstruct inactive voxels under +// COMPRESS_ACTIVE_MASK, producing garbage inactive values. +// +// The test is deliberately structured to maximize the chance that the freed +// stack frame has been overwritten: a non-zero background (3.0f / 5) forces the +// reconstructed inactive value to be wrong if the pointer is stale, and +// COMPRESS_ACTIVE_MASK (flag 0x2, always on by default) is the code path that +// uses the background pointer. +TEST_F(TestCodec, testInactiveValuesAfterReadBuffers) +{ + using namespace openvdb; + using namespace openvdb::io; + + openvdb::io::CodecRegistry::clear(); + openvdb::io::internal::initialize(); + + // Float: non-zero background, active region surrounded by inactive background voxels. + { + const float bg = 3.0f; + FloatGrid::Ptr src = FloatGrid::create(bg); + src->setName("float_bg"); + src->fill(CoordBBox(Coord(0), Coord(15)), 1.0f, /*active=*/true); + src->fill(CoordBBox(Coord(4), Coord(11)), bg, /*active=*/false); + + const std::string path = "testInactiveVals_float.vdb"; + { + io::File f(path); + f.setCompression(COMPRESS_ACTIVE_MASK); + f.write(GridPtrVec{src}); + } + FloatGrid::Ptr result; + { + io::File f(path); + f.open(); + result = gridPtrCast(f.readGrid("float_bg")); + f.close(); + } + ASSERT_TRUE(result); + EXPECT_EQ(result->background(), bg); + FloatGrid::ConstAccessor refAcc = src->getConstAccessor(); + for (FloatGrid::ValueAllCIter it = result->cbeginValueAll(); it; ++it) { + EXPECT_EQ(*it, refAcc.getValue(it.getCoord())); + } + std::remove(path.c_str()); + } + + // Int32: non-zero background (5), verify inactive values round-trip. + { + const int bg = 5; + Int32Grid::Ptr src = Int32Grid::create(bg); + src->setName("int_bg"); + src->fill(CoordBBox(Coord(0), Coord(15)), 99, /*active=*/true); + src->fill(CoordBBox(Coord(4), Coord(11)), bg, /*active=*/false); + + const std::string path = "testInactiveVals_int.vdb"; + { + io::File f(path); + f.setCompression(COMPRESS_ACTIVE_MASK); + f.write(GridPtrVec{src}); + } + Int32Grid::Ptr result; + { + io::File f(path); + f.open(); + result = gridPtrCast(f.readGrid("int_bg")); + f.close(); + } + ASSERT_TRUE(result); + EXPECT_EQ(result->background(), bg); + Int32Grid::ConstAccessor refAcc = src->getConstAccessor(); + for (Int32Grid::ValueAllCIter it = result->cbeginValueAll(); it; ++it) { + EXPECT_EQ(*it, refAcc.getValue(it.getCoord())); + } + std::remove(path.c_str()); + } +} diff --git a/openvdb/openvdb/unittest/TestCount.cc b/openvdb/openvdb/unittest/TestCount.cc index 3ce3f5227b..201ab6efa6 100644 --- a/openvdb/openvdb/unittest/TestCount.cc +++ b/openvdb/openvdb/unittest/TestCount.cc @@ -8,7 +8,6 @@ #include // tools::createLevelSetSphere #include // tools::sdfToFogVolume #include -#include class TestCount: public ::testing::Test @@ -224,8 +223,7 @@ TEST_F(TestCount, testMemUsage) internalNodeMemUsage += sizeof(Coord); for (auto leafIter = internal2Iter->cbeginChildOn(); leafIter; ++leafIter) { - EXPECT_EQ(leafIter->memUsage(), leafIter->memUsageIfLoaded()); - expectedMaxMem += leafIter->memUsageIfLoaded(); + expectedMaxMem += leafIter->memUsage(); ++leafCount; } } @@ -233,54 +231,9 @@ TEST_F(TestCount, testMemUsage) expectedMaxMem += internalNodeMemUsage; - Index64 inCoreMemUsage = tools::memUsage(grid->tree()); - Index64 memUsageIfLoaded = tools::memUsageIfLoaded(grid->tree()); + Index64 memUsage = tools::memUsage(grid->tree()); - EXPECT_EQ(expectedMaxMem, inCoreMemUsage); - EXPECT_EQ(expectedMaxMem, memUsageIfLoaded); - -#ifdef OPENVDB_USE_DELAYED_LOADING - // Write out the grid and read it in with delay-loading. Check the - // expected memory usage values.] - - openvdb::initialize(); - - std::string filename; - - // write out grid to a temp file - { - io::TempFile file; - filename = file.filename(); - io::File fileOut(filename); - fileOut.write({grid}); - } - - io::File fileIn(filename); - fileIn.open(true); // delay-load - auto grids = fileIn.getGrids(); - fileIn.close(); - - grid = GridBase::grid((*grids)[0]); - EXPECT_TRUE(grid); - - inCoreMemUsage = tools::memUsage(grid->tree()); - memUsageIfLoaded = tools::memUsageIfLoaded(grid->tree()); - - EXPECT_EQ(expectedMaxMem, memUsageIfLoaded); - EXPECT_TRUE(inCoreMemUsage < expectedMaxMem); - - // in core memory should be the max memory without the leaf buffers but - // with the FileInfo - - const Index64 leafBuffers = sizeof(FloatGrid::ValueType) * FloatTree::LeafNodeType::SIZE; - const Index64 fileInfo = sizeof(FloatTree::LeafNodeType::Buffer::FileInfo); - const Index64 expectedInCoreMemUsage = expectedMaxMem + (leafCount * (-leafBuffers + fileInfo)); - EXPECT_EQ(expectedInCoreMemUsage, inCoreMemUsage); - - std::remove(filename.c_str()); - - openvdb::uninitialize(); -#endif + EXPECT_EQ(expectedMaxMem, memUsage); } diff --git a/openvdb/openvdb/unittest/TestDelayedLoadMetadata.cc b/openvdb/openvdb/unittest/TestDelayedLoadMetadata.cc deleted file mode 100644 index d1617863e8..0000000000 --- a/openvdb/openvdb/unittest/TestDelayedLoadMetadata.cc +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include - -#include - -class TestDelayedLoadMetadata : public ::testing::Test -{ -}; - - -TEST_F(TestDelayedLoadMetadata, test) -{ - using namespace openvdb::io; - - // registration - - EXPECT_TRUE(!DelayedLoadMetadata::isRegisteredType()); - - DelayedLoadMetadata::registerType(); - - EXPECT_TRUE(DelayedLoadMetadata::isRegisteredType()); - - DelayedLoadMetadata::unregisterType(); - - EXPECT_TRUE(!DelayedLoadMetadata::isRegisteredType()); - - openvdb::initialize(); - - EXPECT_TRUE(DelayedLoadMetadata::isRegisteredType()); - - // construction - - DelayedLoadMetadata metadata; - - EXPECT_TRUE(metadata.empty()); - - metadata.resizeMask(size_t(2)); - - EXPECT_TRUE(!metadata.empty()); - - metadata.setMask(0, DelayedLoadMetadata::MaskType(5)); - metadata.setMask(1, DelayedLoadMetadata::MaskType(-3)); - - EXPECT_EQ(metadata.getMask(0), DelayedLoadMetadata::MaskType(5)); - EXPECT_EQ(metadata.getMask(1), DelayedLoadMetadata::MaskType(-3)); - - metadata.resizeCompressedSize(size_t(3)); - - metadata.setCompressedSize(0, DelayedLoadMetadata::CompressedSizeType(6)); - metadata.setCompressedSize(1, DelayedLoadMetadata::CompressedSizeType(101)); - metadata.setCompressedSize(2, DelayedLoadMetadata::CompressedSizeType(-13522)); - - EXPECT_EQ(metadata.getCompressedSize(0), DelayedLoadMetadata::CompressedSizeType(6)); - EXPECT_EQ(metadata.getCompressedSize(1), DelayedLoadMetadata::CompressedSizeType(101)); - EXPECT_EQ(metadata.getCompressedSize(2), DelayedLoadMetadata::CompressedSizeType(-13522)); - - // copy construction - - DelayedLoadMetadata metadataCopy1(metadata); - - EXPECT_TRUE(!metadataCopy1.empty()); - - EXPECT_EQ(metadataCopy1.getMask(0), DelayedLoadMetadata::MaskType(5)); - EXPECT_EQ(metadataCopy1.getCompressedSize(2), DelayedLoadMetadata::CompressedSizeType(-13522)); - - openvdb::Metadata::Ptr baseMetadataCopy2 = metadata.copy(); - DelayedLoadMetadata::Ptr metadataCopy2 = - openvdb::StaticPtrCast(baseMetadataCopy2); - - EXPECT_EQ(metadataCopy2->getMask(0), DelayedLoadMetadata::MaskType(5)); - EXPECT_EQ(metadataCopy2->getCompressedSize(2), DelayedLoadMetadata::CompressedSizeType(-13522)); - - // I/O - - metadata.clear(); - EXPECT_TRUE(metadata.empty()); - - const size_t headerInitialSize(sizeof(openvdb::Index32)); - const size_t headerCountSize(sizeof(openvdb::Index32)); - const size_t headerMaskSize(sizeof(openvdb::Index32)); - const size_t headerCompressedSize(sizeof(openvdb::Index32)); - const size_t headerTotalSize(headerInitialSize + headerCountSize + headerMaskSize + headerCompressedSize); - - { // empty buffer - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - metadata.write(ss); - EXPECT_EQ(ss.tellp(), std::streampos(headerInitialSize)); - - DelayedLoadMetadata newMetadata; - newMetadata.read(ss); - EXPECT_TRUE(newMetadata.empty()); - } - - { // single value, no compressed sizes - metadata.clear(); - metadata.resizeMask(size_t(1)); - metadata.setMask(0, DelayedLoadMetadata::MaskType(5)); - - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - metadata.write(ss); - std::streampos expectedPos(headerTotalSize + sizeof(int8_t)); - EXPECT_EQ(ss.tellp(), expectedPos); - EXPECT_EQ(static_cast(expectedPos)-headerInitialSize, size_t(metadata.size())); - - DelayedLoadMetadata newMetadata; - newMetadata.read(ss); - EXPECT_TRUE(!newMetadata.empty()); - EXPECT_EQ(newMetadata.getMask(0), DelayedLoadMetadata::MaskType(5)); - } - - { // single value, with compressed sizes - metadata.clear(); - metadata.resizeMask(size_t(1)); - metadata.setMask(0, DelayedLoadMetadata::MaskType(5)); - - metadata.resizeCompressedSize(size_t(1)); - metadata.setCompressedSize(0, DelayedLoadMetadata::CompressedSizeType(-10322)); - - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - metadata.write(ss); - std::streampos expectedPos(headerTotalSize + sizeof(int8_t) + sizeof(int64_t)); - - EXPECT_EQ(expectedPos, ss.tellp()); - EXPECT_EQ(static_cast(ss.tellp())-headerInitialSize, size_t(metadata.size())); - - DelayedLoadMetadata newMetadata; - newMetadata.read(ss); - EXPECT_TRUE(!newMetadata.empty()); - EXPECT_EQ(newMetadata.getMask(0), DelayedLoadMetadata::MaskType(5)); - EXPECT_EQ(newMetadata.getCompressedSize(0), DelayedLoadMetadata::CompressedSizeType(-10322)); - } - - { // larger, but compressible buffer - metadata.clear(); - - const size_t size = 1000; - - const size_t uncompressedBufferSize = (sizeof(int8_t)+sizeof(int64_t))*size; - - metadata.resizeMask(size); - metadata.resizeCompressedSize(size); - for (size_t i = 0; i < size; i++) { - metadata.setMask(i, - DelayedLoadMetadata::MaskType(static_cast((i%32)*2))); - metadata.setCompressedSize(i, - DelayedLoadMetadata::CompressedSizeType(static_cast((i%64)*200))); - } - - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - metadata.write(ss); - - EXPECT_EQ(static_cast(ss.tellp())-headerInitialSize, size_t(metadata.size())); - - std::streampos uncompressedSize(uncompressedBufferSize + headerTotalSize); -#ifdef OPENVDB_USE_BLOSC - // expect a compression ratio of more than 10x - EXPECT_TRUE(ss.tellp() * 10 < uncompressedSize); -#else - EXPECT_TRUE(ss.tellp() == uncompressedSize); -#endif - - DelayedLoadMetadata newMetadata; - newMetadata.read(ss); - EXPECT_EQ(metadata.size(), newMetadata.size()); - for (size_t i = 0; i < size; i++) { - EXPECT_EQ(metadata.getMask(i), newMetadata.getMask(i)); - } - } - - // when read as unknown metadata should be treated as temporary metadata - - { - metadata.clear(); - metadata.resizeMask(size_t(1)); - metadata.setMask(0, DelayedLoadMetadata::MaskType(5)); - - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - - openvdb::MetaMap metamap; - metamap.insertMeta("delayload", metadata); - - EXPECT_EQ(size_t(1), metamap.metaCount()); - - metamap.writeMeta(ss); - - { - openvdb::MetaMap newMetamap; - newMetamap.readMeta(ss); - - EXPECT_EQ(size_t(1), newMetamap.metaCount()); - } - - { - DelayedLoadMetadata::unregisterType(); - - openvdb::MetaMap newMetamap; - newMetamap.readMeta(ss); - - EXPECT_EQ(size_t(0), newMetamap.metaCount()); - } - } -} diff --git a/openvdb/openvdb/unittest/TestFastSweeping.cc b/openvdb/openvdb/unittest/TestFastSweeping.cc index 9bddbf7fbf..226aa35cd8 100644 --- a/openvdb/openvdb/unittest/TestFastSweeping.cc +++ b/openvdb/openvdb/unittest/TestFastSweeping.cc @@ -252,7 +252,7 @@ TEST_F(TestFastSweeping, testMaskSdf) openvdb::initialize();//required whenever I/O of OpenVDB files is performed! const std::string path(TestFastSweeping_DATA_PATH); io::File file( path + "bunny.vdb" ); - file.open(false);//disable delayed loading + file.open(); FloatGrid::Ptr mask = openvdb::gridPtrCast(file.getGrids()->at(0)); //this->writeFile("/tmp/bunny_mask_input.vdb", grid); @@ -595,7 +595,7 @@ TEST_F(TestFastSweeping, velocityExtensionOfFogBunny) openvdb::initialize();//required whenever I/O of OpenVDB files is performed! const std::string path(TestFastSweeping_DATA_PATH); io::File file( path + "bunny.vdb" ); - file.open(false);//disable delayed loading + file.open(); auto grid = openvdb::gridPtrCast(file.getGrids()->at(0)); tools::sdfToFogVolume(*grid); writeFile("/tmp/bunny1_fog_in.vdb", grid); @@ -619,7 +619,7 @@ TEST_F(TestFastSweeping, velocityExtensionOfSdfBunny) using namespace openvdb; const std::string path(TestFastSweeping_DATA_PATH); io::File file( path + "bunny.vdb" ); - file.open(false);//disable delayed loading + file.open(); auto grid = openvdb::gridPtrCast(file.getGrids()->at(0)); writeFile("/tmp/bunny2_sdf_in.vdb", grid); auto bbox = grid->evalActiveVoxelBoundingBox(); diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index cdeb8c8114..9aa7693d6a 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -44,7 +44,7 @@ class TestFile: public ::testing::Test { public: - void SetUp() override {} + void SetUp() override { openvdb::initialize(); } void TearDown() override { openvdb::uninitialize(); } void testHeader(); @@ -53,7 +53,6 @@ class TestFile: public ::testing::Test void testReadGridDescriptors(); void testEmptyGridIO(); void testOpen(); - void testDelayedLoadMetadata(); void testNonVdbOpen(); }; @@ -127,8 +126,6 @@ TestFile::testWriteGrid() tree.setValue(Coord(0, 0, 0), 5); // Add some metadata. - Metadata::clearRegistry(); - StringMetadata::registerType(); const std::string meta0Val, meta1Val("Hello, world."); Metadata::Ptr stringMetadata = Metadata::createMetadata(typeNameAsString()); EXPECT_TRUE(stringMetadata); @@ -157,43 +154,14 @@ TestFile::testWriteGrid() // it doesn't have a header), set the file format version number explicitly. io::setCurrentVersion(istr); - GridBase::Ptr gd2_grid; - EXPECT_THROW(gd2.read(istr), openvdb::LookupError); - - // Register the grid and the transform and the blocks. - GridBase::clearRegistry(); - GridType::registerGrid(); + istr.seekg(0, std::ios_base::beg); + gd2.readHeader(istr); + gd2.readStreamPos(istr); - // Register transform maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); + GridBase::Ptr gd2_grid = Archive::readGrid(gd2, istr); - istr.seekg(0, std::ios_base::beg); - EXPECT_NO_THROW(gd2_grid = gd2.read(istr)); - - EXPECT_EQ(gd.gridName(), gd2.gridName()); - EXPECT_EQ(GridType::gridType(), gd2_grid->type()); - EXPECT_EQ(gd.getGridPos(), gd2.getGridPos()); - EXPECT_EQ(gd.getBlockPos(), gd2.getBlockPos()); - EXPECT_EQ(gd.getEndPos(), gd2.getEndPos()); - - // Position the stream to beginning of the grid storage and read the grid. - gd2.seekToGrid(istr); - Archive::readGridCompression(istr); - gd2_grid->readMeta(istr); - gd2_grid->readTransform(istr); - gd2_grid->readTopology(istr); - - // Remove delay load metadata if it exists. - if ((*gd2_grid)["file_delayed_load"]) { - gd2_grid->removeMeta("file_delayed_load"); - } + // Delay load metadata should not exist. + ASSERT_FALSE(bool((*gd2_grid)["file_delayed_load"])); // Ensure that we have the same metadata. EXPECT_EQ(grid->metaCount(), gd2_grid->metaCount()); @@ -210,23 +178,12 @@ TestFile::testWriteGrid() EXPECT_EQ( grid->baseTree().treeDepth(), gd2_grid->baseTree().treeDepth()); - //EXPECT_EQ(0.1, gd2_grid->getTransform()->getVoxelSizeX()); - //EXPECT_EQ(0.1, gd2_grid->getTransform()->getVoxelSizeY()); - //EXPECT_EQ(0.1, gd2_grid->getTransform()->getVoxelSizeZ()); - - // Read in the data blocks. - gd2.seekToBlocks(istr); - gd2_grid->readBuffers(istr); TreeType::Ptr tree2 = DynamicPtrCast(gd2_grid->baseTreePtr()); EXPECT_TRUE(tree2.get() != nullptr); EXPECT_EQ(10, tree2->getValue(Coord(10, 1, 2))); EXPECT_EQ(5, tree2->getValue(Coord(0, 0, 0))); EXPECT_EQ(1, tree2->getValue(Coord(1000, 1000, 16000))); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); remove("something.vdb2"); } @@ -279,27 +236,15 @@ TestFile::testWriteMultipleGrids() EXPECT_TRUE(gd2.getBlockPos() != 0); EXPECT_TRUE(gd2.getEndPos() != 0); - // register the grid - GridBase::clearRegistry(); - GridType::registerGrid(); - - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Read in the first grid descriptor. GridDescriptor gd_in; std::istringstream istr(ostr.str(), std::ios_base::binary); io::setCurrentVersion(istr); - GridBase::Ptr gd_in_grid; - EXPECT_NO_THROW(gd_in_grid = gd_in.read(istr)); + gd_in.readHeader(istr); + gd_in.readStreamPos(istr); + + GridBase::Ptr gd_in_grid = Archive::readGrid(gd_in, istr); // Ensure read in the right values. EXPECT_EQ(gd.gridName(), gd_in.gridName()); @@ -308,13 +253,6 @@ TestFile::testWriteMultipleGrids() EXPECT_EQ(gd.getBlockPos(), gd_in.getBlockPos()); EXPECT_EQ(gd.getEndPos(), gd_in.getEndPos()); - // Position the stream to beginning of the grid storage and read the grid. - gd_in.seekToGrid(istr); - Archive::readGridCompression(istr); - gd_in_grid->readMeta(istr); - gd_in_grid->readTransform(istr); - gd_in_grid->readTopology(istr); - // Ensure that we have the same topology and transform. EXPECT_EQ( grid->baseTree().leafCount(), gd_in_grid->baseTree().leafCount()); @@ -328,8 +266,6 @@ TestFile::testWriteMultipleGrids() // EXPECT_EQ(0.1, gd_in_grid->getTransform()->getVoxelSizeZ()); // Read in the data blocks. - gd_in.seekToBlocks(istr); - gd_in_grid->readBuffers(istr); TreeType::Ptr grid_in = DynamicPtrCast(gd_in_grid->baseTreePtr()); EXPECT_TRUE(grid_in.get() != nullptr); EXPECT_EQ(10, grid_in->getValue(Coord(10, 1, 2))); @@ -343,8 +279,9 @@ TestFile::testWriteMultipleGrids() gd_in.seekToEnd(istr); GridDescriptor gd2_in; - GridBase::Ptr gd2_in_grid; - EXPECT_NO_THROW(gd2_in_grid = gd2_in.read(istr)); + gd2_in.readHeader(istr); + gd2_in.readStreamPos(istr); + GridBase::Ptr gd2_in_grid = Archive::readGrid(gd2_in, istr); // Ensure that we read in the right values. EXPECT_EQ(gd2.gridName(), gd2_in.gridName()); @@ -353,13 +290,6 @@ TestFile::testWriteMultipleGrids() EXPECT_EQ(gd2.getBlockPos(), gd2_in.getBlockPos()); EXPECT_EQ(gd2.getEndPos(), gd2_in.getEndPos()); - // Position the stream to beginning of the grid storage and read the grid. - gd2_in.seekToGrid(istr); - Archive::readGridCompression(istr); - gd2_in_grid->readMeta(istr); - gd2_in_grid->readTransform(istr); - gd2_in_grid->readTopology(istr); - // Ensure that we have the same topology and transform. EXPECT_EQ( grid2->baseTree().leafCount(), gd2_in_grid->baseTree().leafCount()); @@ -371,19 +301,12 @@ TestFile::testWriteMultipleGrids() // EXPECT_EQ(0.2, gd2_in_grid->getTransform()->getVoxelSizeY()); // EXPECT_EQ(0.2, gd2_in_grid->getTransform()->getVoxelSizeZ()); - // Read in the data blocks. - gd2_in.seekToBlocks(istr); - gd2_in_grid->readBuffers(istr); TreeType::Ptr grid2_in = DynamicPtrCast(gd2_in_grid->baseTreePtr()); EXPECT_TRUE(grid2_in.get() != nullptr); EXPECT_EQ(50, grid2_in->getValue(Coord(1000, 1000, 1000))); EXPECT_EQ(10, grid2_in->getValue(Coord(0, 0, 0))); EXPECT_EQ(2, grid2_in->getValue(Coord(100000, 100000, 16000))); - // Clear registries. - GridBase::clearRegistry(); - - math::MapRegistry::clear(); remove("something.vdb2"); } TEST_F(TestFile, testWriteMultipleGrids) { testWriteMultipleGrids(); } @@ -397,12 +320,6 @@ TEST_F(TestFile, testWriteFloatAsHalf) using TreeType = Vec3STree; using GridType = Grid; - // Register all grid types. - initialize(); - // Ensure that the registry is cleared on exit. - struct Local { static void uninitialize(char*) { openvdb::uninitialize(); } }; - SharedPtr onExit(nullptr, Local::uninitialize); - // Create two test grids. GridType::Ptr grid1 = createGrid(/*bg=*/Vec3s(1, 1, 1)); TreeType& tree1 = grid1->tree(); @@ -463,9 +380,6 @@ TEST_F(TestFile, testWriteInstancedGrids) { using namespace openvdb; - // Register data types. - openvdb::initialize(); - // Remove something.vdb2 when done. We must declare this here before the // other grid smart_ptr's because we re-use them in the test several times. // We will not be able to remove something.vdb2 on Windows if the pointers @@ -622,10 +536,6 @@ TEST_F(TestFile, testWriteInstancedGrids) EXPECT_TRUE(grid.get() != nullptr); density = gridPtrCast(grid)->treePtr(); EXPECT_TRUE(density.get() != nullptr); -#ifdef OPENVDB_USE_DELAYED_LOADING - EXPECT_TRUE(density->unallocatedLeafCount() > 0); - EXPECT_EQ(density->leafCount(), density->unallocatedLeafCount()); -#endif // OPENVDB_USE_DELAYED_LOADING grid = findGridByName(*grids, "density_copy"); EXPECT_TRUE(grid.get() != nullptr); EXPECT_TRUE(gridPtrCast(grid)->treePtr().get() != nullptr); @@ -647,6 +557,77 @@ TEST_F(TestFile, testWriteInstancedGrids) } +// Verify that clipping an instanced grid uses the instance's own transform, +// not the parent's. The bug was that Archive::readGrid() converted the +// world-space bbox to index space with the parent's transform before the +// instance's transform was applied, so the clipped region was wrong when the +// two transforms differed. +TEST_F(TestFile, testReadClippedInstancedGrid) +{ + using namespace openvdb; + + const char* filename = "testReadClippedInstancedGrid.vdb"; + SharedPtr scopedFile(filename, ::remove); + + // Parent grid: voxel size 1.0. Fill index [-5, 5] with value 1. + FloatTree::Ptr tree(new FloatTree(0.0f)); + tree->fill(CoordBBox(Coord(-5), Coord(5)), 1.0f, /*active=*/true); + + GridBase::Ptr parent = FloatGrid::create(tree); + parent->setName("parent"); + parent->setTransform(math::Transform::createLinearTransform(1.0)); + + // Instance grid: same tree, but voxel size 2.0. + // Index coord n → world coord 2n (double the parent's world-space extent). + GridBase::Ptr instance = FloatGrid::create(tree); + instance->setName("instance"); + instance->setTransform(math::Transform::createLinearTransform(2.0)); + + GridPtrVec grids; + grids.push_back(parent); + grids.push_back(instance); + + { + io::File vdbfile(filename); + vdbfile.write(grids); + } + + // World-space clip: [0, 6]. + // Via parent transform (voxel 1.0): index [0, 6] → voxels 0..5 survive. + // Via instance transform (voxel 2.0): index [0, 3] → voxels 0..3 survive. + // The correct answer uses the instance's transform. + const BBoxd clipBox(Vec3d(0.0), Vec3d(6.0)); + + io::File vdbfile(filename); + vdbfile.open(); + + GridBase::Ptr readGrid = vdbfile.readGrid("instance", clipBox); + EXPECT_TRUE(readGrid.get() != nullptr); + FloatGrid::Ptr clipped = gridPtrCast(readGrid); + EXPECT_TRUE(clipped.get() != nullptr); + + const CoordBBox bbox = clipped->evalActiveVoxelBoundingBox(); + // The instance's transform maps world [0,6] to index [0,3]. + EXPECT_EQ(Coord(0, 0, 0), bbox.min()); + EXPECT_EQ(Coord(3, 3, 3), bbox.max()); + + // No active voxels should survive outside [0,3] in any axis. + FloatGrid::ConstAccessor acc = clipped->getConstAccessor(); + for (int i = -5; i <= 5; ++i) { + for (int j = -5; j <= 5; ++j) { + for (int k = -5; k <= 5; ++k) { + const Coord xyz(i, j, k); + if (i >= 0 && j >= 0 && k >= 0 && i <= 3 && j <= 3 && k <= 3) { + EXPECT_EQ(1.0f, acc.getValue(xyz)); + } else { + EXPECT_EQ(0.0f, acc.getValue(xyz)); + } + } + } + } +} + + void TestFile::testReadGridDescriptors() { @@ -687,28 +668,31 @@ TestFile::testReadGridDescriptors() file.writeGrid(gd, grid, ostr, /*seekable=*/true); file.writeGrid(gd2, grid2, ostr, /*seekable=*/true); - // Register the grid and the transform and the blocks. - GridBase::clearRegistry(); - GridType::registerGrid(); - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Read in the grid descriptors. File file2("something.vdb2"); std::istringstream istr(ostr.str(), std::ios_base::binary); io::setCurrentVersion(istr); - file2.readGridDescriptors(istr); + // file2.readGridDescriptors(istr); + //////////////////////////// + file2.mGridDescriptors.clear(); + + for (int32_t i = 0, N = file2.readGridCount(istr); i < N; ++i) { + // Read the grid descriptor. + GridDescriptor gd; + gd.readHeader(istr); + gd.readStreamPos(istr); + + // Add the descriptor to the dictionary. + file2.mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); + + // Skip forward to the next descriptor. + gd.seekToEnd(istr); + } + //////////////////////////// // Compare with the initial grid descriptors. File::NameMapCIter it = file2.findDescriptor("temperature"); - EXPECT_TRUE(it != file2.gridDescriptors().end()); + EXPECT_TRUE(it != file2.mGridDescriptors.end()); GridDescriptor file2gd = it->second; EXPECT_EQ(gd.gridName(), file2gd.gridName()); EXPECT_EQ(gd.getGridPos(), file2gd.getGridPos()); @@ -716,17 +700,13 @@ TestFile::testReadGridDescriptors() EXPECT_EQ(gd.getEndPos(), file2gd.getEndPos()); it = file2.findDescriptor("density"); - EXPECT_TRUE(it != file2.gridDescriptors().end()); + EXPECT_TRUE(it != file2.mGridDescriptors.end()); file2gd = it->second; EXPECT_EQ(gd2.gridName(), file2gd.gridName()); EXPECT_EQ(gd2.getGridPos(), file2gd.getGridPos()); EXPECT_EQ(gd2.getBlockPos(), file2gd.getBlockPos()); EXPECT_EQ(gd2.getEndPos(), file2gd.getEndPos()); - // Clear registries. - GridBase::clearRegistry(); - math::MapRegistry::clear(); - remove("something.vdb2"); } TEST_F(TestFile, testReadGridDescriptors) { testReadGridDescriptors(); } @@ -739,9 +719,6 @@ TEST_F(TestFile, testGridNaming) using TreeType = Int32Tree; - // Register data types. - openvdb::initialize(); - logging::LevelScope suppressLogging{logging::Level::Fatal}; // Create several grids that share a single tree. @@ -954,35 +931,34 @@ TestFile::testEmptyGridIO() EXPECT_EQ(gd.getEndPos(), gd.getBlockPos()); EXPECT_EQ(gd2.getEndPos(), gd2.getBlockPos()); - // Register the grid and the transform and the blocks. - GridBase::clearRegistry(); - GridType::registerGrid(); - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Read in the grid descriptors. File file2(filename); std::istringstream istr(ostr.str(), std::ios_base::binary); io::setCurrentVersion(istr); - file2.readGridDescriptors(istr); + // file2.readGridDescriptors(istr); + //////////////////////////// + file2.mGridDescriptors.clear(); + + for (int32_t i = 0, N = file2.readGridCount(istr); i < N; ++i) { + // Read the grid descriptor. + GridDescriptor gd; + gd.readHeader(istr); + gd.readStreamPos(istr); + + // Add the descriptor to the dictionary. + file2.mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); + + // Skip forward to the next descriptor. + gd.seekToEnd(istr); + } + //////////////////////////// // Compare with the initial grid descriptors. File::NameMapCIter it = file2.findDescriptor("temperature"); - EXPECT_TRUE(it != file2.gridDescriptors().end()); + EXPECT_TRUE(it != file2.mGridDescriptors.end()); GridDescriptor file2gd = it->second; file2gd.seekToGrid(istr); - GridBase::Ptr gd_grid = GridBase::createGrid(file2gd.gridType()); - Archive::readGridCompression(istr); - gd_grid->readMeta(istr); - gd_grid->readTransform(istr); - gd_grid->readTopology(istr); + GridBase::Ptr gd_grid = Archive::readGrid(file2gd, istr); EXPECT_EQ(gd.gridName(), file2gd.gridName()); EXPECT_TRUE(gd_grid.get() != nullptr); EXPECT_EQ(0, int(gd_grid->baseTree().leafCount())); @@ -993,14 +969,10 @@ TestFile::testEmptyGridIO() EXPECT_EQ(gd.getEndPos(), file2gd.getEndPos()); it = file2.findDescriptor("density"); - EXPECT_TRUE(it != file2.gridDescriptors().end()); + EXPECT_TRUE(it != file2.mGridDescriptors.end()); file2gd = it->second; file2gd.seekToGrid(istr); - gd_grid = GridBase::createGrid(file2gd.gridType()); - Archive::readGridCompression(istr); - gd_grid->readMeta(istr); - gd_grid->readTransform(istr); - gd_grid->readTopology(istr); + gd_grid = Archive::readGrid(file2gd, istr); EXPECT_EQ(gd2.gridName(), file2gd.gridName()); EXPECT_TRUE(gd_grid.get() != nullptr); EXPECT_EQ(0, int(gd_grid->baseTree().leafCount())); @@ -1009,10 +981,6 @@ TestFile::testEmptyGridIO() EXPECT_EQ(gd2.getGridPos(), file2gd.getGridPos()); EXPECT_EQ(gd2.getBlockPos(), file2gd.getBlockPos()); EXPECT_EQ(gd2.getEndPos(), file2gd.getEndPos()); - - // Clear registries. - GridBase::clearRegistry(); - math::MapRegistry::clear(); } TEST_F(TestFile, testEmptyGridIO) { testEmptyGridIO(); } @@ -1062,23 +1030,6 @@ void TestFile::testOpen() EXPECT_TRUE(meta.metaValue("author") == "Einstein"); EXPECT_EQ(2009, meta.metaValue("year")); - // Register grid and transform. - GridBase::clearRegistry(); - IntGrid::registerGrid(); - FloatGrid::registerGrid(); - Metadata::clearRegistry(); - StringMetadata::registerType(); - Int32Metadata::registerType(); - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1109,16 +1060,16 @@ void TestFile::testOpen() EXPECT_EQ(2009, vdbfile.getMetadata()->metaValue("year")); // Ensure we got the grid descriptors. - EXPECT_EQ(1, int(vdbfile.gridDescriptors().count("density"))); - EXPECT_EQ(1, int(vdbfile.gridDescriptors().count("temperature"))); + EXPECT_EQ(1, int(vdbfile.mGridDescriptors.count("density"))); + EXPECT_EQ(1, int(vdbfile.mGridDescriptors.count("temperature"))); io::File::NameMapCIter it = vdbfile.findDescriptor("density"); - EXPECT_TRUE(it != vdbfile.gridDescriptors().end()); + EXPECT_TRUE(it != vdbfile.mGridDescriptors.end()); io::GridDescriptor gd = it->second; EXPECT_EQ(IntTree::treeType(), gd.gridType()); it = vdbfile.findDescriptor("temperature"); - EXPECT_TRUE(it != vdbfile.gridDescriptors().end()); + EXPECT_TRUE(it != vdbfile.mGridDescriptors.end()); gd = it->second; EXPECT_EQ(FloatTree::treeType(), gd.gridType()); @@ -1127,16 +1078,11 @@ void TestFile::testOpen() EXPECT_THROW(vdbfile2.open(), openvdb::IoError); EXPECT_THROW(vdbfile2.inputStream(), openvdb::IoError); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); - // Test closing the file. vdbfile.close(); EXPECT_TRUE(vdbfile.isOpen() == false); - EXPECT_TRUE(vdbfile.fileMetadata().get() == nullptr); - EXPECT_EQ(0, int(vdbfile.gridDescriptors().size())); + EXPECT_TRUE(vdbfile.mMeta.get() == nullptr); + EXPECT_EQ(0, int(vdbfile.mGridDescriptors.size())); EXPECT_THROW(vdbfile.inputStream(), openvdb::IoError); remove("something.vdb2"); @@ -1173,11 +1119,6 @@ TEST_F(TestFile, testGetMetadata) meta.insertMeta("author", StringMetadata("Einstein")); meta.insertMeta("year", Int32Metadata(2009)); - // Adjust registry before writing. - Metadata::clearRegistry(); - StringMetadata::registerType(); - Int32Metadata::registerType(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1194,9 +1135,6 @@ TEST_F(TestFile, testGetMetadata) EXPECT_TRUE(meta2->metaValue("author") == "Einstein"); EXPECT_EQ(2009, meta2->metaValue("year")); - // Clear registry. - Metadata::clearRegistry(); - remove("something.vdb2"); } @@ -1241,9 +1179,6 @@ TEST_F(TestFile, testReadAll) grids.push_back(grid1); grids.push_back(grid2); - // Register grid and transform. - openvdb::initialize(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1283,11 +1218,6 @@ TEST_F(TestFile, testReadAll) EXPECT_NEAR(10, temperature->getValue(Coord(0, 0, 0)), /*tolerance=*/0); EXPECT_NEAR(11, temperature->getValue(Coord(0, 100, 0)), /*tolerance=*/0); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); - vdbfile2.close(); remove("something.vdb2"); @@ -1302,11 +1232,6 @@ TEST_F(TestFile, testWriteOpenFile) meta->insertMeta("author", StringMetadata("Einstein")); meta->insertMeta("year", Int32Metadata(2009)); - // Register metadata - Metadata::clearRegistry(); - StringMetadata::registerType(); - Int32Metadata::registerType(); - // Write the metadata out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(GridPtrVec(), *meta); @@ -1337,9 +1262,6 @@ TEST_F(TestFile, testWriteOpenFile) EXPECT_NO_THROW(vdbfile2.write(*grids)); - // Clear registries. - Metadata::clearRegistry(); - remove("something.vdb2"); } @@ -1348,8 +1270,6 @@ TEST_F(TestFile, testReadGridMetadata) { using namespace openvdb; - openvdb::initialize(); - const char* filename = "testReadGridMetadata.vdb2"; SharedPtr scopedFile(filename, ::remove); @@ -1462,10 +1382,8 @@ TEST_F(TestFile, testReadGridMetadata) if ((*statsMetadata)[it->first]) { otherMetadata->removeMeta(it->first); } - // Remove delay load metadata if it exists. - if ((*otherMetadata)["file_delayed_load"]) { - otherMetadata->removeMeta("file_delayed_load"); - } + // Delay load metadata should not exist. + ASSERT_FALSE(bool((*otherMetadata)["file_delayed_load"])); } EXPECT_EQ(srcGrid->str(), otherMetadata->str()); @@ -1521,9 +1439,6 @@ TEST_F(TestFile, testReadGrid) grids.push_back(grid); grids.push_back(grid2); - // Register grid and transform. - openvdb::initialize(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1558,11 +1473,6 @@ TEST_F(TestFile, testReadGrid) EXPECT_NEAR(5,typedDensity->getValue(Coord(0, 0, 0)), /*tolerance=*/0); EXPECT_NEAR(6,typedDensity->getValue(Coord(100, 0, 0)), /*tolerance=*/0); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); - vdbfile2.close(); remove("something.vdb2"); @@ -1613,9 +1523,6 @@ TEST_F(TestFile, testReadClippedGrid) { using namespace openvdb; - // Register types. - openvdb::initialize(); - // World-space clipping region const BBoxd clipBox(Vec3d(4.0, 4.0, -6.0), Vec3d(4.9, 4.9, 6.0)); @@ -1686,246 +1593,6 @@ TEST_F(TestFile, testReadClippedGrid) //////////////////////////////////////// -namespace { - -template struct MultiPassLeafNode; // forward declaration - -// Dummy value type -using MultiPassValue = openvdb::PointIndex; - -// Tree configured to match the default OpenVDB configuration -using MultiPassTree = openvdb::tree::Tree< - openvdb::tree::RootNode< - openvdb::tree::InternalNode< - openvdb::tree::InternalNode< - MultiPassLeafNode, 4>, 5>>>; - -using MultiPassGrid = openvdb::Grid; - - -template -struct MultiPassLeafNode: public openvdb::tree::LeafNode, openvdb::io::MultiPass -{ - // The following had to be copied from the LeafNode class - // to make the derived class compatible with the tree structure. - - using LeafNodeType = MultiPassLeafNode; - using Ptr = openvdb::SharedPtr; - using BaseLeaf = openvdb::tree::LeafNode; - using NodeMaskType = openvdb::util::NodeMask; - using ValueType = T; - using ValueOnCIter = typename BaseLeaf::template ValueIter; - using ChildOnIter = typename BaseLeaf::template ChildIter; - using ChildOnCIter = typename BaseLeaf::template ChildIter< - typename NodeMaskType::OnIterator, const MultiPassLeafNode, typename BaseLeaf::ChildOn>; - - MultiPassLeafNode(const openvdb::Coord& coords, const T& value, bool active = false) - : BaseLeaf(coords, value, active) {} - MultiPassLeafNode(openvdb::PartialCreate, const openvdb::Coord& coords, const T& value, - bool active = false): BaseLeaf(openvdb::PartialCreate(), coords, value, active) {} - MultiPassLeafNode(const MultiPassLeafNode& rhs): BaseLeaf(rhs) {} - - ValueOnCIter cbeginValueOn() const { return ValueOnCIter(this->getValueMask().beginOn(),this); } - ChildOnCIter cbeginChildOn() const { return ChildOnCIter(this->getValueMask().endOn(), this); } - ChildOnIter beginChildOn() { return ChildOnIter(this->getValueMask().endOn(), this); } - - // Methods in use for reading and writing multiple buffers - - void readBuffers(std::istream& is, const openvdb::CoordBBox&, bool fromHalf = false) - { - this->readBuffers(is, fromHalf); - } - - void readBuffers(std::istream& is, bool /*fromHalf*/ = false) - { - const openvdb::io::StreamMetadata::Ptr meta = openvdb::io::getStreamMetadataPtr(is); - if (!meta) { - OPENVDB_THROW(openvdb::IoError, - "Cannot write out a MultiBufferLeaf without StreamMetadata."); - } - - // clamp pass to 16-bit integer - const uint32_t pass(static_cast(meta->pass())); - - // Read in the stored pass number. - uint32_t readPass; - is.read(reinterpret_cast(&readPass), sizeof(uint32_t)); - EXPECT_EQ(pass, readPass); - // Record the pass number. - mReadPasses.push_back(readPass); - - if (pass == 0) { - // Read in the node's origin. - openvdb::Coord origin; - is.read(reinterpret_cast(&origin), sizeof(openvdb::Coord)); - EXPECT_EQ(origin, this->origin()); - } - } - - void writeBuffers(std::ostream& os, bool /*toHalf*/ = false) const - { - const openvdb::io::StreamMetadata::Ptr meta = openvdb::io::getStreamMetadataPtr(os); - if (!meta) { - OPENVDB_THROW(openvdb::IoError, - "Cannot read in a MultiBufferLeaf without StreamMetadata."); - } - - // clamp pass to 16-bit integer - const uint32_t pass(static_cast(meta->pass())); - - // Leaf traversal analysis deduces the number of passes to perform for this leaf - // then updates the leaf traversal value to ensure all passes will be written. - if (meta->countingPasses()) { - if (mNumPasses > pass) meta->setPass(mNumPasses); - return; - } - - // Record the pass number. - EXPECT_TRUE(mWritePassesPtr); - const_cast&>(*mWritePassesPtr).push_back(pass); - - // Write out the pass number. - os.write(reinterpret_cast(&pass), sizeof(uint32_t)); - if (pass == 0) { - // Write out the node's origin and the pass number. - const auto origin = this->origin(); - os.write(reinterpret_cast(&origin), sizeof(openvdb::Coord)); - } - } - - - uint32_t mNumPasses = 0; - // Pointer to external vector in which to record passes as they are written - std::vector* mWritePassesPtr = nullptr; - // Vector in which to record passes as they are read - // (this needs to be internal, because leaf nodes are constructed as a grid is read) - std::vector mReadPasses; -}; // struct MultiPassLeafNode - -} // anonymous namespace - - -TEST_F(TestFile, testMultiPassIO) -{ - using namespace openvdb; - - openvdb::initialize(); - MultiPassGrid::registerGrid(); - - // Create a multi-buffer grid. - const MultiPassGrid::Ptr grid = openvdb::createGrid(); - grid->setName("test"); - grid->setTransform(math::Transform::createLinearTransform(1.0)); - MultiPassGrid::TreeType& tree = grid->tree(); - tree.setValue(Coord(0, 0, 0), 5); - tree.setValue(Coord(0, 10, 0), 5); - EXPECT_EQ(2, int(tree.leafCount())); - - const GridPtrVec grids{grid}; - - // Vector in which to record pass numbers (to ensure blocked ordering) - std::vector writePasses; - { - // Specify the required number of I/O passes for each leaf node. - MultiPassGrid::TreeType::LeafIter leafIter = tree.beginLeaf(); - leafIter->mNumPasses = 3; - leafIter->mWritePassesPtr = &writePasses; - ++leafIter; - leafIter->mNumPasses = 2; - leafIter->mWritePassesPtr = &writePasses; - } - - const char* filename = "testMultiPassIO.vdb"; - SharedPtr scopedFile(filename, ::remove); - { - // Verify that passes are written to a file in the correct order. - io::File(filename).write(grids); - EXPECT_EQ(6, int(writePasses.size())); - EXPECT_EQ(0, writePasses[0]); // leaf 0 - EXPECT_EQ(0, writePasses[1]); // leaf 1 - EXPECT_EQ(1, writePasses[2]); // leaf 0 - EXPECT_EQ(1, writePasses[3]); // leaf 1 - EXPECT_EQ(2, writePasses[4]); // leaf 0 - EXPECT_EQ(2, writePasses[5]); // leaf 1 - } - { - // Verify that passes are read in the correct order. - io::File file(filename); - file.open(); - const auto newGrid = GridBase::grid(file.readGrid("test")); - - auto leafIter = newGrid->tree().beginLeaf(); - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - ++leafIter; - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - } - { - // Verify that when using multi-pass and bbox clipping that each leaf node - // is still being read before being clipped - io::File file(filename); - file.open(); - const auto newGrid = GridBase::grid( - file.readGrid("test", BBoxd(Vec3d(0), Vec3d(1)))); - EXPECT_EQ(Index64(1), newGrid->tree().leafCount()); - - auto leafIter = newGrid->tree().beginLeaf(); - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - ++leafIter; - EXPECT_TRUE(!leafIter); // second leaf node has now been clipped - } - - // Clear the pass data. - writePasses.clear(); - - { - // Verify that passes are written to and read from a non-seekable stream - // in the correct order. - std::ostringstream ostr(std::ios_base::binary); - io::Stream(ostr).write(grids); - - EXPECT_EQ(6, int(writePasses.size())); - EXPECT_EQ(0, writePasses[0]); // leaf 0 - EXPECT_EQ(0, writePasses[1]); // leaf 1 - EXPECT_EQ(1, writePasses[2]); // leaf 0 - EXPECT_EQ(1, writePasses[3]); // leaf 1 - EXPECT_EQ(2, writePasses[4]); // leaf 0 - EXPECT_EQ(2, writePasses[5]); // leaf 1 - - std::istringstream is(ostr.str(), std::ios_base::binary); - io::Stream strm(is); - const auto streamedGrids = strm.getGrids(); - EXPECT_EQ(1, int(streamedGrids->size())); - - const auto newGrid = gridPtrCast(*streamedGrids->begin()); - EXPECT_TRUE(bool(newGrid)); - auto leafIter = newGrid->tree().beginLeaf(); - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - ++leafIter; - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - } -} - - -//////////////////////////////////////// - - TEST_F(TestFile, testHasGrid) { using namespace openvdb; @@ -1967,23 +1634,6 @@ TEST_F(TestFile, testHasGrid) grids.push_back(grid); grids.push_back(grid2); - // Register grid and transform. - GridBase::clearRegistry(); - IntGrid::registerGrid(); - FloatGrid::registerGrid(); - Metadata::clearRegistry(); - StringMetadata::registerType(); - Int32Metadata::registerType(); - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1999,11 +1649,6 @@ TEST_F(TestFile, testHasGrid) EXPECT_TRUE(!vdbfile2.hasGrid("Temperature")); EXPECT_TRUE(!vdbfile2.hasGrid("densitY")); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); - vdbfile2.close(); remove("something.vdb2"); @@ -2049,9 +1694,6 @@ TEST_F(TestFile, testNameIterator) grid->setName("level_set"); grids.push_back(grid); - // Register types. - openvdb::initialize(); - const char* filename = "testNameIterator.vdb2"; SharedPtr scopedFile(filename, ::remove); @@ -2099,9 +1741,6 @@ TEST_F(TestFile, testCompression) using IntGrid = openvdb::Int32Grid; - // Register types. - openvdb::initialize(); - // Create reference grids. IntGrid::Ptr intGrid = IntGrid::create(/*background=*/0); intGrid->fill(CoordBBox(Coord(0), Coord(49)), /*value=*/999, /*active=*/true); @@ -2311,9 +1950,6 @@ TEST_F(TestFile, testAsync) { using namespace openvdb; - // Register types. - openvdb::initialize(); - // Create a grid. FloatGrid::Ptr lsGrid = createLevelSet(); unittest_util::makeSphere(/*dim=*/Coord(100), /*ctr=*/Vec3f(50, 50, 50), /*r=*/20.0, @@ -2431,8 +2067,6 @@ TEST_F(TestFile, testAsync) // (see https://github.com/Blosc/c-blosc/pull/63). TEST_F(TestFile, testBlosc) { - openvdb::initialize(); - const unsigned char rawdata[] = { 0x93, 0xb0, 0x49, 0xaf, 0x62, 0xad, 0xe3, 0xaa, 0xe4, 0xa5, 0x43, 0x20, 0x24, 0x29, 0xc9, 0xaf, 0xee, 0xad, 0x0b, 0xac, 0x3d, 0xa8, 0x1f, 0x99, 0x53, 0x27, @@ -2542,147 +2176,3 @@ TEST_F(TestFile, testBlosc) } } #endif - - -void -TestFile::testDelayedLoadMetadata() -{ - openvdb::initialize(); - - using namespace openvdb; - - io::File file("something.vdb2"); - - // Create a level set grid. - auto lsGrid = createLevelSet(); - lsGrid->setName("sphere"); - unittest_util::makeSphere(/*dim=*/Coord(100), /*ctr=*/Vec3f(50, 50, 50), /*r=*/20.0, - *lsGrid, unittest_util::SPHERE_SPARSE_NARROW_BAND); - - // Write the VDB to a string stream. - std::ostringstream ostr(std::ios_base::binary); - - // Create the grid descriptor out of this grid. - io::GridDescriptor gd(Name("sphere"), lsGrid->type()); - - // Write out the grid. - file.writeGrid(gd, lsGrid, ostr, /*seekable=*/true); - - // Duplicate VDB string stream. - std::ostringstream ostr2(std::ios_base::binary); - - { // Read back in, clip and write out again to verify metadata is rebuilt. - std::istringstream istr(ostr.str(), std::ios_base::binary); - io::setVersion(istr, file.libraryVersion(), file.fileVersion()); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - - const BBoxd clipBbox(Vec3d(-10.0,-10.0,-10.0), Vec3d(10.0,10.0,10.0)); - io::Archive::readGrid(grid, gd2, istr, clipBbox); - - // Verify clipping is working as expected. - EXPECT_TRUE(grid->baseTreePtr()->leafCount() < lsGrid->tree().leafCount()); - - file.writeGrid(gd, grid, ostr2, /*seekable=*/true); - } - - // Since the input is only a fragment of a VDB file (in particular, - // it doesn't have a header), set the file format version number explicitly. - // On read, the delayed load metadata for OpenVDB library versions less than 6.1 - // should be removed to ensure correctness as it possible for the metadata to - // have been treated as unknown and blindly copied over when read and re-written - // using this library version resulting in out-of-sync metadata. - - // By default, DelayedLoadMetadata is dropped from the grid during read so - // as not to be exposed to the user. - - { // read using current library version - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, file.libraryVersion(), file.fileVersion()); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(!((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - // To test the version mechanism, a stream metadata object is created with - // a non-zero test value and set on the input stream. This disables the - // behaviour where the DelayedLoadMetadata is dropped from the grid. - - io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata); - streamMetadata->__setTest(uint32_t(1)); - - { // read using current library version - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, file.libraryVersion(), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - { // read using library version of 5.0 - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, VersionId(5,0), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(!((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - { // read using library version of 4.9 - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, VersionId(4,9), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(!((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - { // read using library version of 6.1 - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, VersionId(6,1), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(!((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - { // read using library version of 6.2 - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, VersionId(6,2), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - remove("something.vdb2"); -} -TEST_F(TestFile, testDelayedLoadMetadata) { testDelayedLoadMetadata(); } - diff --git a/openvdb/openvdb/unittest/TestGrid.cc b/openvdb/openvdb/unittest/TestGrid.cc index 147405c996..724109ce53 100644 --- a/openvdb/openvdb/unittest/TestGrid.cc +++ b/openvdb/openvdb/unittest/TestGrid.cc @@ -65,7 +65,9 @@ class ProxyTree: public openvdb::TreeBase void readBuffers(std::istream& is, const openvdb::CoordBBox&, bool /*saveFloatAsHalf*/=false) override { is.seekg(0); } +#if OPENVDB_ABI_VERSION_NUMBER < 14 void readNonresidentBuffers() const override {} +#endif void readBuffers(std::istream& is, bool /*saveFloatAsHalf*/=false) override { is.seekg(0); } void writeBuffers(std::ostream& os, bool /*saveFloatAsHalf*/=false) const override { os.seekp(0, std::ios::beg); } diff --git a/openvdb/openvdb/unittest/TestGridDescriptor.cc b/openvdb/openvdb/unittest/TestGridDescriptor.cc index c45e8d0433..a39fde278e 100644 --- a/openvdb/openvdb/unittest/TestGridDescriptor.cc +++ b/openvdb/openvdb/unittest/TestGridDescriptor.cc @@ -42,16 +42,15 @@ TEST_F(TestGridDescriptor, testIO) GridDescriptor gd2; - EXPECT_THROW(gd2.read(istr), openvdb::LookupError); - // Register the grid. GridBase::clearRegistry(); GridType::registerGrid(); // seek back and read again. istr.seekg(0, std::ios_base::beg); - GridBase::Ptr grid; - EXPECT_NO_THROW(grid = gd2.read(istr)); + gd2.readHeader(istr); + gd2.readStreamPos(istr); + GridBase::Ptr grid = GridBase::createGrid(gd2.gridType()); EXPECT_EQ(gd.gridName(), gd2.gridName()); EXPECT_EQ(gd.uniqueName(), gd2.uniqueName()); diff --git a/openvdb/openvdb/unittest/TestGridIO.cc b/openvdb/openvdb/unittest/TestGridIO.cc index 184379a030..3f605b86dd 100644 --- a/openvdb/openvdb/unittest/TestGridIO.cc +++ b/openvdb/openvdb/unittest/TestGridIO.cc @@ -3,6 +3,7 @@ #include #include +#include #include #include // for remove() @@ -278,5 +279,10 @@ TEST_F(TestGridIO, testReadAllBool) { readAllTest(); } TEST_F(TestGridIO, testReadAllFloat) { readAllTest(); } TEST_F(TestGridIO, testReadAllHalf) { readAllTest(); } TEST_F(TestGridIO, testReadAllVec3S) { readAllTest(); } -TEST_F(TestGridIO, testReadAllFloat5432) { Float5432Grid::registerGrid(); readAllTest(); } +TEST_F(TestGridIO, testReadAllFloat5432) +{ + Float5432Grid::registerGrid(); + openvdb::io::CodecRegistry::registerCodec>(); + readAllTest(); +} TEST_F(TestGridIO, testCreateWriteReadHalf) { testCreateWriteReadHalf(); } \ No newline at end of file diff --git a/openvdb/openvdb/unittest/TestLeafBool.cc b/openvdb/openvdb/unittest/TestLeafBool.cc index 6acae26ea2..3bb4522001 100644 --- a/openvdb/openvdb/unittest/TestLeafBool.cc +++ b/openvdb/openvdb/unittest/TestLeafBool.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include "util.h" // for unittest_util::makeSphere() #include #include @@ -293,6 +294,60 @@ TEST_F(TestLeafBool, testIO) leaf.setValueOn(openvdb::Coord(0, 1, 0)); leaf.setValueOn(openvdb::Coord(1, 0, 0)); + // read and write topology to disk + + { + // create a grid with the leaf for topology testing + typedef openvdb::Grid, 5>>>> BoolGrid; + BoolGrid::Ptr grid = BoolGrid::create(); + grid->setName("bool_leaf"); + grid->tree().addLeaf(new LeafType(leaf)); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); + + // write to file + { + openvdb::io::File file("leaf_bool.vdb"); + file.write(grids); + file.close(); + } + + // read grid from file + BoolGrid::Ptr gridFromDisk; + { + openvdb::io::File file("leaf_bool.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("bool_leaf"); + file.close(); + + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } + + LeafType* leaf2 = gridFromDisk->tree().probeLeaf(origin); + EXPECT_TRUE(leaf2); + + // check topology and values match + + EXPECT_EQ(origin, leaf2->origin()); + EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(0, 1, 0))); + EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(1, 0, 0))); + EXPECT_TRUE(leaf2->onVoxelCount() == 2); + + remove("leaf_bool.vdb"); + } +} + + +TEST_F(TestLeafBool, testTreeIO) +{ + LeafType leaf(openvdb::Coord(1, 3, 5)); + const openvdb::Coord origin = leaf.origin(); + + leaf.setValueOn(openvdb::Coord(0, 1, 0)); + leaf.setValueOn(openvdb::Coord(1, 0, 0)); + std::ostringstream ostr(std::ios_base::binary); leaf.writeBuffers(ostr); @@ -301,8 +356,6 @@ TEST_F(TestLeafBool, testIO) leaf.setValueOn(openvdb::Coord(0, 1, 1)); std::istringstream istr(ostr.str(), std::ios_base::binary); - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. openvdb::io::setCurrentVersion(istr); leaf.readBuffers(istr); diff --git a/openvdb/openvdb/unittest/TestLeafIO.cc b/openvdb/openvdb/unittest/TestLeafIO.cc index 3b35259f53..7e4cf04575 100644 --- a/openvdb/openvdb/unittest/TestLeafIO.cc +++ b/openvdb/openvdb/unittest/TestLeafIO.cc @@ -2,8 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 #include +#include #include #include +#include #include #include // for toupper() @@ -15,13 +17,73 @@ class TestLeafIO { public: static void testBuffer(); + static void testTreeIO(); }; template void TestLeafIO::testBuffer() { - openvdb::tree::LeafNode leaf(openvdb::Coord(0, 0, 0)); + using LeafT = openvdb::tree::LeafNode; + LeafT leaf(openvdb::Coord(0, 0, 0)); + const openvdb::Coord origin = leaf.origin(); + + leaf.setValueOn(openvdb::Coord(0, 1, 0), T(1)); + leaf.setValueOn(openvdb::Coord(1, 0, 0), T(1)); + + // read and write topology to disk + + { + // create a grid with the leaf for topology testing + typedef openvdb::Grid, 5>>>> GridType; + if (!GridType::isRegistered()) GridType::registerGrid(); + + typename GridType::Ptr grid = GridType::create(); + grid->setName("leaf_io"); + grid->tree().addLeaf(new LeafT(leaf)); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); + + // write to file + { + openvdb::io::File file("leaf_io.vdb"); + file.write(grids); + file.close(); + } + + // read grid from file + typename GridType::Ptr gridFromDisk; + { + openvdb::io::File file("leaf_io.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("leaf_io"); + file.close(); + + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } + + LeafT* leaf2 = gridFromDisk->tree().probeLeaf(origin); + EXPECT_TRUE(leaf2); + + // check topology and values match + + EXPECT_NEAR(T(1), leaf2->getValue(openvdb::Coord(0, 1, 0)), /*tolerance=*/0); + EXPECT_NEAR(T(1), leaf2->getValue(openvdb::Coord(1, 0, 0)), /*tolerance=*/0); + EXPECT_TRUE(leaf2->onVoxelCount() == 2); + + remove("leaf_io.vdb"); + } +} + + +template +void +TestLeafIO::testTreeIO() +{ + using LeafT = openvdb::tree::LeafNode; + LeafT leaf(openvdb::Coord(0, 0, 0)); leaf.setValueOn(openvdb::Coord(0, 1, 0), T(1)); leaf.setValueOn(openvdb::Coord(1, 0, 0), T(1)); @@ -34,9 +96,6 @@ TestLeafIO::testBuffer() leaf.setValueOn(openvdb::Coord(0, 1, 1), T(1)); std::istringstream istr(ostr.str(), std::ios_base::binary); - - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. openvdb::io::setCurrentVersion(istr); leaf.readBuffers(istr); @@ -50,6 +109,9 @@ TestLeafIO::testBuffer() class TestLeafIOTest: public ::testing::Test { +public: + void SetUp() override { openvdb::initialize(); } + void TearDown() override { openvdb::uninitialize(); } }; @@ -62,7 +124,68 @@ TEST_F(TestLeafIOTest, testBufferByte) { TestLeafIO::testBuffer() TEST_F(TestLeafIOTest, testBufferVec3R) { - openvdb::tree::LeafNode leaf(openvdb::Coord(0, 0, 0)); + using LeafT = openvdb::tree::LeafNode; + LeafT leaf(openvdb::Coord(0, 0, 0)); + const openvdb::Coord origin = leaf.origin(); + + leaf.setValueOn(openvdb::Coord(0, 1, 0), openvdb::Vec3R(1, 1, 1)); + leaf.setValueOn(openvdb::Coord(1, 0, 0), openvdb::Vec3R(1, 1, 1)); + + // read and write topology to disk + + { + // create a grid with the leaf for topology testing + typedef openvdb::Grid, 5>>>> GridType; + GridType::Ptr grid = GridType::create(); + grid->setName("leaf_vec3r"); + grid->tree().addLeaf(new LeafT(leaf)); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); + + // write to file + { + openvdb::io::File file("leaf_vec3r.vdb"); + file.write(grids); + file.close(); + } + + // read grid from file + GridType::Ptr gridFromDisk; + { + openvdb::io::File file("leaf_vec3r.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("leaf_vec3r"); + file.close(); + + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } + + LeafT* leaf2 = gridFromDisk->tree().probeLeaf(origin); + EXPECT_TRUE(leaf2); + + // check topology and values match + + EXPECT_TRUE(leaf2->getValue(openvdb::Coord(0, 1, 0)) == openvdb::Vec3R(1, 1, 1)); + EXPECT_TRUE(leaf2->getValue(openvdb::Coord(1, 0, 0)) == openvdb::Vec3R(1, 1, 1)); + EXPECT_TRUE(leaf2->onVoxelCount() == 2); + + remove("leaf_vec3r.vdb"); + } +} + +TEST_F(TestLeafIOTest, testTreeIOInt) { TestLeafIO::testTreeIO(); } +TEST_F(TestLeafIOTest, testTreeIOFloat) { TestLeafIO::testTreeIO(); } +TEST_F(TestLeafIOTest, testTreeIODouble) { TestLeafIO::testTreeIO(); } +TEST_F(TestLeafIOTest, testTreeIOBool) { TestLeafIO::testTreeIO(); } +TEST_F(TestLeafIOTest, testTreeIOByte) { TestLeafIO::testTreeIO(); } + + +TEST_F(TestLeafIOTest, testTreeIOVec3R) +{ + using LeafT = openvdb::tree::LeafNode; + LeafT leaf(openvdb::Coord(0, 0, 0)); leaf.setValueOn(openvdb::Coord(0, 1, 0), openvdb::Vec3R(1, 1, 1)); leaf.setValueOn(openvdb::Coord(1, 0, 0), openvdb::Vec3R(1, 1, 1)); @@ -75,9 +198,6 @@ TEST_F(TestLeafIOTest, testBufferVec3R) leaf.setValueOn(openvdb::Coord(0, 1, 1), openvdb::Vec3R(1, 1, 1)); std::istringstream istr(ostr.str(), std::ios_base::binary); - - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. openvdb::io::setCurrentVersion(istr); leaf.readBuffers(istr); diff --git a/openvdb/openvdb/unittest/TestLeafMask.cc b/openvdb/openvdb/unittest/TestLeafMask.cc index b641ca8836..a9ebab62f3 100644 --- a/openvdb/openvdb/unittest/TestLeafMask.cc +++ b/openvdb/openvdb/unittest/TestLeafMask.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include "util.h" // for unittest_util::makeSphere() #include #include @@ -291,6 +292,60 @@ TEST_F(TestLeafMask, testIO) leaf.setValueOn(openvdb::Coord(0, 1, 0)); leaf.setValueOn(openvdb::Coord(1, 0, 0)); + // read and write topology to disk + + { + // create a grid with the leaf for topology testing + typedef openvdb::Grid, 5>>>> MaskGrid; + MaskGrid::Ptr grid = MaskGrid::create(); + grid->setName("leaf_mask"); + grid->tree().addLeaf(new LeafType(leaf)); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); + + // write to file + { + openvdb::io::File file("leaf_mask.vdb"); + file.write(grids); + file.close(); + } + + // read grid from file + MaskGrid::Ptr gridFromDisk; + { + openvdb::io::File file("leaf_mask.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("leaf_mask"); + file.close(); + + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } + + LeafType* leaf2 = gridFromDisk->tree().probeLeaf(origin); + EXPECT_TRUE(leaf2); + + // check topology and values match + + EXPECT_EQ(origin, leaf2->origin()); + EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(0, 1, 0))); + EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(1, 0, 0))); + EXPECT_TRUE(leaf2->onVoxelCount() == 2); + + remove("leaf_mask.vdb"); + } +} + + +TEST_F(TestLeafMask, testTreeIO) +{ + LeafType leaf(openvdb::Coord(1, 3, 5)); + const openvdb::Coord origin = leaf.origin(); + + leaf.setValueOn(openvdb::Coord(0, 1, 0)); + leaf.setValueOn(openvdb::Coord(1, 0, 0)); + std::ostringstream ostr(std::ios_base::binary); leaf.writeBuffers(ostr); @@ -299,8 +354,6 @@ TEST_F(TestLeafMask, testIO) leaf.setValueOn(openvdb::Coord(0, 1, 1)); std::istringstream istr(ostr.str(), std::ios_base::binary); - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. openvdb::io::setCurrentVersion(istr); leaf.readBuffers(istr); diff --git a/openvdb/openvdb/unittest/TestMeshToVolume.cc b/openvdb/openvdb/unittest/TestMeshToVolume.cc index 5c4ff2ffdf..587c092cf2 100644 --- a/openvdb/openvdb/unittest/TestMeshToVolume.cc +++ b/openvdb/openvdb/unittest/TestMeshToVolume.cc @@ -211,8 +211,8 @@ TEST_F(TestMeshToVolume, testInterrupt) // Should have returned _something_ EXPECT_TRUE(grid); - // Expect to interrupt in under a second + // Expect to interrupt in under two seconds const auto duration = std::chrono::duration_cast(end - start); - EXPECT_LT(duration.count(), 1000); + EXPECT_LT(duration.count(), 2000); } diff --git a/openvdb/openvdb/unittest/TestMultiResGrid.cc b/openvdb/openvdb/unittest/TestMultiResGrid.cc index 32735d508f..3930fbe047 100644 --- a/openvdb/openvdb/unittest/TestMultiResGrid.cc +++ b/openvdb/openvdb/unittest/TestMultiResGrid.cc @@ -208,6 +208,8 @@ TEST_F(TestMultiResGrid, testIO) { using namespace openvdb; + openvdb::initialize(); + const float radius = 1.0f; const Vec3f center(0.0f, 0.0f, 0.0f); const float voxelSize = 0.01f; @@ -243,7 +245,6 @@ TEST_F(TestMultiResGrid, testIO) outputFile.close(); // Read grids - openvdb::initialize(); openvdb::io::File file( filename ); file.open(); GridPtrVecPtr grids = file.getGrids(); @@ -280,7 +281,7 @@ TEST_F(TestMultiResGrid, testModels) << "\" =====================" << std::endl; std::cerr << "Reading \"" << filenames[i] << "\" ..."; io::File file( path + filenames[i] ); - file.open(false);//disable delayed loading + file.open(); FloatGrid::Ptr model = gridPtrCast(file.getGrids()->at(0)); std::cerr << " done\nProcessing \"" << filenames[i] << "\" ..."; timer.start("\nMultiResGrid processing"); diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc new file mode 100644 index 0000000000..0729f6cc98 --- /dev/null +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -0,0 +1,709 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "util.h" // for unittest_util::genPoints + +namespace { + +class PointList { +public: + using PosType = openvdb::Vec3R; + PointList(const std::vector& points) : mPoints(&points) {} + size_t size() const { return mPoints->size(); } + void getPos(size_t n, PosType& xyz) const { xyz = (*mPoints)[n]; } +private: + std::vector const * const mPoints; +}; + +} // namespace + +class TestPointCodec: public ::testing::Test +{ +}; + +TEST_F(TestPointCodec, testPointIndexCodecIO) +{ + using namespace openvdb; + using namespace openvdb::io; + using PointIndexGrid = tools::PointIndexGrid; + + openvdb::initialize(); + CodecRegistry::clear(); + + // Generate points on a unit sphere and build a PointIndexGrid + std::vector points; + unittest_util::genPoints(100, points); + PointList pointList(points); + + const double voxelSize = 0.1; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointIndexGrid::Ptr srcGrid = + tools::createPointIndexGrid(pointList, *transform); + srcGrid->setName("point_index_grid"); + + const std::string rawPath = "testPointIndexCodec_raw.vdb"; + + // Phase 1: write/read without codec + { + io::File f(rawPath); + f.write(GridPtrVec{srcGrid}); + } + + PointIndexGrid::Ptr rawGrid; + { + io::File f(rawPath); + f.open(); + rawGrid = gridPtrCast(f.readGrid("point_index_grid")); + f.close(); + } + ASSERT_TRUE(rawGrid); + + PointIndexGrid::Ptr rawTopo; + { + io::File f(rawPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("point_index_grid")); + rawTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(rawTopo); + EXPECT_EQ(rawTopo->activeVoxelCount(), Index64(97)); + EXPECT_EQ(rawTopo->getName(), std::string("point_index_grid")); + + std::remove(rawPath.c_str()); + + const std::string codecPath = "testPointIndexCodec_codec.vdb"; + + // Phase 2: register codec, write/read with codec + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointIndexGrid::gridType())); + + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + PointIndexGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + codecGrid = gridPtrCast(f.readGrid("point_index_grid")); + f.close(); + } + ASSERT_TRUE(codecGrid); + + // Phase 3: full read comparison + EXPECT_TRUE(srcGrid->tree().hasSameTopology(srcGrid->tree())); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree())); + { + auto codecAcc = codecGrid->getConstAccessor(); + for (PointIndexGrid::ValueOnCIter it = srcGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, codecAcc.getValue(it.getCoord())); + } + } + + // Compare leaf indices arrays + { + auto srcLeafIt = srcGrid->tree().cbeginLeaf(); + auto codecLeafIt = codecGrid->tree().cbeginLeaf(); + for (; srcLeafIt; ++srcLeafIt, ++codecLeafIt) { + ASSERT_TRUE(codecLeafIt); + EXPECT_EQ(srcLeafIt->indices().size(), codecLeafIt->indices().size()); + for (size_t i = 0; i < srcLeafIt->indices().size(); ++i) { + EXPECT_EQ(srcLeafIt->indices()[i], codecLeafIt->indices()[i]); + } + } + EXPECT_TRUE(!codecLeafIt); + } + + // Phase 4: TopologyOnly read + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + PointIndexGrid::Ptr codecTopo; + { + io::File f(codecPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("point_index_grid", topoOpts)); + codecTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(codecTopo); + // TopologyOnly: topology and active state are preserved; voxel values are zero-filled + EXPECT_EQ(codecTopo->tree().leafCount(), srcGrid->tree().leafCount()); + EXPECT_EQ(codecTopo->activeVoxelCount(), srcGrid->activeVoxelCount()); + for (auto leafIt = codecTopo->tree().cbeginLeaf(); leafIt; ++leafIt) { + EXPECT_FALSE(leafIt->buffer().empty()); + for (auto voxIt = leafIt->cbeginValueOn(); voxIt; ++voxIt) { + EXPECT_EQ(*voxIt, PointIndexGrid::ValueType(0)); + } + } + EXPECT_EQ(codecTopo->getName(), std::string("point_index_grid")); + + // Cleanup + CodecRegistry::clear(); + std::remove(codecPath.c_str()); +} + +TEST_F(TestPointCodec, testPointDataCodecIO) +{ + using namespace openvdb; + using namespace openvdb::io; + using namespace openvdb::points; + using PointDataTree = PointDataGrid::TreeType; + + openvdb::initialize(); + CodecRegistry::clear(); + + // Helper: compare P attribute values leaf-by-leaf between two PointDataGrids + auto comparePositions = [](const PointDataGrid& a, const PointDataGrid& b) { + auto aIt = a.tree().cbeginLeaf(); + auto bIt = b.tree().cbeginLeaf(); + for (; aIt && bIt; ++aIt, ++bIt) { + EXPECT_EQ(aIt->pointCount(), bIt->pointCount()); + AttributeHandle aH(aIt->constAttributeArray("P")); + AttributeHandle bH(bIt->constAttributeArray("P")); + for (Index i = 0; i < aIt->pointCount(); ++i) { + const Vec3f av = aH.get(i); + const Vec3f bv = bH.get(i); + EXPECT_NEAR(av.x(), bv.x(), 1e-6f); + EXPECT_NEAR(av.y(), bv.y(), 1e-6f); + EXPECT_NEAR(av.z(), bv.z(), 1e-6f); + } + } + EXPECT_TRUE(!aIt && !bIt); + }; + + // ----------------------------------------------------------------------- + // Section A: Positions only + // ----------------------------------------------------------------------- + { + const std::vector positions = { + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(1.5f, 3.5f, 1.0f), + Vec3f(-1.0f, 6.0f, -2.0f), + Vec3f(1.1f, 1.25f, 0.06f) + }; + + const float voxelSize = 0.5f; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid(positions, *transform); + srcGrid->setName("pdg_positions"); + + const std::string rawPath = "testPDG_A_raw.vdb"; + + // Phase 1: write/read without codec + { + io::File f(rawPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr rawGrid; + { + io::File f(rawPath); + f.open(); + rawGrid = gridPtrCast(f.readGrid("pdg_positions")); + f.close(); + } + ASSERT_TRUE(rawGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(rawGrid->tree())); + + PointDataGrid::Ptr rawTopo; + { + io::File f(rawPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("pdg_positions")); + rawTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(rawTopo); + EXPECT_EQ(rawTopo->activeVoxelCount(), Index64(4)); + + std::remove(rawPath.c_str()); + + const std::string codecPath = "testPDG_A_codec.vdb"; + + // Phase 2: register codec, write/read with codec + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointDataGrid::gridType())); + + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + codecGrid = gridPtrCast(f.readGrid("pdg_positions")); + f.close(); + } + ASSERT_TRUE(codecGrid); + + // Phase 3: compare src vs codec + EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(codecGrid->tree())); + comparePositions(*srcGrid, *codecGrid); + + // Phase 4: TopologyOnly read + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + PointDataGrid::Ptr codecTopo; + { + io::File f(codecPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("pdg_positions", topoOpts)); + codecTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(codecTopo); + // TopologyOnly: topology and active state are preserved; voxel values are zero-filled + EXPECT_EQ(codecTopo->tree().leafCount(), srcGrid->tree().leafCount()); + EXPECT_EQ(codecTopo->activeVoxelCount(), srcGrid->activeVoxelCount()); + for (auto leafIt = codecTopo->tree().cbeginLeaf(); leafIt; ++leafIt) { + for (auto voxIt = leafIt->cbeginValueOn(); voxIt; ++voxIt) { + EXPECT_EQ(*voxIt, PointDataGrid::ValueType(0)); + } + } + + std::remove(codecPath.c_str()); + } + + // ----------------------------------------------------------------------- + // Section B: Multiple attributes + // ----------------------------------------------------------------------- + { + const std::vector positions = { + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(1.5f, 3.5f, 1.0f), + Vec3f(-1.0f, 6.0f, -2.0f), + Vec3f(1.1f, 1.25f, 0.06f) + }; + const std::vector velocities = { + Vec3f(1.0f, 0.0f, 0.0f), + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(0.0f, 0.0f, 1.0f), + Vec3f(1.0f, 1.0f, 0.5f) + }; + const std::vector ids = {0, 1, 2, 3}; + + const float voxelSize = 0.5f; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointAttributeVector posWrapper(positions); + tools::PointIndexGrid::Ptr pointIndexGrid = + tools::createPointIndexGrid(posWrapper, *transform); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid( + *pointIndexGrid, posWrapper, *transform); + srcGrid->setName("pdg_multi"); + + PointDataTree& tree = srcGrid->tree(); + tools::PointIndexTree& indexTree = pointIndexGrid->tree(); + + appendAttribute(tree, "velocity"); + populateAttribute>( + tree, indexTree, "velocity", + PointAttributeVector(velocities)); + + appendAttribute(tree, "id"); + populateAttribute>( + tree, indexTree, "id", + PointAttributeVector(ids)); + + // Verify attribute count on src grid (P, velocity, id) + { + auto leafIt = srcGrid->tree().cbeginLeaf(); + ASSERT_TRUE(leafIt); + EXPECT_EQ(leafIt->attributeSet().size(), size_t(3)); + } + + CodecRegistry::clear(); + + const std::string rawPath = "testPDG_B_raw.vdb"; + + // Phase 1: write/read without codec + { + io::File f(rawPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr rawGrid; + { + io::File f(rawPath); + f.open(); + rawGrid = gridPtrCast(f.readGrid("pdg_multi")); + f.close(); + } + ASSERT_TRUE(rawGrid); + + std::remove(rawPath.c_str()); + + const std::string codecPath = "testPDG_B_codec.vdb"; + + // Phase 2: register codec, write/read with codec + io::internal::initialize(); + + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + codecGrid = gridPtrCast(f.readGrid("pdg_multi")); + f.close(); + } + ASSERT_TRUE(codecGrid); + + EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(codecGrid->tree())); + + // Verify attribute count on codec grid + { + auto leafIt = codecGrid->tree().cbeginLeaf(); + ASSERT_TRUE(leafIt); + EXPECT_EQ(leafIt->attributeSet().size(), size_t(3)); + } + + // Compare all three attributes leaf-by-leaf + { + auto srcIt = srcGrid->tree().cbeginLeaf(); + auto codecIt = codecGrid->tree().cbeginLeaf(); + for (; srcIt && codecIt; ++srcIt, ++codecIt) { + EXPECT_EQ(srcIt->pointCount(), codecIt->pointCount()); + AttributeHandle srcP(srcIt->constAttributeArray("P")); + AttributeHandle codecP(codecIt->constAttributeArray("P")); + AttributeHandle srcVel(srcIt->constAttributeArray("velocity")); + AttributeHandle codecVel(codecIt->constAttributeArray("velocity")); + AttributeHandle srcId(srcIt->constAttributeArray("id")); + AttributeHandle codecId(codecIt->constAttributeArray("id")); + for (Index i = 0; i < srcIt->pointCount(); ++i) { + const Vec3f rp = srcP.get(i); + const Vec3f cp = codecP.get(i); + EXPECT_NEAR(rp.x(), cp.x(), 1e-6f); + EXPECT_NEAR(rp.y(), cp.y(), 1e-6f); + EXPECT_NEAR(rp.z(), cp.z(), 1e-6f); + const Vec3f rv = srcVel.get(i); + const Vec3f cv = codecVel.get(i); + EXPECT_NEAR(rv.x(), cv.x(), 1e-6f); + EXPECT_NEAR(rv.y(), cv.y(), 1e-6f); + EXPECT_NEAR(rv.z(), cv.z(), 1e-6f); + EXPECT_EQ(srcId.get(i), codecId.get(i)); + } + } + EXPECT_TRUE(!srcIt && !codecIt); + } + + CodecRegistry::clear(); + std::remove(codecPath.c_str()); + } + + // ----------------------------------------------------------------------- + // Section C: Shared vs non-shared descriptors + // ----------------------------------------------------------------------- + { + std::vector pts; + unittest_util::genPoints(100, pts); + + std::vector positions; + positions.reserve(pts.size()); + for (const auto& p : pts) { + positions.emplace_back(float(p.x()), float(p.y()), float(p.z())); + } + + const double voxelSize = 0.1; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid(positions, *transform); + srcGrid->setName("pdg_desc"); + + // All leaves should share one Descriptor::Ptr initially + ASSERT_GT(srcGrid->tree().leafCount(), Index32(1)); + { + auto leafIt = srcGrid->tree().cbeginLeaf(); + auto firstDescPtr = leafIt->attributeSet().descriptorPtr(); + for (; leafIt; ++leafIt) { + EXPECT_EQ(leafIt->attributeSet().descriptorPtr(), firstDescPtr); + } + } + + io::internal::initialize(); + + // -- C1: Shared descriptors (default) -- + // All leaves already share one pointer; this exercises the header=1 write path. + const std::string sharedPath = "testPDG_C_shared.vdb"; + { + io::File f(sharedPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr sharedGrid; + { + io::File f(sharedPath); + f.open(); + sharedGrid = gridPtrCast(f.readGrid("pdg_desc")); + f.close(); + } + ASSERT_TRUE(sharedGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(sharedGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(sharedGrid->tree())); + comparePositions(*srcGrid, *sharedGrid); + + // -- C2: After makeDescriptorUnique -- + // makeDescriptorUnique() creates ONE new descriptor and assigns it to every + // leaf, so all leaves still share a single pointer (the new copy). + // The codec still detects matching descriptors and writes header=1; + // this is a regression check that the round-trip remains correct. + makeDescriptorUnique(srcGrid->tree()); + { + auto leafIt = srcGrid->tree().cbeginLeaf(); + auto firstDescPtr = leafIt->attributeSet().descriptorPtr(); + ++leafIt; + if (leafIt) { + // All leaves share the same new pointer + EXPECT_EQ(leafIt->attributeSet().descriptorPtr(), firstDescPtr); + } + } + + const std::string nonSharedPath = "testPDG_C_nonshared.vdb"; + { + io::File f(nonSharedPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr nonSharedGrid; + { + io::File f(nonSharedPath); + f.open(); + nonSharedGrid = gridPtrCast(f.readGrid("pdg_desc")); + f.close(); + } + ASSERT_TRUE(nonSharedGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(nonSharedGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(nonSharedGrid->tree())); + comparePositions(*sharedGrid, *nonSharedGrid); + + // -- C3: Genuinely different descriptors (exercises the header=0 write path) -- + // Add "extra" to all leaves, then drop it from only the first leaf so that + // leaf descriptors differ by value, triggering matching=false in the codec. + appendAttribute(srcGrid->tree(), "extra"); + makeDescriptorUnique(srcGrid->tree()); + srcGrid->setName("pdg_desc_diff"); + + { + auto leafIt = srcGrid->tree().beginLeaf(); + ASSERT_TRUE(leafIt); + const size_t extraIdx = + leafIt->attributeSet().descriptor().find("extra"); + ASSERT_NE(extraIdx, AttributeSet::INVALID_POS); + const std::vector dropIndices = {extraIdx}; + AttributeSet::Descriptor::Ptr newDesc = + leafIt->attributeSet().descriptor().duplicateDrop(dropIndices); + leafIt->dropAttributes( + dropIndices, leafIt->attributeSet().descriptor(), newDesc); + } + + const std::string diffPath = "testPDG_C_diff.vdb"; + { + io::File f(diffPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr diffGrid; + { + io::File f(diffPath); + f.open(); + diffGrid = gridPtrCast(f.readGrid("pdg_desc_diff")); + f.close(); + } + ASSERT_TRUE(diffGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(diffGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(diffGrid->tree())); + + // First leaf has {P} only; remaining leaves have {P, extra} + { + auto diffIt = diffGrid->tree().cbeginLeaf(); + ASSERT_TRUE(diffIt); + EXPECT_EQ(diffIt->attributeSet().size(), size_t(1)); + ++diffIt; + if (diffIt) { + EXPECT_EQ(diffIt->attributeSet().size(), size_t(2)); + } + } + + std::remove(sharedPath.c_str()); + std::remove(nonSharedPath.c_str()); + std::remove(diffPath.c_str()); + } +} + +// A leafless PointDataGrid stores numPasses == 0. The attribute count is +// derived as (numPasses - 4) / 2; without an underflow guard this wraps to a +// huge unsigned value and the attribute loops spin ~4.3e9 times, hanging +// both write and read. This test exercises an empty grid round-trip. +TEST_F(TestPointCodec, testPointDataCodecEmptyGrid) +{ + using namespace openvdb; + using namespace openvdb::io; + using namespace openvdb::points; + + openvdb::initialize(); + CodecRegistry::clear(); + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointDataGrid::gridType())); + + PointDataGrid::Ptr srcGrid = PointDataGrid::create(); + srcGrid->setName("pdg_empty"); + EXPECT_EQ(srcGrid->tree().leafCount(), Index32(0)); + + const std::string codecPath = "testPDG_empty_codec.vdb"; + + { + io::File f(codecPath); + EXPECT_NO_THROW(f.write(GridPtrVec{srcGrid})); + } + + PointDataGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("pdg_empty")); + codecGrid = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(codecGrid); + EXPECT_EQ(codecGrid->tree().leafCount(), Index32(0)); + EXPECT_EQ(codecGrid->activeVoxelCount(), Index64(0)); + + std::remove(codecPath.c_str()); +} + +// Regression test for the fix in AttributeArray::skipPagedBuffers and +// Page::skipBuffers: when the stream is not seekable (written via io::Stream), +// skip must read-and-discard rather than seekg. Without the fix, both code +// paths called seekg unconditionally, corrupting the stream position on +// non-seekable streams. +TEST_F(TestPointCodec, testPointDataCodecSkipNonSeekable) +{ + using namespace openvdb; + using namespace openvdb::io; + using namespace openvdb::points; + + openvdb::initialize(); + CodecRegistry::clear(); + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointDataGrid::gridType())); + + const std::vector positions = { + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(1.5f, 3.5f, 1.0f), + Vec3f(-1.0f, 6.0f, -2.0f), + Vec3f(1.1f, 1.25f, 0.06f) + }; + const std::vector velocities = { + Vec3f(1.0f, 0.0f, 0.0f), + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(0.0f, 0.0f, 1.0f), + Vec3f(1.0f, 1.0f, 0.5f) + }; + const std::vector ids = {0, 1, 2, 3}; + + const float voxelSize = 0.5f; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointAttributeVector posWrapper(positions); + tools::PointIndexGrid::Ptr pointIndexGrid = + tools::createPointIndexGrid(posWrapper, *transform); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid( + *pointIndexGrid, posWrapper, *transform); + srcGrid->setName("pdg_skip"); + + PointDataTree& tree = srcGrid->tree(); + tools::PointIndexTree& indexTree = pointIndexGrid->tree(); + + appendAttribute(tree, "velocity"); + populateAttribute>( + tree, indexTree, "velocity", + PointAttributeVector(velocities)); + + appendAttribute(tree, "id"); + populateAttribute>( + tree, indexTree, "id", + PointAttributeVector(ids)); + + // Write via io::Stream (seekable=false by construction) + std::ostringstream ostr(std::ios_base::binary); + io::Stream(ostr).write(GridPtrVec{srcGrid}); + + // Build ReadOptions requesting only "P" — "velocity" and "id" will be skipped + io::ReadOptions readOptions; + auto typeData = std::make_shared(); + typeData->pointAttributeNames = {"P"}; + readOptions.typeData[PointDataGrid::gridType()] = typeData; + + // Read via io::Stream (non-seekable by construction) with our ReadOptions, so the + // skip path is exercised with seekable == false. + std::istringstream is(ostr.str(), std::ios_base::binary); + io::Stream strm(is, readOptions); + GridPtrVecPtr grids = strm.getGrids(); + ASSERT_TRUE(grids); + ASSERT_EQ(grids->size(), size_t(1)); + + PointDataGrid::Ptr resultGrid = gridPtrCast((*grids)[0]); + ASSERT_TRUE(resultGrid); + + // Only "P" should be present; "velocity" and "id" were skipped + { + auto leafIt = resultGrid->tree().cbeginLeaf(); + ASSERT_TRUE(leafIt); + EXPECT_EQ(leafIt->attributeSet().size(), size_t(1)); + EXPECT_NE(leafIt->attributeSet().find("P"), AttributeSet::INVALID_POS); + EXPECT_EQ(leafIt->attributeSet().find("velocity"), AttributeSet::INVALID_POS); + EXPECT_EQ(leafIt->attributeSet().find("id"), AttributeSet::INVALID_POS); + } + + // All 4 points should be present and readable + EXPECT_EQ(pointCount(resultGrid->tree()), Index64(4)); + { + std::vector readPositions; + for (auto it = resultGrid->tree().cbeginLeaf(); it; ++it) { + AttributeHandle posHandle(it->constAttributeArray("P")); + for (Index i = 0; i < it->pointCount(); ++i) { + readPositions.push_back(posHandle.get(i)); + } + } + EXPECT_EQ(readPositions.size(), size_t(4)); + } +} diff --git a/openvdb/openvdb/unittest/TestPointConversion.cc b/openvdb/openvdb/unittest/TestPointConversion.cc index d6420019cf..8d4132b92d 100644 --- a/openvdb/openvdb/unittest/TestPointConversion.cc +++ b/openvdb/openvdb/unittest/TestPointConversion.cc @@ -1,7 +1,6 @@ // Copyright Contributors to the OpenVDB Project // SPDX-License-Identifier: Apache-2.0 -#include #include #include #include diff --git a/openvdb/openvdb/unittest/TestPointCount.cc b/openvdb/openvdb/unittest/TestPointCount.cc index 58ea887907..0d2332674b 100644 --- a/openvdb/openvdb/unittest/TestPointCount.cc +++ b/openvdb/openvdb/unittest/TestPointCount.cc @@ -3,7 +3,6 @@ #include #include -#include #include #include @@ -314,30 +313,14 @@ TEST_F(TestPointCount, testGroup) GroupFilter groupFilter("test", attributeSet); - bool inCoreOnly; -#ifdef OPENVDB_USE_DELAYED_LOADING - inCoreOnly = true; - - EXPECT_EQ(pointCount(inputTree, NullFilter(), inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, ActiveFilter(), inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, InactiveFilter(), inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, groupFilter, inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, BinaryFilter( - groupFilter, ActiveFilter()), inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, BinaryFilter( - groupFilter, InactiveFilter()), inCoreOnly), Index64(0)); -#endif - - inCoreOnly = false; - - EXPECT_EQ(pointCount(inputTree, NullFilter(), inCoreOnly), Index64(4)); - EXPECT_EQ(pointCount(inputTree, ActiveFilter(), inCoreOnly), Index64(3)); - EXPECT_EQ(pointCount(inputTree, InactiveFilter(), inCoreOnly), Index64(1)); - EXPECT_EQ(pointCount(inputTree, groupFilter, inCoreOnly), Index64(2)); + EXPECT_EQ(pointCount(inputTree, NullFilter()), Index64(4)); + EXPECT_EQ(pointCount(inputTree, ActiveFilter()), Index64(3)); + EXPECT_EQ(pointCount(inputTree, InactiveFilter()), Index64(1)); + EXPECT_EQ(pointCount(inputTree, groupFilter), Index64(2)); EXPECT_EQ(pointCount(inputTree, BinaryFilter( - groupFilter, ActiveFilter()), inCoreOnly), Index64(1)); + groupFilter, ActiveFilter())), Index64(1)); EXPECT_EQ(pointCount(inputTree, BinaryFilter( - groupFilter, InactiveFilter()), inCoreOnly), Index64(1)); + groupFilter, InactiveFilter())), Index64(1)); } std::remove(filename.c_str()); @@ -529,51 +512,6 @@ TEST_F(TestPointCount, testOffsets) fileOut.write(grids); } -#ifdef OPENVDB_USE_DELAYED_LOADING - // test point offsets for a delay-loaded grid - { - io::File fileIn(filename); - fileIn.open(); - - GridPtrVecPtr grids = fileIn.getGrids(); - - fileIn.close(); - - EXPECT_EQ(grids->size(), size_t(1)); - - PointDataGrid::Ptr inputGrid = GridBase::grid((*grids)[0]); - - EXPECT_TRUE(inputGrid); - - PointDataTree& inputTree = inputGrid->tree(); - - std::vector offsets; - std::vector includeGroups; - std::vector excludeGroups; - - MultiGroupFilter filter(includeGroups, excludeGroups, inputTree.cbeginLeaf()->attributeSet()); - Index64 total = pointOffsets(offsets, inputTree, filter, /*inCoreOnly=*/true); - - EXPECT_EQ(offsets.size(), size_t(4)); - EXPECT_EQ(offsets[0], Index64(0)); - EXPECT_EQ(offsets[1], Index64(0)); - EXPECT_EQ(offsets[2], Index64(0)); - EXPECT_EQ(offsets[3], Index64(0)); - EXPECT_EQ(total, Index64(0)); - - offsets.clear(); - - total = pointOffsets(offsets, inputTree, filter, /*inCoreOnly=*/false); - - EXPECT_EQ(offsets.size(), size_t(4)); - EXPECT_EQ(offsets[0], Index64(1)); - EXPECT_EQ(offsets[1], Index64(3)); - EXPECT_EQ(offsets[2], Index64(4)); - EXPECT_EQ(offsets[3], Index64(5)); - EXPECT_EQ(total, Index64(5)); - } -#endif - std::remove(filename.c_str()); } diff --git a/openvdb/openvdb/unittest/TestPointDataLeaf.cc b/openvdb/openvdb/unittest/TestPointDataLeaf.cc index bd96959c45..c5956e2991 100644 --- a/openvdb/openvdb/unittest/TestPointDataLeaf.cc +++ b/openvdb/openvdb/unittest/TestPointDataLeaf.cc @@ -1058,76 +1058,93 @@ TEST_F(TestPointDataLeaf, testIO) // read and write topology to disk { - LeafType leaf2(openvdb::Coord(0, 0, 0)); + // create a grid with the leaf for topology testing + PointDataGrid::Ptr grid = PointDataGrid::create(); + grid->setName("points"); + grid->tree().addLeaf(new LeafType(leaf)); - std::ostringstream ostr(std::ios_base::binary); - leaf.writeTopology(ostr); + openvdb::GridCPtrVec grids; + grids.push_back(grid); - std::istringstream istr(ostr.str(), std::ios_base::binary); - leaf2.readTopology(istr); + // write to file + { + io::File file("leaf_topology.vdb"); + file.write(grids); + file.close(); + } + + // read grid from file + PointDataGrid::Ptr gridFromDisk; + { + io::File file("leaf_topology.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); + file.close(); + + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } + + LeafType* leaf2 = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); + EXPECT_TRUE(leaf2); // check topology matches - EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount()); - EXPECT_TRUE(leaf2.isValueOn(4)); - EXPECT_TRUE(!leaf2.isValueOn(5)); + EXPECT_EQ(leaf.onVoxelCount(), leaf2->onVoxelCount()); + EXPECT_TRUE(leaf2->isValueOn(4)); + EXPECT_TRUE(!leaf2->isValueOn(5)); - // check only topology (values and attributes still empty) + // check that values and attributes are correctly read - EXPECT_EQ(leaf2.getValue(4), ValueType(0)); - EXPECT_EQ(leaf2.attributeSet().size(), size_t(0)); + EXPECT_EQ(leaf2->getValue(4), ValueType(20)); + EXPECT_EQ(leaf2->attributeSet().size(), size_t(2)); + + remove("leaf_topology.vdb"); } // read and write buffers to disk { - LeafType leaf2(openvdb::Coord(0, 0, 0)); - - io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata); + // create a grid with the leaf for buffer testing + PointDataGrid::Ptr grid = PointDataGrid::create(); + grid->setName("points"); + grid->tree().addLeaf(new LeafType(leaf)); - std::ostringstream ostr(std::ios_base::binary); - io::setStreamMetadataPtr(ostr, streamMetadata); - io::setDataCompression(ostr, io::COMPRESS_BLOSC); - leaf.writeTopology(ostr); - for (Index b = 0; b < leaf.buffers(); b++) { - uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b); - streamMetadata->setPass(pass); - leaf.writeBuffers(ostr); - } - { // error checking - streamMetadata->setPass(1000); - leaf.writeBuffers(ostr); + openvdb::GridCPtrVec grids; + grids.push_back(grid); - io::StreamMetadata::Ptr meta; - io::setStreamMetadataPtr(ostr, meta); - EXPECT_THROW(leaf.writeBuffers(ostr), openvdb::IoError); + // write to file + { + io::File file("leaf_buffers.vdb"); + file.write(grids); + file.close(); } - std::istringstream istr(ostr.str(), std::ios_base::binary); - io::setStreamMetadataPtr(istr, streamMetadata); - io::setDataCompression(istr, io::COMPRESS_BLOSC); - - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - io::setCurrentVersion(istr); + // read grid from file + PointDataGrid::Ptr gridFromDisk; + { + io::File file("leaf_buffers.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); + file.close(); - leaf2.readTopology(istr); - for (Index b = 0; b < leaf.buffers(); b++) { - uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b); - streamMetadata->setPass(pass); - leaf2.readBuffers(istr); + gridFromDisk = openvdb::gridPtrCast(baseGrid); } + LeafType* leaf2 = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); + EXPECT_TRUE(leaf2); + // check topology matches - EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount()); - EXPECT_TRUE(leaf2.isValueOn(4)); - EXPECT_TRUE(!leaf2.isValueOn(5)); + EXPECT_EQ(leaf.onVoxelCount(), leaf2->onVoxelCount()); + EXPECT_TRUE(leaf2->isValueOn(4)); + EXPECT_TRUE(!leaf2->isValueOn(5)); - // check only topology (values and attributes still empty) + // check values and attributes are correctly read - EXPECT_EQ(leaf2.getValue(4), ValueType(20)); - EXPECT_EQ(leaf2.attributeSet().size(), size_t(2)); + EXPECT_EQ(leaf2->getValue(4), ValueType(20)); + EXPECT_EQ(leaf2->attributeSet().size(), size_t(2)); + + remove("leaf_buffers.vdb"); } { // test multi-buffer IO @@ -1165,111 +1182,6 @@ TEST_F(TestPointDataLeaf, testIO) EXPECT_TRUE(leaf == *leafFromDisk); } - -#ifdef OPENVDB_USE_DELAYED_LOADING - { // read grids from file and pre-fetch - PointDataGrid::Ptr gridFromDisk; - - { - io::File file("leaf.vdb"); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - gridFromDisk = openvdb::gridPtrCast(baseGrid); - } - - LeafType* leafFromDisk = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); - EXPECT_TRUE(leafFromDisk); - - const AttributeVec3s& position( - AttributeVec3s::cast(leafFromDisk->constAttributeArray("P"))); - const AttributeF& density( - AttributeF::cast(leafFromDisk->constAttributeArray("density"))); - - EXPECT_TRUE(leafFromDisk->buffer().isOutOfCore()); -#ifdef OPENVDB_USE_BLOSC - EXPECT_TRUE(position.isOutOfCore()); - EXPECT_TRUE(density.isOutOfCore()); -#else - // delayed-loading is only available on attribute arrays when using Blosc - EXPECT_TRUE(!position.isOutOfCore()); - EXPECT_TRUE(!density.isOutOfCore()); -#endif - - // prefetch voxel data only - prefetch(gridFromDisk->tree(), /*position=*/false, /*attributes=*/false); - - // ensure out-of-core data is now in-core after pre-fetching - - EXPECT_TRUE(!leafFromDisk->buffer().isOutOfCore()); -#ifdef OPENVDB_USE_BLOSC - EXPECT_TRUE(position.isOutOfCore()); - EXPECT_TRUE(density.isOutOfCore()); -#else - EXPECT_TRUE(!position.isOutOfCore()); - EXPECT_TRUE(!density.isOutOfCore()); -#endif - - { // re-open - io::File file("leaf.vdb"); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - gridFromDisk = openvdb::gridPtrCast(baseGrid); - } - - leafFromDisk = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); - EXPECT_TRUE(leafFromDisk); - - const AttributeVec3s& position2( - AttributeVec3s::cast(leafFromDisk->constAttributeArray("P"))); - const AttributeF& density2( - AttributeF::cast(leafFromDisk->constAttributeArray("density"))); - - // prefetch voxel and position attribute data - prefetch(gridFromDisk->tree(), /*position=*/true, /*attribute=*/false); - - // ensure out-of-core voxel and position data is now in-core after pre-fetching - - EXPECT_TRUE(!leafFromDisk->buffer().isOutOfCore()); - EXPECT_TRUE(!position2.isOutOfCore()); -#ifdef OPENVDB_USE_BLOSC - EXPECT_TRUE(density2.isOutOfCore()); -#else - EXPECT_TRUE(!density2.isOutOfCore()); -#endif - - { // re-open - io::File file("leaf.vdb"); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - gridFromDisk = openvdb::gridPtrCast(baseGrid); - } - - leafFromDisk = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); - EXPECT_TRUE(leafFromDisk); - - const AttributeVec3s& position3( - AttributeVec3s::cast(leafFromDisk->constAttributeArray("P"))); - const AttributeF& density3( - AttributeF::cast(leafFromDisk->constAttributeArray("density"))); - - // prefetch all data - prefetch(gridFromDisk->tree()); - - // ensure out-of-core voxel and position data is now in-core after pre-fetching - - EXPECT_TRUE(!leafFromDisk->buffer().isOutOfCore()); - EXPECT_TRUE(!position3.isOutOfCore()); - EXPECT_TRUE(!density3.isOutOfCore()); - } - - remove("leaf.vdb"); -#endif // OPENVDB_USE_DELAYED_LOADING } { // test multi-buffer IO with varying attribute storage per-leaf @@ -1364,6 +1276,97 @@ TEST_F(TestPointDataLeaf, testIO) } +TEST_F(TestPointDataLeaf, testTreeIO) +{ + using AttributeVec3s = TypedAttributeArray; + using AttributeF = TypedAttributeArray; + + using Descriptor = AttributeSet::Descriptor; + + Descriptor::Ptr descrA = Descriptor::create(AttributeVec3s::attributeType()); + + const size_t size = LeafType::NUM_VOXELS; + + LeafType leaf(openvdb::Coord(0, 0, 0)); + leaf.initializeAttributes(descrA, /*arrayLength=*/size/2); + + descrA = descrA->duplicateAppend("density", AttributeF::attributeType()); + leaf.appendAttribute(leaf.attributeSet().descriptor(), descrA, descrA->find("density")); + + leaf.setOffsetOn(1, 10); + leaf.setOffsetOn(4, 20); + leaf.setOffsetOn(7, 5); + + TypedAttributeArray& attr = + TypedAttributeArray::cast(leaf.attributeArray("density")); + + attr.set(0, 5.0f); + attr.set(50, 2.0f); + attr.set(51, 8.1f); + + { + LeafType leaf2(openvdb::Coord(0, 0, 0)); + + std::ostringstream ostr(std::ios_base::binary); + leaf.writeTopology(ostr); + + std::istringstream istr(ostr.str(), std::ios_base::binary); + leaf2.readTopology(istr); + + EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount()); + EXPECT_TRUE(leaf2.isValueOn(4)); + EXPECT_TRUE(!leaf2.isValueOn(5)); + + EXPECT_EQ(leaf2.getValue(4), ValueType(0)); + EXPECT_EQ(leaf2.attributeSet().size(), size_t(0)); + } + + { + LeafType leaf2(openvdb::Coord(0, 0, 0)); + + io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata); + + std::ostringstream ostr(std::ios_base::binary); + io::setStreamMetadataPtr(ostr, streamMetadata); + io::setDataCompression(ostr, io::COMPRESS_BLOSC); + leaf.writeTopology(ostr); + for (Index b = 0; b < leaf.buffers(); b++) { + uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b); + streamMetadata->setPass(pass); + leaf.writeBuffers(ostr); + } + { + streamMetadata->setPass(1000); + leaf.writeBuffers(ostr); + + io::StreamMetadata::Ptr meta; + io::setStreamMetadataPtr(ostr, meta); + EXPECT_THROW(leaf.writeBuffers(ostr), openvdb::IoError); + } + + std::istringstream istr(ostr.str(), std::ios_base::binary); + io::setStreamMetadataPtr(istr, streamMetadata); + io::setDataCompression(istr, io::COMPRESS_BLOSC); + + io::setCurrentVersion(istr); + + leaf2.readTopology(istr); + for (Index b = 0; b < leaf.buffers(); b++) { + uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b); + streamMetadata->setPass(pass); + leaf2.readBuffers(istr); + } + + EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount()); + EXPECT_TRUE(leaf2.isValueOn(4)); + EXPECT_TRUE(!leaf2.isValueOn(5)); + + EXPECT_EQ(leaf2.getValue(4), ValueType(20)); + EXPECT_EQ(leaf2.attributeSet().size(), size_t(2)); + } +} + + TEST_F(TestPointDataLeaf, testSwap) { using AttributeVec3s = TypedAttributeArray; diff --git a/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc b/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc index a7f85cd33e..7d8997888f 100644 --- a/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc +++ b/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc @@ -2154,9 +2154,6 @@ TEST_F(TestPointRasterizeFrustum, testStreaming) auto leaf = points->tree().cbeginLeaf(); EXPECT_TRUE(leaf); -#ifdef OPENVDB_USE_DELAYED_LOADING - EXPECT_TRUE(leaf->buffer().isOutOfCore()); -#endif using Rasterizer = FrustumRasterizer; using Settings = FrustumRasterizerSettings; @@ -2327,25 +2324,15 @@ TEST_F(TestPointRasterizeFrustum, testStreaming) auto points2 = points->deepCopy(); points2->setTransform(transform); -#ifdef OPENVDB_USE_DELAYED_LOADING - // verify both grids are out-of-core - - EXPECT_TRUE(points->tree().cbeginLeaf()->buffer().isOutOfCore()); - EXPECT_TRUE(points2->tree().cbeginLeaf()->buffer().isOutOfCore()); -#endif - #ifndef ONLY_RASTER_FLOAT // memory tests - if (io::Archive::isDelayedLoadingEnabled() && io::Archive::hasBloscCompression()) { + if (io::Archive::hasBloscCompression()) { FloatGrid::Ptr density1, density2, density3; Vec3SGrid::Ptr velocity1, velocity2, velocity3; const size_t mb = 1024*1024; - const size_t tinyMemory = static_cast(0.1*mb); - - size_t initialMemory; { // memory test 1 - retain caches and streaming disabled Rasterizer rasterizer(settings); @@ -2353,114 +2340,25 @@ TEST_F(TestPointRasterizeFrustum, testStreaming) rasterizer.addPoints(points, /*stream=*/false); rasterizer.addPoints(points2, /*stream=*/false); - initialMemory = rasterizer.memUsage(); + // memory usage should be around 80-100MB throughout rasterization - EXPECT_TRUE(initialMemory > size_t(4*mb) && initialMemory < size_t(16*mb)); + EXPECT_TRUE(rasterizer.memUsage() > size_t(80*mb) && + rasterizer.memUsage() < size_t(100*mb)); EXPECT_EQ(size_t(2), rasterizer.size()); velocity1 = rasterizer.rasterizeAttribute("v"); EXPECT_EQ(Index64(219780), velocity1->activeVoxelCount()); - EXPECT_TRUE(rasterizer.memUsage() > size_t(71*mb) && - rasterizer.memUsage() < size_t(91*mb)); + EXPECT_TRUE(rasterizer.memUsage() > size_t(80*mb) && + rasterizer.memUsage() < size_t(100*mb)); density1 = rasterizer.rasterizeDensity("density"); EXPECT_EQ(Index64(219780), density1->activeVoxelCount()); - // no data is discarded so expect a fairly high memory footprint - EXPECT_TRUE(rasterizer.memUsage() > size_t(80*mb) && rasterizer.memUsage() < size_t(100*mb)); } - - { // memory test 2 - retain caches and streaming enabled - - { // reopen file and deep copy while setting transform - io::File file(filename); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - points = openvdb::gridPtrCast(baseGrid); - points2 = points->deepCopy(); - points2->setTransform(transform); - } - - Rasterizer rasterizer(settings); - - rasterizer.addPoints(points, /*stream=*/true); - rasterizer.addPoints(points2, /*stream=*/true); - - EXPECT_EQ(initialMemory, rasterizer.memUsage()); - - EXPECT_EQ(size_t(2), rasterizer.size()); - - velocity2 = rasterizer.rasterizeAttribute("v"); - EXPECT_EQ(Index64(219780), velocity2->activeVoxelCount()); - - size_t postRasterMemory = rasterizer.memUsage(); - - EXPECT_TRUE(postRasterMemory > size_t(70*mb) && postRasterMemory < size_t(85*mb)); - - density2 = rasterizer.rasterizeDensity("density"); - EXPECT_EQ(Index64(219780), density2->activeVoxelCount()); - - // as data is being streamed, second attribute shouldn't change memory usage very much - - EXPECT_TRUE(rasterizer.memUsage() < (postRasterMemory + tinyMemory)); - } - - { // memory test 3 - release caches and streaming enabled - - { // reopen file and deep copy while setting transform - io::File file(filename); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - points = openvdb::gridPtrCast(baseGrid); - points2 = points->deepCopy(); - points2->setTransform(transform); - } - - auto points3 = points->deepCopy(); - auto points4 = points2->deepCopy(); - - Settings settings2(*frustum); - settings2.threshold = 0.0f; - - Mask mask2(*frustum, nullptr, BBoxd(), /*clipToFrustum=*/false); - - Rasterizer rasterizer(settings2, mask2); - - rasterizer.addPoints(points, /*stream=*/true); - rasterizer.addPoints(points2, /*stream=*/true); - - EXPECT_EQ(initialMemory, rasterizer.memUsage()); - EXPECT_EQ(size_t(2), rasterizer.size()); - - density3 = rasterizer.rasterizeDensity("density", RasterMode::ACCUMULATE, true); - EXPECT_EQ(Index64(219780), density3->activeVoxelCount()); - - // all voxel data, attribute data and caches are being discarded, - // so memory after rasterizing shouldn't change very much - - EXPECT_TRUE(rasterizer.memUsage() < (initialMemory + tinyMemory)); - - // deep-copies of delay-loaded point grids need to be used for repeat rasterization - - rasterizer.clear(); - rasterizer.addPoints(points3, /*stream=*/true); - rasterizer.addPoints(points4, /*stream=*/true); - - EXPECT_EQ(size_t(2), rasterizer.size()); - - EXPECT_TRUE(rasterizer.memUsage() < (initialMemory + tinyMemory)); - - velocity3 = rasterizer.rasterizeAttribute("v", RasterMode::ACCUMULATE, true); - EXPECT_EQ(Index64(219780), velocity3->activeVoxelCount()); - } } #endif diff --git a/openvdb/openvdb/unittest/TestStream.cc b/openvdb/openvdb/unittest/TestStream.cc index cc9bde5b75..6146b8f96e 100644 --- a/openvdb/openvdb/unittest/TestStream.cc +++ b/openvdb/openvdb/unittest/TestStream.cc @@ -36,26 +36,7 @@ class TestStream: public ::testing::Test void TestStream::SetUp() { - openvdb::uninitialize(); - - openvdb::Int32Grid::registerGrid(); - openvdb::FloatGrid::registerGrid(); - - openvdb::StringMetadata::registerType(); - openvdb::Int32Metadata::registerType(); - openvdb::Int64Metadata::registerType(); - openvdb::Vec3IMetadata::registerType(); - openvdb::io::DelayedLoadMetadata::registerType(); - - // Register maps - openvdb::math::MapRegistry::clear(); - openvdb::math::AffineMap::registerMap(); - openvdb::math::ScaleMap::registerMap(); - openvdb::math::UniformScaleMap::registerMap(); - openvdb::math::TranslationMap::registerMap(); - openvdb::math::ScaleTranslateMap::registerMap(); - openvdb::math::UniformScaleTranslateMap::registerMap(); - openvdb::math::NonlinearFrustumMap::registerMap(); + openvdb::initialize(); } @@ -235,3 +216,65 @@ TestStream::testFileReadFromStream() verifyTestGrids(grids, meta); } TEST_F(TestStream, testFileReadFromStream) { testFileReadFromStream(); } + + +TEST_F(TestStream, testUnsupportedReadModes) +{ + using namespace openvdb; + + Int32Grid::Ptr grid1 = Int32Grid::create(0); + grid1->setName("first"); + grid1->tree().setValue(Coord(0, 0, 0), 1); + + FloatGrid::Ptr grid2 = FloatGrid::create(0.0f); + grid2->setName("second"); + grid2->tree().setValue(Coord(1, 2, 3), 2.0f); + + std::ostringstream ostr(std::ios_base::binary); + io::Stream(ostr).write(GridPtrVec{grid1, grid2}); + + // A stream is read sequentially, so read modes that leave part of a grid + // unread cannot be supported - the next grid header would be read from + // the wrong offset. + for (auto readMode: {io::ReadMode::MetadataOnly, io::ReadMode::TopologyOnly}) { + io::ReadOptions readOptions; + readOptions.readMode = readMode; + + std::istringstream is(ostr.str(), std::ios_base::binary); + EXPECT_THROW({ io::Stream strm(is, readOptions); }, ValueError); + } + + // Modes that read every byte of each grid are supported. + for (auto readMode: {io::ReadMode::Original, io::ReadMode::Half, + io::ReadMode::Bool, io::ReadMode::Mask}) + { + io::ReadOptions readOptions; + readOptions.readMode = readMode; + + std::istringstream is(ostr.str(), std::ios_base::binary); + io::Stream strm(is, readOptions); + + GridPtrVecPtr grids = strm.getGrids(); + ASSERT_TRUE(grids); + ASSERT_EQ(grids->size(), size_t(2)); + EXPECT_EQ((*grids)[0]->getName(), std::string("first")); + EXPECT_EQ((*grids)[1]->getName(), std::string("second")); + } +} + + +TEST_F(TestStream, testAssignmentPreservesArchiveFlags) +{ + using namespace openvdb; + + std::ostringstream os1(std::ios_base::binary), os2(std::ios_base::binary); + io::Stream src(os1); + src.setCompression(io::COMPRESS_ZIP); + src.setInstancingEnabled(false); + + io::Stream dst(os2); + dst = src; + + EXPECT_EQ(src.compression(), dst.compression()); + EXPECT_EQ(src.isInstancingEnabled(), dst.isInstancingEnabled()); +} diff --git a/openvdb/openvdb/unittest/TestStreamCompression.cc b/openvdb/openvdb/unittest/TestStreamCompression.cc index ceb8bcf96b..27af0bf40d 100644 --- a/openvdb/openvdb/unittest/TestStreamCompression.cc +++ b/openvdb/openvdb/unittest/TestStreamCompression.cc @@ -8,31 +8,6 @@ #include -#ifdef OPENVDB_USE_DELAYED_LOADING -#ifdef __clang__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-macros" -#endif -// Boost.Interprocess uses a header-only portion of Boost.DateTime -#define BOOST_DATE_TIME_NO_LIB -#ifdef __clang__ -#pragma GCC diagnostic pop -#endif -#include -#include -#include -#include - -#ifdef _WIN32 -#include // open_existing_file(), close_file() -#include -#else -#include // for struct stat -#include // for stat() -#include // for unlink() -#endif -#endif // OPENVDB_USE_DELAYED_LOADING - #include #include #include // for std::iota() @@ -41,6 +16,10 @@ #include #endif +#ifdef _WIN32 +#include +#endif + using namespace openvdb; using namespace openvdb::compression; @@ -467,98 +446,6 @@ TestStreamCompression::testPagedStreams() EXPECT_EQ(fileout.tellp(), std::streampos(values.size()+sizeof(int)*pages)); #endif - -#ifdef OPENVDB_USE_DELAYED_LOADING - auto mappedFile = TestMappedFile::create(filename); - - // read - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - EXPECT_EQ(filein.tellg(), std::streampos(0)); - - PagedInputStream istreamSizeOnly(filein); - istreamSizeOnly.setSizeOnly(true); - - std::vector handles; - - for (size_t i = 0; i < values.size(); i += increment) { - if (size_t(i+increment) > values.size()) { - handles.push_back(istreamSizeOnly.createHandle(values.size() - i)); - } - else { - handles.push_back(istreamSizeOnly.createHandle(increment)); - } - } - -#ifdef OPENVDB_USE_BLOSC - // two integers - compressed size and uncompressed size - EXPECT_EQ(filein.tellg(), std::streampos(pages*sizeof(int)*2)); -#else - // one integer - uncompressed size - EXPECT_EQ(filein.tellg(), std::streampos(pages*sizeof(int))); -#endif - - PagedInputStream istream(filein); - - int pageHandle = 0; - - for (size_t i = 0; i < values.size(); i += increment) { - if (size_t(i+increment) > values.size()) { - istream.read(handles[pageHandle++], values.size() - i); - } - else { - istream.read(handles[pageHandle++], increment); - } - } - - // first three handles live in the same page - - Page& page0 = handles[0]->page(); - Page& page1 = handles[1]->page(); - Page& page2 = handles[2]->page(); - Page& page3 = handles[3]->page(); - - EXPECT_TRUE(page0.isOutOfCore()); - EXPECT_TRUE(page1.isOutOfCore()); - EXPECT_TRUE(page2.isOutOfCore()); - EXPECT_TRUE(page3.isOutOfCore()); - - handles[0]->read(); - - // store the Page shared_ptr - - Page::Ptr page = handles[0]->mPage; - - // verify use count is four (one plus three handles) - - EXPECT_EQ(page.use_count(), long(4)); - - // on reading from the first handle, all pages referenced - // in the first three handles are in-core - - EXPECT_TRUE(!page0.isOutOfCore()); - EXPECT_TRUE(!page1.isOutOfCore()); - EXPECT_TRUE(!page2.isOutOfCore()); - EXPECT_TRUE(page3.isOutOfCore()); - - handles[1]->read(); - - EXPECT_TRUE(handles[0]->mPage); - - handles[2]->read(); - - handles.erase(handles.begin()); - handles.erase(handles.begin()); - handles.erase(handles.begin()); - - // after all three handles have been read, - // page should have just one use count (itself) - - EXPECT_EQ(page.use_count(), long(1)); - -#endif // OPENVDB_USE_DELAYED_LOADING } std::remove(filename.c_str()); } diff --git a/openvdb/openvdb/unittest/TestTools.cc b/openvdb/openvdb/unittest/TestTools.cc index df17050a06..4ffc1fccd1 100644 --- a/openvdb/openvdb/unittest/TestTools.cc +++ b/openvdb/openvdb/unittest/TestTools.cc @@ -1762,7 +1762,7 @@ TEST_F(TestTools, testPrune) util::CpuTimer timer; initialize();//required whenever I/O of OpenVDB files is performed! io::File sourceFile("/usr/pic1/Data/OpenVDB/LevelSetModels/crawler.vdb"); - sourceFile.open(false);//disable delayed loading + sourcefile.open(); FloatGrid::Ptr grid = gridPtrCast(sourceFile.getGrids()->at(0)); const Index64 leafCount = grid->tree().leafCount(); @@ -1775,7 +1775,7 @@ TEST_F(TestTools, testPrune) util::CpuTimer timer; initialize();//required whenever I/O of OpenVDB files is performed! io::File sourceFile("/usr/pic1/Data/OpenVDB/LevelSetModels/crawler.vdb"); - sourceFile.open(false);//disable delayed loading + sourcefile.open(); FloatGrid::Ptr grid = gridPtrCast(sourceFile.getGrids()->at(0)); const Index64 leafCount = grid->tree().leafCount(); diff --git a/openvdb/openvdb/unittest/TestTree.cc b/openvdb/openvdb/unittest/TestTree.cc index a5ac2afade..624906b199 100644 --- a/openvdb/openvdb/unittest/TestTree.cc +++ b/openvdb/openvdb/unittest/TestTree.cc @@ -3,10 +3,12 @@ #include #include +#include #include #include // for tools::setValueOnMin(), et al. #include #include // for io::RealToHalf +#include #include // for Abs() #include #include @@ -104,6 +106,14 @@ TEST_F(TestTree, testChangeBackground) TEST_F(TestTree, testHalf) { + // explicitly register these grid types as they are not registered by default + openvdb::Grid::registerGrid(); + openvdb::Grid::registerGrid(); + + // Register scalar codecs for all vec2 tree types which are not registered by default + openvdb::io::CodecRegistry::registerCodec>>(); + openvdb::io::CodecRegistry::registerCodec>>(); + testWriteHalf(); testWriteHalf(); testWriteHalf(); @@ -125,27 +135,54 @@ TestTree::testWriteHalf() using GridType = openvdb::Grid; using ValueT = typename TreeType::ValueType; ValueT background(5); - GridType grid(background); + typename GridType::Ptr grid = GridType::create(background); + grid->setName("density"); unittest_util::makeSphere(openvdb::Coord(64, 64, 64), openvdb::Vec3f(35, 30, 40), - /*radius=*/10, grid, + /*radius=*/10, *grid, /*dx=*/1.0f, unittest_util::SPHERE_DENSE); - EXPECT_TRUE(!grid.tree().empty()); + EXPECT_TRUE(!grid->tree().empty()); + + // find stream and grid header position for save as float + + size_t headerSize = 0; + { + std::ostringstream outFull(std::ios_base::binary); + openvdb::io::Stream streamFull(outFull); + streamFull.write({}); + openvdb::io::GridDescriptor gd(grid->getName(), grid->type()); + gd.writeHeader(outFull); + headerSize = outFull.str().size(); + } + + // find stream and grid header position for save as half + + size_t halfHeaderSize = 0; + { + std::ostringstream outHalf(std::ios_base::binary); + openvdb::io::Stream streamHalf(outHalf); + streamHalf.write({}); + openvdb::io::GridDescriptor gdHalf(grid->getName(), grid->type(), /*half=*/true); + gdHalf.writeHeader(outHalf); + halfHeaderSize = outHalf.str().size(); + } // Write grid blocks in both float and half formats. std::ostringstream outFull(std::ios_base::binary); - grid.setSaveFloatAsHalf(false); - grid.writeBuffers(outFull); + openvdb::io::Stream streamFull(outFull); + grid->setSaveFloatAsHalf(false); + streamFull.write({grid}); outFull.flush(); - const size_t fullBytes = outFull.str().size(); + const size_t fullBytes = outFull.str().size() - headerSize; if (fullBytes == 0) FAIL() << "wrote empty full float buffers"; std::ostringstream outHalf(std::ios_base::binary); - grid.setSaveFloatAsHalf(true); - grid.writeBuffers(outHalf); + openvdb::io::Stream streamHalf(outHalf); + grid->setSaveFloatAsHalf(true); + streamHalf.write({grid}); outHalf.flush(); - const size_t halfBytes = outHalf.str().size(); + const size_t halfBytes = outHalf.str().size() - halfHeaderSize; if (halfBytes == 0) FAIL() << "wrote empty half float buffers"; if (openvdb::io::RealToHalf::isReal) { @@ -166,22 +203,26 @@ TestTree::testWriteHalf() // then write it out again in half float format. Verify that the resulting file // is identical to the original half float file. { - openvdb::Grid gridCopy(grid); - gridCopy.setSaveFloatAsHalf(true); std::istringstream is(outHalf.str(), std::ios_base::binary); + openvdb::io::Stream streamHalf2(is); - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(is); - - gridCopy.readBuffers(is); + openvdb::GridPtrVecPtr grids = streamHalf2.getGrids(); + openvdb::GridBase::Ptr gridCopy = (*grids)[0]; + gridCopy->setSaveFloatAsHalf(true); std::ostringstream outDiff(std::ios_base::binary); - gridCopy.writeBuffers(outDiff); + openvdb::io::Stream streamDiff(outDiff); + streamDiff.write({gridCopy}); outDiff.flush(); - if (outHalf.str() != outDiff.str()) { - FAIL() << "half-from-full and half-from-half buffers differ"; + // Compare the two buffers, skipping the header region + { + const std::string s1 = outHalf.str().substr(halfHeaderSize); + const std::string s2 = outDiff.str().substr(halfHeaderSize); + if (s1 != s2) + { + FAIL() << "half-from-full and half-from-half buffers differ"; + } } } } @@ -731,7 +772,68 @@ TEST_F(TestTree, testIterators) TEST_F(TestTree, testIO) { - const char* filename = "testIO.dbg"; + using TreeType = openvdb::tree::Tree; + using GridType = openvdb::Grid; + + const char* filename = "testIO.vdb"; + openvdb::SharedPtr scopedFile(filename, ::remove); + { + ValueType background=5.0f; + GridType::Ptr grid = GridType::create(background); + grid->setName("test_grid"); + grid->tree().setValueOn(openvdb::Coord(5,10,20),0.234f); + grid->tree().setValueOn(openvdb::Coord(50000,20000,30000),4.5678f); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); + + openvdb::io::File file(filename); + file.write(grids); + file.close(); + } + { + ValueType background=2.0f; + GridType::Ptr grid = GridType::create(background); + ASSERT_DOUBLES_EXACTLY_EQUAL(background, grid->tree().getValue(openvdb::Coord(5,10,20))); + + { + openvdb::io::File file(filename); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("test_grid"); + file.close(); + + grid = openvdb::gridPtrCast(baseGrid); + EXPECT_TRUE(grid.get() != nullptr); + } + + ASSERT_DOUBLES_EXACTLY_EQUAL(0.234f, grid->tree().getValue(openvdb::Coord(5,10,20))); + ASSERT_DOUBLES_EXACTLY_EQUAL(5.0f, grid->tree().getValue(openvdb::Coord(5,11,20))); + ValueType sum=0.0f; + for (RootNodeType::ChildOnIter root_iter = grid->tree().root().beginChildOn(); + root_iter.test(); ++root_iter) + { + for (InternalNodeType2::ChildOnIter internal_iter2 = root_iter->beginChildOn(); + internal_iter2.test(); ++internal_iter2) + { + for (InternalNodeType1::ChildOnIter internal_iter1 = + internal_iter2->beginChildOn(); internal_iter1.test(); ++internal_iter1) + { + for (LeafNodeType::ValueOnIter block_iter = + internal_iter1->beginValueOn(); block_iter.test(); ++block_iter) + { + sum += *block_iter; + } + } + } + } + ASSERT_DOUBLES_EXACTLY_EQUAL(sum, (0.234f + 4.5678f)); + } +} + + +TEST_F(TestTree, testTreeIO) +{ + const char* filename = "testTreeIO.dbg"; openvdb::SharedPtr scopedFile(filename, ::remove); { ValueType background=5.0f; @@ -750,8 +852,6 @@ TEST_F(TestTree, testIO) ASSERT_DOUBLES_EXACTLY_EQUAL(background, root_node.getValue(openvdb::Coord(5,10,20))); { std::ifstream is(filename, std::ios_base::binary); - // Since the test file doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. openvdb::io::setCurrentVersion(is); root_node.readTopology(is); root_node.readBuffers(is); diff --git a/openvdb/openvdb/unittest/util.h b/openvdb/openvdb/unittest/util.h index b830e3da88..1903e27b46 100644 --- a/openvdb/openvdb/unittest/util.h +++ b/openvdb/openvdb/unittest/util.h @@ -10,18 +10,6 @@ #include // for pruneLevelSet #include -#ifdef OPENVDB_USE_DELAYED_LOADING -/// @brief io::MappedFile has a private constructor, so declare a class that acts as the friend -class TestMappedFile -{ -public: - static openvdb::io::MappedFile::Ptr create(const std::string& filename) - { - return openvdb::SharedPtr(new openvdb::io::MappedFile(filename)); - } -}; -#endif - namespace unittest_util { diff --git a/openvdb/openvdb/version.h.in b/openvdb/openvdb/version.h.in index f284b58252..df2cf27b55 100644 --- a/openvdb/openvdb/version.h.in +++ b/openvdb/openvdb/version.h.in @@ -138,11 +138,6 @@ #cmakedefine OPENVDB_USE_ZLIB #endif -/* Denotes whether VDB was built with Delayed Loading support */ -#ifndef OPENVDB_USE_DELAYED_LOADING -#cmakedefine OPENVDB_USE_DELAYED_LOADING -#endif - /* Denotes whether VDB was built asserts enabled in VDB code */ #ifndef OPENVDB_ENABLE_ASSERTS #cmakedefine OPENVDB_ENABLE_ASSERTS diff --git a/openvdb_ax/openvdb_ax/compiler/PointExecutable.cc b/openvdb_ax/openvdb_ax/compiler/PointExecutable.cc index 3598fb3adb..7b704e8524 100644 --- a/openvdb_ax/openvdb_ax/compiler/PointExecutable.cc +++ b/openvdb_ax/openvdb_ax/compiler/PointExecutable.cc @@ -362,7 +362,6 @@ struct PointFunctionArguments // @todo if the array is shared we should probably make it unique? if (mData.mUseBufferKernel) { - const_cast(array).loadData(); const char* data = array.constDataAsByteArray(); void* ptr = static_cast(const_cast(data)); mHandlesOrBuffers.emplace_back(ptr); @@ -387,14 +386,13 @@ struct PointFunctionArguments array.expand(); if (mData.mUseBufferKernel) { - array.loadData(); const char* data = array.constDataAsByteArray(); void* ptr = static_cast(const_cast(data)); mHandlesOrBuffers.emplace_back(ptr); const codegen::Codec* codec = codegen::getCodec(ast::tokens::tokenFromTypeString(array.valueType()), array.codecType()); if (codec) flag |= codec->flag(); - OPENVDB_ASSERT(array.isDataLoaded() && !array.isUniform()); + OPENVDB_ASSERT(!array.isUniform()); } else { typename WriteHandle::UniquePtr handle(new WriteHandle(leaf, Index(pos))); diff --git a/openvdb_cmd/vdb_ax/main.cc b/openvdb_cmd/vdb_ax/main.cc index 9543d9fe6c..b62c4559a7 100644 --- a/openvdb_cmd/vdb_ax/main.cc +++ b/openvdb_cmd/vdb_ax/main.cc @@ -791,9 +791,7 @@ main(int argc, char *argv[]) if (opts.mMode.get() == openvdb::ax::VDB_AX_MODE::Execute) { // read vdb file data for - axlog("[INFO] Reading VDB data" - << (openvdb::io::Archive::isDelayedLoadingEnabled() ? - " (delay-load)" : "") << '\n'); + axlog("[INFO] Reading VDB data" << '\n'); for (const auto& filename : opts.mInputVDBFiles.get()) { openvdb::io::File file(filename); try { diff --git a/openvdb_cmd/vdb_print/main.cc b/openvdb_cmd/vdb_print/main.cc index e093838768..3a329d4089 100644 --- a/openvdb_cmd/vdb_print/main.cc +++ b/openvdb_cmd/vdb_print/main.cc @@ -209,14 +209,9 @@ printShortListing(const StringVec& filenames, bool metadata) // Print the grid's size, in bytes - // no support for memUsageIfLoaded until ABI >= 10 for points::PointDataGrid types using ListT = openvdb::GridTypes; grid->apply([&](const auto& typed){ - // @todo combine these methods to avoid iterating across the tree twice - const openvdb::Index64 incore = openvdb::tools::memUsage(typed.tree()); - const openvdb::Index64 total = openvdb::tools::memUsageIfLoaded(typed.tree()); - - std::cout << " " << std::right << std::setw(6) << bytesAsString(incore) << " (In Core)"; + const openvdb::Index64 total = openvdb::tools::memUsage(typed.tree()); std::cout << " " << std::right << std::setw(6) << bytesAsString(total) << " (Total)"; }); diff --git a/openvdb_cmd/vdb_render/main.cc b/openvdb_cmd/vdb_render/main.cc index 127d8720b7..2281405978 100644 --- a/openvdb_cmd/vdb_render/main.cc +++ b/openvdb_cmd/vdb_render/main.cc @@ -768,7 +768,7 @@ main(int argc, char *argv[]) } } else { // If no grid was specified by name, retrieve the first float grid from the file. - file.open(/*delayLoad=*/false); + file.open(); openvdb::io::File::NameIterator it = file.beginName(); openvdb::GridPtrVecPtr grids = file.readAllGridMetadata(); for (size_t i = 0; i < grids->size(); ++i, ++it) { diff --git a/openvdb_cmd/vdb_tool/include/Geometry.h b/openvdb_cmd/vdb_tool/include/Geometry.h index 09f0a5fc96..0a916327d8 100644 --- a/openvdb_cmd/vdb_tool/include/Geometry.h +++ b/openvdb_cmd/vdb_tool/include/Geometry.h @@ -1098,7 +1098,7 @@ void Geometry::readVDB(const std::string &fileName) { initialize(); io::File file(fileName); - file.open();// enables delayed loading by default + file.open(); GridPtrVecPtr meta = file.readAllGridMetadata(); for (auto m : *meta) { if (m->isType()) { diff --git a/openvdb_cmd/vdb_tool/include/Tool.h b/openvdb_cmd/vdb_tool/include/Tool.h index 2833bf1505..bc9824c4e6 100644 --- a/openvdb_cmd/vdb_tool/include/Tool.h +++ b/openvdb_cmd/vdb_tool/include/Tool.h @@ -622,8 +622,7 @@ void Tool::init() mParser.addAction( {"read", "import", "load", "i"}, "Read one or more geometry or VDB files from disk or STDIN.", {{"files", "", "{file|stdin}.{obj|ply|abc|stl|off|pts|xyz|e57|vdb|nvdb|gltf|glb|geo|usd|usda|usdc|usdz}", "list of files or the input stream, e.g. file.vdb,stdin.vdb. Note that \"files=\" is optional since any argument without \"=\" is intrepreted as a file and appended to \"files\""}, - {"grids", "*", "*|grid_name,...", "list of VDB grids name to be imported (defaults to \"*\", i.e. import all available grids)"}, - {"delayed", "true", "1|0|true|false", "toggle delayed loading of VDB grids (enabled by default). This option is ignored by other file types"}}, + {"grids", "*", "*|grid_name,...", "list of VDB grids name to be imported (defaults to \"*\", i.e. import all available grids)"}}, [](){}, [&](){this->read();}, 0);// anonymous options are treated as to the first option,i.e. "files" mParser.addAction( @@ -1466,7 +1465,7 @@ void Tool::readVDB(const std::string &fileName) } else { if (mParser.verbose) mTimer.start("Reading VDB grid(s) from file named \""+fileName+"\""); io::File file(fileName); - file.open(mParser.get("delayed")); + file.open(); grids = file.getGrids(); } const size_t count = mGrid.size(); diff --git a/openvdb_houdini/openvdb_houdini/CMakeLists.txt b/openvdb_houdini/openvdb_houdini/CMakeLists.txt index 50d9ed4cab..1a8ae34dd0 100644 --- a/openvdb_houdini/openvdb_houdini/CMakeLists.txt +++ b/openvdb_houdini/openvdb_houdini/CMakeLists.txt @@ -123,6 +123,8 @@ set(OPENVDB_HOUDINI_ICON_INSTALL_PREFIX ${OPENVDB_HOUDINI_INSTALL_PREFIX}/config CACHE PATH "Install path for the OpenVDB Houdini node icons") set(OPENVDB_HOUDINI_PYTHON_INSTALL_PREFIX ${OPENVDB_HOUDINI_INSTALL_PREFIX}/python2.7libs CACHE PATH "Install path for the OpenVDB Houdini startup script") +set(OPENVDB_HOUDINI_GLSL_INSTALL_PREFIX ${OPENVDB_HOUDINI_INSTALL_PREFIX}/glsl + CACHE PATH "Install path for the OpenVDB Houdini GLSL shaders") ######################################################################### @@ -395,6 +397,9 @@ install(FILES install(DIRECTORY help/ DESTINATION ${OPENVDB_HOUDINI_HELP_INSTALL_PREFIX}) +install(DIRECTORY glsl/ + DESTINATION ${OPENVDB_HOUDINI_GLSL_INSTALL_PREFIX}) + if(OPENVDB_INSTALL_HOUDINI_PYTHONRC) install(FILES pythonrc.py diff --git a/openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc b/openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc index 54802cd36c..38bc5e1b25 100644 --- a/openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc +++ b/openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc @@ -108,7 +108,7 @@ GEO_VDBTranslator::fileStat(const char *filename, GA_Stat &stat, uint /*level*/) try { openvdb::io::File file(filename); - file.open(/*delayLoad*/false); + file.open(); int nprim = 0; UT_BoundingBox bbox; @@ -204,7 +204,7 @@ GEO_VDBTranslator::fileLoad(GEO_Detail *geogdp, UT_IStream &is, bool /*ate_magic try { // Create and open a VDB file, but don't read any grids yet. - openvdb::io::Stream file(*stdstream, /*delayLoad*/false); + openvdb::io::Stream file(*stdstream); // Read the file-level metadata into global attributes. openvdb::MetaMap::Ptr fileMetadata = file.getMetadata(); diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index 9608005294..7869ab94a7 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -7,8 +7,6 @@ /// /// @brief GR Render Hook and Primitive for VDB PointDataGrid -#include - #include #include #include @@ -30,6 +28,18 @@ #include #include #include +#include + +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support +#include +#include +#include +#include +#include +#include +#include +#include +#endif #include #include @@ -39,19 +49,19 @@ #include #include -#if UT_VERSION_INT < 0x14000000 // Below 20.0, there is no RE_RenderContext -#define RE_RenderContext RE_Render * -#endif - //////////////////////////////////////// static RE_ShaderHandle theMarkerDecorShader("decor/GL32/point_marker.prog"); static RE_ShaderHandle theNormalDecorShader("decor/GL32/point_normal.prog"); static RE_ShaderHandle theVelocityDecorShader("decor/GL32/user_point_vector3.prog"); -static RE_ShaderHandle theLineShader("basic/GL32/wire_color.prog"); static RE_ShaderHandle thePixelShader("particle/GL32/pixel.prog"); static RE_ShaderHandle thePointShader("particle/GL32/point.prog"); +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support +static RV_ShaderProgram* theVkPointShader = nullptr; +static RV_ShaderProgram* theVkVelocityShader = nullptr; +#endif + /// @note An additional scale for velocity trails to accurately match /// the visualization of velocity for Houdini points #define VELOCITY_DECOR_SCALE -0.041f; @@ -120,11 +130,8 @@ class GR_PrimVDBPoints : public GR_Primitive /// return true if the primitive is in or overlaps the view frustum. /// always returning true will effectively disable frustum culling. - bool inViewFrustum(const UT_Matrix4D &objviewproj -#if (UT_VERSION_INT >= 0x1105014e) // 17.5.334 or later - , const UT_BoundingBoxD *bbox -#endif - ) override; + bool inViewFrustum(const UT_Matrix4D &objviewproj, + const UT_BoundingBoxD *bbox) override; /// Called whenever the primitive is required to render, which may be more /// than one time per viewport redraw (beauty, shadow passes, wireframe-over) @@ -145,10 +152,6 @@ class GR_PrimVDBPoints : public GR_Primitive const openvdb::points::PointDataGrid& grid, const RE_CacheVersion& version); - void updateWireBuffer(RE_Render* r, - const openvdb::points::PointDataGrid& grid, - const RE_CacheVersion& version); - bool updateVec3Buffer(RE_Render* r, const openvdb::points::PointDataGrid& grid, const std::string& attributeName, @@ -162,12 +165,35 @@ class GR_PrimVDBPoints : public GR_Primitive void removeBuffer(const std::string& name); +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support + void updatePosBufferVk(RV_Render* r, + const openvdb::points::PointDataGrid& grid, + const RE_CacheVersion& version); + + bool updateVec3BufferVk(RV_Render* r, + const openvdb::points::PointDataGrid& grid, + const std::string& attributeName, + const std::string& bufferName, + const RE_CacheVersion& version); +#endif + private: UT_UniquePtr myGeo; - UT_UniquePtr myWire; bool mDefaultPointColor = true; openvdb::Vec3f mCentroid{0, 0, 0}; openvdb::BBoxd mBbox; +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support + UT_UniquePtr myGeoVk; + UT_UniquePtr myVkObjectSet; + UT_UniquePtr myVkObjectBlock; + UT_UniquePtr myVkDrawingSet; + UT_UniquePtr myVkGeoBlock; + // velocity shader uniform state (separate to avoid clobbering point shader) + UT_UniquePtr myVkVelObjectSet; + UT_UniquePtr myVkVelObjectBlock; + bool mHasNormals = false; + bool mHasVelocity = false; +#endif }; @@ -252,10 +278,15 @@ bool patchShader(RE_Render* r, RE_ShaderHandle& shader, RE_ShaderType type, shader->getShaderSource(r, source, type); const int version = shader->getCodeVersion(); + // normalize whitespace (collapse tabs and runs of spaces to single space) + + source.substitute("\t", " "); + while (source.substitute(" ", " ")) {} + // patch the shader to replace the strings for (const auto& stringPair : stringReplacements) { - source.substitute(stringPair.first.c_str(), stringPair.second.c_str(), /*all=*/true); + source.substitute(stringPair.first.c_str(), stringPair.second.c_str()); } // patch the shader to insert the strings @@ -312,9 +343,6 @@ void patchShaderNoRedeclarations(RE_Render* r, RE_ShaderHandle& shader) { static const std::vector stringReplacements { - StringPair("\t", " "), - StringPair(" ", " "), - StringPair(" ", " "), StringPair("uniform vec2 glH_DepthProject;", "//uniform vec2 glH_DepthProject;"), StringPair("uniform vec2 glH_ScreenSize", "//uniform vec2 glH_ScreenSize") }; @@ -354,84 +382,6 @@ GR_PrimVDBPoints::acceptPrimitive(GT_PrimitiveType, return GR_NOT_PROCESSED; } -namespace gr_primitive_internal -{ - -struct FillGPUBuffersLeafBoxes -{ - FillGPUBuffersLeafBoxes(UT_Vector3H* buffer, - const std::vector& coords, - const openvdb::math::Transform& transform, - const openvdb::Vec3f& positionOffset) - : mBuffer(buffer) - , mCoords(coords) - , mTransform(transform) - , mPositionOffset(positionOffset) { } - - void operator()(const tbb::blocked_range& range) const - { - std::vector corners; - corners.reserve(8); - - for (size_t n = range.begin(), N = range.end(); n != N; ++n) { - const openvdb::Coord& origin = mCoords[n]; - - // define 8 corners - - corners.clear(); - - const openvdb::Vec3f pos000 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 0.0, 0.0)) - mPositionOffset; - corners.emplace_back(pos000.x(), pos000.y(), pos000.z()); - const openvdb::Vec3f pos001 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 0.0, 8.0)) - mPositionOffset; - corners.emplace_back(pos001.x(), pos001.y(), pos001.z()); - const openvdb::Vec3f pos010 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 8.0, 0.0)) - mPositionOffset; - corners.emplace_back(pos010.x(), pos010.y(), pos010.z()); - const openvdb::Vec3f pos011 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 8.0, 8.0)) - mPositionOffset; - corners.emplace_back(pos011.x(), pos011.y(), pos011.z()); - const openvdb::Vec3f pos100 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 0.0, 0.0)) - mPositionOffset; - corners.emplace_back(pos100.x(), pos100.y(), pos100.z()); - const openvdb::Vec3f pos101 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 0.0, 8.0)) - mPositionOffset; - corners.emplace_back(pos101.x(), pos101.y(), pos101.z()); - const openvdb::Vec3f pos110 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 8.0, 0.0)) - mPositionOffset; - corners.emplace_back(pos110.x(), pos110.y(), pos110.z()); - const openvdb::Vec3f pos111 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 8.0, 8.0)) - mPositionOffset; - corners.emplace_back(pos111.x(), pos111.y(), pos111.z()); - - openvdb::Index64 offset = n*8*3; - - // Z axis - - mBuffer[offset++] = corners[0]; mBuffer[offset++] = corners[1]; - mBuffer[offset++] = corners[2]; mBuffer[offset++] = corners[3]; - mBuffer[offset++] = corners[4]; mBuffer[offset++] = corners[5]; - mBuffer[offset++] = corners[6]; mBuffer[offset++] = corners[7]; - - // Y axis - - mBuffer[offset++] = corners[0]; mBuffer[offset++] = corners[2]; - mBuffer[offset++] = corners[1]; mBuffer[offset++] = corners[3]; - mBuffer[offset++] = corners[4]; mBuffer[offset++] = corners[6]; - mBuffer[offset++] = corners[5]; mBuffer[offset++] = corners[7]; - - // X axis - - mBuffer[offset++] = corners[0]; mBuffer[offset++] = corners[4]; - mBuffer[offset++] = corners[1]; mBuffer[offset++] = corners[5]; - mBuffer[offset++] = corners[2]; mBuffer[offset++] = corners[6]; - mBuffer[offset++] = corners[3]; mBuffer[offset++] = corners[7]; - } - } - - ////////// - - UT_Vector3H* mBuffer; - const std::vector& mCoords; - const openvdb::math::Transform& mTransform; - const openvdb::Vec3f mPositionOffset; -}; // class FillGPUBuffersLeafBoxes - -} // namespace gr_primitive_internal - void GR_PrimVDBPoints::computeCentroid(const openvdb::points::PointDataGrid& grid) { @@ -530,12 +480,49 @@ struct VectorAttribute UT_Vector3H* mBuffer; }; // struct VectorAttribute +struct VectorAttribute4H +{ + using ValueType = Vec3f; + + struct Handle + { + Handle(VectorAttribute4H& attribute) + : mBuffer(attribute.mBuffer) { } + + template + void set(openvdb::Index offset, + openvdb::Index /*stride*/, + const openvdb::math::Vec3& value) + { + mBuffer[offset] = UT_Vector4H( + fpreal16(float(value.x())), + fpreal16(float(value.y())), + fpreal16(float(value.z())), + fpreal16(0)); + } + + private: + UT_Vector4H* mBuffer; + }; // struct Handle + + VectorAttribute4H(UT_Vector4H* buffer) + : mBuffer(buffer) { } + + void expand() { } + void compact() { } + +private: + UT_Vector4H* mBuffer; +}; // struct VectorAttribute4H + void GR_PrimVDBPoints::updatePosBuffer(RE_Render* r, const openvdb::points::PointDataGrid& grid, const RE_CacheVersion& version) { - const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3); + const GR_RenderVersion renderVersion = getRenderVersion(); + const bool gl3 = (renderVersion == GR_RENDER_GL3 + || renderVersion == GR_RENDER_GL4); // Initialize the geometry with the proper name for the GL cache if (!myGeo) myGeo.reset(new RE_Geometry); @@ -563,10 +550,10 @@ GR_PrimVDBPoints::updatePosBuffer(RE_Render* r, int numPoints = 0; if (useGroup) { GroupFilter filter(groupName, iter->attributeSet()); - numPoints = static_cast(pointCount(tree, filter, /*inCoreOnly=*/true)); + numPoints = static_cast(pointCount(tree, filter)); } else { NullFilter filter; - numPoints = static_cast(pointCount(tree, filter, /*inCoreOnly=*/true)); + numPoints = static_cast(pointCount(tree, filter)); } if (numPoints == 0) return; @@ -596,13 +583,13 @@ GR_PrimVDBPoints::updatePosBuffer(RE_Render* r, MultiGroupFilter filter(includeGroups, excludeGroups, iter->attributeSet()); std::vector offsets; - pointOffsets(offsets, grid.tree(), filter, /*inCoreOnly=*/true); + pointOffsets(offsets, grid.tree(), filter); UT_UniquePtr pdata(new UT_Vector3F[numPoints]); PositionAttribute positionAttribute(pdata.get(), mCentroid); convertPointDataGridPosition(positionAttribute, grid, offsets, - /*startOffset=*/ 0, filter, /*inCoreOnly=*/true); + /*startOffset=*/ 0, filter); posGeo->setArray(r, pdata.get()); posGeo->setCacheVersion(version); @@ -632,96 +619,42 @@ GR_PrimVDBPoints::updatePosBuffer(RE_Render* r, } void -GR_PrimVDBPoints::updateWireBuffer(RE_Render *r, - const openvdb::points::PointDataGrid& grid, - const RE_CacheVersion& version) +GR_PrimVDBPoints::update(RE_RenderContext r, + const GT_PrimitiveHandle &primh, + const GR_UpdateParms &p) { - const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3); - - // Initialize the geometry with the proper name for the GL cache - if (!myWire) myWire.reset(new RE_Geometry); - myWire->cacheBuffers(getCacheName()); - - using GridType = openvdb::points::PointDataGrid; - using TreeType = GridType::TreeType; - using LeafNode = TreeType::LeafNodeType; - - const TreeType& tree = grid.tree(); - if (tree.leafCount() == 0) return; - - // count up total points ignoring any leaf nodes that are out of core - - size_t outOfCoreLeaves = 0; - for (auto iter = tree.cbeginLeaf(); iter; ++iter) { - if (iter->buffer().isOutOfCore()) outOfCoreLeaves++; - } - - if (outOfCoreLeaves == 0) return; - - // Initialize the number of points for the wireframe box per leaf. - - int numPoints = static_cast(outOfCoreLeaves*8*3); - myWire->setNumPoints(numPoints); - - // fetch wireframe position, if its cache version matches, no upload is required. - - RE_VertexArray* posWire = myWire->findCachedAttrib(r, "P", RE_GPU_FLOAT16, 3, RE_ARRAY_POINT, true); - - if (posWire->getCacheVersion() != version) +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support + if (r.isVulkan()) { - using gr_primitive_internal::FillGPUBuffersLeafBoxes; - - // fill the wire data - - UT_UniquePtr data(new UT_Vector3H[numPoints]); - - std::vector coords; - - for (auto iter = tree.cbeginLeaf(); iter; ++iter) { - const LeafNode& leaf = *iter; - - // skip in-core leaf nodes (for use when delay loading VDBs) - if (!leaf.buffer().isOutOfCore()) continue; - - coords.push_back(leaf.origin()); + if (p.reason & (GR_GEO_CHANGED | GR_GEO_TOPOLOGY_CHANGED)) + { + const GT_PrimVDB& gt_primVDB = + static_cast(*primh); + + const openvdb::GridBase* grid = + const_cast(gt_primVDB).getGrid(); + + const openvdb::points::PointDataGrid& pointDataGrid = + static_cast(*grid); + + computeCentroid(pointDataGrid); + computeBbox(pointDataGrid); + + RV_Render* rv = r.vkRender(); + updatePosBufferVk(rv, pointDataGrid, p.geo_version); + mDefaultPointColor = !updateVec3BufferVk( + rv, pointDataGrid, "Cd", "Cd", p.geo_version); + updateVec3BufferVk( + rv, pointDataGrid, "N", "N", p.geo_version); + updateVec3BufferVk( + rv, pointDataGrid, "v", "V", p.geo_version); } - - FillGPUBuffersLeafBoxes fill(data.get(), coords, grid.transform(), mCentroid); - const tbb::blocked_range range(0, coords.size()); - tbb::parallel_for(range, fill); - - posWire->setArray(r, data.get()); - posWire->setCacheVersion(version); - } - - if (gl3) - { - // Extra constant inputs for the GL3 default shaders we are using. - - fpreal32 uv[2] = { 0.0, 0.0 }; - fpreal32 alpha = 1.0; - fpreal32 pnt = 0.0; - UT_Matrix4F instance; - instance.identity(); - - myWire->createConstAttribute(r, "uv", RE_GPU_FLOAT32, 2, uv); - myWire->createConstAttribute(r, "Alpha", RE_GPU_FLOAT32, 1, &alpha); - myWire->createConstAttribute(r, "pointSelection", RE_GPU_FLOAT32, 1,&pnt); - myWire->createConstAttribute(r, "instmat", RE_GPU_MATRIX4, 1, - instance.data()); + return; } +#endif - myWire->connectAllPrims(r, RE_GEO_WIRE_IDX, RE_PRIM_LINES, nullptr, true); -} - -void -GR_PrimVDBPoints::update(RE_RenderContext r, - const GT_PrimitiveHandle &primh, - const GR_UpdateParms &p) -{ // patch the point shaders at run-time to add an offset (does nothing if already patched) - patchShaderVertexOffset(r, theLineShader); patchShaderVertexOffset(r, thePixelShader); patchShaderVertexOffset(r, thePointShader); @@ -749,27 +682,19 @@ GR_PrimVDBPoints::update(RE_RenderContext r, computeCentroid(pointDataGrid); computeBbox(pointDataGrid); updatePosBuffer(r, pointDataGrid, p.geo_version); - updateWireBuffer(r, pointDataGrid, p.geo_version); mDefaultPointColor = !updateVec3Buffer(r, pointDataGrid, "Cd", "Cd", p.geo_version); } } bool -GR_PrimVDBPoints::inViewFrustum(const UT_Matrix4D& objviewproj -#if (UT_VERSION_INT >= 0x1105014e) // 17.5.334 or later - , const UT_BoundingBoxD *passed_bbox -#endif - ) +GR_PrimVDBPoints::inViewFrustum(const UT_Matrix4D& objviewproj, + const UT_BoundingBoxD *passed_bbox) { const UT_BoundingBoxD bbox( mBbox.min().x(), mBbox.min().y(), mBbox.min().z(), mBbox.max().x(), mBbox.max().y(), mBbox.max().z()); -#if (UT_VERSION_INT >= 0x1105014e) // 17.5.334 or later - return GR_Utils::inViewFrustum(passed_bbox ? *passed_bbox :bbox, + return GR_Utils::inViewFrustum(passed_bbox ? *passed_bbox : bbox, objviewproj); -#else - return GR_Utils::inViewFrustum(bbox, objviewproj); -#endif } bool @@ -826,12 +751,12 @@ GR_PrimVDBPoints::updateVec3Buffer(RE_Render* r, MultiGroupFilter filter(includeGroups, excludeGroups, iter->attributeSet()); std::vector offsets; - pointOffsets(offsets, grid.tree(), filter, /*inCoreOnly=*/true); + pointOffsets(offsets, grid.tree(), filter); VectorAttribute typedAttribute(data.get()); convertPointDataGridAttribute(typedAttribute, grid.tree(), offsets, /*startOffset=*/ 0, static_cast(index), /*stride=*/1, - filter, /*inCoreOnly=*/true); + filter); } bufferGeo->setArray(r, data.get()); @@ -864,14 +789,294 @@ GR_PrimVDBPoints::removeBuffer(const std::string& name) myGeo->clearAttribute(name.c_str()); } + +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support + +void +GR_PrimVDBPoints::updatePosBufferVk(RV_Render* r, + const openvdb::points::PointDataGrid& grid, + const RE_CacheVersion& version) +{ + using GridType = openvdb::points::PointDataGrid; + using TreeType = GridType::TreeType; + using AttributeSet = openvdb::points::AttributeSet; + + const TreeType& tree = grid.tree(); + auto iter = tree.cbeginLeaf(); + if (!iter) return; + + const AttributeSet::Descriptor& descriptor = + iter->attributeSet().descriptor(); + + // check if group viewport is in use + + const openvdb::StringMetadata::ConstPtr s = + grid.getMetadata( + openvdb_houdini::META_GROUP_VIEWPORT); + + const std::string groupName = s ? s->value() : ""; + const bool useGroup = + !groupName.empty() && descriptor.hasGroup(groupName); + + // count up total points + + int numPoints = 0; + if (useGroup) { + GroupFilter filter(groupName, iter->attributeSet()); + numPoints = static_cast(pointCount(tree, filter)); + } else { + NullFilter filter; + numPoints = static_cast(pointCount(tree, filter)); + } + + if (numPoints == 0) return; + + const size_t positionIndex = descriptor.find("P"); + if (positionIndex == AttributeSet::INVALID_POS) return; + + // check for decoration attributes + + mHasNormals = (descriptor.find("N") != AttributeSet::INVALID_POS); + mHasVelocity = (descriptor.find("v") != AttributeSet::INVALID_POS); + + // (re)create the VK geometry + + myGeoVk.reset(new RV_Geometry); + myGeoVk->setName("vdb_points"); + myGeoVk->setNumPoints(numPoints); + myGeoVk->createAttribute("P", RV_GPU_FLOAT32, 3); + myGeoVk->createAttribute("Cd", RV_GPU_FLOAT16, 4); + if (mHasNormals) + myGeoVk->createAttribute("N", RV_GPU_FLOAT16, 4); + if (mHasVelocity) + myGeoVk->createAttribute("V", RV_GPU_FLOAT16, 4); + myGeoVk->createConstant("pointSelection", RV_GPU_UINT32, 1); + myGeoVk->connectAllPrims(0, RV_PRIM_POINTS); + if (mHasVelocity) + myGeoVk->connectAllPrims(1, RV_PRIM_PATCHES, /*patch_size=*/1); + myGeoVk->populateBuffers(r); + + // set pointSelection to zero (no selection) + + myGeoVk->setAttributeConstValue("pointSelection", 0.0); + + // fill the position buffer (world-space, no offset subtraction) + + { + std::vector includeGroups, excludeGroups; + if (useGroup) includeGroups.emplace_back(groupName); + + MultiGroupFilter filter( + includeGroups, excludeGroups, iter->attributeSet()); + + std::vector offsets; + pointOffsets(offsets, grid.tree(), filter); + + UT_UniquePtr pdata(new UT_Vector3F[numPoints]); + + // use zero offset so positions are stored in world space + PositionAttribute positionAttribute( + pdata.get(), Vec3f(0, 0, 0)); + convertPointDataGridPosition( + positionAttribute, grid, offsets, + /*startOffset=*/ 0, filter); + + const exint byteSize = + static_cast(numPoints) * sizeof(UT_Vector3F); + myGeoVk->getAttribute("P")->uploadData(r, pdata.get(), byteSize); + } + + // set a default Cd (will be overwritten if Cd attribute exists) + + myGeoVk->setAttributeConstVecValue( + "Cd", UT_Vector4F(0.6f, 0.6f, 0.5f, 1.0f)); + +} + + +bool +GR_PrimVDBPoints::updateVec3BufferVk(RV_Render* r, + const openvdb::points::PointDataGrid& grid, + const std::string& attributeName, + const std::string& bufferName, + const RE_CacheVersion& version) +{ + if (!myGeoVk) return false; + + using GridType = openvdb::points::PointDataGrid; + using TreeType = GridType::TreeType; + using AttributeSet = openvdb::points::AttributeSet; + + const TreeType& tree = grid.tree(); + auto iter = tree.cbeginLeaf(); + if (!iter) return false; + + const int numPoints = static_cast(myGeoVk->getNumPoints()); + if (numPoints == 0) return false; + + const AttributeSet::Descriptor& descriptor = + iter->attributeSet().descriptor(); + const size_t index = descriptor.find(attributeName); + + if (index == AttributeSet::INVALID_POS) return false; + + const openvdb::Name& type = descriptor.type(index).first; + if (type != "vec3s") return false; + + // check if group viewport is in use + + const openvdb::StringMetadata::ConstPtr s = + grid.getMetadata( + openvdb_houdini::META_GROUP_VIEWPORT); + + const std::string groupName = s ? s->value() : ""; + const bool useGroup = + !groupName.empty() && descriptor.hasGroup(groupName); + + std::vector includeGroups, excludeGroups; + if (useGroup) includeGroups.emplace_back(groupName); + + MultiGroupFilter filter( + includeGroups, excludeGroups, iter->attributeSet()); + + std::vector offsets; + pointOffsets(offsets, grid.tree(), filter); + + // read vec3s attribute data directly into 4-component float16 + // (VK_FORMAT_R16G16B16A16_SFLOAT is mandatory; 3-component is not) + + UT_UniquePtr data(new UT_Vector4H[numPoints]); + + VectorAttribute4H typedAttribute(data.get()); + convertPointDataGridAttribute(typedAttribute, grid.tree(), offsets, + /*startOffset=*/ 0, static_cast(index), + /*stride=*/1, filter); + + const exint byteSize = + static_cast(numPoints) * sizeof(UT_Vector4H); + RV_VKBuffer* buf = myGeoVk->getAttribute(bufferName.c_str()); + if (buf) { + buf->uploadData(r, data.get(), byteSize); + } + + return true; +} + +#endif // UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) + + void GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_DrawParms dp) { - if (!myGeo && !myWire) return; +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support + if (r.isVulkan()) + { + if (!myGeoVk) return; + + RV_Render* rv = r.vkRender(); + GR_Uniforms* uniforms = r.uniforms(); + + // initialize the VK point shader on first use + + if (!theVkPointShader) { + theVkPointShader = RV_ShaderProgram::loadShaderProgram( + rv->instance(), "openvdb/VK/points.prog"); + if (!theVkPointShader) return; + } + + // set the shader + + rv->setShader(theVkPointShader); + + // bind global uniform blocks (pass info, object transforms) + + if (uniforms) { + bool globalBound = uniforms->bindRVGlobalBlock(rv, theVkPointShader); + + // bind set 1 (glH_Object) for per-object transform uniforms + if (theVkPointShader->hasSet(1)) { + const auto* objBinding = + theVkPointShader->getBinding(1, 0); + if (objBinding) { + if (!myVkObjectBlock) { + myVkObjectBlock.reset( + RV_ShaderBlock::create( + rv->instance(), *objBinding)); + } + if (!myVkObjectSet || + !theVkPointShader->isSetCompatible( + *myVkObjectSet)) { + myVkObjectSet = + theVkPointShader->createSet( + rv->instance(), 1); + } + uniforms->assignRVBlock( + rv, myVkObjectBlock.get(), theVkPointShader); + // override DecorationScale with the user's + // point size from display options + const GR_CommonDispOption& commonOpts = + dp.opts->common(); + myVkObjectBlock->bindFloat( + "DecorationScale", + static_cast(commonOpts.pointSize())); + myVkObjectBlock->uploadBuffer(rv); + myVkObjectSet->attachBufferBlock( + rv->instance(), "Object", + myVkObjectBlock.get()); + rv->bindSet( + myVkObjectSet.get(), theVkPointShader); + } + } + + // bind set 2 (auto-injected Geometry uniform block) + if (theVkPointShader->hasSet(2)) { + const auto* geoBinding = + theVkPointShader->getBinding(2, 2); + if (geoBinding) { + if (!myVkGeoBlock) { + myVkGeoBlock.reset( + RV_ShaderBlock::create( + rv->instance(), *geoBinding)); + } + if (!myVkDrawingSet || + !theVkPointShader->isSetCompatible( + *myVkDrawingSet)) { + myVkDrawingSet = + theVkPointShader->createSet( + rv->instance(), 2); + } + myVkGeoBlock->uploadBuffer(rv); + myVkDrawingSet->attachBufferBlock( + rv->instance(), "Geometry", + myVkGeoBlock.get()); + rv->bindSet( + myVkDrawingSet.get(), theVkPointShader); + } + } + + } + + // draw - must be inside beginRendering/endRendering block + + bool beganRendering = false; + if (!rv->isRendering()) { + beganRendering = rv->beginRendering(); + } + + rv->draw(myGeoVk.get(), 0); + + if (beganRendering) { + rv->endRendering(); + } + return; + } +#endif - const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3); + if (!myGeo) return; - if (!gl3) return; + const GR_RenderVersion renderVersion = getRenderVersion(); + if (renderVersion != GR_RENDER_GL3 && renderVersion != GR_RENDER_GL4) + return; const GR_CommonDispOption& commonOpts = dp.opts->common(); @@ -911,35 +1116,133 @@ GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_D r->popShader(); } - - // draw leaf bboxes - - if (myWire && myWire->getNumPoints() > 0) { - - // bind the shader - - r->pushShader(); - r->bindShader(theLineShader); - - // bind the position offset - - UT_Vector3F positionOffset(mCentroid.x(), mCentroid.y(), mCentroid.z()); - theLineShader->bindVector(r, "offset", positionOffset); - - fpreal32 constcol[3] = { 0.6f, 0.6f, 0.6f }; - myWire->createConstAttribute(r, "Cd", RE_GPU_FLOAT32, 3, constcol); - - r->pushLineWidth(commonOpts.wireWidth()); - myWire->draw(r, RE_GEO_WIRE_IDX); - r->popLineWidth(); - r->popShader(); - } } void GR_PrimVDBPoints::renderDecoration(RE_RenderContext r, GR_Decoration decor, const GR_DecorationParms& p) { +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support + if (r.isVulkan()) + { + if (!myGeoVk) return; + + if (decor == GR_POINT_MARKER) + { + drawDecorationForGeo(r, myGeoVk.get(), decor, p.opts, + GR_DECOR_RENDER_FLAG_NONE, + /*overlay=*/false, /*override_vis=*/false, + /*instance_group=*/-1, GR_SELECT_NONE, + GR_DecorationRender::PRIM_POINT); + } + else if (decor == GR_POINT_NORMAL && mHasNormals) + { + drawDecorationForGeo(r, myGeoVk.get(), decor, p.opts, + GR_DECOR_RENDER_FLAG_NONE, + /*overlay=*/false, /*override_vis=*/false, + /*instance_group=*/-1, GR_SELECT_NONE, + GR_DecorationRender::PRIM_POINT); + } + else if (decor == GR_POINT_VELOCITY && mHasVelocity) + { + // GPU-based velocity trail rendering using tessellation. + // The VK decoration system requires GR_GeoRenderVK for its + // built-in tessellation shaders, which custom GR_Primitive + // subclasses don't have. Instead, we use a custom velocity + // shader that reads P and V as vertex inputs and generates + // isolines via tessellation on the GPU. + + RV_Render* rv = r.vkRender(); + GR_Uniforms* uniforms = r.uniforms(); + + // load the velocity tessellation shader on first use + + if (!theVkVelocityShader) { + theVkVelocityShader = + RV_ShaderProgram::loadShaderProgram( + rv->instance(), + "openvdb/VK/velocity.prog"); + if (!theVkVelocityShader) return; + } + + rv->setShader(theVkVelocityShader); + + // compute velocity scale and trail color + + const GR_CommonDispOption& commonOpts = + p.opts->common(); + const float velocityScale = + static_cast(commonOpts.vectorScale()) + * -0.041f; + + UT_Color trailColor = + commonOpts.getColor(GR_POINT_TRAIL_COLOR); + float cr, cg, cb; + trailColor.getRGB(&cr, &cg, &cb); + + // bind uniform blocks + + if (uniforms) { + uniforms->bindRVGlobalBlock( + rv, theVkVelocityShader); + + // bind set 1 (glH_Object) with velocity overrides + if (theVkVelocityShader->hasSet(1)) { + const auto* objBinding = + theVkVelocityShader->getBinding(1, 0); + if (objBinding) { + if (!myVkVelObjectBlock) { + myVkVelObjectBlock.reset( + RV_ShaderBlock::create( + rv->instance(), + *objBinding)); + } + if (!myVkVelObjectSet || + !theVkVelocityShader->isSetCompatible( + *myVkVelObjectSet)) { + myVkVelObjectSet = + theVkVelocityShader->createSet( + rv->instance(), 1); + } + // fill with viewport transforms + uniforms->assignRVBlock(rv, + myVkVelObjectBlock.get(), + theVkVelocityShader); + // override decoration scale and wire color + myVkVelObjectBlock->bindFloat( + "DecorationScale", velocityScale); + myVkVelObjectBlock->bindVector( + "WireColor", + UT_Vector4F(cr, cg, cb, 1.0f)); + myVkVelObjectBlock->uploadBuffer(rv); + myVkVelObjectSet->attachBufferBlock( + rv->instance(), "Object", + myVkVelObjectBlock.get()); + rv->bindSet(myVkVelObjectSet.get(), + theVkVelocityShader); + } + } + } + + // draw using PATCHES connection group (index 1) + + bool beganRendering = false; + if (!rv->isRendering()) { + beganRendering = rv->beginRendering(); + } + + rv->draw(myGeoVk.get(), 1); + + if (beganRendering) { + rv->endRendering(); + } + } + return; + } +#endif + + if (!myGeo) return; + // just render native GR_Primitive decorations if position not available const RE_VertexArray* const position = myGeo->getAttribute("P"); diff --git a/openvdb_houdini/openvdb_houdini/PointUtils.cc b/openvdb_houdini/openvdb_houdini/PointUtils.cc index e8070f0265..fdae57ccf1 100644 --- a/openvdb_houdini/openvdb_houdini/PointUtils.cc +++ b/openvdb_houdini/openvdb_houdini/PointUtils.cc @@ -1073,8 +1073,7 @@ convertPointDataGridToHoudini( const PointDataGrid& grid, const std::vector& attributes, const std::vector& includeGroups, - const std::vector& excludeGroups, - const bool inCoreOnly) + const std::vector& excludeGroups) { using namespace openvdb::math; @@ -1096,14 +1095,14 @@ convertPointDataGridToHoudini( // obtain cumulative point offsets and total points std::vector offsets; MultiGroupFilter filter(includeGroups, excludeGroups, leafIter->attributeSet()); - const Index64 total = pointOffsets(offsets, tree, filter, inCoreOnly); + const Index64 total = pointOffsets(offsets, tree, filter); // a block's global offset is needed to transform its point offsets to global offsets const Index64 startOffset = detail.appendPointBlock(total); HoudiniWriteAttribute positionAttribute(*detail.getP()); convertPointDataGridPosition(positionAttribute, grid, offsets, startOffset, - filter, inCoreOnly); + filter); // add other point attributes to the hdk detail const AttributeSet::Descriptor::NameToPosMap& nameToPosMap = descriptor.map(); @@ -1186,87 +1185,87 @@ convertPointDataGridToHoudini( if (valueType == "string") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "bool") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "int8") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "int16") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "int32") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "int64") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "float") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "double") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "vec3i") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "vec3s") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "vec3d") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "quats") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "quatd") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "mat3s") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "mat3d") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "mat4s") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "mat4d") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else { throw std::runtime_error("Unknown Attribute Type for Conversion: " + valueType); @@ -1288,7 +1287,7 @@ convertPointDataGridToHoudini( attributeSet.groupIndex(name); HoudiniGroup group(*pointGroup, startOffset, total); - convertPointDataGridGroup(group, tree, offsets, startOffset, index, filter, inCoreOnly); + convertPointDataGridGroup(group, tree, offsets, startOffset, index, filter); } } @@ -1615,32 +1614,7 @@ collectPointInfo(const PointDataGrid& grid, const PointDataTree& tree = grid.constTree(); - // iterate through all leaf nodes to find out if all are out-of-core - bool allOutOfCore = true; - for (auto iter = tree.cbeginLeaf(); iter; ++iter) { - if (!iter->buffer().isOutOfCore()) { - allOutOfCore = false; - break; - } - } - - openvdb::Index64 totalPointCount = 0; - - // it is more technically correct to rely on the voxel count as this may be - // out of sync with the attribute size, however for faster node preview when - // the voxel buffers are all out-of-core, count up the sizes of the first - // attribute array instead - - if (allOutOfCore) { - for (auto iter = tree.cbeginLeaf(); iter; ++iter) { - if (iter->attributeSet().size() > 0) { - totalPointCount += iter->constAttributeArray(0).size(); - } - } - } - else { - totalPointCount = openvdb::points::pointCount(tree); - } + openvdb::Index64 totalPointCount = openvdb::points::pointCount(tree); std::ostringstream os; os << openvdb::util::formattedInt(totalPointCount); @@ -1674,16 +1648,8 @@ collectPointInfo(const PointDataGrid& grid, os << it.first << "("; - // for faster node preview when all the voxel buffers are out-of-core, - // don't load the group arrays to display the group sizes, just print - // "out-of-core" instead @todo - put the group sizes into the grid - // metadata on write for this use case - - if (allOutOfCore) os << "out-of-core"; - else { - const openvdb::points::GroupFilter filter(it.first, attributeSet); - os << openvdb::util::formattedInt(pointCount(tree, filter)); - } + const openvdb::points::GroupFilter filter(it.first, attributeSet); + os << openvdb::util::formattedInt(pointCount(tree, filter)); os << ")"; } diff --git a/openvdb_houdini/openvdb_houdini/PointUtils.h b/openvdb_houdini/openvdb_houdini/PointUtils.h index 57dbf6306e..89fe54d941 100644 --- a/openvdb_houdini/openvdb_houdini/PointUtils.h +++ b/openvdb_houdini/openvdb_houdini/PointUtils.h @@ -109,7 +109,6 @@ convertHoudiniToPointDataGrid( /// (empty vector defaults to all) /// @param excludeGroups a vector of VDB Points groups to be excluded /// (empty vector defaults to none) -/// @param inCoreOnly true if out-of-core leaf nodes are to be ignored OPENVDB_HOUDINI_API void convertPointDataGridToHoudini( @@ -117,8 +116,7 @@ convertPointDataGridToHoudini( const openvdb::points::PointDataGrid& grid, const std::vector& attributes = {}, const std::vector& includeGroups = {}, - const std::vector& excludeGroups = {}, - const bool inCoreOnly = false); + const std::vector& excludeGroups = {}); /// @brief Populate VDB Points grid metadata from Houdini detail attributes diff --git a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc index e5abf786d3..665882b9ae 100644 --- a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc +++ b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc @@ -693,23 +693,10 @@ SOP_OpenVDB_Points_Convert::Cache::cookVDBSop(OP_Context& context) // all attributes should be converted const std::vector emptyNameVector; - // if all point data is being converted, sequentially pre-fetch any out-of-core - // data for faster performance when using delayed-loading - - const bool allData = emptyNameVector.empty() && - includeGroups.empty() && - excludeGroups.empty(); - for (const PointDataGrid::ConstPtr &grid : pointGrids) { GU_Detail geo; - // if all the data is being loaded, prefetch it for faster load performance - - if (allData) { - prefetch(grid->tree()); - } - // perform conversion hvdb::convertPointDataGridToHoudini( diff --git a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc index 2dcaf0aa26..d1a11f007c 100644 --- a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc +++ b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc @@ -516,59 +516,24 @@ struct GridsToRasterize struct Grid { - private: - static bool isTreeOutOfCore(const TreeType& tree) - { - using LeafManagerT = openvdb::tree::LeafManager; - using LeafRangeT = typename LeafManagerT::LeafRange; - LeafManagerT leafManager(tree); - return tbb::parallel_reduce(leafManager.leafRange(), true, - [] (const LeafRangeT& range, bool result) -> bool { - for (const auto& leaf : range) { - if (!leaf.buffer().isOutOfCore()) return false; - } - return result; - }, - [] (bool n, bool m) -> bool { return n || m; }); - } - public: explicit Grid(const ConstPtr& grid) - : mGrid(grid) - , mOutOfCore(isTreeOutOfCore(grid->constTree())) { } - - inline bool isOutOfCore() const { return mOutOfCore; } + : mGrid(grid) { } template void addToRasterizer(RasterizerT& rasterizer) { - if (mOutOfCore) { - auto newGrid = mGrid->deepCopy(); - rasterizer.addPoints(newGrid, /*stream=*/true); - } - else { - rasterizer.addPoints(mGrid); - } + rasterizer.addPoints(mGrid); } private: ConstPtr mGrid; - const bool mOutOfCore; }; // struct Grid void push_back(ConstPtr& grid) { mGrids.emplace_back(grid); } size_t size() const { return mGrids.size(); } bool empty() const { return mGrids.empty(); } - // return true if any of the grids is out-of-core - bool streaming() const - { - for (auto& grid : mGrids) { - if (grid.isOutOfCore()) return true; - } - return false; - } - template void addGridToRasterizer(RasterizerT& rasterizer, size_t index) { @@ -885,8 +850,7 @@ Rasterizing can be performed into cartesian and frustum grids (otherwise referre grids). For frustum grids, approximations are used by default to accelerate the rasterization.\n\ This can be disabled to achieve a more accurate result.\n\ \n\ -This node supports streaming of VDB points and attributes if one or more delayed-load VDB is\n\ -provided as an input.\n\ +This node supports streaming of VDB points and attributes\n\ \n\ @related\n\ - [OpenVDB Rasterize Points|Node:sop/DW_OpenVDBRasterizePoints]\n\ @@ -1257,8 +1221,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) iterations = 1; } - bool streaming = pointGrids.streaming(); - for (size_t i = 0; i < iterations; i++) { if (!mergeVDBPoints) { @@ -1274,12 +1236,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) if (std::find(vectorAttribNames.begin(), vectorAttribNames.end(), velocityAttribute) != vectorAttribNames.end()) { velocity = rasterizer.rasterizeAttribute(velocityAttribute, vectorMode, reduceMemory, scale, rasterGroups); - - // need to deep-copy input grids again if caches are being discarded - if (streaming && reduceMemory) { - if (mergeVDBPoints) pointGrids.addGridsToRasterizer(rasterizer); - else pointGrids.addGridToRasterizer(rasterizer, i); - } } // rasterize density @@ -1287,12 +1243,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) if (createDensity) { auto density = rasterizer.rasterizeDensity(densityAttribute, densityMode, reduceMemory, densityScale, rasterGroups); outputGrids.push_back(density); - - // need to deep-copy input grids again if caches are being discarded - if (streaming && reduceMemory) { - if (mergeVDBPoints) pointGrids.addGridsToRasterizer(rasterizer); - else pointGrids.addGridToRasterizer(rasterizer, i); - } } // rasterize mask @@ -1307,12 +1257,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) for (const auto& name : scalarAttribNames) { auto scalar = rasterizer.rasterizeAttribute(name, scalarMode, reduceMemory, scale, rasterGroups); outputGrids.push_back(scalar); - - // need to deep-copy input grids again if caches are being discarded - if (streaming && reduceMemory) { - if (mergeVDBPoints) pointGrids.addGridsToRasterizer(rasterizer); - else pointGrids.addGridToRasterizer(rasterizer, i); - } } // rasterize vector attributes @@ -1327,12 +1271,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) else { auto vector = rasterizer.rasterizeAttribute(name, vectorMode, reduceMemory, scale, rasterGroups); outputGrids.push_back(vector); - - // need to deep-copy input grids again if caches are being discarded - if (streaming && reduceMemory) { - if (mergeVDBPoints) pointGrids.addGridsToRasterizer(rasterizer); - else pointGrids.addGridToRasterizer(rasterizer, i); - } } } } diff --git a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Read.cc b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Read.cc index 7b11986b40..887a0a6e11 100644 --- a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Read.cc +++ b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Read.cc @@ -204,40 +204,20 @@ newSopOperator(OP_OperatorTable* table) .setCallbackFunc(&reloadCB) .setTooltip("Reread the VDB file.")); - parms.add(hutil::ParmFactory(PRM_SEPARATOR, "sep1", "Sep")); - - // Delayed loading - parms.add(hutil::ParmFactory(PRM_TOGGLE, "delayload", "Delay Loading") - .setDefault(PRMoneDefaults) - .setTooltip( - "Don't allocate memory for or read voxel values until the values" - " are actually accessed.\n\n" - "Delayed loading can significantly lower memory usage, but\n" - "note that viewport visualization of a volume usually requires\n" - "the entire volume to be loaded into memory.")); - - // Localization file size slider - parms.add(hutil::ParmFactory(PRM_FLT_J, "copylimit", "Copy If Smaller Than") + // Obsolete parameters + hutil::ParmList obsoleteParms; + obsoleteParms.add(hutil::ParmFactory(PRM_SEPARATOR, "sep1", "Sep")); + obsoleteParms.add(hutil::ParmFactory(PRM_TOGGLE, "delayload", "Delay Loading") + .setDefault(PRMoneDefaults)); + obsoleteParms.add(hutil::ParmFactory(PRM_FLT_J, "copylimit", "Copy If Smaller Than") .setTypeExtended(PRM_TYPE_JOIN_PAIR) - .setDefault(0.5f) - .setRange(PRM_RANGE_RESTRICTED, 0, PRM_RANGE_UI, 10) - .setTooltip( - "When delayed loading is enabled, a file must not be modified on disk before\n" - "it has been fully read. For safety, files smaller than the given size (in GB)\n" - "will be copied to a private, temporary location (either $OPENVDB_TEMP_DIR,\n" - "$TMPDIR or a system default temp directory).") - .setDocumentation( - "When delayed loading is enabled, a file must not be modified on disk before" - " it has been fully read. For safety, files smaller than the given size (in GB)" - " will be copied to a private, temporary location (either `$OPENVDB_TEMP_DIR`," - " `$TMPDIR` or a system default temp directory).")); - - parms.add(hutil::ParmFactory(PRM_LABEL, "copylimitlabel", "GB") - .setDocumentation(nullptr)); + .setDefault(0.5f)); + obsoleteParms.add(hutil::ParmFactory(PRM_LABEL, "copylimitlabel", "GB")); // Register this operator. hvdb::OpenVDBOpFactory("VDB Read", SOP_OpenVDB_Read::factory, parms, *table) .setNativeName("") + .setObsoleteParms(obsoleteParms) .addOptionalInput("Optional Bounding Geometry") .setDocumentation("\ #icon: COMMON/openvdb\n\ @@ -248,13 +228,7 @@ newSopOperator(OP_OperatorTable* table) @overview\n\ \n\ This node reads VDB volumes from a `.vdb` file.\n\ -It is usually preferable to use Houdini's native [File|Node:sop/file] node,\n\ -however unlike the native node, this node allows one to take advantage of\n\ -delayed loading, meaning that only those portions of a volume that are\n\ -actually accessed in a scene get loaded into memory.\n\ -Delayed loading can significantly reduce memory usage when working\n\ -with large volumes (but note that viewport visualization of a volume\n\ -usually requires the entire volume to be loaded into memory).\n\ +It is usually preferable to use Houdini's native [File|Node:sop/file] node.\n\ \n\ @related\n\ - [OpenVDB Write|Node:sop/DW_OpenVDBWrite]\n\ @@ -297,10 +271,6 @@ SOP_OpenVDB_Read::updateParmsFlags() changed |= enableParm("group", bool(evalInt("enable_grouping", 0, t))); - const bool delayedLoad = evalInt("delayload", 0, t); - changed |= enableParm("copylimit", delayedLoad); - changed |= enableParm("copylimitlabel", delayedLoad); - return changed; } @@ -344,10 +314,6 @@ SOP_OpenVDB_Read::cookVDBSop(OP_Context& context) //} } - const bool delayedLoad = evalInt("delayload", 0, t); - const openvdb::Index64 copyMaxBytes = - openvdb::Index64(1.0e9 * evalFloat("copylimit", 0, t)); - openvdb::BBoxd clipBBox; bool clip = evalInt("clip", 0, t); if (clip) { @@ -370,8 +336,7 @@ SOP_OpenVDB_Read::cookVDBSop(OP_Context& context) openvdb::MetaMap::Ptr fileMetadata; try { // Open the VDB file, but don't read any grids yet. - file.setCopyMaxBytes(copyMaxBytes); - file.open(delayedLoad); + file.open(); // Read the file-level metadata. fileMetadata = file.getMetadata(); diff --git a/openvdb_houdini/openvdb_houdini/VRAY_OpenVDB_Points.cc b/openvdb_houdini/openvdb_houdini/VRAY_OpenVDB_Points.cc index 89c808f406..a14b7b514a 100644 --- a/openvdb_houdini/openvdb_houdini/VRAY_OpenVDB_Points.cc +++ b/openvdb_houdini/openvdb_houdini/VRAY_OpenVDB_Points.cc @@ -351,7 +351,7 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) { struct Local { - static GridVecPtr loadGrids(const std::string& filename, const bool stream) + static GridVecPtr loadGrids(const std::string& filename) { GridVecPtr grids; @@ -367,11 +367,6 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) if (baseGrid->isType()) { auto grid = StaticPtrCast(file.readGrid(*iter)); assert(grid); - if (stream) { - // enable streaming mode to auto-collapse attributes - // on read for improved memory efficiency - points::setStreamingMode(grid->tree(), /*on=*/true); - } grids.push_back(grid); } } @@ -389,8 +384,6 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) import("file", mFilename); - int streamData; - import("streamdata", &streamData, 1); import("attrmask", mAttrStr); float fps; @@ -430,7 +423,7 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) } } - mGridPtrs = Local::loadGrids(mFilename.toStdString(), streamData ? true : false); + mGridPtrs = Local::loadGrids(mFilename.toStdString()); // extract which groups to include and exclude UT_StringHolder groupStr; @@ -447,11 +440,6 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) static_cast(vdbBox.max().y()), static_cast(vdbBox.max().z())); - // if streaming the data, re-open the file now that the bounding box has been computed - if (streamData) { - mGridPtrs = Local::loadGrids(mFilename.toStdString(), true); - } - return 1; } diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.frag b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.frag new file mode 100644 index 0000000000..22229d6310 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.frag @@ -0,0 +1,28 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB Points fragment shader for Vulkan. +// Outputs the interpolated color from the vertex shader. +// Uses gl_PointCoord to discard fragments outside a circular disc +// with smooth anti-aliased edges. + +layout(location = 0) in vec4 pnt_color; + +layout(location = 0) out vec4 color_out; + +void main() +{ + if (pnt_color.a == 0.0) + discard; + + // circular point sprite with anti-aliased edge + vec2 coord = gl_PointCoord * 2.0 - 1.0; + float r2 = dot(coord, coord); + if (r2 > 1.0) + discard; + + // smooth edge over the outermost ~1 pixel + float alpha = 1.0 - smoothstep(0.8, 1.0, r2); + + color_out = vec4(pnt_color.rgb, pnt_color.a * alpha); +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.prog b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.prog new file mode 100644 index 0000000000..ee21815c92 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.prog @@ -0,0 +1,6 @@ +#name VDB Points +#version 450 +#output color 0 + +points.vert +points.frag diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.vert b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.vert new file mode 100644 index 0000000000..5257e48d01 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.vert @@ -0,0 +1,28 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// Minimal VDB Points vertex shader for Vulkan. +// Transforms position P by the standard Houdini view/projection matrices, +// passes Cd (color) to the fragment shader. +// +// Uses direct vertex inputs (not ATTRIB macros) because the geometry is +// populated with createAttribute() which creates VBOs, not SSBOs. + +layout(location = 0) in vec3 P; +layout(location = 1) in vec3 Cd; + +layout(location = 0) out vec4 pnt_color; + +layout(set=0, binding=0) +#using glH_PassInfo + +layout(set=1, binding=0) +#using glH_Object + +void main() +{ + vec4 pos = glH_Object.ObjView * vec4(P, 1.0); + gl_Position = glH_PassInfo.Projection * pos; + gl_PointSize = glH_Object.DecorationScale; + pnt_color = vec4(Cd, 1.0); +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.frag b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.frag new file mode 100644 index 0000000000..16d45e8293 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.frag @@ -0,0 +1,26 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB velocity decoration fragment shader. +// Applies a fade effect along the velocity trail using the u parameter +// from the tessellation evaluation shader. Trail color comes from +// glH_Object.WireColor which is set by the C++ code to the trail color. + +layout(location = 0) in float fsU; + +layout(location = 0) out vec4 color_out; + +layout(set=1, binding=0) +#using glH_Object + +void main() +{ + vec4 col = glH_Object.WireColor; + + // cubic fade from opaque (tip) to transparent (tail) + float a = 1.0 - fsU; + a = 1.0 - a * a * a; + a *= col.a; + + color_out = vec4(col.rgb, a); +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.prog b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.prog new file mode 100644 index 0000000000..3470f6fba5 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.prog @@ -0,0 +1,8 @@ +#name VDB Velocity Lines +#version 450 +#output color 0 + +velocity.vert +velocity.tcs +velocity.tes +velocity.frag diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tcs b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tcs new file mode 100644 index 0000000000..3bf7e2bfbb --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tcs @@ -0,0 +1,46 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB velocity decoration tessellation control shader. +// Takes a single control point (patch size 1) and computes the +// start and end positions of the velocity vector in clip space. +// Tessellation generates one isoline segment (2 vertices). + +layout(vertices = 1) out; + +layout(location = 0) in parms +{ + vec3 velocity; +} tsIn[]; + +layout(location = 0) out patch nparms +{ + vec4 startPos; + vec4 endPos; +} tsOut; + +layout(set=0, binding=0) +#using glH_PassInfo + +layout(set=1, binding=0) +#using glH_Object + +void main() +{ + vec3 vel = tsIn[0].velocity * glH_Object.DecorationScale; + + // start position in clip space + tsOut.startPos = glH_PassInfo.Projection * gl_in[0].gl_Position; + + // end position in clip space (add scaled velocity in view space) + tsOut.endPos = glH_PassInfo.Projection * + (gl_in[0].gl_Position + vec4(vel, 0.0)); + + // one isoline segment = 2 vertices + gl_TessLevelInner[0] = 0.0; + gl_TessLevelInner[1] = 0.0; + gl_TessLevelOuter[0] = 1.0; + gl_TessLevelOuter[1] = 1.0; + gl_TessLevelOuter[2] = 1.0; + gl_TessLevelOuter[3] = 1.0; +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tes b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tes new file mode 100644 index 0000000000..0eea5f48e4 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tes @@ -0,0 +1,23 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB velocity decoration tessellation evaluation shader. +// Generates isoline vertices by interpolating between start and end +// positions. The u parameter drives a fade effect in the fragment shader. + +layout(isolines) in; + +layout(location = 0) in patch nparms +{ + vec4 startPos; + vec4 endPos; +} tsIn; + +layout(location = 0) out float fsU; + +void main() +{ + float t = gl_TessCoord.x; + fsU = 1.0 - t; + gl_Position = mix(tsIn.startPos, tsIn.endPos, t); +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.vert b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.vert new file mode 100644 index 0000000000..bc2f0a0a16 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.vert @@ -0,0 +1,30 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB velocity decoration vertex shader for Vulkan. +// Reads P (position) and V (velocity) as vertex inputs. +// Transforms position to view space. Velocity is passed through +// in view space for the tessellation stage to generate isolines. + +layout(location = 0) in vec3 P; +layout(location = 1) in vec3 V; + +layout(location = 0) out parms +{ + vec3 velocity; +} vsOut; + +layout(set=0, binding=0) +#using glH_PassInfo + +layout(set=1, binding=0) +#using glH_Object + +void main() +{ + // position in view space (projection applied in TCS) + gl_Position = glH_Object.ObjView * vec4(P, 1.0); + + // velocity direction in view space + vsOut.velocity = mat3(glH_Object.ObjView) * V; +} diff --git a/openvdb_houdini/openvdb_houdini/reference/GEO_PrimVDB.cc b/openvdb_houdini/openvdb_houdini/reference/GEO_PrimVDB.cc index 76a2e0c01c..ca8c43b210 100644 --- a/openvdb_houdini/openvdb_houdini/reference/GEO_PrimVDB.cc +++ b/openvdb_houdini/openvdb_houdini/reference/GEO_PrimVDB.cc @@ -2263,7 +2263,7 @@ GEO_PrimVDB::loadVDB(UT_JSONParser &p, bool as_shmem) try { SYS_SharedMemoryInputStream is_shm(*shmem); - openvdb::io::Stream vis(is_shm, /*delayLoad*/false); + openvdb::io::Stream vis(is_shm); openvdb::GridPtrVecPtr grids = vis.getGrids(); int count = (grids ? grids->size() : 0); @@ -2303,7 +2303,7 @@ GEO_PrimVDB::loadVDB(UT_JSONParser &p, bool as_shmem) { UT_JSONParser::TiledStream is(p); - openvdb::io::Stream vis(is, /*delayLoad*/false); + openvdb::io::Stream vis(is); openvdb::GridPtrVecPtr grids = vis.getGrids(); diff --git a/pendingchanges/codecs.txt b/pendingchanges/codecs.txt new file mode 100644 index 0000000000..1e049f78e4 --- /dev/null +++ b/pendingchanges/codecs.txt @@ -0,0 +1,7 @@ +OpenVDB: + Features: + - Introduced new openvdb::codecs subsystem and CodecRegistry for stream encoding and decoding. + - Added support for implicit codec conversion fallback (e.g. float to half, scalar to mask) + and ReadDiagnostics. + API changes: + - Extracted Point Data I/O function overloads from PointDataGrid.h into new openvdb/points/PointDataIO.h header. diff --git a/pendingchanges/delayedloading.txt b/pendingchanges/delayedloading.txt new file mode 100644 index 0000000000..6a445fe74f --- /dev/null +++ b/pendingchanges/delayedloading.txt @@ -0,0 +1,6 @@ +OpenVDB: + Highlights: + - Completely removed support for out-of-core / delayed loading across OpenVDB core, + including removal of the OPENVDB_USE_DELAYED_LOADING build option, DelayedLoadMetadata, + and TempFile. This was done to improve performance, reduce codebase complexity and to + entirely eliminate the optional Boost dependency. diff --git a/pendingchanges/houdinivulkan.txt b/pendingchanges/houdinivulkan.txt new file mode 100644 index 0000000000..b64d663f0a --- /dev/null +++ b/pendingchanges/houdinivulkan.txt @@ -0,0 +1,3 @@ +OpenVDB: + Houdini: + - Add new Vulkan viewport support for VDB Points primitives and remove support for delayed loading. diff --git a/pendingchanges/iooptions.txt b/pendingchanges/iooptions.txt new file mode 100644 index 0000000000..ca67c833f5 --- /dev/null +++ b/pendingchanges/iooptions.txt @@ -0,0 +1,6 @@ +OpenVDB: + API changes: + - Added optional trailing WriteOptions parameter to io::File::write() and io::Stream::write(). + No changes in behavior. + - Added optional trailing ReadOptions parameter to io::File::getGrids() and io::File::readGrid(). + No changes in behavior. diff --git a/pendingchanges/ioreadconversion.txt b/pendingchanges/ioreadconversion.txt new file mode 100644 index 0000000000..46cddca5db --- /dev/null +++ b/pendingchanges/ioreadconversion.txt @@ -0,0 +1,5 @@ +OpenVDB: + New features: + - io::ReadOptions now supports read-time grid type conversion, allowing a grid + to be read as a different registered type than the one stored in the file + (for example, reading a FloatGrid as a HalfGrid). diff --git a/pendingchanges/iorefactor.txt b/pendingchanges/iorefactor.txt new file mode 100644 index 0000000000..331fd2ba22 --- /dev/null +++ b/pendingchanges/iorefactor.txt @@ -0,0 +1,8 @@ +OpenVDB: + API changes: + - GridDescriptor::read() has been deprecated, use GridDescriptor::readHeader() + followed by GridDescriptor::readStreamPos() instead. + + Improvements: + - Large refactor of I/O classes, many private and protected member functions have + been modified. No change in behavior for io::File and io::Stream. diff --git a/pendingchanges/removedelayedloading.txt b/pendingchanges/removedelayedloading.txt new file mode 100644 index 0000000000..7d31734800 --- /dev/null +++ b/pendingchanges/removedelayedloading.txt @@ -0,0 +1,20 @@ +OpenVDB: + API changes: + - Delayed loading has been removed. The OPENVDB_USE_DELAYED_LOADING CMake option, + MappedFile support and DelayedLoadMetadata have all been removed. + - io::File::open() and io::Stream constructors that accepted delayed-loading + parameters are deprecated and now behave as their non-delayed-loading + equivalents. + - Out-of-core tree APIs are deprecated, including LeafBuffer::isOutOfCore(), + memUsageIfLoaded() and readNonResidentBuffers(). The latter are now no-ops + and memUsageIfLoaded() forwards to memUsage(). + - Free functions that previously accepted an inCoreOnly parameter are deprecated + in favor of overloads without that parameter. + + Improvements: + - OpenVDB no longer depends on Boost. The boost::iostreams and Boost + interprocess optional dependencies have been removed along with io::TempFile. + + Build: + - Removed the OPENVDB_USE_DELAYED_LOADING CMake option. + - OpenVDB no longer requires or links against Boost. diff --git a/pendingchanges/removeleafiotests.txt b/pendingchanges/removeleafiotests.txt new file mode 100644 index 0000000000..481a910de8 --- /dev/null +++ b/pendingchanges/removeleafiotests.txt @@ -0,0 +1,3 @@ +OpenVDB: + Improvements: + - Remove direct testing of Tree I/O methods in favor of indirect testing using the Grid I/O. diff --git a/pendingchanges/treebaseio.txt b/pendingchanges/treebaseio.txt new file mode 100644 index 0000000000..260862ee03 --- /dev/null +++ b/pendingchanges/treebaseio.txt @@ -0,0 +1,4 @@ + OpenVDB: + API changes: + - TreeBase::readTopology() and TreeBase::writeTopology() are now pure virtual. Derived + classes of TreeBase now need to read and write int32_t(1) to remain backwards-compatible. diff --git a/pyproject.toml b/pyproject.toml index b42bcd5c80..33a942f382 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ wheel.packages = [] OPENVDB_CORE_STATIC="OFF" USE_EXPLICIT_INSTANTIATION="OFF" DISABLE_DEPENDENCY_VERSION_CHECKS="ON" -OPENVDB_USE_DELAYED_LOADING="OFF" OPENVDB_BUILD_PYTHON_MODULE="ON" USE_NUMPY="ON"