From 9298b75ad61f7a47719519b5630851c524ad7b76 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 30 Jan 2026 12:47:03 -0800 Subject: [PATCH 01/70] Remove the OPENVDB_USE_DELAYED_LOADING flag and all the code guarded by it Signed-off-by: Dan Bailey --- .github/workflows/weekly.yml | 4 +- CMakeLists.txt | 2 - cmake/FindOpenVDB.cmake | 18 - openvdb/openvdb/CMakeLists.txt | 39 - openvdb/openvdb/io/Archive.cc | 164 +--- openvdb/openvdb/io/File.cc | 106 +-- openvdb/openvdb/io/File.h | 29 +- openvdb/openvdb/io/Stream.cc | 52 -- openvdb/openvdb/io/TempFile.cc | 144 --- openvdb/openvdb/io/TempFile.h | 54 -- openvdb/openvdb/io/io.h | 60 -- openvdb/openvdb/points/AttributeArray.h | 27 +- openvdb/openvdb/points/StreamCompression.cc | 83 +- openvdb/openvdb/points/StreamCompression.h | 3 - openvdb/openvdb/tree/LeafBuffer.h | 186 +--- openvdb/openvdb/tree/LeafNode.h | 41 +- .../openvdb/unittest/TestAttributeArray.cc | 821 ------------------ openvdb/openvdb/unittest/TestCount.cc | 44 - openvdb/openvdb/unittest/TestFile.cc | 4 - .../openvdb/unittest/TestPointConversion.cc | 1 - openvdb/openvdb/unittest/TestPointCount.cc | 59 -- openvdb/openvdb/unittest/TestPointDataLeaf.cc | 105 --- .../unittest/TestPointRasterizeFrustum.cc | 113 +-- .../openvdb/unittest/TestStreamCompression.cc | 121 +-- openvdb/openvdb/unittest/util.h | 12 - openvdb/openvdb/version.h.in | 5 - pyproject.toml | 1 - 27 files changed, 51 insertions(+), 2247 deletions(-) delete mode 100644 openvdb/openvdb/io/TempFile.cc delete mode 100644 openvdb/openvdb/io/TempFile.h diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index 353303e54c..614cb37de9 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -146,7 +146,7 @@ jobs: matrix: config: - { name: 'all', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=ON -DUSE_ZLIB=ON -DUSE_EXR=ON -DUSE_PNG=ON' } - - { name: 'lite', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF -DOPENVDB_USE_DELAYED_LOADING=OFF' } + - { name: 'lite', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF' } - { name: 'half', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_IMATH_HALF=ON' } - { name: 'sse', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_SIMD=SSE42' } - { name: 'avx', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_SIMD=AVX' } @@ -216,7 +216,6 @@ jobs: --components=\"core,axcore,python,bin,render,test,axbin\" --cargs=\' -DCMAKE_CXX_STANDARD=20 - -DOPENVDB_USE_DELAYED_LOADING=OFF -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install ${{ matrix.config.cmake }} \' @@ -421,7 +420,6 @@ jobs: --components="${{ matrix.config.components }}" --cargs=\' -A x64 -G \"Visual Studio 17 2022\" - -DOPENVDB_USE_DELAYED_LOADING=OFF -DUSE_BLOSC=OFF \ -DUSE_ZLIB=OFF \ -DVCPKG_TARGET_TRIPLET=${VCPKG_DEFAULT_TRIPLET} diff --git a/CMakeLists.txt b/CMakeLists.txt index 089381aa41..c66ad21fa7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,7 +80,6 @@ option(OPENVDB_BUILD_NANOVDB "Build the NanoVDB library. Turns ON if USE_NANOVDB # Global options option(OPENVDB_ENABLE_RPATH "Build with RPATH information" ON) option(OPENVDB_ENABLE_ASSERTS "Build with asserts in OpenVDB code enabled" OFF) -option(OPENVDB_USE_DELAYED_LOADING "Build the core OpenVDB library with delayed-loading." ON) option(OPENVDB_CXX_STRICT "Enable or disable pre-defined compiler warnings" OFF) cmake_dependent_option(OPENVDB_INSTALL_CMAKE_MODULES "Install the provided OpenVDB CMake modules when building the core library" @@ -249,7 +248,6 @@ mark_as_advanced( OPENVDB_BUILD_HOUDINI_ABITESTS OPENVDB_CXX_STRICT OPENVDB_ENABLE_RPATH - OPENVDB_USE_DELAYED_LOADING OPENVDB_FUTURE_DEPRECATION OPENVDB_SIMD SYSTEM_LIBRARY_PATHS diff --git a/cmake/FindOpenVDB.cmake b/cmake/FindOpenVDB.cmake index 400afd2544..17a4c2da53 100644 --- a/cmake/FindOpenVDB.cmake +++ b/cmake/FindOpenVDB.cmake @@ -64,8 +64,6 @@ This will define the following variables: True if the OpenVDB Library has been built with log4cplus support ``OpenVDB_USES_IMATH_HALF`` True if the OpenVDB Library has been built with Imath half support -``OpenVDB_USES_DELAYED_LOADING`` - True if the OpenVDB Library has been built with delayed-loading ``OpenVDB_ABI`` Set if this module was able to determine the ABI number the located OpenVDB Library was built against. Unset otherwise. @@ -522,7 +520,6 @@ set(OpenVDB_USES_BLOSC ${USE_BLOSC}) set(OpenVDB_USES_ZLIB ${USE_ZLIB}) set(OpenVDB_USES_LOG4CPLUS ${USE_LOG4CPLUS}) set(OpenVDB_USES_IMATH_HALF ${USE_IMATH_HALF}) -set(OpenVDB_USES_DELAYED_LOADING ${OPENVDB_USE_DELAYED_LOADING}) set(OpenVDB_DEFINITIONS) if(WIN32) @@ -555,7 +552,6 @@ if(_OPENVDB_HAS_NEW_VERSION_HEADER) OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_IMATH_HALF" OpenVDB_USES_IMATH_HALF) OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_BLOSC" OpenVDB_USES_BLOSC) OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_ZLIB" OpenVDB_USES_ZLIB) - OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_DELAYED_LOADING" OpenVDB_USES_DELAYED_LOADING) elseif(NOT OPENVDB_USE_STATIC_LIBS) # Use GetPrerequisites to see which libraries this OpenVDB lib has linked to # which we can query for optional deps. This basically runs ldd/otoll/objdump @@ -602,11 +598,6 @@ elseif(NOT OPENVDB_USE_STATIC_LIBS) if(NOT ${_HAS_DEP} EQUAL -1) set(OpenVDB_USES_IMATH_HALF ON) endif() - - string(FIND ${PREREQUISITE} "boost_iostreams" _HAS_DEP) - if(NOT ${_HAS_DEP} EQUAL -1) - set(OpenVDB_USES_DELAYED_LOADING ON) - endif() endforeach() unset(_OPENVDB_PREREQUISITE_LIST) @@ -628,10 +619,6 @@ if(OpenVDB_USES_IMATH_HALF) find_package(Imath REQUIRED CONFIG) endif() -if(OpenVDB_USES_DELAYED_LOADING) - find_package(Boost REQUIRED COMPONENTS iostreams) -endif() - if(UNIX) find_package(Threads REQUIRED) endif() @@ -643,11 +630,6 @@ endif() set(_OPENVDB_VISIBLE_DEPENDENCIES "") -if(OpenVDB_USES_DELAYED_LOADING) - list(APPEND _OPENVDB_VISIBLE_DEPENDENCIES Boost::iostreams) - list(APPEND OpenVDB_DEFINITIONS OPENVDB_USE_DELAYED_LOADING) -endif() - if(OpenVDB_USES_IMATH_HALF) list(APPEND _OPENVDB_VISIBLE_DEPENDENCIES Imath::Imath) endif() diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 8348d0d527..c496509a72 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -116,19 +116,6 @@ endif() # Collect and configure lib dependencies -if(OPENVDB_USE_DELAYED_LOADING) - find_package(Boost ${MINIMUM_BOOST_VERSION} REQUIRED COMPONENTS iostreams) - - if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_BOOST_VERSION) - # The X.Y.Z boost version value isn't available until CMake 3.14 - set(FULL_BOOST_VERSION "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}") - if(${FULL_BOOST_VERSION} VERSION_LESS FUTURE_MINIMUM_BOOST_VERSION) - message(DEPRECATION "Support for Boost versions < ${FUTURE_MINIMUM_BOOST_VERSION} " - "is deprecated and will be removed.") - endif() - endif() -endif() - find_package(TBB ${MINIMUM_TBB_VERSION} REQUIRED COMPONENTS tbb) if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_TBB_VERSION) if(${TBB_VERSION} VERSION_LESS FUTURE_MINIMUM_TBB_VERSION) @@ -251,27 +238,6 @@ if(UNIX) list(APPEND OPENVDB_CORE_DEPENDENT_LIBS Threads::Threads) endif() -# Pull in Boost last as houdini's boost (hboost) is fully namespaced and libs -# are renamed too. Boost can be pulled in at any time, but do it last so that, -# if it's in a shared place (like /usr/local) it doesn't accidently pull in -# other headers. - -if(OPENVDB_USE_DELAYED_LOADING) - list(APPEND OPENVDB_CORE_DEPENDENT_LIBS Boost::iostreams) - if(WIN32) - # Boost headers contain #pragma commands on Windows which causes Boost - # libraries to be linked in automatically. Custom boost installations - # may find that these naming conventions don't always match and can - # cause linker errors. This option disables this feature of Boost. Note - # -DBOOST_ALL_NO_LIB can also be provided manually. - if(OPENVDB_DISABLE_BOOST_IMPLICIT_LINKING) - list(APPEND OPENVDB_CORE_DEPENDENT_LIBS - Boost::disable_autolinking # add -DBOOST_ALL_NO_LIB - ) - endif() - endif() -endif() - ########################################################################## ##### Core library configuration @@ -369,7 +335,6 @@ set(OPENVDB_LIBRARY_SOURCE_FILES io/GridDescriptor.cc io/Queue.cc io/Stream.cc - io/TempFile.cc math/Maps.cc math/Proximity.cc math/QuantizedUnitVec.cc @@ -409,7 +374,6 @@ set(OPENVDB_LIBRARY_IO_INCLUDE_FILES io/io.h io/Queue.h io/Stream.h - io/TempFile.h ) set(OPENVDB_LIBRARY_MATH_INCLUDE_FILES @@ -670,9 +634,6 @@ endif() if(USE_LOG4CPLUS) list(APPEND OPENVDB_CORE_PUBLIC_DEFINES -DOPENVDB_USE_LOG4CPLUS) endif() -if(OPENVDB_USE_DELAYED_LOADING) - list(APPEND OPENVDB_CORE_PUBLIC_DEFINES -DOPENVDB_USE_DELAYED_LOADING) -endif() ########################################################################## diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 4431d69299..03576b2566 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -13,35 +13,6 @@ #include #include -#ifdef OPENVDB_USE_DELAYED_LOADING -// Boost.Interprocess uses a header-only portion of Boost.DateTime -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunused-macros" -#endif -#define BOOST_DATE_TIME_NO_LIB -#ifdef __clang__ -#pragma clang diagnostic pop -#endif -#include -#include -#include -#include - -#ifdef _WIN32 -#include // open_existing_file(), close_file() -extern "C" __declspec(dllimport) bool __stdcall GetFileTime( - void* fh, void* ctime, void* atime, void* mtime); -// boost::interprocess::detail was renamed to boost::interprocess::ipcdetail in Boost 1.48. -// Ensure that both namespaces exist. -namespace boost { namespace interprocess { namespace detail {} namespace ipcdetail {} } } -#else -#include // for struct stat -#include // for stat() -#include // for unlink() -#endif -#endif // OPENVDB_USE_DELAYED_LOADING - #include #include // for std::find_if() @@ -443,117 +414,6 @@ operator<<(std::ostream& os, const StreamMetadata::AuxDataMap& auxData) //////////////////////////////////////// -#ifdef OPENVDB_USE_DELAYED_LOADING - - -// Memory-mapping a VDB file permits threaded input (and output, potentially, -// though that might not be practical for compressed files or files containing -// multiple grids). In particular, a memory-mapped file can be loaded lazily, -// meaning that the voxel buffers of the leaf nodes of a grid's tree are not allocated -// until they are actually accessed. When access to its buffer is requested, -// a leaf node allocates memory for the buffer and then streams in (and decompresses) -// its contents from the memory map, starting from a stream offset that was recorded -// at the time the node was constructed. The memory map must persist as long as -// there are unloaded leaf nodes; this is ensured by storing a shared pointer -// to the map in each unloaded node. - -class MappedFile::Impl -{ -public: - Impl(const std::string& filename, bool autoDelete) - : mMap(filename.c_str(), boost::interprocess::read_only) - , mRegion(mMap, boost::interprocess::read_only) - , mAutoDelete(autoDelete) - { - if (mAutoDelete) { -#ifndef _WIN32 - // On Unix systems, unlink the file so that it gets deleted once it is closed. - ::unlink(mMap.get_name()); -#endif - } - } - - ~Impl() - { - std::string filename; - if (const char* s = mMap.get_name()) filename = s; - OPENVDB_LOG_DEBUG_RUNTIME("closing memory-mapped file " << filename); - if (mNotifier) mNotifier(filename); - if (mAutoDelete) { - if (!boost::interprocess::file_mapping::remove(filename.c_str())) { - if (errno != ENOENT) { - // Warn if the file exists but couldn't be removed. - std::string mesg = getErrorString(); - if (!mesg.empty()) mesg = " (" + mesg + ")"; - OPENVDB_LOG_WARN("failed to remove temporary file " << filename << mesg); - } - } - } - } - - boost::interprocess::file_mapping mMap; - boost::interprocess::mapped_region mRegion; - bool mAutoDelete; - Notifier mNotifier; -#if OPENVDB_ABI_VERSION_NUMBER <= 12 - mutable std::atomic mLastWriteTime; -#endif - -private: - Impl(const Impl&); // not copyable - Impl& operator=(const Impl&); // not copyable -}; - - -MappedFile::MappedFile(const std::string& filename, bool autoDelete): - mImpl(new Impl(filename, autoDelete)) -{ -} - - -MappedFile::~MappedFile() -{ -} - - -std::string -MappedFile::filename() const -{ - std::string result; - if (const char* s = mImpl->mMap.get_name()) result = s; - return result; -} - - -SharedPtr -MappedFile::createBuffer() const -{ - return SharedPtr{ - new boost::iostreams::stream_buffer{ - static_cast(mImpl->mRegion.get_address()), mImpl->mRegion.get_size()}}; -} - - -void -MappedFile::setNotifier(const Notifier& notifier) -{ - mImpl->mNotifier = notifier; -} - - -void -MappedFile::clearNotifier() -{ - mImpl->mNotifier = nullptr; -} - - -#endif // OPENVDB_USE_DELAYED_LOADING - - -//////////////////////////////////////// - - std::string getErrorString(int errorNum) { @@ -886,25 +746,6 @@ setGridBackgroundValuePtr(std::ios_base& strm, const void* background) } -#ifdef OPENVDB_USE_DELAYED_LOADING -MappedFile::Ptr -getMappedFilePtr(std::ios_base& strm) -{ - if (const void* ptr = strm.pword(GetSteamState().mappedFile)) { - return *static_cast(ptr); - } - return MappedFile::Ptr(); -} - - -void -setMappedFilePtr(std::ios_base& strm, io::MappedFile::Ptr& mappedFile) -{ - strm.pword(GetSteamState().mappedFile) = &mappedFile; -} -#endif // OPENVDB_USE_DELAYED_LOADING - - StreamMetadata::Ptr getStreamMetadataPtr(std::ios_base& strm) { @@ -1120,14 +961,11 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co bool Archive::isDelayedLoadingEnabled() { -#ifdef OPENVDB_USE_DELAYED_LOADING - return (nullptr == std::getenv("OPENVDB_DISABLE_DELAYED_LOAD")); -#else return false; -#endif } + namespace { struct NoBBox {}; diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index aee16c8879..dfb0c797e1 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -5,20 +5,11 @@ #include "File.h" -#include "TempFile.h" #include #include #include #include -#ifdef OPENVDB_USE_DELAYED_LOADING -#include -#ifndef _WIN32 -#include -#include -#endif -#endif // OPENVDB_USE_DELAYED_LOADING - #include // stat() #include // for getenv(), strtoul() @@ -99,14 +90,6 @@ struct File::Impl Archive::NamedGridMap mNamedGrids; // All grids stored in the file (used only when mHasGridOffsets is false) GridPtrVecPtr mGrids; -#ifdef OPENVDB_USE_DELAYED_LOADING - // The memory-mapped file - MappedFile::Ptr mFileMapping; - // The buffer for the input stream, if it is a memory-mapped file - SharedPtr mStreamBuf; - // File size limit for copying during delayed loading - Index64 mCopyMaxBytes; -#endif }; // class File::Impl @@ -117,9 +100,6 @@ File::File(const std::string& filename): mImpl(new Impl) { mImpl->mFilename = filename; mImpl->mIsOpen = false; -#ifdef OPENVDB_USE_DELAYED_LOADING - mImpl->mCopyMaxBytes = Impl::getDefaultCopyMaxBytes(); -#endif setInputHasGridOffsets(true); } @@ -146,9 +126,6 @@ File::operator=(const File& other) mImpl->mFilename = otherImpl.mFilename; mImpl->mMeta = otherImpl.mMeta; mImpl->mIsOpen = false; // don't want two file objects reading from the same stream -#ifdef OPENVDB_USE_DELAYED_LOADING - mImpl->mCopyMaxBytes = otherImpl.mCopyMaxBytes; -#endif mImpl->mGridDescriptors = otherImpl.mGridDescriptors; mImpl->mNamedGrids = otherImpl.mNamedGrids; mImpl->mGrids = otherImpl.mGrids; @@ -252,22 +229,6 @@ File::getSize() const } -#ifdef OPENVDB_USE_DELAYED_LOADING -Index64 -File::copyMaxBytes() const -{ - return mImpl->mCopyMaxBytes; -} - - -void -File::setCopyMaxBytes(Index64 bytes) -{ - mImpl->mCopyMaxBytes = bytes; -} -#endif - - //////////////////////////////////////// @@ -279,11 +240,7 @@ File::isOpen() const bool -#ifdef OPENVDB_USE_DELAYED_LOADING -File::open(bool delayLoad, const MappedFile::Notifier& notifier) -#else -File::open(bool /*delayLoad = true*/) -#endif // OPENVDB_USE_DELAYED_LOADING +File::open() { if (isOpen()) { OPENVDB_THROW(IoError, filename() << " is already open"); @@ -292,53 +249,8 @@ File::open(bool /*delayLoad = true*/) // Open the file. std::unique_ptr newStream; - SharedPtr newStreamBuf; -#ifdef OPENVDB_USE_DELAYED_LOADING - MappedFile::Ptr newFileMapping; - if (!delayLoad || !Archive::isDelayedLoadingEnabled()) { -#endif - newStream.reset(new std::ifstream( - filename().c_str(), std::ios_base::in | std::ios_base::binary)); -#ifdef OPENVDB_USE_DELAYED_LOADING - } else { - bool isTempFile = false; - std::string fname = filename(); - if (getSize() < copyMaxBytes()) { - // If the file is not too large, make a temporary private copy of it - // and open the copy instead. The original file can then be modified - // or removed without affecting delayed load. - try { - TempFile tempFile; - std::ifstream fstrm(filename().c_str(), - std::ios_base::in | std::ios_base::binary); - boost::iostreams::copy(fstrm, tempFile); - fname = tempFile.filename(); - isTempFile = true; - } catch (std::exception& e) { - std::string mesg; - if (e.what()) mesg = std::string(" (") + e.what() + ")"; - OPENVDB_LOG_WARN("failed to create a temporary copy of " << filename() - << " for delayed loading" << mesg - << "; will read directly from " << filename() << " instead"); - } - } - - // While the file is open, its mapping, stream buffer and stream - // must all be maintained. Once the file is closed, the buffer and - // the stream can be discarded, but the mapping needs to persist - // if any grids were lazily loaded. - try { - newFileMapping.reset(new MappedFile(fname, /*autoDelete=*/isTempFile)); - newStreamBuf = newFileMapping->createBuffer(); - newStream.reset(new std::istream(newStreamBuf.get())); - } catch (std::exception& e) { - std::ostringstream ostr; - ostr << "could not open file " << filename(); - if (e.what() != nullptr) ostr << " (" << e.what() << ")"; - OPENVDB_THROW(IoError, ostr.str()); - } - } -#endif // OPENVDB_USE_DELAYED_LOADING + newStream.reset(new std::ifstream( + filename().c_str(), std::ios_base::in | std::ios_base::binary)); if (newStream->fail()) { OPENVDB_THROW(IoError, "could not open file " << filename()); @@ -356,11 +268,6 @@ File::open(bool /*delayLoad = true*/) throw; } -#ifdef OPENVDB_USE_DELAYED_LOADING - mImpl->mFileMapping = newFileMapping; - if (mImpl->mFileMapping) mImpl->mFileMapping->setNotifier(notifier); - mImpl->mStreamBuf = newStreamBuf; -#endif mImpl->mInStream.swap(newStream); // Tag the input stream with the file format and library version numbers @@ -371,9 +278,6 @@ File::open(bool /*delayLoad = true*/) Archive::setFormatVersion(inputStream()); Archive::setLibraryVersion(inputStream()); Archive::setDataCompression(inputStream()); -#ifdef OPENVDB_USE_DELAYED_LOADING - io::setMappedFilePtr(inputStream(), mImpl->mFileMapping); -#endif // Read in the VDB metadata. mImpl->mMeta = MetaMap::Ptr(new MetaMap); @@ -422,10 +326,6 @@ File::close() mImpl->mNamedGrids.clear(); mImpl->mInStream.reset(); mImpl->mStreamMetadata.reset(); -#ifdef OPENVDB_USE_DELAYED_LOADING - mImpl->mStreamBuf.reset(); - mImpl->mFileMapping.reset(); -#endif mImpl->mIsOpen = false; setInputHasGridOffsets(true); diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index 41dd6f9694..dbde5b6d51 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -54,20 +54,11 @@ class OPENVDB_API File: public Archive /// @details The file does not necessarily exist on disk yet. const std::string& filename() const; -#ifdef OPENVDB_USE_DELAYED_LOADING /// @brief Open the file, read the file header and the file-level metadata, /// and populate the grid descriptors, but do not load any grids into memory. - /// @details If @a delayLoad is true, map the file into memory and enable delayed loading - /// of grids, and if a notifier is provided, call it when the file gets unmapped. - /// @note Define the environment variable @c OPENVDB_DISABLE_DELAYED_LOAD to disable - /// delayed loading unconditionally. /// @throw IoError if the file is not a valid VDB file. /// @return @c true if the file's UUID has changed since it was last read. - /// @see setCopyMaxBytes - bool open(bool delayLoad = true, const MappedFile::Notifier& = MappedFile::Notifier()); -#else - bool open(bool /*delayLoad*/ = false); -#endif + bool open(); /// Return @c true if the file has been opened for reading. bool isOpen() const; @@ -79,24 +70,6 @@ class OPENVDB_API File: public Archive /// @throw IoError if the file size cannot be determined. Index64 getSize() const; -#ifdef OPENVDB_USE_DELAYED_LOADING - /// @brief Return the size in bytes above which this file will not be - /// automatically copied during delayed loading. - Index64 copyMaxBytes() const; - /// @brief If this file is opened with delayed loading enabled, make a private copy - /// of the file if its size in bytes is less than the specified value. - /// @details Making a private copy ensures that the file can't change on disk - /// before it has been fully read. - /// @warning If the file is larger than this size, it is the user's responsibility - /// to ensure that it does not change on disk before it has been fully read. - /// Undefined behavior and/or a crash might result otherwise. - /// @note Copying is enabled by default, but it can be disabled for individual files - /// by setting the maximum size to zero bytes. A default size limit can be specified - /// by setting the environment variable @c OPENVDB_DELAYED_LOAD_COPY_MAX_BYTES - /// to the desired number of bytes. - void setCopyMaxBytes(Index64 bytes); -#endif - /// Return @c true if a grid of the given name exists in this file. bool hasGrid(const Name&) const; diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 662a2f4cd9..4ba3fce816 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -5,14 +5,9 @@ #include "File.h" ///< @todo refactor #include "GridDescriptor.h" -#include "TempFile.h" #include #include -#ifdef OPENVDB_USE_DELAYED_LOADING -#include -#endif - #include // for remove() #include // for std::bind() #include @@ -49,59 +44,12 @@ struct Stream::Impl //////////////////////////////////////// -#ifdef OPENVDB_USE_DELAYED_LOADING - -namespace { - -/// @todo Use MappedFile auto-deletion instead. -void -removeTempFile(const std::string expectedFilename, const std::string& filename) -{ - if (filename == expectedFilename) { - if (0 != std::remove(filename.c_str())) { - std::string mesg = getErrorString(); - if (!mesg.empty()) mesg = " (" + mesg + ")"; - OPENVDB_LOG_WARN("failed to remove temporary file " << filename << mesg); - } - } -} - -} - -#endif // OPENVDB_USE_DELAYED_LOADING - - Stream::Stream(std::istream& is, bool delayLoad): mImpl(new Impl) { if (!is) return; (void) delayLoad; -#ifdef OPENVDB_USE_DELAYED_LOADING - if (delayLoad && Archive::isDelayedLoadingEnabled()) { - // Copy the contents of the stream to a temporary private file - // and open the file instead. - std::unique_ptr tempFile; - try { - tempFile.reset(new TempFile); - } catch (std::exception& e) { - std::string mesg; - if (e.what()) mesg = std::string(" (") + e.what() + ")"; - OPENVDB_LOG_WARN("failed to create a temporary file for delayed loading" << mesg - << "; will read directly from the input stream instead"); - } - if (tempFile) { - boost::iostreams::copy(is, *tempFile); - const std::string& filename = tempFile->filename(); - mImpl->mFile.reset(new File(filename)); - mImpl->mFile->setCopyMaxBytes(0); // don't make a copy of the temporary file - /// @todo Need to pass auto-deletion flag to MappedFile. - mImpl->mFile->open(delayLoad, - std::bind(&removeTempFile, filename, std::placeholders::_1)); - } - } -#endif // OPENVDB_USE_DELAYED_LOADING - if (!mImpl->mFile) { readHeader(is); diff --git a/openvdb/openvdb/io/TempFile.cc b/openvdb/openvdb/io/TempFile.cc deleted file mode 100644 index 290f2e6b9c..0000000000 --- a/openvdb/openvdb/io/TempFile.cc +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -/// @file TempFile.cc - -#ifdef OPENVDB_USE_DELAYED_LOADING - -#include "TempFile.h" - -#include -#ifndef _WIN32 -#include -#include -#include // for std::getenv(), mkstemp() -#include // for mode_t -#include // for mkdir(), umask() -#include // for access() -#else -#include // for std::filebuf -#endif -#include // for std::tmpnam(), L_tmpnam, P_tmpdir -#include -#include -#include -#include - - -namespace openvdb { -OPENVDB_USE_VERSION_NAMESPACE -namespace OPENVDB_VERSION_NAME { -namespace io { - -struct TempFile::TempFileImpl -{ - const std::string& filename() const { return mPath; } - - bool is_open() const { return mBuffer.is_open(); } - - /// @internal boost::filesystem::unique_path(), etc. might be useful here, - /// but as of 9/2014, Houdini ships without the Boost.Filesystem library, - /// which makes it much less convenient to use that library. -#ifndef _WIN32 - TempFileImpl(std::ostream& os): mFileDescr(-1) { this->init(os); } - - void init(std::ostream& os) - { - std::string fn = this->getTempDir() + "/openvdb_temp_XXXXXX"; - std::vector fnbuf(fn.begin(), fn.end()); - fnbuf.push_back(char(0)); - - //const mode_t savedMode = ::umask(~(S_IRUSR | S_IWUSR)); - mFileDescr = ::mkstemp(&fnbuf[0]); - //::umask(savedMode); - if (mFileDescr < 0) { - OPENVDB_THROW(IoError, "failed to generate temporary file"); - } - - mPath.assign(&fnbuf[0]); - - mDevice = DeviceType(mFileDescr, boost::iostreams::never_close_handle); - mBuffer.open(mDevice); - os.rdbuf(&mBuffer); - - if (!os.good()) { - OPENVDB_THROW(IoError, "failed to open temporary file " + mPath); - } - } - - void close() { mBuffer.close(); if (mFileDescr >= 0) ::close(mFileDescr); } - - static std::string getTempDir() - { - if (const char* dir = std::getenv("OPENVDB_TEMP_DIR")) { - if (0 != ::access(dir, F_OK)) { -#ifdef _WIN32 - ::mkdir(dir); -#else - ::mkdir(dir, S_IRUSR | S_IWUSR | S_IXUSR); -#endif - if (0 != ::access(dir, F_OK)) { - OPENVDB_THROW(IoError, - "failed to create OPENVDB_TEMP_DIR (" + std::string(dir) + ")"); - } - } - return dir; - } - if (const char* dir = std::getenv("TMPDIR")) return dir; - return P_tmpdir; - } - - using DeviceType = boost::iostreams::file_descriptor_sink; - using BufferType = boost::iostreams::stream_buffer; - - std::string mPath; - DeviceType mDevice; - BufferType mBuffer; - int mFileDescr; -#else // _WIN32 - // Use only standard library routines; no POSIX. - - TempFileImpl(std::ostream& os) { this->init(os); } - - void init(std::ostream& os) - { - char fnbuf[L_tmpnam]; - const char* filename = std::tmpnam(fnbuf); - if (!filename) { - OPENVDB_THROW(IoError, "failed to generate name for temporary file"); - } - /// @todo This is not safe, since another process could open a file - /// with this name before we do. Unfortunately, there is no safe, - /// portable way to create a temporary file. - mPath = filename; - - const std::ios_base::openmode mode = (std::ios_base::out | std::ios_base::binary); - os.rdbuf(mBuffer.open(mPath.c_str(), mode)); - if (!os.good()) { - OPENVDB_THROW(IoError, "failed to open temporary file " + mPath); - } - } - - void close() { mBuffer.close(); } - - std::string mPath; - std::filebuf mBuffer; -#endif // _WIN32 - -private: - TempFileImpl(const TempFileImpl&); // disable copying - TempFileImpl& operator=(const TempFileImpl&); // disable assignment -}; - - -TempFile::TempFile(): std::ostream(nullptr), mImpl(new TempFileImpl(*this)) {} -TempFile::~TempFile() { this->close(); } -const std::string& TempFile::filename() const { return mImpl->filename(); } -bool TempFile::is_open() const { return mImpl->is_open(); } -void TempFile::close() { mImpl->close(); } - -} // namespace io -} // namespace OPENVDB_VERSION_NAME -} // namespace openvdb - -#endif // OPENVDB_USE_DELAYED_LOADING diff --git a/openvdb/openvdb/io/TempFile.h b/openvdb/openvdb/io/TempFile.h deleted file mode 100644 index cbbdec2250..0000000000 --- a/openvdb/openvdb/io/TempFile.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -/// @file TempFile.h - -#ifdef OPENVDB_USE_DELAYED_LOADING - -#ifndef OPENVDB_IO_TEMPFILE_HAS_BEEN_INCLUDED -#define OPENVDB_IO_TEMPFILE_HAS_BEEN_INCLUDED - -#include -#include -#include - - -namespace openvdb { -OPENVDB_USE_VERSION_NAMESPACE -namespace OPENVDB_VERSION_NAME { -namespace io { - -/// Output stream to a unique temporary file -class OPENVDB_API TempFile: public std::ostream -{ -public: - /// @brief Create and open a unique file. - /// @details On UNIX systems, the file is created in the directory specified by - /// the environment variable @c OPENVDB_TEMP_DIR, if that variable is defined, - /// or else in the directory specified by @c TMPDIR, if that variable is defined. - /// Otherwise (and on non-UNIX systems), the file is created in the system default - /// temporary directory. - TempFile(); - ~TempFile(); - - /// Return the path to the temporary file. - const std::string& filename() const; - - /// Return @c true if the file is open for writing. - bool is_open() const; - - /// Close the file. - void close(); - -private: - struct TempFileImpl; - std::unique_ptr mImpl; -}; - -} // namespace io -} // namespace OPENVDB_VERSION_NAME -} // namespace openvdb - -#endif // OPENVDB_IO_TEMPFILE_HAS_BEEN_INCLUDED - -#endif // OPENVDB_USE_DELAYED_LOADING diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index ff3adb6d26..40bce111d5 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -127,56 +127,6 @@ struct MultiPass {}; //////////////////////////////////////// -class File; - -#ifdef OPENVDB_USE_DELAYED_LOADING - -/// @brief Handle to control the lifetime of a memory-mapped .vdb file -class OPENVDB_API MappedFile -{ -public: - using Ptr = SharedPtr; - - ~MappedFile(); - MappedFile(const MappedFile&) = delete; // not copyable - MappedFile& operator=(const MappedFile&) = delete; - - /// Return the filename of the mapped file. - std::string filename() const; - - /// @brief Return a new stream buffer for the mapped file. - /// @details Typical usage is - /// @code - /// openvdb::io::MappedFile::Ptr mappedFile = ...; - /// auto buf = mappedFile->createBuffer(); - /// std::istream istrm{buf.get()}; - /// // Read from istrm... - /// @endcode - /// The buffer must persist as long as the stream is open. - SharedPtr createBuffer() const; - - using Notifier = std::function; - /// @brief Register a function that will be called with this file's name - /// when the file is unmapped. - void setNotifier(const Notifier&); - /// Deregister the notifier. - void clearNotifier(); - -private: - friend class File; - friend class ::TestMappedFile; - - explicit MappedFile(const std::string& filename, bool autoDelete = false); - - class Impl; - std::unique_ptr mImpl; -}; // class MappedFile - -#endif // OPENVDB_USE_DELAYED_LOADING - -//////////////////////////////////////// - - /// Return a string (possibly empty) describing the given system error code. std::string getErrorString(int errorNum); @@ -249,16 +199,6 @@ OPENVDB_API bool getWriteGridStatsMetadata(std::ios_base&); /// and store them as grid metadata when writing to the given stream. OPENVDB_API void setWriteGridStatsMetadata(std::ios_base&, bool writeGridStats); -#ifdef OPENVDB_USE_DELAYED_LOADING -/// @brief Return a shared pointer to the memory-mapped file with which the given stream -/// is associated, or a null pointer if the stream is not associated with a memory-mapped file. -OPENVDB_API SharedPtr getMappedFilePtr(std::ios_base&); -/// @brief Associate the given stream with (a shared pointer to) a memory-mapped file. -/// @note The shared pointer object (not just the io::MappedFile object to which it points) -/// must remain valid until the file is closed. -OPENVDB_API void setMappedFilePtr(std::ios_base&, SharedPtr&); -#endif // OPENVDB_USE_DELAYED_LOADING - /// @brief Return a shared pointer to an object that stores metadata (file format, /// compression scheme, etc.) for use when reading from or writing to the given stream. OPENVDB_API SharedPtr getStreamMetadataPtr(std::ios_base&); diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index 9fba9c4c53..46d0b02318 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -1663,13 +1663,6 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu return; } -#ifdef OPENVDB_USE_DELAYED_LOADING - // If this array is being read from a memory-mapped file, delay loading of its data - // until the data is actually accessed. - io::MappedFile::Ptr mappedFile = io::getMappedFilePtr(is.getInputStream()); - const bool delayLoad = (mappedFile.get() != nullptr); -#endif - if (is.sizeOnly()) { size_t compressedBytes(mCompressedBytes); @@ -1686,22 +1679,12 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu this->deallocate(); -#ifdef OPENVDB_USE_DELAYED_LOADING - this->setOutOfCore(delayLoad); - is.read(mPageHandle, std::streamsize(mPageHandle->size()), delayLoad); -#else + this->setOutOfCore(false); is.read(mPageHandle, std::streamsize(mPageHandle->size()), false); -#endif // OPENVDB_USE_DELAYED_LOADING - -#ifdef OPENVDB_USE_DELAYED_LOADING - if (!delayLoad) { -#endif - std::unique_ptr buffer = mPageHandle->read(); - mData.reset(reinterpret_cast(buffer.release())); - mPageHandle.reset(); -#ifdef OPENVDB_USE_DELAYED_LOADING - } -#endif + + std::unique_ptr buffer = mPageHandle->read(); + mData.reset(reinterpret_cast(buffer.release())); + mPageHandle.reset(); // clear page state diff --git a/openvdb/openvdb/points/StreamCompression.cc b/openvdb/openvdb/points/StreamCompression.cc index 3f24b1b0e6..fde60d9f93 100644 --- a/openvdb/openvdb/points/StreamCompression.cc +++ b/openvdb/openvdb/points/StreamCompression.cc @@ -297,10 +297,6 @@ Page::uncompressedBytes() const const char* Page::buffer(const int index) const { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) this->load(); -#endif - return mData.get() + index; } @@ -336,51 +332,23 @@ Page::readBuffers(std::istream&is, bool delayed) bool isCompressed = mInfo->compressedBytes > 0; -#ifdef OPENVDB_USE_DELAYED_LOADING - io::MappedFile::Ptr mappedFile = io::getMappedFilePtr(is); - - if (delayed && mappedFile) { - SharedPtr meta = io::getStreamMetadataPtr(is); - OPENVDB_ASSERT(meta); - - std::streamoff filepos = is.tellg(); + std::unique_ptr buffer(new char[ + (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)]); + is.read(buffer.get(), (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)); - // seek over the page - is.seekg((isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes), - std::ios_base::cur); - - mInfo->mappedFile = mappedFile; - mInfo->meta = meta; - mInfo->filepos = filepos; - - OPENVDB_ASSERT(mInfo->mappedFile); + if (mInfo->compressedBytes > 0) { + this->decompress(buffer); + } else { + this->copy(buffer, -static_cast(mInfo->compressedBytes)); } - else { -#endif - std::unique_ptr buffer(new char[ - (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)]); - is.read(buffer.get(), (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)); - - if (mInfo->compressedBytes > 0) { - this->decompress(buffer); - } else { - this->copy(buffer, -static_cast(mInfo->compressedBytes)); - } - mInfo.reset(); -#ifdef OPENVDB_USE_DELAYED_LOADING - } -#endif + mInfo.reset(); } bool Page::isOutOfCore() const { -#ifdef OPENVDB_USE_DELAYED_LOADING - return bool(mInfo); -#else return false; -#endif } @@ -409,41 +377,6 @@ Page::decompress(const std::unique_ptr& temp) void Page::doLoad() const { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (!this->isOutOfCore()) return; - - Page* self = const_cast(this); - - // This lock will be contended at most once, after which this buffer - // will no longer be out-of-core. - tbb::spin_mutex::scoped_lock lock(self->mMutex); - if (!this->isOutOfCore()) return; - - OPENVDB_ASSERT(self->mInfo); - - int compressedBytes = static_cast(self->mInfo->compressedBytes); - bool compressed = compressedBytes > 0; - if (!compressed) compressedBytes = -compressedBytes; - - OPENVDB_ASSERT(compressedBytes); - - std::unique_ptr temp(new char[compressedBytes]); - - OPENVDB_ASSERT(self->mInfo->mappedFile); - SharedPtr buf = self->mInfo->mappedFile->createBuffer(); - OPENVDB_ASSERT(buf); - - std::istream is(buf.get()); - io::setStreamMetadataPtr(is, self->mInfo->meta, /*transfer=*/true); - is.seekg(self->mInfo->filepos); - - is.read(temp.get(), compressedBytes); - - if (compressed) self->decompress(temp); - else self->copy(temp, compressedBytes); - - self->mInfo.reset(); -#endif } diff --git a/openvdb/openvdb/points/StreamCompression.h b/openvdb/openvdb/points/StreamCompression.h index c7dcd4a8e9..d63e9c1116 100644 --- a/openvdb/openvdb/points/StreamCompression.h +++ b/openvdb/openvdb/points/StreamCompression.h @@ -114,9 +114,6 @@ class OPENVDB_API Page private: struct Info { -#ifdef OPENVDB_USE_DELAYED_LOADING - io::MappedFile::Ptr mappedFile; -#endif SharedPtr meta; std::streamoff filepos; long compressedBytes; diff --git a/openvdb/openvdb/tree/LeafBuffer.h b/openvdb/openvdb/tree/LeafBuffer.h index a684db8da6..671781a129 100644 --- a/openvdb/openvdb/tree/LeafBuffer.h +++ b/openvdb/openvdb/tree/LeafBuffer.h @@ -36,23 +36,9 @@ class LeafBuffer using NodeMaskType = util::NodeMask; static const Index SIZE = 1 << 3 * Log2Dim; -#ifdef OPENVDB_USE_DELAYED_LOADING - struct FileInfo - { - FileInfo(): bufpos(0) , maskpos(0) {} - std::streamoff bufpos; - std::streamoff maskpos; - io::MappedFile::Ptr mapping; - SharedPtr meta; - }; -#endif - /// Default constructor inline LeafBuffer(): mData(new ValueType[SIZE]) { -#ifdef OPENVDB_USE_DELAYED_LOADING - mOutOfCore = 0; -#endif } /// Construct a buffer populated with the specified value. explicit inline LeafBuffer(const ValueType&); @@ -61,9 +47,6 @@ class LeafBuffer /// Construct a buffer but don't allocate memory for the full array of values. LeafBuffer(PartialCreate, const ValueType&): mData(nullptr) { -#ifdef OPENVDB_USE_DELAYED_LOADING - mOutOfCore = 0; -#endif } /// Destructor inline ~LeafBuffer(); @@ -71,23 +54,13 @@ class LeafBuffer /// Return @c true if this buffer's values have not yet been read from disk. bool isOutOfCore() const { -#ifdef OPENVDB_USE_DELAYED_LOADING - return bool(mOutOfCore); -#else return false; -#endif } /// Return @c true if memory for this buffer has not yet been allocated. bool empty() const { return !mData || this->isOutOfCore(); } /// Allocate memory for this buffer if it has not already been allocated. bool allocate() { if (mData == nullptr) mData = new ValueType[SIZE]; return true; } -#ifdef OPENVDB_USE_DELAYED_LOADING - /// Enable out-of-core in the LeafBuffer. - void enableOutOfCore(SharedPtr& meta, const std::streamoff& bufpos, - io::MappedFile::Ptr& mappedFile, const std::streamoff& maskpos); -#endif - /// Populate this buffer with a constant value. inline void fill(const ValueType&); @@ -142,31 +115,18 @@ class LeafBuffer inline void setOutOfCore(bool b) { (void) b; -#ifdef OPENVDB_USE_DELAYED_LOADING - mOutOfCore = b; -#endif } // To facilitate inlining in the common case in which the buffer is in-core, // the loading logic is split into a separate function, doLoad(). inline void loadValues() const { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) this->doLoad(); -#endif } inline void doLoad() const; inline bool detachFromFile(); using FlagsType = std::atomic; -#ifdef OPENVDB_USE_DELAYED_LOADING - union { - ValueType* mData; - FileInfo* mFileInfo; - }; -#else ValueType* mData; -#endif FlagsType mOutOfCore; // interpreted as bool; extra bits reserved for future use tbb::spin_mutex mMutex; // 1 byte //int8_t mReserved[3]; // padding for alignment @@ -185,9 +145,6 @@ inline LeafBuffer::LeafBuffer(const ValueType& val) : mData(new ValueType[SIZE]) { -#ifdef OPENVDB_USE_DELAYED_LOADING - mOutOfCore = 0; -#endif this->fill(val); } @@ -196,15 +153,7 @@ template inline LeafBuffer::~LeafBuffer() { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) { - this->detachFromFile(); - } else { - this->deallocate(); - } -#else this->deallocate(); -#endif } @@ -212,25 +161,14 @@ template inline LeafBuffer::LeafBuffer(const LeafBuffer& other) : mData(nullptr) -#ifdef OPENVDB_USE_DELAYED_LOADING - , mOutOfCore(other.mOutOfCore.load()) -#endif { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (other.isOutOfCore()) { - mFileInfo = new FileInfo(*other.mFileInfo); - } else { -#endif - if (other.mData != nullptr) { - this->allocate(); - ValueType* target = mData; - const ValueType* source = other.mData; - Index n = SIZE; - while (n--) *target++ = *source++; - } -#ifdef OPENVDB_USE_DELAYED_LOADING + if (other.mData != nullptr) { + this->allocate(); + ValueType* target = mData; + const ValueType* source = other.mData; + Index n = SIZE; + while (n--) *target++ = *source++; } -#endif } @@ -249,47 +187,17 @@ inline LeafBuffer& LeafBuffer::operator=(const LeafBuffer& other) { if (&other != this) { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) { - this->detachFromFile(); - } else { - if (other.isOutOfCore()) this->deallocate(); - } - if (other.isOutOfCore()) { - mOutOfCore.store(other.mOutOfCore.load(std::memory_order_acquire), - std::memory_order_release); - mFileInfo = new FileInfo(*other.mFileInfo); - } else { -#endif - if (other.mData != nullptr) { - this->allocate(); - ValueType* target = mData; - const ValueType* source = other.mData; - Index n = SIZE; - while (n--) *target++ = *source++; - } -#ifdef OPENVDB_USE_DELAYED_LOADING + if (other.mData != nullptr) { + this->allocate(); + ValueType* target = mData; + const ValueType* source = other.mData; + Index n = SIZE; + while (n--) *target++ = *source++; } -#endif } return *this; } -#ifdef OPENVDB_USE_DELAYED_LOADING -template -void LeafBuffer::enableOutOfCore( - SharedPtr& meta, const std::streamoff& bufpos, - io::MappedFile::Ptr& mappedFile, const std::streamoff& maskpos) -{ - this->setOutOfCore(true); - mFileInfo = new FileInfo; - mFileInfo->meta = meta; - mFileInfo->bufpos = bufpos; - mFileInfo->mapping = mappedFile; - mFileInfo->maskpos = maskpos; -} -#endif - template inline void LeafBuffer::fill(const ValueType& val) @@ -323,15 +231,6 @@ inline void LeafBuffer::swap(LeafBuffer& other) { std::swap(mData, other.mData); -#ifdef OPENVDB_USE_DELAYED_LOADING - // Two atomics can't be swapped because it would require hardware support: - // https://en.wikipedia.org/wiki/Double_compare-and-swap - // Note that there's a window in which other.mOutOfCore could be written - // between our load from it and our store to it. - auto tmp = other.mOutOfCore.load(std::memory_order_acquire); - tmp = mOutOfCore.exchange(std::move(tmp)); - other.mOutOfCore.store(std::move(tmp), std::memory_order_release); -#endif } @@ -340,14 +239,7 @@ inline Index LeafBuffer::memUsage() const { size_t n = sizeof(*this); -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) n += sizeof(FileInfo); - else { -#endif - if (mData) n += SIZE * sizeof(ValueType); -#ifdef OPENVDB_USE_DELAYED_LOADING - } -#endif + if (mData) n += SIZE * sizeof(ValueType); return static_cast(n); } @@ -369,10 +261,6 @@ LeafBuffer::data() const this->loadValues(); if (mData == nullptr) { LeafBuffer* self = const_cast(this); -#ifdef OPENVDB_USE_DELAYED_LOADING - // This lock will be contended at most once. - tbb::spin_mutex::scoped_lock lock(self->mMutex); -#endif if (mData == nullptr) self->mData = new ValueType[SIZE]; } return mData; @@ -384,10 +272,6 @@ LeafBuffer::data() { this->loadValues(); if (mData == nullptr) { -#ifdef OPENVDB_USE_DELAYED_LOADING - // This lock will be contended at most once. - tbb::spin_mutex::scoped_lock lock(mMutex); -#endif if (mData == nullptr) mData = new ValueType[SIZE]; } return mData; @@ -413,9 +297,6 @@ LeafBuffer::deallocate() { if (mData != nullptr) { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) return false; -#endif delete[] mData; mData = nullptr; return true; @@ -428,39 +309,6 @@ template inline void LeafBuffer::doLoad() const { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (!this->isOutOfCore()) return; - - LeafBuffer* self = const_cast*>(this); - - // This lock will be contended at most once, after which this buffer - // will no longer be out-of-core. - tbb::spin_mutex::scoped_lock lock(self->mMutex); - if (!this->isOutOfCore()) return; - - std::unique_ptr info(self->mFileInfo); - OPENVDB_ASSERT(info.get() != nullptr); - OPENVDB_ASSERT(info->mapping.get() != nullptr); - OPENVDB_ASSERT(info->meta.get() != nullptr); - - /// @todo For now, we have to clear the mData pointer in order for allocate() to take effect. - self->mData = nullptr; - self->allocate(); - - SharedPtr buf = info->mapping->createBuffer(); - std::istream is(buf.get()); - - io::setStreamMetadataPtr(is, info->meta, /*transfer=*/true); - - NodeMaskType mask; - is.seekg(info->maskpos); - mask.load(is); - - is.seekg(info->bufpos); - io::readCompressedValues(is, self->mData, SIZE, mask, io::getHalfFloat(is)); - - self->setOutOfCore(false); -#endif } @@ -468,14 +316,6 @@ template inline bool LeafBuffer::detachFromFile() { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) { - delete mFileInfo; - mFileInfo = nullptr; - this->setOutOfCore(false); - return true; - } -#endif return false; } diff --git a/openvdb/openvdb/tree/LeafNode.h b/openvdb/openvdb/tree/LeafNode.h index 9e7fbbf7c4..6894d17391 100644 --- a/openvdb/openvdb/tree/LeafNode.h +++ b/openvdb/openvdb/tree/LeafNode.h @@ -1375,10 +1375,6 @@ LeafNode::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo SharedPtr meta = io::getStreamMetadataPtr(is); const bool seekable = meta && meta->seekable(); -#ifdef OPENVDB_USE_DELAYED_LOADING - std::streamoff maskpos = is.tellg(); -#endif - if (seekable) { // Seek over the value mask. mValueMask.seek(is); @@ -1396,35 +1392,16 @@ LeafNode::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo mValueMask.setOff(); mBuffer.setOutOfCore(false); } else { -#ifdef OPENVDB_USE_DELAYED_LOADING - // If this node lies completely inside the clipping region and it is being read - // from a memory-mapped file, delay loading of its buffer until the buffer - // is actually accessed. (If this node requires clipping, its buffer - // must be accessed and therefore must be loaded.) - io::MappedFile::Ptr mappedFile = io::getMappedFilePtr(is); - const bool delayLoad = ((mappedFile.get() != nullptr) && clipBBox.isInside(nodeBBox)); - - if (delayLoad) { - // Save the offset to the value mask (maskpos), because the in-memory copy - // might change before the value buffer gets read. - mBuffer.enableOutOfCore(meta, is.tellg(), mappedFile, maskpos); - // Skip over voxel values. - skipCompressedValues(seekable, is, fromHalf); - } else { -#endif - mBuffer.allocate(); - io::readCompressedValues(is, mBuffer.mData, SIZE, mValueMask, fromHalf); - mBuffer.setOutOfCore(false); - - // Get this tree's background value. - T background = zeroVal(); - if (const void* bgPtr = io::getGridBackgroundValuePtr(is)) { - background = *static_cast(bgPtr); - } - this->clip(clipBBox, background); -#ifdef OPENVDB_USE_DELAYED_LOADING + mBuffer.allocate(); + io::readCompressedValues(is, mBuffer.mData, SIZE, mValueMask, fromHalf); + mBuffer.setOutOfCore(false); + + // Get this tree's background value. + T background = zeroVal(); + if (const void* bgPtr = io::getGridBackgroundValuePtr(is)) { + background = *static_cast(bgPtr); } -#endif + this->clip(clipBBox, background); } if (numBuffers > 1) { diff --git a/openvdb/openvdb/unittest/TestAttributeArray.cc b/openvdb/openvdb/unittest/TestAttributeArray.cc index 0b949c3ece..dc360e6fb1 100644 --- a/openvdb/openvdb/unittest/TestAttributeArray.cc +++ b/openvdb/openvdb/unittest/TestAttributeArray.cc @@ -12,31 +12,6 @@ #include -#ifdef OPENVDB_USE_DELAYED_LOADING -#ifdef __clang__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-macros" -#endif -// Boost.Interprocess uses a header-only portion of Boost.DateTime -#define BOOST_DATE_TIME_NO_LIB -#ifdef __clang__ -#pragma GCC diagnostic pop -#endif -#include -#include - -#ifdef _WIN32 -#include // open_existing_file(), close_file() -// boost::interprocess::detail was renamed to boost::interprocess::ipcdetail in Boost 1.48. -// Ensure that both namespaces exist. -namespace boost { namespace interprocess { namespace detail {} namespace ipcdetail {} } } -#include -#else -#include // for struct stat -#include // for stat() -#endif -#endif // OPENVDB_USE_DELAYED_LOADING - #include #include @@ -113,7 +88,6 @@ class TestAttributeArray: public ::testing::Test void testRegistry(); void testAccessorEval(); - void testDelayedLoad(); }; // class TestAttributeArray @@ -1400,801 +1374,6 @@ TEST_F(TestAttributeArray, testStrided) } } -#ifdef OPENVDB_USE_DELAYED_LOADING -void -TestAttributeArray::testDelayedLoad() -{ - using AttributeArrayI = TypedAttributeArray; - using AttributeArrayF = TypedAttributeArray; - - AttributeArrayI::registerType(); - AttributeArrayF::registerType(); - - SharedPtr mappedFile; - - io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata); - - std::string tempDir; - if (const char* dir = std::getenv("TMPDIR")) tempDir = dir; -#ifdef _WIN32 - if (tempDir.empty()) { - char tempDirBuffer[MAX_PATH+1]; - int tempDirLen = GetTempPath(MAX_PATH+1, tempDirBuffer); - EXPECT_TRUE(tempDirLen > 0 && tempDirLen <= MAX_PATH); - tempDir = tempDirBuffer; - } -#else - if (tempDir.empty()) tempDir = P_tmpdir; -#endif - - { // IO - const Index count = 50; - AttributeArrayI attrA(count); - - for (unsigned i = 0; i < unsigned(count); ++i) { - attrA.set(i, int(i)); - } - - AttributeArrayF attrA2(count); - - std::string filename; - - // write out attribute array to a temp file - { - filename = tempDir + "/openvdb_delayed1"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - attrA.writeMetadata(fileout, false, /*paged=*/true); - compression::PagedOutputStream outputStreamSize(fileout); - outputStreamSize.setSizeOnly(true); - attrA.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout); - outputStream.setSizeOnly(false); - attrA.writePagedBuffers(outputStream, false); - outputStream.flush(); - - attrA2.writeMetadata(fileout, false, /*paged=*/true); - compression::PagedOutputStream outputStreamSize2(fileout); - outputStreamSize2.setSizeOnly(true); - attrA2.writePagedBuffers(outputStreamSize2, false); - outputStreamSize2.flush(); - compression::PagedOutputStream outputStream2(fileout); - outputStream2.setSizeOnly(false); - attrA2.writePagedBuffers(outputStream2, false); - outputStream2.flush(); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check manual loading of data - { - AttributeArrayI attrB; - AttributeArrayF attrB2; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(matchingNamePairs(attrA.type(), attrB.type())); - EXPECT_EQ(attrA.size(), attrB.size()); - EXPECT_EQ(attrA.isUniform(), attrB.isUniform()); - EXPECT_EQ(attrA.isTransient(), attrB.isTransient()); - EXPECT_EQ(attrA.isHidden(), attrB.isHidden()); - - AttributeArrayI attrBcopy(attrB); - AttributeArrayI attrBequal = attrB; - - EXPECT_TRUE(attrB.isOutOfCore()); - EXPECT_TRUE(attrBcopy.isOutOfCore()); - EXPECT_TRUE(attrBequal.isOutOfCore()); - - EXPECT_TRUE(!static_cast(attrB).isDataLoaded()); - EXPECT_TRUE(!static_cast(attrBcopy).isDataLoaded()); - EXPECT_TRUE(!static_cast(attrBequal).isDataLoaded()); - - attrB.loadData(); - attrBcopy.loadData(); - attrBequal.loadData(); - - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrBcopy.isOutOfCore()); - EXPECT_TRUE(!attrBequal.isOutOfCore()); - - EXPECT_TRUE(static_cast(attrB).isDataLoaded()); - EXPECT_TRUE(static_cast(attrBcopy).isDataLoaded()); - EXPECT_TRUE(static_cast(attrBequal).isDataLoaded()); - - EXPECT_EQ(attrA.memUsage(), attrB.memUsage()); - EXPECT_EQ(attrA.memUsage(), attrBcopy.memUsage()); - EXPECT_EQ(attrA.memUsage(), attrBequal.memUsage()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - EXPECT_EQ(attrA.get(i), attrBcopy.get(i)); - EXPECT_EQ(attrA.get(i), attrBequal.get(i)); - } - - attrB2.readMetadata(filein); - compression::PagedInputStream inputStream2(filein); - inputStream2.setSizeOnly(true); - attrB2.readPagedBuffers(inputStream2); - inputStream2.setSizeOnly(false); - attrB2.readPagedBuffers(inputStream2); - - EXPECT_TRUE(matchingNamePairs(attrA2.type(), attrB2.type())); - EXPECT_EQ(attrA2.size(), attrB2.size()); - EXPECT_EQ(attrA2.isUniform(), attrB2.isUniform()); - EXPECT_EQ(attrA2.isTransient(), attrB2.isTransient()); - EXPECT_EQ(attrA2.isHidden(), attrB2.isHidden()); - - AttributeArrayF attrB2copy(attrB2); - AttributeArrayF attrB2equal = attrB2; - - EXPECT_TRUE(attrB2.isOutOfCore()); - EXPECT_TRUE(attrB2copy.isOutOfCore()); - EXPECT_TRUE(attrB2equal.isOutOfCore()); - attrB2.loadData(); - attrB2copy.loadData(); - attrB2equal.loadData(); - - EXPECT_TRUE(!attrB2.isOutOfCore()); - EXPECT_TRUE(!attrB2copy.isOutOfCore()); - EXPECT_TRUE(!attrB2equal.isOutOfCore()); - - EXPECT_EQ(attrA2.memUsage(), attrB2.memUsage()); - EXPECT_EQ(attrA2.memUsage(), attrB2copy.memUsage()); - EXPECT_EQ(attrA2.memUsage(), attrB2equal.memUsage()); - - EXPECT_EQ(attrA2.get(0), attrB2.get(0)); - EXPECT_EQ(attrA2.get(0), attrB2copy.get(0)); - EXPECT_EQ(attrA2.get(0), attrB2equal.get(0)); - } - - // read in using delayed load and check fill() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - EXPECT_TRUE(!attrB.isUniform()); - - attrB.fill(5); - - EXPECT_TRUE(!attrB.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(5, attrB.get(i)); - } - } - - // read in using delayed load and check streaming (write handle) - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - EXPECT_TRUE(!attrB.isUniform()); - - attrB.setStreaming(true); - - { - AttributeWriteHandle handle(attrB); - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrB.isUniform()); - } - - EXPECT_TRUE(!attrB.isUniform()); - } - - // read in using delayed load and check streaming (read handle) - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - EXPECT_TRUE(!attrB.isUniform()); - - attrB.setStreaming(true); - - { - AttributeHandle handle(attrB); - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrB.isUniform()); - } - - EXPECT_TRUE(attrB.isUniform()); - } - - // read in using delayed load and check implicit load through get() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - attrB.get(0); - - EXPECT_TRUE(!attrB.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - } - } - - // read in using delayed load and check implicit load through compress() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - } - - // read in using delayed load and check copy and assignment constructors - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - AttributeArrayI attrC(attrB); - AttributeArrayI attrD = attrB; - - EXPECT_TRUE(attrB.isOutOfCore()); - EXPECT_TRUE(attrC.isOutOfCore()); - EXPECT_TRUE(attrD.isOutOfCore()); - - attrB.loadData(); - attrC.loadData(); - attrD.loadData(); - - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrC.isOutOfCore()); - EXPECT_TRUE(!attrD.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - EXPECT_EQ(attrA.get(i), attrC.get(i)); - EXPECT_EQ(attrA.get(i), attrD.get(i)); - } - } - - // read in using delayed load and check implicit load through AttributeHandle - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - AttributeHandle handle(attrB); - - EXPECT_TRUE(!attrB.isOutOfCore()); - } - - // read in using delayed load and check detaching of file (using collapse()) - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - EXPECT_TRUE(!attrB.isUniform()); - - attrB.collapse(); - - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(attrB.isUniform()); - - EXPECT_EQ(0, attrB.get(0)); - } - - // read in and write out using delayed load to check writing out-of-core attributes - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - std::string filename2 = tempDir + "/openvdb_delayed5"; - std::ofstream fileout2(filename2.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout2, streamMetadata); - io::setDataCompression(fileout2, io::COMPRESS_BLOSC); - - attrB.writeMetadata(fileout2, false, /*paged=*/true); - compression::PagedOutputStream outputStreamSize(fileout2); - outputStreamSize.setSizeOnly(true); - attrB.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout2); - outputStream.setSizeOnly(false); - attrB.writePagedBuffers(outputStream, false); - outputStream.flush(); - - fileout2.close(); - - AttributeArrayI attrB2; - - std::ifstream filein2(filename2.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein2, streamMetadata); - io::setMappedFilePtr(filein2, mappedFile); - - attrB2.readMetadata(filein2); - compression::PagedInputStream inputStream2(filein2); - inputStream2.setSizeOnly(true); - attrB2.readPagedBuffers(inputStream2); - inputStream2.setSizeOnly(false); - attrB2.readPagedBuffers(inputStream2); - - EXPECT_TRUE(attrB2.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrB.get(i), attrB2.get(i)); - } - - filein2.close(); - } - - // Clean up temp files. - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - - AttributeArrayI attrUniform(count); - - // write out uniform attribute array to a temp file - { - filename = tempDir + "/openvdb_delayed2"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - attrUniform.writeMetadata(fileout, false, /*paged=*/true); - - compression::PagedOutputStream outputStreamSize(fileout); - outputStreamSize.setSizeOnly(true); - attrUniform.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout); - outputStream.setSizeOnly(false); - attrUniform.writePagedBuffers(outputStream, false); - outputStream.flush(); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check fill() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isUniform()); - - attrB.fill(5); - - EXPECT_TRUE(attrB.isUniform()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(5, attrB.get(i)); - } - } - - AttributeArrayI attrStrided(count, /*stride=*/3); - - EXPECT_EQ(Index(3), attrStrided.stride()); - - // Clean up temp files. - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - - // write out strided attribute array to a temp file - { - filename = tempDir + "/openvdb_delayed3"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - attrStrided.writeMetadata(fileout, false, /*paged=*/true); - - compression::PagedOutputStream outputStreamSize(fileout); - outputStreamSize.setSizeOnly(true); - attrStrided.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout); - outputStream.setSizeOnly(false); - attrStrided.writePagedBuffers(outputStream, false); - outputStream.flush(); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check fill() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_EQ(Index(3), attrB.stride()); - } - - // Clean up temp files. - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - - // write out compressed attribute array to a temp file - { - filename = tempDir + "/openvdb_delayed4"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - attrA.writeMetadata(fileout, false, /*paged=*/true); - - compression::PagedOutputStream outputStreamSize(fileout); - outputStreamSize.setSizeOnly(true); - attrA.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout); - outputStream.setSizeOnly(false); - attrA.writePagedBuffers(outputStream, false); - outputStream.flush(); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check manual loading of data - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - attrB.loadData(); - EXPECT_TRUE(!attrB.isOutOfCore()); - - EXPECT_EQ(attrA.memUsage(), attrB.memUsage()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - } - } - - // read in using delayed load and check partial read state - { - std::unique_ptr attrB(new AttributeArrayI); - - EXPECT_TRUE(!(attrB->flags() & AttributeArray::PARTIALREAD)); - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB->readMetadata(filein); - - // PARTIALREAD flag should now be set - EXPECT_TRUE(attrB->flags() & AttributeArray::PARTIALREAD); - - // copy-construct and assign AttributeArray - AttributeArrayI attrC(*attrB); - EXPECT_TRUE(attrC.flags() & AttributeArray::PARTIALREAD); - AttributeArrayI attrD = *attrB; - EXPECT_TRUE(attrD.flags() & AttributeArray::PARTIALREAD); - - // verify deleting attrB is safe - attrB.reset(); - - // verify data is not valid - EXPECT_TRUE(!attrC.validData()); - - { // attempting to write a partially-read AttributeArray throws - std::string filename = tempDir + "/openvdb_partial1"; - ScopedFile f(filename); - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - EXPECT_THROW(attrC.writeMetadata(fileout, false, /*paged=*/true), IoError); - } - - // continue loading with copy-constructed AttributeArray - - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrC.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrC.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrC.isOutOfCore()); - attrC.loadData(); - EXPECT_TRUE(!attrC.isOutOfCore()); - - // verify data is now valid - EXPECT_TRUE(attrC.validData()); - - EXPECT_EQ(attrA.memUsage(), attrC.memUsage()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrC.get(i)); - } - } - - // read in using delayed load and check implicit load through get() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - attrB.get(0); - - EXPECT_TRUE(!attrB.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - } - } - -#ifdef OPENVDB_USE_BLOSC - // read in using delayed load and check copy and assignment constructors - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - AttributeArrayI attrC(attrB); - AttributeArrayI attrD = attrB; - - EXPECT_TRUE(attrB.isOutOfCore()); - EXPECT_TRUE(attrC.isOutOfCore()); - EXPECT_TRUE(attrD.isOutOfCore()); - - attrB.loadData(); - attrC.loadData(); - attrD.loadData(); - - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrC.isOutOfCore()); - EXPECT_TRUE(!attrD.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - EXPECT_EQ(attrA.get(i), attrC.get(i)); - EXPECT_EQ(attrA.get(i), attrD.get(i)); - } - } - - // read in using delayed load and check implicit load through AttributeHandle - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - AttributeHandle handle(attrB); - - EXPECT_TRUE(!attrB.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), handle.get(i)); - } - } -#endif - - // Clean up temp files. - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - - // write out invalid serialization flags as metadata to a temp file - { - filename = tempDir + "/openvdb_delayed5"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - // write out unknown serialization flags to check forwards-compatibility - - Index64 bytes(0); - uint8_t flags(0); - uint8_t serializationFlags(Int16(0x10)); - Index size(0); - - fileout.write(reinterpret_cast(&bytes), sizeof(Index64)); - fileout.write(reinterpret_cast(&flags), sizeof(uint8_t)); - fileout.write(reinterpret_cast(&serializationFlags), sizeof(uint8_t)); - fileout.write(reinterpret_cast(&size), sizeof(Index)); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check metadata fail due to serialization flags - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - EXPECT_THROW(attrB.readMetadata(filein), openvdb::IoError); - } - - // cleanup temp files - - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - } -} -TEST_F(TestAttributeArray, testDelayedLoad) { testDelayedLoad(); } -#endif - TEST_F(TestAttributeArray, testDefaultValue) { diff --git a/openvdb/openvdb/unittest/TestCount.cc b/openvdb/openvdb/unittest/TestCount.cc index 3ce3f5227b..0509b71359 100644 --- a/openvdb/openvdb/unittest/TestCount.cc +++ b/openvdb/openvdb/unittest/TestCount.cc @@ -8,7 +8,6 @@ #include // tools::createLevelSetSphere #include // tools::sdfToFogVolume #include -#include class TestCount: public ::testing::Test @@ -238,49 +237,6 @@ TEST_F(TestCount, testMemUsage) EXPECT_EQ(expectedMaxMem, inCoreMemUsage); EXPECT_EQ(expectedMaxMem, memUsageIfLoaded); - -#ifdef OPENVDB_USE_DELAYED_LOADING - // Write out the grid and read it in with delay-loading. Check the - // expected memory usage values.] - - openvdb::initialize(); - - std::string filename; - - // write out grid to a temp file - { - io::TempFile file; - filename = file.filename(); - io::File fileOut(filename); - fileOut.write({grid}); - } - - io::File fileIn(filename); - fileIn.open(true); // delay-load - auto grids = fileIn.getGrids(); - fileIn.close(); - - grid = GridBase::grid((*grids)[0]); - EXPECT_TRUE(grid); - - inCoreMemUsage = tools::memUsage(grid->tree()); - memUsageIfLoaded = tools::memUsageIfLoaded(grid->tree()); - - EXPECT_EQ(expectedMaxMem, memUsageIfLoaded); - EXPECT_TRUE(inCoreMemUsage < expectedMaxMem); - - // in core memory should be the max memory without the leaf buffers but - // with the FileInfo - - const Index64 leafBuffers = sizeof(FloatGrid::ValueType) * FloatTree::LeafNodeType::SIZE; - const Index64 fileInfo = sizeof(FloatTree::LeafNodeType::Buffer::FileInfo); - const Index64 expectedInCoreMemUsage = expectedMaxMem + (leafCount * (-leafBuffers + fileInfo)); - EXPECT_EQ(expectedInCoreMemUsage, inCoreMemUsage); - - std::remove(filename.c_str()); - - openvdb::uninitialize(); -#endif } diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index dd344cb515..0d9c9a3e8a 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -622,10 +622,6 @@ TEST_F(TestFile, testWriteInstancedGrids) EXPECT_TRUE(grid.get() != nullptr); density = gridPtrCast(grid)->treePtr(); EXPECT_TRUE(density.get() != nullptr); -#ifdef OPENVDB_USE_DELAYED_LOADING - EXPECT_TRUE(density->unallocatedLeafCount() > 0); - EXPECT_EQ(density->leafCount(), density->unallocatedLeafCount()); -#endif // OPENVDB_USE_DELAYED_LOADING grid = findGridByName(*grids, "density_copy"); EXPECT_TRUE(grid.get() != nullptr); EXPECT_TRUE(gridPtrCast(grid)->treePtr().get() != nullptr); diff --git a/openvdb/openvdb/unittest/TestPointConversion.cc b/openvdb/openvdb/unittest/TestPointConversion.cc index d6420019cf..8d4132b92d 100644 --- a/openvdb/openvdb/unittest/TestPointConversion.cc +++ b/openvdb/openvdb/unittest/TestPointConversion.cc @@ -1,7 +1,6 @@ // Copyright Contributors to the OpenVDB Project // SPDX-License-Identifier: Apache-2.0 -#include #include #include #include diff --git a/openvdb/openvdb/unittest/TestPointCount.cc b/openvdb/openvdb/unittest/TestPointCount.cc index 58ea887907..f02661456c 100644 --- a/openvdb/openvdb/unittest/TestPointCount.cc +++ b/openvdb/openvdb/unittest/TestPointCount.cc @@ -3,7 +3,6 @@ #include #include -#include #include #include @@ -315,19 +314,6 @@ TEST_F(TestPointCount, testGroup) GroupFilter groupFilter("test", attributeSet); bool inCoreOnly; -#ifdef OPENVDB_USE_DELAYED_LOADING - inCoreOnly = true; - - EXPECT_EQ(pointCount(inputTree, NullFilter(), inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, ActiveFilter(), inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, InactiveFilter(), inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, groupFilter, inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, BinaryFilter( - groupFilter, ActiveFilter()), inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, BinaryFilter( - groupFilter, InactiveFilter()), inCoreOnly), Index64(0)); -#endif - inCoreOnly = false; EXPECT_EQ(pointCount(inputTree, NullFilter(), inCoreOnly), Index64(4)); @@ -529,51 +515,6 @@ TEST_F(TestPointCount, testOffsets) fileOut.write(grids); } -#ifdef OPENVDB_USE_DELAYED_LOADING - // test point offsets for a delay-loaded grid - { - io::File fileIn(filename); - fileIn.open(); - - GridPtrVecPtr grids = fileIn.getGrids(); - - fileIn.close(); - - EXPECT_EQ(grids->size(), size_t(1)); - - PointDataGrid::Ptr inputGrid = GridBase::grid((*grids)[0]); - - EXPECT_TRUE(inputGrid); - - PointDataTree& inputTree = inputGrid->tree(); - - std::vector offsets; - std::vector includeGroups; - std::vector excludeGroups; - - MultiGroupFilter filter(includeGroups, excludeGroups, inputTree.cbeginLeaf()->attributeSet()); - Index64 total = pointOffsets(offsets, inputTree, filter, /*inCoreOnly=*/true); - - EXPECT_EQ(offsets.size(), size_t(4)); - EXPECT_EQ(offsets[0], Index64(0)); - EXPECT_EQ(offsets[1], Index64(0)); - EXPECT_EQ(offsets[2], Index64(0)); - EXPECT_EQ(offsets[3], Index64(0)); - EXPECT_EQ(total, Index64(0)); - - offsets.clear(); - - total = pointOffsets(offsets, inputTree, filter, /*inCoreOnly=*/false); - - EXPECT_EQ(offsets.size(), size_t(4)); - EXPECT_EQ(offsets[0], Index64(1)); - EXPECT_EQ(offsets[1], Index64(3)); - EXPECT_EQ(offsets[2], Index64(4)); - EXPECT_EQ(offsets[3], Index64(5)); - EXPECT_EQ(total, Index64(5)); - } -#endif - std::remove(filename.c_str()); } diff --git a/openvdb/openvdb/unittest/TestPointDataLeaf.cc b/openvdb/openvdb/unittest/TestPointDataLeaf.cc index bd96959c45..651b3174d9 100644 --- a/openvdb/openvdb/unittest/TestPointDataLeaf.cc +++ b/openvdb/openvdb/unittest/TestPointDataLeaf.cc @@ -1165,111 +1165,6 @@ TEST_F(TestPointDataLeaf, testIO) EXPECT_TRUE(leaf == *leafFromDisk); } - -#ifdef OPENVDB_USE_DELAYED_LOADING - { // read grids from file and pre-fetch - PointDataGrid::Ptr gridFromDisk; - - { - io::File file("leaf.vdb"); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - gridFromDisk = openvdb::gridPtrCast(baseGrid); - } - - LeafType* leafFromDisk = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); - EXPECT_TRUE(leafFromDisk); - - const AttributeVec3s& position( - AttributeVec3s::cast(leafFromDisk->constAttributeArray("P"))); - const AttributeF& density( - AttributeF::cast(leafFromDisk->constAttributeArray("density"))); - - EXPECT_TRUE(leafFromDisk->buffer().isOutOfCore()); -#ifdef OPENVDB_USE_BLOSC - EXPECT_TRUE(position.isOutOfCore()); - EXPECT_TRUE(density.isOutOfCore()); -#else - // delayed-loading is only available on attribute arrays when using Blosc - EXPECT_TRUE(!position.isOutOfCore()); - EXPECT_TRUE(!density.isOutOfCore()); -#endif - - // prefetch voxel data only - prefetch(gridFromDisk->tree(), /*position=*/false, /*attributes=*/false); - - // ensure out-of-core data is now in-core after pre-fetching - - EXPECT_TRUE(!leafFromDisk->buffer().isOutOfCore()); -#ifdef OPENVDB_USE_BLOSC - EXPECT_TRUE(position.isOutOfCore()); - EXPECT_TRUE(density.isOutOfCore()); -#else - EXPECT_TRUE(!position.isOutOfCore()); - EXPECT_TRUE(!density.isOutOfCore()); -#endif - - { // re-open - io::File file("leaf.vdb"); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - gridFromDisk = openvdb::gridPtrCast(baseGrid); - } - - leafFromDisk = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); - EXPECT_TRUE(leafFromDisk); - - const AttributeVec3s& position2( - AttributeVec3s::cast(leafFromDisk->constAttributeArray("P"))); - const AttributeF& density2( - AttributeF::cast(leafFromDisk->constAttributeArray("density"))); - - // prefetch voxel and position attribute data - prefetch(gridFromDisk->tree(), /*position=*/true, /*attribute=*/false); - - // ensure out-of-core voxel and position data is now in-core after pre-fetching - - EXPECT_TRUE(!leafFromDisk->buffer().isOutOfCore()); - EXPECT_TRUE(!position2.isOutOfCore()); -#ifdef OPENVDB_USE_BLOSC - EXPECT_TRUE(density2.isOutOfCore()); -#else - EXPECT_TRUE(!density2.isOutOfCore()); -#endif - - { // re-open - io::File file("leaf.vdb"); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - gridFromDisk = openvdb::gridPtrCast(baseGrid); - } - - leafFromDisk = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); - EXPECT_TRUE(leafFromDisk); - - const AttributeVec3s& position3( - AttributeVec3s::cast(leafFromDisk->constAttributeArray("P"))); - const AttributeF& density3( - AttributeF::cast(leafFromDisk->constAttributeArray("density"))); - - // prefetch all data - prefetch(gridFromDisk->tree()); - - // ensure out-of-core voxel and position data is now in-core after pre-fetching - - EXPECT_TRUE(!leafFromDisk->buffer().isOutOfCore()); - EXPECT_TRUE(!position3.isOutOfCore()); - EXPECT_TRUE(!density3.isOutOfCore()); - } - - remove("leaf.vdb"); -#endif // OPENVDB_USE_DELAYED_LOADING } { // test multi-buffer IO with varying attribute storage per-leaf diff --git a/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc b/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc index a7f85cd33e..572b66dd2b 100644 --- a/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc +++ b/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc @@ -2154,9 +2154,6 @@ TEST_F(TestPointRasterizeFrustum, testStreaming) auto leaf = points->tree().cbeginLeaf(); EXPECT_TRUE(leaf); -#ifdef OPENVDB_USE_DELAYED_LOADING - EXPECT_TRUE(leaf->buffer().isOutOfCore()); -#endif using Rasterizer = FrustumRasterizer; using Settings = FrustumRasterizerSettings; @@ -2327,17 +2324,10 @@ TEST_F(TestPointRasterizeFrustum, testStreaming) auto points2 = points->deepCopy(); points2->setTransform(transform); -#ifdef OPENVDB_USE_DELAYED_LOADING - // verify both grids are out-of-core - - EXPECT_TRUE(points->tree().cbeginLeaf()->buffer().isOutOfCore()); - EXPECT_TRUE(points2->tree().cbeginLeaf()->buffer().isOutOfCore()); -#endif - #ifndef ONLY_RASTER_FLOAT // memory tests - if (io::Archive::isDelayedLoadingEnabled() && io::Archive::hasBloscCompression()) { + if (io::Archive::hasBloscCompression()) { FloatGrid::Ptr density1, density2, density3; Vec3SGrid::Ptr velocity1, velocity2, velocity3; @@ -2345,122 +2335,31 @@ TEST_F(TestPointRasterizeFrustum, testStreaming) const size_t mb = 1024*1024; const size_t tinyMemory = static_cast(0.1*mb); - size_t initialMemory; - { // memory test 1 - retain caches and streaming disabled Rasterizer rasterizer(settings); rasterizer.addPoints(points, /*stream=*/false); rasterizer.addPoints(points2, /*stream=*/false); - initialMemory = rasterizer.memUsage(); + // memory usage should be around 80-100MB throughout rasterization - EXPECT_TRUE(initialMemory > size_t(4*mb) && initialMemory < size_t(16*mb)); + EXPECT_TRUE(rasterizer.memUsage() > size_t(80*mb) && + rasterizer.memUsage() < size_t(100*mb)); EXPECT_EQ(size_t(2), rasterizer.size()); velocity1 = rasterizer.rasterizeAttribute("v"); EXPECT_EQ(Index64(219780), velocity1->activeVoxelCount()); - EXPECT_TRUE(rasterizer.memUsage() > size_t(71*mb) && - rasterizer.memUsage() < size_t(91*mb)); + EXPECT_TRUE(rasterizer.memUsage() > size_t(80*mb) && + rasterizer.memUsage() < size_t(100*mb)); density1 = rasterizer.rasterizeDensity("density"); EXPECT_EQ(Index64(219780), density1->activeVoxelCount()); - // no data is discarded so expect a fairly high memory footprint - EXPECT_TRUE(rasterizer.memUsage() > size_t(80*mb) && rasterizer.memUsage() < size_t(100*mb)); } - - { // memory test 2 - retain caches and streaming enabled - - { // reopen file and deep copy while setting transform - io::File file(filename); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - points = openvdb::gridPtrCast(baseGrid); - points2 = points->deepCopy(); - points2->setTransform(transform); - } - - Rasterizer rasterizer(settings); - - rasterizer.addPoints(points, /*stream=*/true); - rasterizer.addPoints(points2, /*stream=*/true); - - EXPECT_EQ(initialMemory, rasterizer.memUsage()); - - EXPECT_EQ(size_t(2), rasterizer.size()); - - velocity2 = rasterizer.rasterizeAttribute("v"); - EXPECT_EQ(Index64(219780), velocity2->activeVoxelCount()); - - size_t postRasterMemory = rasterizer.memUsage(); - - EXPECT_TRUE(postRasterMemory > size_t(70*mb) && postRasterMemory < size_t(85*mb)); - - density2 = rasterizer.rasterizeDensity("density"); - EXPECT_EQ(Index64(219780), density2->activeVoxelCount()); - - // as data is being streamed, second attribute shouldn't change memory usage very much - - EXPECT_TRUE(rasterizer.memUsage() < (postRasterMemory + tinyMemory)); - } - - { // memory test 3 - release caches and streaming enabled - - { // reopen file and deep copy while setting transform - io::File file(filename); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - points = openvdb::gridPtrCast(baseGrid); - points2 = points->deepCopy(); - points2->setTransform(transform); - } - - auto points3 = points->deepCopy(); - auto points4 = points2->deepCopy(); - - Settings settings2(*frustum); - settings2.threshold = 0.0f; - - Mask mask2(*frustum, nullptr, BBoxd(), /*clipToFrustum=*/false); - - Rasterizer rasterizer(settings2, mask2); - - rasterizer.addPoints(points, /*stream=*/true); - rasterizer.addPoints(points2, /*stream=*/true); - - EXPECT_EQ(initialMemory, rasterizer.memUsage()); - EXPECT_EQ(size_t(2), rasterizer.size()); - - density3 = rasterizer.rasterizeDensity("density", RasterMode::ACCUMULATE, true); - EXPECT_EQ(Index64(219780), density3->activeVoxelCount()); - - // all voxel data, attribute data and caches are being discarded, - // so memory after rasterizing shouldn't change very much - - EXPECT_TRUE(rasterizer.memUsage() < (initialMemory + tinyMemory)); - - // deep-copies of delay-loaded point grids need to be used for repeat rasterization - - rasterizer.clear(); - rasterizer.addPoints(points3, /*stream=*/true); - rasterizer.addPoints(points4, /*stream=*/true); - - EXPECT_EQ(size_t(2), rasterizer.size()); - - EXPECT_TRUE(rasterizer.memUsage() < (initialMemory + tinyMemory)); - - velocity3 = rasterizer.rasterizeAttribute("v", RasterMode::ACCUMULATE, true); - EXPECT_EQ(Index64(219780), velocity3->activeVoxelCount()); - } } #endif diff --git a/openvdb/openvdb/unittest/TestStreamCompression.cc b/openvdb/openvdb/unittest/TestStreamCompression.cc index ceb8bcf96b..27af0bf40d 100644 --- a/openvdb/openvdb/unittest/TestStreamCompression.cc +++ b/openvdb/openvdb/unittest/TestStreamCompression.cc @@ -8,31 +8,6 @@ #include -#ifdef OPENVDB_USE_DELAYED_LOADING -#ifdef __clang__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-macros" -#endif -// Boost.Interprocess uses a header-only portion of Boost.DateTime -#define BOOST_DATE_TIME_NO_LIB -#ifdef __clang__ -#pragma GCC diagnostic pop -#endif -#include -#include -#include -#include - -#ifdef _WIN32 -#include // open_existing_file(), close_file() -#include -#else -#include // for struct stat -#include // for stat() -#include // for unlink() -#endif -#endif // OPENVDB_USE_DELAYED_LOADING - #include #include #include // for std::iota() @@ -41,6 +16,10 @@ #include #endif +#ifdef _WIN32 +#include +#endif + using namespace openvdb; using namespace openvdb::compression; @@ -467,98 +446,6 @@ TestStreamCompression::testPagedStreams() EXPECT_EQ(fileout.tellp(), std::streampos(values.size()+sizeof(int)*pages)); #endif - -#ifdef OPENVDB_USE_DELAYED_LOADING - auto mappedFile = TestMappedFile::create(filename); - - // read - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - EXPECT_EQ(filein.tellg(), std::streampos(0)); - - PagedInputStream istreamSizeOnly(filein); - istreamSizeOnly.setSizeOnly(true); - - std::vector handles; - - for (size_t i = 0; i < values.size(); i += increment) { - if (size_t(i+increment) > values.size()) { - handles.push_back(istreamSizeOnly.createHandle(values.size() - i)); - } - else { - handles.push_back(istreamSizeOnly.createHandle(increment)); - } - } - -#ifdef OPENVDB_USE_BLOSC - // two integers - compressed size and uncompressed size - EXPECT_EQ(filein.tellg(), std::streampos(pages*sizeof(int)*2)); -#else - // one integer - uncompressed size - EXPECT_EQ(filein.tellg(), std::streampos(pages*sizeof(int))); -#endif - - PagedInputStream istream(filein); - - int pageHandle = 0; - - for (size_t i = 0; i < values.size(); i += increment) { - if (size_t(i+increment) > values.size()) { - istream.read(handles[pageHandle++], values.size() - i); - } - else { - istream.read(handles[pageHandle++], increment); - } - } - - // first three handles live in the same page - - Page& page0 = handles[0]->page(); - Page& page1 = handles[1]->page(); - Page& page2 = handles[2]->page(); - Page& page3 = handles[3]->page(); - - EXPECT_TRUE(page0.isOutOfCore()); - EXPECT_TRUE(page1.isOutOfCore()); - EXPECT_TRUE(page2.isOutOfCore()); - EXPECT_TRUE(page3.isOutOfCore()); - - handles[0]->read(); - - // store the Page shared_ptr - - Page::Ptr page = handles[0]->mPage; - - // verify use count is four (one plus three handles) - - EXPECT_EQ(page.use_count(), long(4)); - - // on reading from the first handle, all pages referenced - // in the first three handles are in-core - - EXPECT_TRUE(!page0.isOutOfCore()); - EXPECT_TRUE(!page1.isOutOfCore()); - EXPECT_TRUE(!page2.isOutOfCore()); - EXPECT_TRUE(page3.isOutOfCore()); - - handles[1]->read(); - - EXPECT_TRUE(handles[0]->mPage); - - handles[2]->read(); - - handles.erase(handles.begin()); - handles.erase(handles.begin()); - handles.erase(handles.begin()); - - // after all three handles have been read, - // page should have just one use count (itself) - - EXPECT_EQ(page.use_count(), long(1)); - -#endif // OPENVDB_USE_DELAYED_LOADING } std::remove(filename.c_str()); } diff --git a/openvdb/openvdb/unittest/util.h b/openvdb/openvdb/unittest/util.h index d2aeadea69..2527c6c8b9 100644 --- a/openvdb/openvdb/unittest/util.h +++ b/openvdb/openvdb/unittest/util.h @@ -10,18 +10,6 @@ #include // for pruneLevelSet #include -#ifdef OPENVDB_USE_DELAYED_LOADING -/// @brief io::MappedFile has a private constructor, so declare a class that acts as the friend -class TestMappedFile -{ -public: - static openvdb::io::MappedFile::Ptr create(const std::string& filename) - { - return openvdb::SharedPtr(new openvdb::io::MappedFile(filename)); - } -}; -#endif - namespace unittest_util { diff --git a/openvdb/openvdb/version.h.in b/openvdb/openvdb/version.h.in index b5838936a2..3cd8043661 100644 --- a/openvdb/openvdb/version.h.in +++ b/openvdb/openvdb/version.h.in @@ -138,11 +138,6 @@ #cmakedefine OPENVDB_USE_ZLIB #endif -/* Denotes whether VDB was built with Delayed Loading support */ -#ifndef OPENVDB_USE_DELAYED_LOADING -#cmakedefine OPENVDB_USE_DELAYED_LOADING -#endif - /* Denotes whether VDB was built asserts enabled in VDB code */ #ifndef OPENVDB_ENABLE_ASSERTS #cmakedefine OPENVDB_ENABLE_ASSERTS diff --git a/pyproject.toml b/pyproject.toml index b42bcd5c80..33a942f382 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ wheel.packages = [] OPENVDB_CORE_STATIC="OFF" USE_EXPLICIT_INSTANTIATION="OFF" DISABLE_DEPENDENCY_VERSION_CHECKS="ON" -OPENVDB_USE_DELAYED_LOADING="OFF" OPENVDB_BUILD_PYTHON_MODULE="ON" USE_NUMPY="ON" From 27f2841e2bf9959266d9527c46acf86d7134d3e3 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 30 Jan 2026 12:32:05 -0800 Subject: [PATCH 02/70] Remove Delayed Load Metadata seek optimization Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 2 - openvdb/openvdb/io/Archive.cc | 111 +------ openvdb/openvdb/io/Compression.h | 46 +-- openvdb/openvdb/io/DelayedLoadMetadata.cc | 300 ------------------ openvdb/openvdb/io/DelayedLoadMetadata.h | 102 ------ openvdb/openvdb/io/File.cc | 2 +- openvdb/openvdb/openvdb.cc | 1 - openvdb/openvdb/openvdb.h | 6 +- openvdb/openvdb/unittest/CMakeLists.txt | 1 - .../unittest/TestDelayedLoadMetadata.cc | 206 ------------ openvdb/openvdb/unittest/TestFile.cc | 157 +-------- openvdb/openvdb/unittest/TestStream.cc | 1 - 12 files changed, 25 insertions(+), 910 deletions(-) delete mode 100644 openvdb/openvdb/io/DelayedLoadMetadata.cc delete mode 100644 openvdb/openvdb/io/DelayedLoadMetadata.h delete mode 100644 openvdb/openvdb/unittest/TestDelayedLoadMetadata.cc diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index c496509a72..91fc5fcfd7 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -330,7 +330,6 @@ set(OPENVDB_LIBRARY_SOURCE_FILES Grid.cc io/Archive.cc io/Compression.cc - io/DelayedLoadMetadata.cc io/File.cc io/GridDescriptor.cc io/Queue.cc @@ -368,7 +367,6 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES set(OPENVDB_LIBRARY_IO_INCLUDE_FILES io/Archive.h io/Compression.h - io/DelayedLoadMetadata.h io/File.h io/GridDescriptor.h io/io.h diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 03576b2566..0e48fdae04 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -4,7 +4,6 @@ #include "Archive.h" #include "GridDescriptor.h" -#include "DelayedLoadMetadata.h" #include "io.h" #include @@ -180,7 +179,7 @@ struct StreamMetadata::Impl uint32_t mPass = 0; MetaMap mGridMetadata; AuxDataMap mAuxData; - bool mDelayedLoadMeta = DelayedLoadMetadata::isRegisteredType(); + bool mDelayedLoadMeta = false; uint64_t mLeaf = 0; uint32_t mTest = 0; // for testing only }; // struct StreamMetadata @@ -277,7 +276,6 @@ StreamMetadata::str() const ostr << "compression: " << compressionToString(compression()) << "\n"; ostr << "half_float: " << halfFloat() << "\n"; ostr << "seekable: " << seekable() << "\n"; - ostr << "delayed_load_meta: " << delayedLoadMeta() << "\n"; ostr << "pass: " << pass() << "\n"; ostr << "counting_passes: " << countingPasses() << "\n"; ostr << "write_grid_stats_metadata: " << writeGridStats() << "\n"; @@ -310,73 +308,6 @@ writeAsType(std::ostream& os, const std::any& val) return false; } -struct PopulateDelayedLoadMetadataOp -{ - DelayedLoadMetadata& metadata; - uint32_t compression; - - PopulateDelayedLoadMetadataOp(DelayedLoadMetadata& _metadata, uint32_t _compression) - : metadata(_metadata) - , compression(_compression) { } - - template - void operator()(const GridT& grid) const - { - using TreeT = typename GridT::TreeType; - using ValueT = typename TreeT::ValueType; - using LeafT = typename TreeT::LeafNodeType; - using MaskT = typename LeafT::NodeMaskType; - - const TreeT& tree = grid.constTree(); - const Index64 leafCount = tree.leafCount(); - - // early exit if not leaf nodes - if (leafCount == Index64(0)) return; - - metadata.resizeMask(leafCount); - - if (compression & (COMPRESS_BLOSC | COMPRESS_ZIP)) { - metadata.resizeCompressedSize(leafCount); - } - - const auto background = tree.background(); - const bool saveFloatAsHalf = grid.saveFloatAsHalf(); - - tree::LeafManager leafManager(tree); - - leafManager.foreach( - [&](const LeafT& leaf, size_t idx) { - // set mask value - MaskCompress maskCompressData( - leaf.valueMask(), /*childMask=*/MaskT(), leaf.buffer().data(), background); - metadata.setMask(idx, maskCompressData.metadata); - - if (compression & (COMPRESS_BLOSC | COMPRESS_ZIP)) { - // set compressed size value - size_t sizeBytes(8); - size_t compressedSize = io::writeCompressedValuesSize( - leaf.buffer().data(), LeafT::SIZE, - leaf.valueMask(), maskCompressData.metadata, saveFloatAsHalf, compression); - metadata.setCompressedSize(idx, compressedSize+sizeBytes); - } - } - ); - } -}; - -bool populateDelayedLoadMetadata(DelayedLoadMetadata& metadata, - const GridBase& gridBase, uint32_t compression) -{ - PopulateDelayedLoadMetadataOp op(metadata, compression); - - using AllowedTypes = TypeList< - Int32Grid, Int64Grid, - FloatGrid, DoubleGrid, - Vec3IGrid, Vec3SGrid, Vec3DGrid>; - - return gridBase.apply(op); -} - } // unnamed namespace std::ostream& @@ -1016,17 +947,9 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const //grid->insertMeta(GridBase::META_FILE_COMPRESSION, // StringMetadata(compressionToString(c))); - const VersionId version = getLibraryVersion(is); - if (version.first < 6 || (version.first == 6 && version.second <= 1)) { - // If delay load metadata exists, but the file format version does not support - // delay load metadata, this likely means the original grid was read and then - // written using a prior version of OpenVDB and ABI>=5 where unknown metadata - // can be blindly copied. This means that it is possible for the metadata to - // no longer be in sync with the grid, so we remove it to ensure correctness. - - if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { - grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } + // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists + if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { + grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); } streamMetadata->gridMetadata() = static_cast(*grid); @@ -1036,15 +959,6 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const // reset leaf value to zero streamMetadata->setLeaf(0); - // drop DelayedLoadMetadata from the grid as it is only useful for IO - // a stream metadata non-zero value disables this behaviour for testing - - if (streamMetadata->__test() == uint32_t(0)) { - if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { - grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } - } - grid->readTransform(is); if (!gd.isInstance()) { grid->readTopology(is); @@ -1231,25 +1145,18 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, // Save the compression settings for this grid. setGridCompression(os, *grid); - // copy grid and add delay load metadata - const auto copyOfGrid = grid->copyGrid(); // shallow copy - const auto nonConstCopyOfGrid = ConstPtrCast(copyOfGrid); - nonConstCopyOfGrid->insertMeta(GridBase::META_FILE_DELAYED_LOAD, - DelayedLoadMetadata()); - DelayedLoadMetadata::Ptr delayLoadMeta = - nonConstCopyOfGrid->getMetadata(GridBase::META_FILE_DELAYED_LOAD); - if (!populateDelayedLoadMetadata(*delayLoadMeta, *grid, compression())) { - nonConstCopyOfGrid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } - // Save the grid's metadata and transform. if (getWriteGridStatsMetadata(os)) { // Compute and add grid statistics metadata. + const auto copyOfGrid = grid->copyGrid(); // shallow copy + const auto nonConstCopyOfGrid = ConstPtrCast(copyOfGrid); nonConstCopyOfGrid->addStatsMetadata(); nonConstCopyOfGrid->insertMeta(GridBase::META_FILE_COMPRESSION, StringMetadata(compressionToString(getDataCompression(os)))); + copyOfGrid->writeMeta(os); + } else { + grid->writeMeta(os); } - copyOfGrid->writeMeta(os); grid->writeTransform(os); // Save the grid's structure. diff --git a/openvdb/openvdb/io/Compression.h b/openvdb/openvdb/io/Compression.h index 44da4d8bcc..85bbf6b09f 100644 --- a/openvdb/openvdb/io/Compression.h +++ b/openvdb/openvdb/io/Compression.h @@ -9,7 +9,6 @@ #include // for negative() #include #include "io.h" // for getDataCompression(), etc. -#include "DelayedLoadMetadata.h" #include #include #include @@ -236,28 +235,20 @@ OPENVDB_API void bloscFromStream(std::istream&, char* data, size_t numBytes); /// @param count the number of elements to read in /// @param compression whether and how the data is compressed (either COMPRESS_NONE, /// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC) -/// @param metadata optional pointer to a DelayedLoadMetadata object that stores -/// the size of the compressed buffer -/// @param metadataOffset offset into DelayedLoadMetadata, ignored if pointer is null /// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled /// without Blosc support. /// @details This default implementation is instantiated only for types /// whose size can be determined by the sizeof() operator. template inline void -readData(std::istream& is, T* data, Index count, uint32_t compression, - DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) +readData(std::istream& is, T* data, Index count, uint32_t compression) { const bool seek = data == nullptr; if (seek) { OPENVDB_ASSERT(!getStreamMetadataPtr(is) || getStreamMetadataPtr(is)->seekable()); } - const bool hasCompression = compression & (COMPRESS_BLOSC | COMPRESS_ZIP); - if (metadata && seek && hasCompression) { - size_t compressedSize = metadata->getCompressedSize(metadataOffset); - is.seekg(compressedSize, std::ios_base::cur); - } else if (compression & COMPRESS_BLOSC) { + if (compression & COMPRESS_BLOSC) { bloscFromStream(is, reinterpret_cast(data), sizeof(T) * count); } else if (compression & COMPRESS_ZIP) { unzipFromStream(is, reinterpret_cast(data), sizeof(T) * count); @@ -271,8 +262,7 @@ readData(std::istream& is, T* data, Index count, uint32_t compression, /// Specialization for std::string input template<> inline void -readData(std::istream& is, std::string* data, Index count, uint32_t /*compression*/, - DelayedLoadMetadata* /*metadata*/, size_t /*metadataOffset*/) +readData(std::istream& is, std::string* data, Index count, uint32_t /*compression*/) { for (Index i = 0; i < count; ++i) { size_t len = 0; @@ -294,25 +284,22 @@ template struct HalfReader; /// Partial specialization for non-floating-point types (no half to float promotion) template struct HalfReader { - static inline void read(std::istream& is, T* data, Index count, uint32_t compression, - DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) { - readData(is, data, count, compression, metadata, metadataOffset); + static inline void read(std::istream& is, T* data, Index count, uint32_t compression) { + readData(is, data, count, compression); } }; /// Partial specialization for floating-point types template struct HalfReader { using HalfT = typename RealToHalf::HalfT; - static inline void read(std::istream& is, T* data, Index count, uint32_t compression, - DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) { + static inline void read(std::istream& is, T* data, Index count, uint32_t compression) { if (count < 1) return; if (data == nullptr) { // seek mode - pass through null pointer - readData(is, nullptr, count, compression, metadata, metadataOffset); + readData(is, nullptr, count, compression); } else { std::vector halfData(count); // temp buffer into which to read half float values - readData(is, reinterpret_cast(&halfData[0]), count, compression, - metadata, metadataOffset); + readData(is, reinterpret_cast(&halfData[0]), count, compression); // Copy half float values from the temporary buffer to the full float output array. std::copy(halfData.begin(), halfData.end(), data); } @@ -476,25 +463,12 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, const bool seek = (destBuf == nullptr); OPENVDB_ASSERT(!seek || (!meta || meta->seekable())); - // Get delayed load metadata if it exists - - DelayedLoadMetadata::Ptr delayLoadMeta; - uint64_t leafIndex(0); - if (seek && meta && meta->delayedLoadMeta()) { - delayLoadMeta = - meta->gridMetadata().getMetadata("file_delayed_load"); - leafIndex = meta->leaf(); - } - int8_t metadata = NO_MASK_AND_ALL_VALS; // Read the flag that specifies what, if any, additional metadata // (selection mask and/or inactive value(s)) is saved. if (seek && !maskCompressed) { is.seekg(/*bytes=*/1, std::ios_base::cur); - } else if (seek && delayLoadMeta) { - metadata = delayLoadMeta->getMask(leafIndex); - is.seekg(/*bytes=*/1, std::ios_base::cur); } else { is.read(reinterpret_cast(&metadata), /*bytes=*/1); } @@ -560,10 +534,10 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, // Read in the buffer. if (fromHalf) { HalfReader::isReal, ValueT>::read( - is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex); + is, (seek ? nullptr : tempBuf), tempCount, compression); } else { readData( - is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex); + is, (seek ? nullptr : tempBuf), tempCount, compression); } // If mask compression is enabled and the number of active values read into diff --git a/openvdb/openvdb/io/DelayedLoadMetadata.cc b/openvdb/openvdb/io/DelayedLoadMetadata.cc deleted file mode 100644 index 1f958b0a4c..0000000000 --- a/openvdb/openvdb/io/DelayedLoadMetadata.cc +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -#include "DelayedLoadMetadata.h" - -#include -#include - -#ifdef OPENVDB_USE_BLOSC -#include - -namespace { - -inline size_t padMask(size_t bytes) -{ - return size_t(std::ceil(static_cast(bytes+1) / - sizeof(openvdb::io::DelayedLoadMetadata::MaskType))); -} - -inline size_t padCompressedSize(size_t bytes) -{ - return size_t(std::ceil(static_cast(bytes+1) / - sizeof(openvdb::io::DelayedLoadMetadata::CompressedSizeType))); -} - -} // namespace - -#endif - -namespace openvdb { -OPENVDB_USE_VERSION_NAMESPACE -namespace OPENVDB_VERSION_NAME { -namespace io { - -DelayedLoadMetadata::DelayedLoadMetadata(const DelayedLoadMetadata& other) - : Metadata() - , mMask(other.mMask) - , mCompressedSize(other.mCompressedSize) -{ -} - -Name DelayedLoadMetadata::typeName() const -{ - return DelayedLoadMetadata::staticTypeName(); -} - -Metadata::Ptr DelayedLoadMetadata::copy() const -{ - Metadata::Ptr metadata(new DelayedLoadMetadata()); - metadata->copy(*this); - return metadata; -} - -void DelayedLoadMetadata::copy(const Metadata& other) -{ - const DelayedLoadMetadata* t = dynamic_cast(&other); - if (t == nullptr) OPENVDB_THROW(TypeError, "Incompatible type during copy"); - mMask = t->mMask; - mCompressedSize = t->mCompressedSize; -} - -std::string DelayedLoadMetadata::str() const -{ - return ""; -} - -bool DelayedLoadMetadata::asBool() const -{ - return false; -} - -Index32 DelayedLoadMetadata::size() const -{ - if (mMask.empty() && mCompressedSize.empty()) return Index32(0); - - // count - size_t size = sizeof(Index32); - - { // mask - size += sizeof(Index32); - size_t compressedSize = compression::bloscCompressedSize( - reinterpret_cast(mMask.data()), mMask.size()*sizeof(MaskType)); - - if (compressedSize > 0) size += compressedSize; - else size += mMask.size()*sizeof(MaskType); - } - { // compressed size - size += sizeof(Index32); - if (!mCompressedSize.empty()) { - size_t compressedSize = compression::bloscCompressedSize( - reinterpret_cast(mCompressedSize.data()), mCompressedSize.size()*sizeof(CompressedSizeType)); - - if (compressedSize > 0) size += compressedSize; - else size += mCompressedSize.size()*sizeof(CompressedSizeType); - } - } - - return static_cast(size); -} - -void DelayedLoadMetadata::clear() -{ - mMask.clear(); - mCompressedSize.clear(); -} - -bool DelayedLoadMetadata::empty() const -{ - return mMask.empty() && mCompressedSize.empty(); -} - -void DelayedLoadMetadata::resizeMask(size_t size) -{ - mMask.resize(size); -} - -void DelayedLoadMetadata::resizeCompressedSize(size_t size) -{ - mCompressedSize.resize(size); -} - -DelayedLoadMetadata::MaskType DelayedLoadMetadata::getMask(size_t index) const -{ - OPENVDB_ASSERT(DelayedLoadMetadata::isRegisteredType()); - OPENVDB_ASSERT(index < mMask.size()); - return mMask[index]; -} - -void DelayedLoadMetadata::setMask(size_t index, const MaskType& value) -{ - OPENVDB_ASSERT(index < mMask.size()); - mMask[index] = value; -} - -DelayedLoadMetadata::CompressedSizeType DelayedLoadMetadata::getCompressedSize(size_t index) const -{ - OPENVDB_ASSERT(DelayedLoadMetadata::isRegisteredType()); - OPENVDB_ASSERT(index < mCompressedSize.size()); - return mCompressedSize[index]; -} - -void DelayedLoadMetadata::setCompressedSize(size_t index, const CompressedSizeType& value) -{ - OPENVDB_ASSERT(index < mCompressedSize.size()); - mCompressedSize[index] = value; -} - -void DelayedLoadMetadata::readValue(std::istream& is, Index32 numBytes) -{ - if (numBytes == 0) return; - - // initial header size - size_t total = sizeof(Index32); - - Index32 count = 0; - is.read(reinterpret_cast(&count), sizeof(Index32)); - total += sizeof(Index32); - - Index32 bytes = 0; - is.read(reinterpret_cast(&bytes), sizeof(Index32)); - total += sizeof(Index32); - - if (bytes > Index32(0)) { - std::unique_ptr compressedBuffer(new char[bytes]); - is.read(reinterpret_cast(compressedBuffer.get()), bytes); - - total += bytes; - -#ifdef OPENVDB_USE_BLOSC - // pad to include BLOSC_MAX_OVERHEAD - size_t uncompressedBytes = openvdb::compression::bloscUncompressedSize(compressedBuffer.get()); - const size_t paddedCount = padMask(uncompressedBytes + BLOSC_MAX_OVERHEAD); - - mMask.reserve(paddedCount); - mMask.resize(count); - - // resize should never modify capacity for smaller vector sizes - OPENVDB_ASSERT(mMask.capacity() >= paddedCount); - - compression::bloscDecompress(reinterpret_cast(mMask.data()), count*sizeof(MaskType), mMask.capacity()*sizeof(MaskType), compressedBuffer.get()); -#endif - } else { - mMask.resize(count); - is.read(reinterpret_cast(mMask.data()), count*sizeof(MaskType)); - total += count*sizeof(MaskType); - } - - is.read(reinterpret_cast(&bytes), sizeof(Index32)); - - if (bytes != std::numeric_limits::max()) { - if (bytes > Index32(0)) { - std::unique_ptr compressedBuffer(new char[bytes]); - is.read(reinterpret_cast(compressedBuffer.get()), bytes); - - total += size_t(bytes); - -#ifdef OPENVDB_USE_BLOSC - // pad to include BLOSC_MAX_OVERHEAD - size_t uncompressedBytes = openvdb::compression::bloscUncompressedSize(compressedBuffer.get()); - const size_t paddedCount = padCompressedSize(uncompressedBytes + BLOSC_MAX_OVERHEAD); - - mCompressedSize.reserve(paddedCount); - mCompressedSize.resize(count); - - // resize should never modify capacity for smaller vector sizes - OPENVDB_ASSERT(mCompressedSize.capacity() >= paddedCount); - - compression::bloscDecompress(reinterpret_cast(mCompressedSize.data()), count*sizeof(CompressedSizeType), mCompressedSize.capacity()*sizeof(CompressedSizeType), compressedBuffer.get()); -#endif - } else { - mCompressedSize.resize(count); - is.read(reinterpret_cast(mCompressedSize.data()), count*sizeof(CompressedSizeType)); - total += count*sizeof(CompressedSizeType); - } - } - - Index32 totalBytes = static_cast(total); - - if (totalBytes < numBytes) { - // Read and discard any unknown bytes at the end of the metadata for forwards-compatibility - // (without seeking, because the stream might not be seekable). - const size_t BUFFER_SIZE = 1024; - std::vector buffer(BUFFER_SIZE); - for (Index32 bytesRemaining = numBytes - totalBytes; bytesRemaining > 0; ) { - const Index32 bytesToSkip = std::min(bytesRemaining, BUFFER_SIZE); - is.read(&buffer[0], bytesToSkip); - bytesRemaining -= bytesToSkip; - } - } -} - -void DelayedLoadMetadata::writeValue(std::ostream& os) const -{ - // metadata has a limit of 2^32 bytes - OPENVDB_ASSERT(mMask.size() < std::numeric_limits::max()); - OPENVDB_ASSERT(mCompressedSize.size() < std::numeric_limits::max()); - - if (mMask.empty() && mCompressedSize.empty()) return; - - OPENVDB_ASSERT(mCompressedSize.empty() || (mMask.size() == mCompressedSize.size())); - - Index32 count = static_cast(mMask.size()); - os.write(reinterpret_cast(&count), sizeof(Index32)); - - const Index32 zeroSize(0); - const Index32 maxSize(std::numeric_limits::max()); - - { // mask buffer - size_t compressedBytes(0); - std::unique_ptr compressedBuffer; - if (compression::bloscCanCompress()) { - compressedBuffer = compression::bloscCompress( - reinterpret_cast(mMask.data()), - mMask.size()*sizeof(MaskType), compressedBytes, /*resize=*/false); - } - - if (compressedBuffer) { - OPENVDB_ASSERT(compressedBytes < std::numeric_limits::max()); - Index32 bytes(static_cast(compressedBytes)); - os.write(reinterpret_cast(&bytes), sizeof(Index32)); - os.write(reinterpret_cast(compressedBuffer.get()), compressedBytes); - } - else { - os.write(reinterpret_cast(&zeroSize), sizeof(Index32)); - os.write(reinterpret_cast(mMask.data()), - mMask.size()*sizeof(MaskType)); - } - } - - // compressed size buffer - - if (mCompressedSize.empty()) { - // write out maximum Index32 value to denote no compressed sizes stored - os.write(reinterpret_cast(&maxSize), sizeof(Index32)); - } else { - size_t compressedBytes(0); - std::unique_ptr compressedBuffer; - if (compression::bloscCanCompress()) { - compressedBuffer = compression::bloscCompress( - reinterpret_cast(mCompressedSize.data()), - mCompressedSize.size()*sizeof(CompressedSizeType), compressedBytes, /*resize=*/false); - } - - if (compressedBuffer) { - OPENVDB_ASSERT(compressedBytes < std::numeric_limits::max()); - Index32 bytes(static_cast(compressedBytes)); - os.write(reinterpret_cast(&bytes), sizeof(Index32)); - os.write(reinterpret_cast(compressedBuffer.get()), compressedBytes); - } - else { - os.write(reinterpret_cast(&zeroSize), sizeof(Index32)); - os.write(reinterpret_cast(mCompressedSize.data()), - mCompressedSize.size()*sizeof(CompressedSizeType)); - } - } -} - -} // namespace io -} // namespace OPENVDB_VERSION_NAME -} // namespace openvdb diff --git a/openvdb/openvdb/io/DelayedLoadMetadata.h b/openvdb/openvdb/io/DelayedLoadMetadata.h deleted file mode 100644 index 8ba9d6f519..0000000000 --- a/openvdb/openvdb/io/DelayedLoadMetadata.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OPENVDB_DELAYED_LOAD_METADATA_HAS_BEEN_INCLUDED -#define OPENVDB_DELAYED_LOAD_METADATA_HAS_BEEN_INCLUDED - -#include -#include -#include -#include -#include - - -namespace openvdb { -OPENVDB_USE_VERSION_NAMESPACE -namespace OPENVDB_VERSION_NAME { -namespace io { - -/// @brief Store a buffer of data that can be optionally used -/// during reading for faster delayed-load I/O performance -class OPENVDB_API DelayedLoadMetadata: public Metadata -{ -public: - using Ptr = SharedPtr; - using ConstPtr = SharedPtr; - using MaskType = int8_t; - using CompressedSizeType = int64_t; - - DelayedLoadMetadata() = default; - DelayedLoadMetadata(const DelayedLoadMetadata& other); - ~DelayedLoadMetadata() override = default; - - Name typeName() const override; - Metadata::Ptr copy() const override; - void copy(const Metadata&) override; - std::string str() const override; - bool asBool() const override; - Index32 size() const override; - - static Name staticTypeName() { return "__delayedload"; } - - static Metadata::Ptr createMetadata() - { - Metadata::Ptr ret(new DelayedLoadMetadata); - return ret; - } - - static void registerType() - { - Metadata::registerType(DelayedLoadMetadata::staticTypeName(), - DelayedLoadMetadata::createMetadata); - } - - static void unregisterType() - { - Metadata::unregisterType(DelayedLoadMetadata::staticTypeName()); - } - - static bool isRegisteredType() - { - return Metadata::isRegisteredType(DelayedLoadMetadata::staticTypeName()); - } - - /// @brief Delete the contents of the mask and compressed size arrays - void clear(); - /// @brief Return @c true if both arrays are empty - bool empty() const; - - /// @brief Resize the mask array - void resizeMask(size_t size); - /// @brief Resize the compressed size array - void resizeCompressedSize(size_t size); - - /// @brief Return the mask value for a specific index - /// @note throws if index is out-of-range or DelayedLoadMask not registered - MaskType getMask(size_t index) const; - /// @brief Set the mask value for a specific index - /// @note throws if index is out-of-range - void setMask(size_t index, const MaskType& value); - - /// @brief Return the compressed size value for a specific index - /// @note throws if index is out-of-range or DelayedLoadMask not registered - CompressedSizeType getCompressedSize(size_t index) const; - /// @brief Set the compressed size value for a specific index - /// @note throws if index is out-of-range - void setCompressedSize(size_t index, const CompressedSizeType& value); - -protected: - void readValue(std::istream&, Index32 numBytes) override; - void writeValue(std::ostream&) const override; - -private: - std::vector mMask; - std::vector mCompressedSize; -}; // class DelayedLoadMetadata - - -} // namespace io -} // namespace OPENVDB_VERSION_NAME -} // namespace openvdb - -#endif // OPENVDB_DELAYED_LOAD_METADATA_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index dfb0c797e1..218dcbb7e9 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -727,7 +727,7 @@ File::readGridPartial(GridBase::Ptr grid, std::istream& is, readGridCompression(is); grid->readMeta(is); - // drop DelayedLoadMetadata from the grid as it is only useful for IO + // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); } diff --git a/openvdb/openvdb/openvdb.cc b/openvdb/openvdb/openvdb.cc index d4691c36a7..f830054849 100644 --- a/openvdb/openvdb/openvdb.cc +++ b/openvdb/openvdb/openvdb.cc @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 #include "openvdb.h" -#include "io/DelayedLoadMetadata.h" #include "points/PointDataGrid.h" #include "tools/PointIndexGrid.h" #include "util/logging.h" diff --git a/openvdb/openvdb/openvdb.h b/openvdb/openvdb/openvdb.h index a4db4b7337..8ce06d4167 100644 --- a/openvdb/openvdb/openvdb.h +++ b/openvdb/openvdb/openvdb.h @@ -46,9 +46,6 @@ OPENVDB_API void initialize(); OPENVDB_API void uninitialize(); -// foward declare some default types -namespace io { class DelayedLoadMetadata; } - /// Common tree types using BoolTree = tree::Tree4::Type; using DoubleTree = tree::Tree4::Type; @@ -222,8 +219,7 @@ using MetaTypes = TypeList< Vec4SMetadata, Vec4DMetadata, Mat4SMetadata, - Mat4DMetadata, - io::DelayedLoadMetadata>; + Mat4DMetadata>; } // namespace OPENVDB_VERSION_NAME diff --git a/openvdb/openvdb/unittest/CMakeLists.txt b/openvdb/openvdb/unittest/CMakeLists.txt index 3878f95b65..990849d809 100644 --- a/openvdb/openvdb/unittest/CMakeLists.txt +++ b/openvdb/openvdb/unittest/CMakeLists.txt @@ -87,7 +87,6 @@ else() TestCount.cc TestCpt.cc TestCurl.cc - TestDelayedLoadMetadata.cc TestDense.cc TestDenseSparseTools.cc TestDiagnostics.cc diff --git a/openvdb/openvdb/unittest/TestDelayedLoadMetadata.cc b/openvdb/openvdb/unittest/TestDelayedLoadMetadata.cc deleted file mode 100644 index d1617863e8..0000000000 --- a/openvdb/openvdb/unittest/TestDelayedLoadMetadata.cc +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include - -#include - -class TestDelayedLoadMetadata : public ::testing::Test -{ -}; - - -TEST_F(TestDelayedLoadMetadata, test) -{ - using namespace openvdb::io; - - // registration - - EXPECT_TRUE(!DelayedLoadMetadata::isRegisteredType()); - - DelayedLoadMetadata::registerType(); - - EXPECT_TRUE(DelayedLoadMetadata::isRegisteredType()); - - DelayedLoadMetadata::unregisterType(); - - EXPECT_TRUE(!DelayedLoadMetadata::isRegisteredType()); - - openvdb::initialize(); - - EXPECT_TRUE(DelayedLoadMetadata::isRegisteredType()); - - // construction - - DelayedLoadMetadata metadata; - - EXPECT_TRUE(metadata.empty()); - - metadata.resizeMask(size_t(2)); - - EXPECT_TRUE(!metadata.empty()); - - metadata.setMask(0, DelayedLoadMetadata::MaskType(5)); - metadata.setMask(1, DelayedLoadMetadata::MaskType(-3)); - - EXPECT_EQ(metadata.getMask(0), DelayedLoadMetadata::MaskType(5)); - EXPECT_EQ(metadata.getMask(1), DelayedLoadMetadata::MaskType(-3)); - - metadata.resizeCompressedSize(size_t(3)); - - metadata.setCompressedSize(0, DelayedLoadMetadata::CompressedSizeType(6)); - metadata.setCompressedSize(1, DelayedLoadMetadata::CompressedSizeType(101)); - metadata.setCompressedSize(2, DelayedLoadMetadata::CompressedSizeType(-13522)); - - EXPECT_EQ(metadata.getCompressedSize(0), DelayedLoadMetadata::CompressedSizeType(6)); - EXPECT_EQ(metadata.getCompressedSize(1), DelayedLoadMetadata::CompressedSizeType(101)); - EXPECT_EQ(metadata.getCompressedSize(2), DelayedLoadMetadata::CompressedSizeType(-13522)); - - // copy construction - - DelayedLoadMetadata metadataCopy1(metadata); - - EXPECT_TRUE(!metadataCopy1.empty()); - - EXPECT_EQ(metadataCopy1.getMask(0), DelayedLoadMetadata::MaskType(5)); - EXPECT_EQ(metadataCopy1.getCompressedSize(2), DelayedLoadMetadata::CompressedSizeType(-13522)); - - openvdb::Metadata::Ptr baseMetadataCopy2 = metadata.copy(); - DelayedLoadMetadata::Ptr metadataCopy2 = - openvdb::StaticPtrCast(baseMetadataCopy2); - - EXPECT_EQ(metadataCopy2->getMask(0), DelayedLoadMetadata::MaskType(5)); - EXPECT_EQ(metadataCopy2->getCompressedSize(2), DelayedLoadMetadata::CompressedSizeType(-13522)); - - // I/O - - metadata.clear(); - EXPECT_TRUE(metadata.empty()); - - const size_t headerInitialSize(sizeof(openvdb::Index32)); - const size_t headerCountSize(sizeof(openvdb::Index32)); - const size_t headerMaskSize(sizeof(openvdb::Index32)); - const size_t headerCompressedSize(sizeof(openvdb::Index32)); - const size_t headerTotalSize(headerInitialSize + headerCountSize + headerMaskSize + headerCompressedSize); - - { // empty buffer - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - metadata.write(ss); - EXPECT_EQ(ss.tellp(), std::streampos(headerInitialSize)); - - DelayedLoadMetadata newMetadata; - newMetadata.read(ss); - EXPECT_TRUE(newMetadata.empty()); - } - - { // single value, no compressed sizes - metadata.clear(); - metadata.resizeMask(size_t(1)); - metadata.setMask(0, DelayedLoadMetadata::MaskType(5)); - - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - metadata.write(ss); - std::streampos expectedPos(headerTotalSize + sizeof(int8_t)); - EXPECT_EQ(ss.tellp(), expectedPos); - EXPECT_EQ(static_cast(expectedPos)-headerInitialSize, size_t(metadata.size())); - - DelayedLoadMetadata newMetadata; - newMetadata.read(ss); - EXPECT_TRUE(!newMetadata.empty()); - EXPECT_EQ(newMetadata.getMask(0), DelayedLoadMetadata::MaskType(5)); - } - - { // single value, with compressed sizes - metadata.clear(); - metadata.resizeMask(size_t(1)); - metadata.setMask(0, DelayedLoadMetadata::MaskType(5)); - - metadata.resizeCompressedSize(size_t(1)); - metadata.setCompressedSize(0, DelayedLoadMetadata::CompressedSizeType(-10322)); - - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - metadata.write(ss); - std::streampos expectedPos(headerTotalSize + sizeof(int8_t) + sizeof(int64_t)); - - EXPECT_EQ(expectedPos, ss.tellp()); - EXPECT_EQ(static_cast(ss.tellp())-headerInitialSize, size_t(metadata.size())); - - DelayedLoadMetadata newMetadata; - newMetadata.read(ss); - EXPECT_TRUE(!newMetadata.empty()); - EXPECT_EQ(newMetadata.getMask(0), DelayedLoadMetadata::MaskType(5)); - EXPECT_EQ(newMetadata.getCompressedSize(0), DelayedLoadMetadata::CompressedSizeType(-10322)); - } - - { // larger, but compressible buffer - metadata.clear(); - - const size_t size = 1000; - - const size_t uncompressedBufferSize = (sizeof(int8_t)+sizeof(int64_t))*size; - - metadata.resizeMask(size); - metadata.resizeCompressedSize(size); - for (size_t i = 0; i < size; i++) { - metadata.setMask(i, - DelayedLoadMetadata::MaskType(static_cast((i%32)*2))); - metadata.setCompressedSize(i, - DelayedLoadMetadata::CompressedSizeType(static_cast((i%64)*200))); - } - - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - metadata.write(ss); - - EXPECT_EQ(static_cast(ss.tellp())-headerInitialSize, size_t(metadata.size())); - - std::streampos uncompressedSize(uncompressedBufferSize + headerTotalSize); -#ifdef OPENVDB_USE_BLOSC - // expect a compression ratio of more than 10x - EXPECT_TRUE(ss.tellp() * 10 < uncompressedSize); -#else - EXPECT_TRUE(ss.tellp() == uncompressedSize); -#endif - - DelayedLoadMetadata newMetadata; - newMetadata.read(ss); - EXPECT_EQ(metadata.size(), newMetadata.size()); - for (size_t i = 0; i < size; i++) { - EXPECT_EQ(metadata.getMask(i), newMetadata.getMask(i)); - } - } - - // when read as unknown metadata should be treated as temporary metadata - - { - metadata.clear(); - metadata.resizeMask(size_t(1)); - metadata.setMask(0, DelayedLoadMetadata::MaskType(5)); - - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - - openvdb::MetaMap metamap; - metamap.insertMeta("delayload", metadata); - - EXPECT_EQ(size_t(1), metamap.metaCount()); - - metamap.writeMeta(ss); - - { - openvdb::MetaMap newMetamap; - newMetamap.readMeta(ss); - - EXPECT_EQ(size_t(1), newMetamap.metaCount()); - } - - { - DelayedLoadMetadata::unregisterType(); - - openvdb::MetaMap newMetamap; - newMetamap.readMeta(ss); - - EXPECT_EQ(size_t(0), newMetamap.metaCount()); - } - } -} diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index 0d9c9a3e8a..40206d1beb 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -53,7 +53,6 @@ class TestFile: public ::testing::Test void testReadGridDescriptors(); void testEmptyGridIO(); void testOpen(); - void testDelayedLoadMetadata(); void testNonVdbOpen(); }; @@ -190,10 +189,8 @@ TestFile::testWriteGrid() gd2_grid->readTransform(istr); gd2_grid->readTopology(istr); - // Remove delay load metadata if it exists. - if ((*gd2_grid)["file_delayed_load"]) { - gd2_grid->removeMeta("file_delayed_load"); - } + // Delay load metadata should not exist. + ASSERT_FALSE(bool((*gd2_grid)["file_delayed_load"])); // Ensure that we have the same metadata. EXPECT_EQ(grid->metaCount(), gd2_grid->metaCount()); @@ -1450,10 +1447,8 @@ TEST_F(TestFile, testReadGridMetadata) if ((*statsMetadata)[it->first]) { otherMetadata->removeMeta(it->first); } - // Remove delay load metadata if it exists. - if ((*otherMetadata)["file_delayed_load"]) { - otherMetadata->removeMeta("file_delayed_load"); - } + // Delay load metadata should not exist. + ASSERT_FALSE(bool((*otherMetadata)["file_delayed_load"])); } EXPECT_EQ(srcGrid->str(), otherMetadata->str()); @@ -2530,147 +2525,3 @@ TEST_F(TestFile, testBlosc) } } #endif - - -void -TestFile::testDelayedLoadMetadata() -{ - openvdb::initialize(); - - using namespace openvdb; - - io::File file("something.vdb2"); - - // Create a level set grid. - auto lsGrid = createLevelSet(); - lsGrid->setName("sphere"); - unittest_util::makeSphere(/*dim=*/Coord(100), /*ctr=*/Vec3f(50, 50, 50), /*r=*/20.0, - *lsGrid, unittest_util::SPHERE_SPARSE_NARROW_BAND); - - // Write the VDB to a string stream. - std::ostringstream ostr(std::ios_base::binary); - - // Create the grid descriptor out of this grid. - io::GridDescriptor gd(Name("sphere"), lsGrid->type()); - - // Write out the grid. - file.writeGrid(gd, lsGrid, ostr, /*seekable=*/true); - - // Duplicate VDB string stream. - std::ostringstream ostr2(std::ios_base::binary); - - { // Read back in, clip and write out again to verify metadata is rebuilt. - std::istringstream istr(ostr.str(), std::ios_base::binary); - io::setVersion(istr, file.libraryVersion(), file.fileVersion()); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - - const BBoxd clipBbox(Vec3d(-10.0,-10.0,-10.0), Vec3d(10.0,10.0,10.0)); - io::Archive::readGrid(grid, gd2, istr, clipBbox); - - // Verify clipping is working as expected. - EXPECT_TRUE(grid->baseTreePtr()->leafCount() < lsGrid->tree().leafCount()); - - file.writeGrid(gd, grid, ostr2, /*seekable=*/true); - } - - // Since the input is only a fragment of a VDB file (in particular, - // it doesn't have a header), set the file format version number explicitly. - // On read, the delayed load metadata for OpenVDB library versions less than 6.1 - // should be removed to ensure correctness as it possible for the metadata to - // have been treated as unknown and blindly copied over when read and re-written - // using this library version resulting in out-of-sync metadata. - - // By default, DelayedLoadMetadata is dropped from the grid during read so - // as not to be exposed to the user. - - { // read using current library version - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, file.libraryVersion(), file.fileVersion()); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(!((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - // To test the version mechanism, a stream metadata object is created with - // a non-zero test value and set on the input stream. This disables the - // behaviour where the DelayedLoadMetadata is dropped from the grid. - - io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata); - streamMetadata->__setTest(uint32_t(1)); - - { // read using current library version - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, file.libraryVersion(), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - { // read using library version of 5.0 - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, VersionId(5,0), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(!((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - { // read using library version of 4.9 - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, VersionId(4,9), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(!((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - { // read using library version of 6.1 - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, VersionId(6,1), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(!((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - { // read using library version of 6.2 - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, VersionId(6,2), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - remove("something.vdb2"); -} -TEST_F(TestFile, testDelayedLoadMetadata) { testDelayedLoadMetadata(); } - diff --git a/openvdb/openvdb/unittest/TestStream.cc b/openvdb/openvdb/unittest/TestStream.cc index cc9bde5b75..8183317ea9 100644 --- a/openvdb/openvdb/unittest/TestStream.cc +++ b/openvdb/openvdb/unittest/TestStream.cc @@ -45,7 +45,6 @@ TestStream::SetUp() openvdb::Int32Metadata::registerType(); openvdb::Int64Metadata::registerType(); openvdb::Vec3IMetadata::registerType(); - openvdb::io::DelayedLoadMetadata::registerType(); // Register maps openvdb::math::MapRegistry::clear(); From d6074750e52c3689e7c42d0732d331585497273a Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 30 Jan 2026 14:15:03 -0800 Subject: [PATCH 03/70] File::open() and Stream() deprecations Signed-off-by: Dan Bailey --- .../nanovdb/cmd/convert/nanovdb_convert.cc | 2 +- .../coarsen_nanovdb_cuda.cpp | 2 +- .../dilate_nanovdb_cuda.cpp | 2 +- .../merge_nanovdb_cuda.cpp | 4 +- .../examples/ex_raytrace_level_set/openvdb.cc | 2 +- .../refine_nanovdb_cuda.cpp | 2 +- nanovdb/nanovdb/unittest/TestOpenVDB.cc | 8 +-- openvdb/openvdb/io/File.h | 10 +++ openvdb/openvdb/io/Stream.cc | 71 +++++++++---------- openvdb/openvdb/io/Stream.h | 10 +-- openvdb/openvdb/io/io.h | 42 +++++++++++ openvdb/openvdb/unittest/TestFastSweeping.cc | 6 +- openvdb/openvdb/unittest/TestMultiResGrid.cc | 2 +- openvdb/openvdb/unittest/TestTools.cc | 4 +- openvdb_cmd/vdb_render/main.cc | 2 +- openvdb_cmd/vdb_tool/include/Geometry.h | 2 +- openvdb_cmd/vdb_tool/include/Tool.h | 5 +- .../openvdb_houdini/GEO_VDBTranslator.cc | 4 +- .../openvdb_houdini/reference/GEO_PrimVDB.cc | 4 +- 19 files changed, 116 insertions(+), 68 deletions(-) diff --git a/nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc b/nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc index 2298508a93..bab45dccd6 100644 --- a/nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc +++ b/nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc @@ -257,7 +257,7 @@ int main(int argc, char* argv[]) } if (verbose) std::cout << "Opening OpenVDB file named \"" << inputFile << "\"" << std::endl; openvdb::io::File file(inputFile); - file.open(false); //disable delayed loading + file.open(); if (gridName.empty()) {// convert all grid in the file auto grids = file.getGrids(); std::vector > handles; diff --git a/nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp b/nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp index 8826a3000c..7154e02740 100644 --- a/nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp +++ b/nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp @@ -50,7 +50,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read input VDB file"); openvdb::initialize(); openvdb::io::File inFile(argv[1]); - inFile.open(false); // disable delayed loading + inFile.open(); auto baseGrids = inFile.getGrids(); inFile.close(); auto grid = openvdb::gridPtrCast(baseGrids->at(0)); diff --git a/nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp b/nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp index 6f6000f03c..6e65006934 100644 --- a/nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp +++ b/nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp @@ -44,7 +44,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read input VDB file"); openvdb::initialize(); openvdb::io::File inFile(argv[1]); - inFile.open(false); // disable delayed loading + inFile.open(); auto baseGrids = inFile.getGrids(); inFile.close(); auto grid = openvdb::gridPtrCast(baseGrids->at(0)); diff --git a/nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp b/nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp index 8b6626558b..e8cba39e02 100644 --- a/nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp +++ b/nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp @@ -41,7 +41,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read first input VDB file"); openvdb::initialize(); openvdb::io::File inFile1(argv[1]); - inFile1.open(false); // disable delayed loading + inFile1.open(); auto baseGrids1 = inFile1.getGrids(); inFile1.close(); auto grid1 = openvdb::gridPtrCast(baseGrids1->at(0)); @@ -51,7 +51,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read second input VDB file"); openvdb::initialize(); openvdb::io::File inFile2(argv[2]); - inFile2.open(false); // disable delayed loading + inFile2.open(); auto baseGrids2 = inFile2.getGrids(); inFile2.close(); auto grid2 = openvdb::gridPtrCast(baseGrids2->at(0)); diff --git a/nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc b/nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc index b8ca62c838..7b8f12256e 100644 --- a/nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc +++ b/nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc @@ -37,7 +37,7 @@ void runOpenVDB(nanovdb::GridHandle& handle, int nu openvdb::initialize(); std::string filename = "C:/Users/william/Downloads/dragon.vdb"; openvdb::io::File file(filename); - file.open(false); //disable delayed loading + file.open(); auto srcGrid = file.readGrid("ls_dragon"); std::cout << "Loading OpenVDB grid[" << srcGrid->getName() << "]...\n"; #endif diff --git a/nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp b/nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp index 19795c5dfd..f86980a72b 100644 --- a/nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp +++ b/nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp @@ -81,7 +81,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read input VDB file"); openvdb::initialize(); openvdb::io::File inFile(argv[1]); - inFile.open(false); // disable delayed loading + inFile.open(); auto baseGrids = inFile.getGrids(); inFile.close(); auto grid = openvdb::gridPtrCast(baseGrids->at(0)); diff --git a/nanovdb/nanovdb/unittest/TestOpenVDB.cc b/nanovdb/nanovdb/unittest/TestOpenVDB.cc index fd47a7e017..8ada169df9 100644 --- a/nanovdb/nanovdb/unittest/TestOpenVDB.cc +++ b/nanovdb/nanovdb/unittest/TestOpenVDB.cc @@ -140,7 +140,7 @@ class TestOpenVDB : public ::testing::Test mTimer.start("Reading grid from the file \"" + fileName + "\""); try { openvdb::io::File file(fileName); - file.open(false); //disable delayed loading + file.open(); grid = openvdb::gridPtrCast(file.readGrid(file.beginName().gridName())); } catch(const std::exception& e) { std::cerr << "An exception occurred: \"" << e.what() << "\"" << std::endl; @@ -2860,7 +2860,7 @@ TEST_F(TestOpenVDB, LevelSetFiles) //mTimer.start("\nReading grid from the file \"" + fileName + "\""); try { openvdb::io::File file(fileName); - file.open(false); //disable delayed loading + file.open(); auto srcGrid = openvdb::gridPtrCast(file.readGrid(file.beginName().gridName())); const size_t pos = fileName.find_last_of("/\\") + 1; @@ -2938,7 +2938,7 @@ TEST_F(TestOpenVDB, FogFiles) //mTimer.start("Reading grid from the file \"" + fileName + "\""); try { openvdb::io::File file(fileName); - file.open(false); //disable delayed loading + file.open(); auto srcGrid = openvdb::gridPtrCast(file.readGrid(file.beginName().gridName())); const size_t pos = fileName.find_last_of("/\\") + 1; @@ -3008,7 +3008,7 @@ TEST_F(TestOpenVDB, PointFiles) //mTimer.start("Reading grid from the file \"" + fileName + "\""); try { openvdb::io::File file(fileName); - file.open(false); //disable delayed loading + file.open(); auto srcGrid = openvdb::gridPtrCast(file.readGrid(file.beginName().gridName())); //std::cerr << "Read PointDataGrid named \"" << srcGrid->getName() << "\"" << std::endl; diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index dbde5b6d51..be8ab89968 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -60,6 +60,11 @@ class OPENVDB_API File: public Archive /// @return @c true if the file's UUID has changed since it was last read. bool open(); + OPENVDB_DEPRECATED_MESSAGE("Use File::open() instead.This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool open(bool /*delayLoad*/) { return open(); } + OPENVDB_DEPRECATED_MESSAGE("Use File::open() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool open(bool /*delayLoad*/, const MappedFile::Notifier& /*notifier*/) { return open(); } + /// Return @c true if the file has been opened for reading. bool isOpen() const; @@ -70,6 +75,11 @@ class OPENVDB_API File: public Archive /// @throw IoError if the file size cannot be determined. Index64 getSize() const; + OPENVDB_DEPRECATED_MESSAGE("Always returns 0. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 copyMaxBytes() const { return 0; } + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void setCopyMaxBytes(Index64 /*bytes*/) { } + /// Return @c true if a grid of the given name exists in this file. bool hasGrid(const Name&) const; diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 4ba3fce816..e3ff05c2ed 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -44,47 +44,44 @@ struct Stream::Impl //////////////////////////////////////// -Stream::Stream(std::istream& is, bool delayLoad): mImpl(new Impl) +Stream::Stream(std::istream& is): mImpl(new Impl) { if (!is) return; - (void) delayLoad; - - if (!mImpl->mFile) { - readHeader(is); - - // Tag the input stream with the library and file format version numbers - // and the compression options specified in the header. - StreamMetadata::Ptr streamMetadata(new StreamMetadata); - io::setStreamMetadataPtr(is, streamMetadata, /*transfer=*/false); - io::setVersion(is, libraryVersion(), fileVersion()); - io::setDataCompression(is, compression()); - - // Read in the VDB metadata. - mImpl->mMeta.reset(new MetaMap); - mImpl->mMeta->readMeta(is); - - // Read in the number of grids. - const int32_t gridCount = readGridCount(is); - - // Read in all grids and insert them into mGrids. - mImpl->mGrids.reset(new GridPtrVec); - std::vector descriptors; - descriptors.reserve(gridCount); - Archive::NamedGridMap namedGrids; - for (int32_t i = 0; i < gridCount; ++i) { - GridDescriptor gd; - gd.read(is); - descriptors.push_back(gd); - GridBase::Ptr grid = readGrid(gd, is); - mImpl->mGrids->push_back(grid); - namedGrids[gd.uniqueName()] = grid; - } + // Delayed loading has been removed - always read directly from the stream + readHeader(is); + + // Tag the input stream with the library and file format version numbers + // and the compression options specified in the header. + StreamMetadata::Ptr streamMetadata(new StreamMetadata); + io::setStreamMetadataPtr(is, streamMetadata, /*transfer=*/false); + io::setVersion(is, libraryVersion(), fileVersion()); + io::setDataCompression(is, compression()); + + // Read in the VDB metadata. + mImpl->mMeta.reset(new MetaMap); + mImpl->mMeta->readMeta(is); + + // Read in the number of grids. + const int32_t gridCount = readGridCount(is); + + // Read in all grids and insert them into mGrids. + mImpl->mGrids.reset(new GridPtrVec); + std::vector descriptors; + descriptors.reserve(gridCount); + Archive::NamedGridMap namedGrids; + for (int32_t i = 0; i < gridCount; ++i) { + GridDescriptor gd; + gd.read(is); + descriptors.push_back(gd); + GridBase::Ptr grid = readGrid(gd, is); + mImpl->mGrids->push_back(grid); + namedGrids[gd.uniqueName()] = grid; + } - // Connect instances (grids that share trees with other grids). - for (size_t i = 0, N = descriptors.size(); i < N; ++i) { - Archive::connectInstance(descriptors[i], namedGrids); - } + // Connect instances (grids that share trees with other grids). + for (size_t i = 0, N = descriptors.size(); i < N; ++i) { + Archive::connectInstance(descriptors[i], namedGrids); } } diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index e2978dc20d..480f5bf074 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -22,11 +22,11 @@ class OPENVDB_API Stream: public Archive { public: /// @brief Read grids from an input stream. - /// @details If @a delayLoad is true, map the contents of the input stream - /// into memory and enable delayed loading of grids. - /// @note Define the environment variable @c OPENVDB_DISABLE_DELAYED_LOAD - /// to disable delayed loading unconditionally. - explicit Stream(std::istream&, bool delayLoad = true); + /// @param is The input stream to read from + explicit Stream(std::istream&); + + OPENVDB_DEPRECATED_MESSAGE("Use Stream(std::istream&) instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Stream(std::istream& is, bool /*delayLoad*/) : Stream(is) { } /// Construct an archive for stream output. Stream(); diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index 40bce111d5..cd1c460aaf 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -127,6 +127,43 @@ struct MultiPass {}; //////////////////////////////////////// +class File; + + +// This class is deprecated and will be removed. Delayed loading is no longer supported. +// It is retained for API backwards compatibility only, but all methods are no-ops. +class OPENVDB_API MappedFile +{ +public: + using Ptr = SharedPtr; + using Notifier = std::function; + + OPENVDB_DEPRECATED_MESSAGE("This class is deprecated and will be removed. Delayed loading is no longer supported.") + explicit MappedFile(const std::string& /*filename*/, bool /*autoDelete*/ = false) {} + + /// @brief Destructor + ~MappedFile() = default; + + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + + OPENVDB_DEPRECATED_MESSAGE("Always returns an empty string. This method is deprecated and will be removed. Delayed loading is no longer supported.") + std::string filename() const { return std::string(); } + + OPENVDB_DEPRECATED_MESSAGE("Always returns a null stream buffer pointer. This method is deprecated and will be removed. Delayed loading is no longer supported.") + SharedPtr createBuffer() const { return SharedPtr(); } + + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void setNotifier(const Notifier&) {} + + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void clearNotifier() {} +}; // class MappedFile + + +//////////////////////////////////////// + + /// Return a string (possibly empty) describing the given system error code. std::string getErrorString(int errorNum); @@ -199,6 +236,11 @@ OPENVDB_API bool getWriteGridStatsMetadata(std::ios_base&); /// and store them as grid metadata when writing to the given stream. OPENVDB_API void setWriteGridStatsMetadata(std::ios_base&, bool writeGridStats); +OPENVDB_DEPRECATED_MESSAGE("Always returns a null pointer. This function is deprecated and will be removed. Delayed loading is no longer supported.") +inline SharedPtr getMappedFilePtr(std::ios_base&) { return SharedPtr(); } +OPENVDB_DEPRECATED_MESSAGE("This function is deprecated and will be removed. Delayed loading is no longer supported.") +inline void setMappedFilePtr(std::ios_base&, SharedPtr&) { } + /// @brief Return a shared pointer to an object that stores metadata (file format, /// compression scheme, etc.) for use when reading from or writing to the given stream. OPENVDB_API SharedPtr getStreamMetadataPtr(std::ios_base&); diff --git a/openvdb/openvdb/unittest/TestFastSweeping.cc b/openvdb/openvdb/unittest/TestFastSweeping.cc index 9bddbf7fbf..226aa35cd8 100644 --- a/openvdb/openvdb/unittest/TestFastSweeping.cc +++ b/openvdb/openvdb/unittest/TestFastSweeping.cc @@ -252,7 +252,7 @@ TEST_F(TestFastSweeping, testMaskSdf) openvdb::initialize();//required whenever I/O of OpenVDB files is performed! const std::string path(TestFastSweeping_DATA_PATH); io::File file( path + "bunny.vdb" ); - file.open(false);//disable delayed loading + file.open(); FloatGrid::Ptr mask = openvdb::gridPtrCast(file.getGrids()->at(0)); //this->writeFile("/tmp/bunny_mask_input.vdb", grid); @@ -595,7 +595,7 @@ TEST_F(TestFastSweeping, velocityExtensionOfFogBunny) openvdb::initialize();//required whenever I/O of OpenVDB files is performed! const std::string path(TestFastSweeping_DATA_PATH); io::File file( path + "bunny.vdb" ); - file.open(false);//disable delayed loading + file.open(); auto grid = openvdb::gridPtrCast(file.getGrids()->at(0)); tools::sdfToFogVolume(*grid); writeFile("/tmp/bunny1_fog_in.vdb", grid); @@ -619,7 +619,7 @@ TEST_F(TestFastSweeping, velocityExtensionOfSdfBunny) using namespace openvdb; const std::string path(TestFastSweeping_DATA_PATH); io::File file( path + "bunny.vdb" ); - file.open(false);//disable delayed loading + file.open(); auto grid = openvdb::gridPtrCast(file.getGrids()->at(0)); writeFile("/tmp/bunny2_sdf_in.vdb", grid); auto bbox = grid->evalActiveVoxelBoundingBox(); diff --git a/openvdb/openvdb/unittest/TestMultiResGrid.cc b/openvdb/openvdb/unittest/TestMultiResGrid.cc index 32735d508f..d30cb0a28d 100644 --- a/openvdb/openvdb/unittest/TestMultiResGrid.cc +++ b/openvdb/openvdb/unittest/TestMultiResGrid.cc @@ -280,7 +280,7 @@ TEST_F(TestMultiResGrid, testModels) << "\" =====================" << std::endl; std::cerr << "Reading \"" << filenames[i] << "\" ..."; io::File file( path + filenames[i] ); - file.open(false);//disable delayed loading + file.open(); FloatGrid::Ptr model = gridPtrCast(file.getGrids()->at(0)); std::cerr << " done\nProcessing \"" << filenames[i] << "\" ..."; timer.start("\nMultiResGrid processing"); diff --git a/openvdb/openvdb/unittest/TestTools.cc b/openvdb/openvdb/unittest/TestTools.cc index df17050a06..4ffc1fccd1 100644 --- a/openvdb/openvdb/unittest/TestTools.cc +++ b/openvdb/openvdb/unittest/TestTools.cc @@ -1762,7 +1762,7 @@ TEST_F(TestTools, testPrune) util::CpuTimer timer; initialize();//required whenever I/O of OpenVDB files is performed! io::File sourceFile("/usr/pic1/Data/OpenVDB/LevelSetModels/crawler.vdb"); - sourceFile.open(false);//disable delayed loading + sourcefile.open(); FloatGrid::Ptr grid = gridPtrCast(sourceFile.getGrids()->at(0)); const Index64 leafCount = grid->tree().leafCount(); @@ -1775,7 +1775,7 @@ TEST_F(TestTools, testPrune) util::CpuTimer timer; initialize();//required whenever I/O of OpenVDB files is performed! io::File sourceFile("/usr/pic1/Data/OpenVDB/LevelSetModels/crawler.vdb"); - sourceFile.open(false);//disable delayed loading + sourcefile.open(); FloatGrid::Ptr grid = gridPtrCast(sourceFile.getGrids()->at(0)); const Index64 leafCount = grid->tree().leafCount(); diff --git a/openvdb_cmd/vdb_render/main.cc b/openvdb_cmd/vdb_render/main.cc index 127d8720b7..2281405978 100644 --- a/openvdb_cmd/vdb_render/main.cc +++ b/openvdb_cmd/vdb_render/main.cc @@ -768,7 +768,7 @@ main(int argc, char *argv[]) } } else { // If no grid was specified by name, retrieve the first float grid from the file. - file.open(/*delayLoad=*/false); + file.open(); openvdb::io::File::NameIterator it = file.beginName(); openvdb::GridPtrVecPtr grids = file.readAllGridMetadata(); for (size_t i = 0; i < grids->size(); ++i, ++it) { diff --git a/openvdb_cmd/vdb_tool/include/Geometry.h b/openvdb_cmd/vdb_tool/include/Geometry.h index ebd497b3bb..627bf6213e 100644 --- a/openvdb_cmd/vdb_tool/include/Geometry.h +++ b/openvdb_cmd/vdb_tool/include/Geometry.h @@ -824,7 +824,7 @@ void Geometry::readVDB(const std::string &fileName) { initialize(); io::File file(fileName); - file.open();// enables delayed loading by default + file.open(); GridPtrVecPtr meta = file.readAllGridMetadata(); for (auto m : *meta) { if (m->isType()) { diff --git a/openvdb_cmd/vdb_tool/include/Tool.h b/openvdb_cmd/vdb_tool/include/Tool.h index e558be1923..b6d5e3aaef 100644 --- a/openvdb_cmd/vdb_tool/include/Tool.h +++ b/openvdb_cmd/vdb_tool/include/Tool.h @@ -370,8 +370,7 @@ void Tool::init() mParser.addAction( "read", "i", "Read one or more geometry or VDB files from disk or STDIN.", {{"files", "", "{file|stdin}.{abc|obj|ply|stl|off|vdb}", "list of files or the input stream, e.g. file.vdb,stdin.vdb. Note that \"files=\" is optional since any argument without \"=\" is intrepreted as a file and appended to \"files\""}, - {"grids", "*", "*|grid_name,...", "list of VDB grids name to be imported (defaults to \"*\", i.e. import all available grids)"}, - {"delayed", "true", "1|0|true|false", "toggle delayed loading of VDB grids (enabled by default). This option is ignored by other file types"}}, + {"grids", "*", "*|grid_name,...", "list of VDB grids name to be imported (defaults to \"*\", i.e. import all available grids)"}}, [](){}, [&](){this->read();}, 0);// anonymous options are treated as to the first option,i.e. "files" mParser.addAction( @@ -1023,7 +1022,7 @@ void Tool::readVDB(const std::string &fileName) } else { if (mParser.verbose) mTimer.start("Reading VDB grid(s) from file named \""+fileName+"\""); io::File file(fileName); - file.open(mParser.get("delayed")); + file.open(); grids = file.getGrids(); } const size_t count = mGrid.size(); diff --git a/openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc b/openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc index 54802cd36c..38bc5e1b25 100644 --- a/openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc +++ b/openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc @@ -108,7 +108,7 @@ GEO_VDBTranslator::fileStat(const char *filename, GA_Stat &stat, uint /*level*/) try { openvdb::io::File file(filename); - file.open(/*delayLoad*/false); + file.open(); int nprim = 0; UT_BoundingBox bbox; @@ -204,7 +204,7 @@ GEO_VDBTranslator::fileLoad(GEO_Detail *geogdp, UT_IStream &is, bool /*ate_magic try { // Create and open a VDB file, but don't read any grids yet. - openvdb::io::Stream file(*stdstream, /*delayLoad*/false); + openvdb::io::Stream file(*stdstream); // Read the file-level metadata into global attributes. openvdb::MetaMap::Ptr fileMetadata = file.getMetadata(); diff --git a/openvdb_houdini/openvdb_houdini/reference/GEO_PrimVDB.cc b/openvdb_houdini/openvdb_houdini/reference/GEO_PrimVDB.cc index 13df7c3fbf..76ef00eae8 100644 --- a/openvdb_houdini/openvdb_houdini/reference/GEO_PrimVDB.cc +++ b/openvdb_houdini/openvdb_houdini/reference/GEO_PrimVDB.cc @@ -3112,7 +3112,7 @@ GEO_PrimVDB::loadVDB(UT_JSONParser &p, bool as_shmem) try { SYS_SharedMemoryInputStream is_shm(*shmem); - openvdb::io::Stream vis(is_shm, /*delayLoad*/false); + openvdb::io::Stream vis(is_shm); openvdb::GridPtrVecPtr grids = vis.getGrids(); int count = (grids ? grids->size() : 0); @@ -3152,7 +3152,7 @@ GEO_PrimVDB::loadVDB(UT_JSONParser &p, bool as_shmem) { UT_JSONParser::TiledStream is(p); - openvdb::io::Stream vis(is, /*delayLoad*/false); + openvdb::io::Stream vis(is); openvdb::GridPtrVecPtr grids = vis.getGrids(); From 4c8551bc1ddc7e9e56a8d74d023aee54d0b7fd19 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 30 Jan 2026 14:16:16 -0800 Subject: [PATCH 04/70] Adapt all tooling to remove support for out-of-core data Signed-off-by: Dan Bailey --- openvdb/openvdb/Grid.h | 26 +-- openvdb/openvdb/io/Archive.cc | 20 --- openvdb/openvdb/io/Archive.h | 8 +- openvdb/openvdb/io/File.cc | 14 +- openvdb/openvdb/io/Stream.cc | 7 +- openvdb/openvdb/io/io.h | 3 - openvdb/openvdb/points/AttributeArray.cc | 3 - openvdb/openvdb/points/AttributeArray.h | 158 ++---------------- openvdb/openvdb/points/AttributeGroup.cc | 8 - openvdb/openvdb/points/AttributeSet.cc | 11 -- openvdb/openvdb/points/AttributeSet.h | 6 +- openvdb/openvdb/points/PointConversion.h | 44 ++++- openvdb/openvdb/points/PointCount.h | 19 ++- openvdb/openvdb/points/PointDataGrid.h | 79 +-------- openvdb/openvdb/points/StreamCompression.cc | 20 --- openvdb/openvdb/points/StreamCompression.h | 12 +- .../openvdb/points/impl/PointConversionImpl.h | 86 ++++++---- openvdb/openvdb/points/impl/PointCountImpl.h | 31 +++- openvdb/openvdb/points/impl/PointMoveImpl.h | 4 - .../points/impl/PointRasterizeFrustumImpl.h | 2 +- .../impl/PrincipalComponentAnalysisImpl.h | 3 - openvdb/openvdb/tools/Count.h | 35 +--- openvdb/openvdb/tools/Merge.h | 8 +- openvdb/openvdb/tools/PointIndexGrid.h | 10 +- openvdb/openvdb/tools/VolumeToMesh.h | 4 +- openvdb/openvdb/tree/LeafBuffer.h | 79 ++------- openvdb/openvdb/tree/LeafNode.h | 26 +-- openvdb/openvdb/tree/LeafNodeBool.h | 12 +- openvdb/openvdb/tree/LeafNodeMask.h | 12 +- openvdb/openvdb/tree/Tree.h | 37 +--- openvdb/openvdb/unittest/TestCount.cc | 9 +- openvdb/openvdb/unittest/TestMeshToVolume.cc | 4 +- openvdb/openvdb/unittest/TestPointCount.cc | 15 +- .../unittest/TestPointRasterizeFrustum.cc | 1 - .../openvdb_ax/compiler/PointExecutable.cc | 4 +- openvdb_cmd/vdb_ax/main.cc | 4 +- openvdb_cmd/vdb_print/main.cc | 7 +- openvdb_cmd/vdb_tool/include/Tool.h | 2 +- .../openvdb_houdini/GR_PrimVDBPoints.cc | 27 +-- openvdb_houdini/openvdb_houdini/PointUtils.cc | 82 +++------ openvdb_houdini/openvdb_houdini/PointUtils.h | 4 +- .../SOP_OpenVDB_Points_Convert.cc | 13 -- .../SOP_OpenVDB_Rasterize_Frustum.cc | 68 +------- .../openvdb_houdini/SOP_OpenVDB_Read.cc | 57 ++----- .../openvdb_houdini/VRAY_OpenVDB_Points.cc | 16 +- 45 files changed, 275 insertions(+), 825 deletions(-) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index cb0a48e4e2..ecb18f9b66 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -457,12 +457,10 @@ class OPENVDB_API GridBase: public MetaMap virtual void readBuffers(std::istream&) = 0; /// Read all of this grid's data buffers that intersect the given index-space bounding box. virtual void readBuffers(std::istream&, const CoordBBox&) = 0; - /// @brief Read all of this grid's data buffers that are not yet resident in memory - /// (because delayed loading is in effect). - /// @details If this grid was read from a memory-mapped file, this operation - /// disconnects the grid from the file. - /// @sa io::File::open, io::MappedFile + + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void readNonresidentBuffers() const = 0; + /// Write out all data buffers for this grid. virtual void writeBuffers(std::ostream&) const = 0; @@ -939,12 +937,10 @@ class Grid: public GridBase void readBuffers(std::istream&) override; /// Read all of this grid's data buffers that intersect the given index-space bounding box. void readBuffers(std::istream&, const CoordBBox&) override; - /// @brief Read all of this grid's data buffers that are not yet resident in memory - /// (because delayed loading is in effect). - /// @details If this grid was read from a memory-mapped file, this operation - /// disconnects the grid from the file. - /// @sa io::File::open, io::MappedFile - void readNonresidentBuffers() const override; + + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void readNonresidentBuffers() const override { } + /// Write out all data buffers for this grid. void writeBuffers(std::ostream&) const override; @@ -1670,14 +1666,6 @@ Grid::readBuffers(std::istream& is, const CoordBBox& bbox) } -template -inline void -Grid::readNonresidentBuffers() const -{ - tree().readNonresidentBuffers(); -} - - template inline void Grid::writeBuffers(std::ostream& os) const diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 0e48fdae04..7ddd4f48b8 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -244,7 +244,6 @@ bool StreamMetadata::seekable() const { return mImpl->mSeekabl bool StreamMetadata::delayedLoadMeta() const { return mImpl->mDelayedLoadMeta; } bool StreamMetadata::countingPasses() const { return mImpl->mCountingPasses; } uint32_t StreamMetadata::pass() const { return mImpl->mPass; } -uint64_t StreamMetadata::leaf() const { return mImpl->mLeaf; } MetaMap& StreamMetadata::gridMetadata() { return mImpl->mGridMetadata; } const MetaMap& StreamMetadata::gridMetadata() const { return mImpl->mGridMetadata; } uint32_t StreamMetadata::__test() const { return mImpl->mTest; } @@ -262,7 +261,6 @@ void StreamMetadata::setWriteGridStats(bool b) { mImpl->mWriteGridStats void StreamMetadata::setSeekable(bool b) { mImpl->mSeekable = b; } void StreamMetadata::setCountingPasses(bool b) { mImpl->mCountingPasses = b; } void StreamMetadata::setPass(uint32_t i) { mImpl->mPass = i; } -void StreamMetadata::setLeaf(uint64_t i) { mImpl->mLeaf = i; } void StreamMetadata::__setTest(uint32_t t) { mImpl->mTest = t; } std::string @@ -888,15 +886,6 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co //////////////////////////////////////// -//static -bool -Archive::isDelayedLoadingEnabled() -{ - return false; -} - - - namespace { struct NoBBox {}; @@ -941,12 +930,6 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const grid->readMeta(is); - // Add a description of the compression settings to the grid as metadata. - /// @todo Would this be useful? - //const uint32_t c = getDataCompression(is); - //grid->insertMeta(GridBase::META_FILE_COMPRESSION, - // StringMetadata(compressionToString(c))); - // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); @@ -956,9 +939,6 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const const GridClass gridClass = grid->getGridClass(); io::setGridClass(is, gridClass); - // reset leaf value to zero - streamMetadata->setLeaf(0); - grid->readTransform(is); if (!gd.isInstance()) { grid->readTopology(is); diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index 4e4e680273..87f31b14eb 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -91,12 +91,8 @@ class OPENVDB_API Archive /// @brief Write the grids in the given container to this archive's output stream. virtual void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const {} - /// @brief Return @c true if delayed loading is enabled. - /// @details If enabled, delayed loading can be disabled for individual files, - /// but not vice-versa. - /// @note Define the environment variable @c OPENVDB_DISABLE_DELAYED_LOAD - /// to disable delayed loading unconditionally. - static bool isDelayedLoadingEnabled(); + /// @brief Return @c false (delayed loading has been removed). + static bool isDelayedLoadingEnabled() { return false; } protected: /// @brief Return @c true if the input stream contains grid offsets diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 218dcbb7e9..4cbda52eb6 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -28,8 +28,6 @@ namespace io { // Implementation details of the File class struct File::Impl { - enum { DEFAULT_COPY_MAX_BYTES = 500000000 }; // 500 MB - struct NoBBox {}; // Common implementation of the various File::readGrid() overloads, @@ -64,16 +62,6 @@ struct File::Impl file.Archive::readGrid(grid, gd, file.inputStream(), worldBBox); } - static Index64 getDefaultCopyMaxBytes() - { - Index64 result = DEFAULT_COPY_MAX_BYTES; - if (const char* s = std::getenv("OPENVDB_DELAYED_LOAD_COPY_MAX_BYTES")) { - char* endptr = nullptr; - result = std::strtoul(s, &endptr, /*base=*/10); - } - return result; - } - std::string mFilename; // The file-level metadata MetaMap::Ptr mMeta; @@ -247,7 +235,7 @@ File::open() } mImpl->mInStream.reset(); - // Open the file. + // Open the file using standard I/O (delayed loading has been removed) std::unique_ptr newStream; newStream.reset(new std::ifstream( filename().c_str(), std::ios_base::in | std::ios_base::binary)); diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index e3ff05c2ed..be3a003515 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -170,9 +170,7 @@ MetaMap::Ptr Stream::getMetadata() const { MetaMap::Ptr result; - if (mImpl->mFile) { - result = mImpl->mFile->getMetadata(); - } else if (mImpl->mMeta) { + if (mImpl->mMeta) { // Return a deep copy of the file-level metadata // that was read when this object was constructed. result.reset(new MetaMap(*mImpl->mMeta)); @@ -184,9 +182,6 @@ Stream::getMetadata() const GridPtrVecPtr Stream::getGrids() { - if (mImpl->mFile) { - return mImpl->mFile->getGrids(); - } return mImpl->mGrids; } diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index cd1c460aaf..f2ac342ff7 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -76,9 +76,6 @@ class OPENVDB_API StreamMetadata uint32_t pass() const; void setPass(uint32_t); - uint64_t leaf() const; - void setLeaf(uint64_t); - //@{ /// @brief Return a (reference to a) copy of the metadata of the grid /// currently being read or written. diff --git a/openvdb/openvdb/points/AttributeArray.cc b/openvdb/openvdb/points/AttributeArray.cc index a24fc5a2bd..7d83051b52 100644 --- a/openvdb/openvdb/points/AttributeArray.cc +++ b/openvdb/openvdb/points/AttributeArray.cc @@ -190,9 +190,6 @@ AttributeArray::setConstantStride(bool state) bool AttributeArray::operator==(const AttributeArray& other) const { - this->loadData(); - other.loadData(); - if (this->mUsePagedRead != other.mUsePagedRead || this->mFlags != other.mFlags) return false; return this->isEqual(other); diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index 46d0b02318..bbfba7b156 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -130,7 +130,7 @@ class OPENVDB_API AttributeArray template friend class AttributeHandle; - AttributeArray(): mPageHandle() { mOutOfCore = 0; } + AttributeArray(): mPageHandle() { } virtual ~AttributeArray() { // if this AttributeArray has been partially read, zero the compressed bytes, @@ -189,10 +189,7 @@ class OPENVDB_API AttributeArray /// Return the number of bytes of memory used by this attribute. virtual size_t memUsage() const = 0; - /// Return the number of bytes of memory used by this attribute array once it - /// has been deserialized (this may be different to memUsage() if delay-loading - /// is in use). Note that this method does NOT consider the fact that a - /// uniform attribute could be expanded and only deals with delay-loading. + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual size_t memUsageIfLoaded() const = 0; /// Create a new attribute array of the given (registered) type, length and stride. @@ -315,10 +312,10 @@ class OPENVDB_API AttributeArray /// @param outputTransient if true, write out transient attributes virtual void writePagedBuffers(compression::PagedOutputStream&, bool outputTransient) const = 0; - /// Ensures all data is in-core + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void loadData() const = 0; - /// Return @c true if all data has been loaded + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual bool isDataLoaded() const = 0; /// Check the compressed bytes and flags. If they are equal, perform a deeper @@ -366,7 +363,7 @@ class OPENVDB_API AttributeArray mutable tbb::spin_mutex mMutex; uint8_t mFlags = 0; uint8_t mUsePagedRead = 0; - std::atomic mOutOfCore; // interpreted as bool + std::atomic mOutOfCore{0}; // interpreted as bool /// used for out-of-core, paged reading union { compression::PageHandle::Ptr mPageHandle; @@ -611,11 +608,8 @@ class TypedAttributeArray final: public AttributeArray /// Return the number of bytes of memory used by this attribute. size_t memUsage() const override; - /// Return the number of bytes of memory used by this attribute array once it - /// has been deserialized (this may be different to memUsage() if delay-loading - /// is in use). Note that this method does NOT consider the fact that a - /// uniform attribute could be expanded and only deals with delay-loading. - size_t memUsageIfLoaded() const override; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + size_t memUsageIfLoaded() const override { return memUsage(); } /// Return the value at index @a n (assumes in-core) ValueType getUnsafe(Index n) const; @@ -696,14 +690,14 @@ class TypedAttributeArray final: public AttributeArray /// @param outputTransient if true, write out transient attributes void writePagedBuffers(compression::PagedOutputStream& os, bool outputTransient) const override; - /// Return @c true if this buffer's values have not yet been read from disk. - inline bool isOutOfCore() const; + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool isOutOfCore() const { return false; } - /// Ensures all data is in-core - void loadData() const override; + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void loadData() const override { } - /// Return @c true if all data has been loaded - bool isDataLoaded() const override; + OPENVDB_DEPRECATED_MESSAGE("Always returns true. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool isDataLoaded() const override { return true; } /// Return the raw data buffer inline const StorageType* constData() const { return this->data(); } @@ -716,18 +710,13 @@ class TypedAttributeArray final: public AttributeArray inline const StorageType* data() const { OPENVDB_ASSERT(validData()); return mData.get(); } /// Verify that data is not out-of-core or in a partially-read state - inline bool validData() const { return !(isOutOfCore() || (flags() & PARTIALREAD)); } + inline bool validData() const { return !((flags() & PARTIALREAD)); } private: friend class ::TestAttributeArray; TypedAttributeArray(const TypedAttributeArray&, const tbb::spin_mutex::scoped_lock&); - /// Load data from memory-mapped file. - inline void doLoad() const; - /// Load data from memory-mapped file (unsafe as this function is not protected by a mutex). - inline void doLoadUnsafe() const; - /// Toggle out-of-core state inline void setOutOfCore(const bool); @@ -962,8 +951,6 @@ void AttributeArray::doCopyValues(const AttributeArray& sourceArray, const IterT { // ensure both arrays have float-float or integer-integer value types OPENVDB_ASSERT(sourceArray.valueTypeIsFloatingPoint() == this->valueTypeIsFloatingPoint()); - // ensure both arrays have been loaded from disk (if delay-loaded) - OPENVDB_ASSERT(sourceArray.isDataLoaded() && this->isDataLoaded()); // ensure storage size * stride matches on both arrays OPENVDB_ASSERT(this->storageTypeSize()*this->stride() == sourceArray.storageTypeSize()*sourceArray.stride()); @@ -1024,10 +1011,6 @@ void AttributeArray::copyValues(const AttributeArray& sourceArray, const IterT& OPENVDB_THROW(TypeError, "Cannot copy array data due to mis-match in storage type sizes."); } - // ensure both arrays have been loaded from disk - sourceArray.loadData(); - this->loadData(); - // if the target array is uniform, expand it first this->expand(); @@ -1206,8 +1189,6 @@ template size_t TypedAttributeArray::arrayMemUsage() const { - if (this->isOutOfCore()) return 0; - return (mIsUniform ? 1 : this->dataSize()) * sizeof(StorageType); } @@ -1232,11 +1213,6 @@ template void TypedAttributeArray::deallocate() { - // detach from file if delay-loaded - if (this->isOutOfCore()) { - this->setOutOfCore(false); - this->mPageHandle.reset(); - } if (mData) mData.reset(); } @@ -1304,14 +1280,6 @@ TypedAttributeArray::memUsage() const } -template -size_t -TypedAttributeArray::memUsageIfLoaded() const -{ - return sizeof(*this) + (mIsUniform ? 1 : this->dataSize()) * sizeof(StorageType); -} - - template typename TypedAttributeArray::ValueType TypedAttributeArray::getUnsafe(Index n) const @@ -1329,7 +1297,6 @@ typename TypedAttributeArray::ValueType TypedAttributeArray::get(Index n) const { if (n >= this->dataSize()) OPENVDB_THROW(IndexError, "Out-of-range access."); - if (this->isOutOfCore()) this->doLoad(); return this->getUnsafe(n); } @@ -1366,7 +1333,6 @@ void TypedAttributeArray::setUnsafe(Index n, const ValueType& val) { OPENVDB_ASSERT(n < this->dataSize()); - OPENVDB_ASSERT(!this->isOutOfCore()); OPENVDB_ASSERT(!this->isUniform()); // this unsafe method assumes the data is not uniform, however if it is, this redirects the index @@ -1381,7 +1347,6 @@ void TypedAttributeArray::set(Index n, const ValueType& val) { if (n >= this->dataSize()) OPENVDB_THROW(IndexError, "Out-of-range access."); - if (this->isOutOfCore()) this->doLoad(); if (this->isUniform()) this->expand(); this->setUnsafe(n, val); @@ -1486,12 +1451,6 @@ template void TypedAttributeArray::fill(const ValueType& value) { - if (this->isOutOfCore()) { - tbb::spin_mutex::scoped_lock lock(mMutex); - this->deallocate(); - this->allocate(); - } - const Index size = mIsUniform ? 1 : this->dataSize(); for (Index i = 0; i < size; ++i) { Codec::encode(value, this->data()[i]); @@ -1507,54 +1466,6 @@ TypedAttributeArray::fill(AttributeArray* array, const Value } -template -bool -TypedAttributeArray::isOutOfCore() const -{ - return mOutOfCore; -} - - -template -void -TypedAttributeArray::setOutOfCore(const bool b) -{ - mOutOfCore = b; -} - - -template -void -TypedAttributeArray::doLoad() const -{ - if (!(this->isOutOfCore())) return; - - TypedAttributeArray* self = - const_cast*>(this); - - // This lock will be contended at most once, after which this buffer - // will no longer be out-of-core. - tbb::spin_mutex::scoped_lock lock(self->mMutex); - this->doLoadUnsafe(); -} - - -template -void -TypedAttributeArray::loadData() const -{ - this->doLoad(); -} - - -template -bool -TypedAttributeArray::isDataLoaded() const -{ - return !this->isOutOfCore(); -} - - template void TypedAttributeArray::read(std::istream& is) @@ -1679,9 +1590,7 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu this->deallocate(); - this->setOutOfCore(false); is.read(mPageHandle, std::streamsize(mPageHandle->size()), false); - std::unique_ptr buffer = mPageHandle->read(); mData.reset(reinterpret_cast(buffer.release())); mPageHandle.reset(); @@ -1727,9 +1636,6 @@ TypedAttributeArray::writeMetadata(std::ostream& os, bool ou bool bloscCompression = io::getDataCompression(os) & io::COMPRESS_BLOSC; - // any compressed data needs to be loaded if out-of-core - if (bloscCompression) this->doLoad(); - size_t compressedBytes = 0; if (!strideOfOne) @@ -1778,8 +1684,6 @@ TypedAttributeArray::writeBuffers(std::ostream& os, bool out OPENVDB_THROW(IoError, "Cannot write out a partially-read AttributeArray."); } - this->doLoad(); - if (this->isUniform()) { os.write(reinterpret_cast(this->data()), sizeof(StorageType)); } @@ -1827,37 +1731,10 @@ TypedAttributeArray::writePagedBuffers(compression::PagedOut OPENVDB_THROW(IoError, "Cannot write out a partially-read AttributeArray."); } - this->doLoad(); - os.write(reinterpret_cast(this->data()), this->arrayMemUsage()); } -template -void -TypedAttributeArray::doLoadUnsafe() const -{ - if (!(this->isOutOfCore())) return; - - // this function expects the mutex to already be locked - - auto* self = const_cast*>(this); - - OPENVDB_ASSERT(self->mPageHandle); - OPENVDB_ASSERT(!(self->mFlags & PARTIALREAD)); - - std::unique_ptr buffer = self->mPageHandle->read(); - - self->mData.reset(reinterpret_cast(buffer.release())); - - self->mPageHandle.reset(); - - // clear all write and out-of-core flags - - self->mOutOfCore = false; -} - - template AttributeArray::AccessorBasePtr TypedAttributeArray::getAccessor() const @@ -1884,9 +1761,6 @@ TypedAttributeArray::isEqual(const AttributeArray& other) co this->mIsUniform != otherT->mIsUniform || this->attributeType() != this->attributeType()) return false; - this->doLoad(); - otherT->doLoad(); - const StorageType *target = this->data(), *source = otherT->data(); if (!target && !source) return true; if (!target || !source) return false; @@ -1978,10 +1852,6 @@ AttributeHandle::AttributeHandle(const AttributeArray& arr OPENVDB_THROW(TypeError, "Cannot bind handle due to incompatible type of AttributeArray."); } - // load data if delay-loaded - - mArray->loadData(); - // bind getter and setter methods AttributeArray::AccessorBasePtr accessor = mArray->getAccessor(); diff --git a/openvdb/openvdb/points/AttributeGroup.cc b/openvdb/openvdb/points/AttributeGroup.cc index 4f022bae7d..431a5ac73a 100644 --- a/openvdb/openvdb/points/AttributeGroup.cc +++ b/openvdb/openvdb/points/AttributeGroup.cc @@ -23,10 +23,6 @@ GroupHandle::GroupHandle(const GroupAttributeArray& array, const GroupType& offs , mBitMask(static_cast(1 << offset)) { OPENVDB_ASSERT(isGroup(mArray)); - - // load data if delay-loaded - - mArray.loadData(); } @@ -36,10 +32,6 @@ GroupHandle::GroupHandle(const GroupAttributeArray& array, const GroupType& bitM , mBitMask(bitMask) { OPENVDB_ASSERT(isGroup(mArray)); - - // load data if delay-loaded - - mArray.loadData(); } diff --git a/openvdb/openvdb/points/AttributeSet.cc b/openvdb/openvdb/points/AttributeSet.cc index 1eebe436ff..71cc065151 100644 --- a/openvdb/openvdb/points/AttributeSet.cc +++ b/openvdb/openvdb/points/AttributeSet.cc @@ -140,17 +140,6 @@ AttributeSet::memUsage() const } -size_t -AttributeSet::memUsageIfLoaded() const -{ - size_t bytes = sizeof(*this) + mDescr->memUsage(); - for (const auto& attr : mAttrs) { - bytes += attr->memUsageIfLoaded(); - } - return bytes; -} - - size_t AttributeSet::find(const std::string& name) const { diff --git a/openvdb/openvdb/points/AttributeSet.h b/openvdb/openvdb/points/AttributeSet.h index 3137eb7413..84cbeb3eaa 100644 --- a/openvdb/openvdb/points/AttributeSet.h +++ b/openvdb/openvdb/points/AttributeSet.h @@ -114,10 +114,8 @@ class OPENVDB_API AttributeSet /// Return the number of bytes of memory used by this attribute set. size_t memUsage() const; - /// Return the number of bytes of memory used by this attribute set once it - /// has been deserialized (this may be different to memUsage() if delay-loading - /// is in use). - size_t memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + size_t memUsageIfLoaded() const { return memUsage(); } /// @brief Return the position of the attribute array whose name is @a name, /// or @c INVALID_POS if no match is found. diff --git a/openvdb/openvdb/points/PointConversion.h b/openvdb/openvdb/points/PointConversion.h index dbdfd97b6d..a77df2f267 100644 --- a/openvdb/openvdb/points/PointConversion.h +++ b/openvdb/openvdb/points/PointConversion.h @@ -134,7 +134,6 @@ populateAttribute( PointDataTreeT& tree, /// @param pointOffsets a vector of cumulative point offsets for each leaf /// @param startOffset a value to shift all the point offsets by /// @param filter an index filter -/// @param inCoreOnly true if out-of-core leaf nodes are to be ignored /// template @@ -143,8 +142,17 @@ convertPointDataGridPosition( PositionAttribute& positionAttribute, const PointDataGridT& grid, const std::vector& pointOffsets, const Index64 startOffset, - const FilterT& filter = NullFilter(), - const bool inCoreOnly = false); + const FilterT& filter = NullFilter()); + +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridPosition() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +inline void +convertPointDataGridPosition( PositionAttribute& positionAttribute, + const PointDataGridT& grid, + const std::vector& pointOffsets, + const Index64 startOffset, + const FilterT& filter, + const bool /*inCoreOnly*/); /// @brief Convert the attribute from a PointDataGrid @@ -156,7 +164,6 @@ convertPointDataGridPosition( PositionAttribute& positionAttribute, /// @param arrayIndex the index in the Descriptor of the array to be converted. /// @param stride the stride of the attribute /// @param filter an index filter -/// @param inCoreOnly true if out-of-core leaf nodes are to be ignored template inline void convertPointDataGridAttribute( TypedAttribute& attribute, @@ -165,8 +172,19 @@ convertPointDataGridAttribute( TypedAttribute& attribute, const Index64 startOffset, const unsigned arrayIndex, const Index stride = 1, - const FilterT& filter = NullFilter(), - const bool inCoreOnly = false); + const FilterT& filter = NullFilter()); + +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridAttribute() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +inline void +convertPointDataGridAttribute( TypedAttribute& attribute, + const PointDataTreeT& tree, + const std::vector& pointOffsets, + const Index64 startOffset, + const unsigned arrayIndex, + const Index stride, + const FilterT& filter, + const bool /*inCoreOnly*/); /// @brief Convert the group from a PointDataGrid @@ -177,18 +195,26 @@ convertPointDataGridAttribute( TypedAttribute& attribute, /// @param startOffset a value to shift all the point offsets by /// @param index the group index to be converted. /// @param filter an index filter -/// @param inCoreOnly true if out-of-core leaf nodes are to be ignored /// +template +inline void +convertPointDataGridGroup( Group& group, + const PointDataTreeT& tree, + const std::vector& pointOffsets, + const Index64 startOffset, + const AttributeSet::Descriptor::GroupIndex index, + const FilterT& filter = NullFilter()); template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridGroup() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") inline void convertPointDataGridGroup( Group& group, const PointDataTreeT& tree, const std::vector& pointOffsets, const Index64 startOffset, const AttributeSet::Descriptor::GroupIndex index, - const FilterT& filter = NullFilter(), - const bool inCoreOnly = false); + const FilterT& filter, + const bool /*inCoreOnly*/); // for internal use only - this traits class extracts T::value_type if defined, // otherwise falls back to using Vec3R diff --git a/openvdb/openvdb/points/PointCount.h b/openvdb/openvdb/points/PointCount.h index d3fef689db..32ae569b5a 100644 --- a/openvdb/openvdb/points/PointCount.h +++ b/openvdb/openvdb/points/PointCount.h @@ -28,28 +28,39 @@ namespace points { /// @brief Count the total number of points in a PointDataTree /// @param tree the PointDataTree in which to count the points /// @param filter an optional index filter -/// @param inCoreOnly if true, points in out-of-core leaf nodes are not counted /// @param threaded enable or disable threading (threading is enabled by default) template inline Index64 pointCount( const PointDataTreeT& tree, const FilterT& filter = NullFilter(), - const bool inCoreOnly = false, const bool threaded = true); +template +OPENVDB_DEPRECATED_MESSAGE("Use pointCount() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +inline Index64 pointCount( const PointDataTreeT& tree, + const FilterT& filter, + const bool /*inCoreOnly*/, + const bool threaded); + /// @brief Populate an array of cumulative point offsets per leaf node. /// @param pointOffsets array of offsets to be populated /// @param tree the PointDataTree from which to populate the offsets /// @param filter an optional index filter -/// @param inCoreOnly if true, points in out-of-core leaf nodes are ignored /// @param threaded enable or disable threading (threading is enabled by default) /// @return The final cumulative point offset. template inline Index64 pointOffsets(std::vector& pointOffsets, const PointDataTreeT& tree, const FilterT& filter = NullFilter(), - const bool inCoreOnly = false, const bool threaded = true); +template +OPENVDB_DEPRECATED_MESSAGE("Use pointOffsets() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +inline Index64 pointOffsets(std::vector& pointOffsets, + const PointDataTreeT& tree, + const FilterT& filter, + const bool /*inCoreOnly*/, + const bool threaded); + /// @brief Generate a new grid with voxel values to store the number of points per voxel /// @param grid the PointDataGrid to use to compute the count grid /// @param filter an optional index filter diff --git a/openvdb/openvdb/points/PointDataGrid.h b/openvdb/openvdb/points/PointDataGrid.h index f3f72eab56..fcb10bdb2a 100644 --- a/openvdb/openvdb/points/PointDataGrid.h +++ b/openvdb/openvdb/points/PointDataGrid.h @@ -218,19 +218,15 @@ makeDescriptorUnique(PointDataTreeT& tree); /// /// @note Multiple threads cannot safely access the same AttributeArray when using streaming. template +OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") inline void -setStreamingMode(PointDataTreeT& tree, bool on = true); +setStreamingMode(PointDataTreeT&, bool /*on*/ = true) { } -/// @brief Sequentially pre-fetch all delayed-load voxel and attribute data from disk in order -/// to accelerate subsequent random access. -/// -/// @param tree the PointDataTree. -/// @param position if enabled, prefetch the position attribute (default is on) -/// @param otherAttributes if enabled, prefetch all other attributes (default is on) template +OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") inline void -prefetch(PointDataTreeT& tree, bool position = true, bool otherAttributes = true); +prefetch(PointDataTreeT&, bool /*position*/ = true, bool /*otherAttributes*/ = true) { } //////////////////////////////////////// @@ -504,7 +500,8 @@ class PointDataLeafNode : public tree::LeafNode, io::MultiPass { Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } void evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels = true) const; @@ -1522,13 +1519,6 @@ PointDataLeafNode::memUsage() const return BaseLeaf::memUsage() + mAttributeSet->memUsage(); } -template -inline Index64 -PointDataLeafNode::memUsageIfLoaded() const -{ - return BaseLeaf::memUsageIfLoaded() + mAttributeSet->memUsageIfLoaded(); -} - template inline void PointDataLeafNode::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const @@ -1598,63 +1588,6 @@ makeDescriptorUnique(PointDataTreeT& tree) } -template -inline void -setStreamingMode(PointDataTreeT& tree, bool on) -{ - auto leafIter = tree.beginLeaf(); - for (; leafIter; ++leafIter) { - for (size_t i = 0; i < leafIter->attributeSet().size(); i++) { - leafIter->attributeArray(i).setStreaming(on); - } - } -} - - -template -inline void -prefetch(PointDataTreeT& tree, bool position, bool otherAttributes) -{ - // NOTE: the following is intentionally not multi-threaded, as the I/O - // is faster if done in the order in which it is stored in the file - - auto leaf = tree.cbeginLeaf(); - if (!leaf) return; - - const auto& attributeSet = leaf->attributeSet(); - - // pre-fetch leaf data - - for ( ; leaf; ++leaf) { - leaf->buffer().data(); - } - - // pre-fetch position attribute data (position will typically have index 0) - - size_t positionIndex = attributeSet.find("P"); - - if (position && positionIndex != AttributeSet::INVALID_POS) { - for (leaf = tree.cbeginLeaf(); leaf; ++leaf) { - OPENVDB_ASSERT(leaf->hasAttribute(positionIndex)); - leaf->constAttributeArray(positionIndex).loadData(); - } - } - - // pre-fetch other attribute data - - if (otherAttributes) { - const size_t attributes = attributeSet.size(); - for (size_t attributeIndex = 0; attributeIndex < attributes; attributeIndex++) { - if (attributeIndex == positionIndex) continue; - for (leaf = tree.cbeginLeaf(); leaf; ++leaf) { - OPENVDB_ASSERT(leaf->hasAttribute(attributeIndex)); - leaf->constAttributeArray(attributeIndex).loadData(); - } - } - } -} - - namespace internal { /// @brief Global registration of point data-related types diff --git a/openvdb/openvdb/points/StreamCompression.cc b/openvdb/openvdb/points/StreamCompression.cc index fde60d9f93..5760f3b4ae 100644 --- a/openvdb/openvdb/points/StreamCompression.cc +++ b/openvdb/openvdb/points/StreamCompression.cc @@ -279,13 +279,6 @@ bloscDecompress(const char*, const size_t, const bool) //////////////////////////////////////// -void -Page::load() const -{ - this->doLoad(); -} - - long Page::uncompressedBytes() const { @@ -345,13 +338,6 @@ Page::readBuffers(std::istream&is, bool delayed) } -bool -Page::isOutOfCore() const -{ - return false; -} - - void Page::copy(const std::unique_ptr& temp, int pageSize) { @@ -374,12 +360,6 @@ Page::decompress(const std::unique_ptr& temp) } -void -Page::doLoad() const -{ -} - - //////////////////////////////////////// diff --git a/openvdb/openvdb/points/StreamCompression.h b/openvdb/openvdb/points/StreamCompression.h index d63e9c1116..d983ea2404 100644 --- a/openvdb/openvdb/points/StreamCompression.h +++ b/openvdb/openvdb/points/StreamCompression.h @@ -114,7 +114,6 @@ class OPENVDB_API Page private: struct Info { - SharedPtr meta; std::streamoff filepos; long compressedBytes; long uncompressedBytes; @@ -125,8 +124,8 @@ class OPENVDB_API Page Page() = default; - /// @brief load the Page into memory - void load() const; + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") + void load() const { } /// @brief Uncompressed bytes of the Paged data, available /// when the header has been read. @@ -143,8 +142,8 @@ class OPENVDB_API Page /// pointers will be stored to load the data lazily. void readBuffers(std::istream&, bool delayed); - /// @brief Test if the data is out-of-core - bool isOutOfCore() const; + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool isOutOfCore() const { return false; } private: /// @brief Convenience method to store a copy of the supplied buffer @@ -153,9 +152,6 @@ class OPENVDB_API Page /// @brief Decompress and store the supplied data void decompress(const std::unique_ptr& temp); - /// @brief Thread-safe loading of the data - void doLoad() const; - std::unique_ptr mInfo = std::unique_ptr(new Info); std::unique_ptr mData; tbb::spin_mutex mMutex; diff --git a/openvdb/openvdb/points/impl/PointConversionImpl.h b/openvdb/openvdb/points/impl/PointConversionImpl.h index 22fc09a5a0..bab3771610 100644 --- a/openvdb/openvdb/points/impl/PointConversionImpl.h +++ b/openvdb/openvdb/points/impl/PointConversionImpl.h @@ -133,15 +133,13 @@ struct ConvertPointDataGridPositionOp { const Index64 startOffset, const math::Transform& transform, const size_t index, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) : mAttribute(attribute) , mPointOffsets(pointOffsets) , mStartOffset(startOffset) , mTransform(transform) , mIndex(index) , mFilter(filter) - , mInCoreOnly(inCoreOnly) { // only accept Vec3f as ValueType static_assert(VecTraits::Size == 3 && @@ -169,8 +167,6 @@ struct ConvertPointDataGridPositionOp { OPENVDB_ASSERT(leaf.pos() < mPointOffsets.size()); - if (mInCoreOnly && leaf->buffer().isOutOfCore()) continue; - Index64 offset = mStartOffset; if (leaf.pos() > 0) offset += mPointOffsets[leaf.pos() - 1]; @@ -196,7 +192,6 @@ struct ConvertPointDataGridPositionOp { const math::Transform& mTransform; const size_t mIndex; const FilterT& mFilter; - const bool mInCoreOnly; }; // ConvertPointDataGridPositionOp @@ -215,15 +210,13 @@ struct ConvertPointDataGridAttributeOp { const Index64 startOffset, const size_t index, const Index stride, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) : mAttribute(attribute) , mPointOffsets(pointOffsets) , mStartOffset(startOffset) , mIndex(index) , mStride(stride) - , mFilter(filter) - , mInCoreOnly(inCoreOnly) { } + , mFilter(filter) { } template void convert(IterT& iter, HandleT& targetHandle, @@ -257,8 +250,6 @@ struct ConvertPointDataGridAttributeOp { OPENVDB_ASSERT(leaf.pos() < mPointOffsets.size()); - if (mInCoreOnly && leaf->buffer().isOutOfCore()) continue; - Index64 offset = mStartOffset; if (leaf.pos() > 0) offset += mPointOffsets[leaf.pos() - 1]; @@ -284,7 +275,6 @@ struct ConvertPointDataGridAttributeOp { const size_t mIndex; const Index mStride; const FilterT& mFilter; - const bool mInCoreOnly; }; // ConvertPointDataGridAttributeOp template @@ -299,14 +289,12 @@ struct ConvertPointDataGridGroupOp { const std::vector& pointOffsets, const Index64 startOffset, const AttributeSet::Descriptor::GroupIndex index, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) : mGroup(group) , mPointOffsets(pointOffsets) , mStartOffset(startOffset) , mIndex(index) - , mFilter(filter) - , mInCoreOnly(inCoreOnly) { } + , mFilter(filter) { } template void convert(IterT& iter, const GroupAttributeArray& groupArray, Index64& offset) const @@ -337,8 +325,6 @@ struct ConvertPointDataGridGroupOp { OPENVDB_ASSERT(leaf.pos() < mPointOffsets.size()); - if (mInCoreOnly && leaf->buffer().isOutOfCore()) continue; - Index64 offset = mStartOffset; if (leaf.pos() > 0) offset += mPointOffsets[leaf.pos() - 1]; @@ -365,7 +351,6 @@ struct ConvertPointDataGridGroupOp { const Index64 mStartOffset; const GroupIndex mIndex; const FilterT& mFilter; - const bool mInCoreOnly; }; // ConvertPointDataGridGroupOp template @@ -582,8 +567,7 @@ convertPointDataGridPosition( PositionAttribute& positionAttribute, const PointDataGridT& grid, const std::vector& pointOffsets, const Index64 startOffset, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) { using TreeType = typename PointDataGridT::TreeType; using LeafManagerT = typename tree::LeafManager; @@ -601,12 +585,26 @@ convertPointDataGridPosition( PositionAttribute& positionAttribute, LeafManagerT leafManager(tree); ConvertPointDataGridPositionOp convert( positionAttribute, pointOffsets, startOffset, grid.transform(), positionIndex, - filter, inCoreOnly); + filter); tbb::parallel_for(leafManager.leafRange(), convert); positionAttribute.compact(); } +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridPosition() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +void +convertPointDataGridPosition( PositionAttribute& positionAttribute, + const PointDataGridT& grid, + const std::vector& pointOffsets, + const Index64 startOffset, + const FilterT& filter, + const bool /*inCoreOnly*/) +{ + convertPointDataGridPosition(positionAttribute, grid, pointOffsets, startOffset, filter); +} + + //////////////////////////////////////// @@ -618,8 +616,7 @@ convertPointDataGridAttribute( TypedAttribute& attribute, const Index64 startOffset, const unsigned arrayIndex, const Index stride, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) { using LeafManagerT = typename tree::LeafManager; @@ -633,12 +630,28 @@ convertPointDataGridAttribute( TypedAttribute& attribute, LeafManagerT leafManager(tree); ConvertPointDataGridAttributeOp convert( attribute, pointOffsets, startOffset, arrayIndex, stride, - filter, inCoreOnly); + filter); tbb::parallel_for(leafManager.leafRange(), convert); attribute.compact(); } +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridAttribute() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +void +convertPointDataGridAttribute( TypedAttribute& attribute, + const PointDataTreeT& tree, + const std::vector& pointOffsets, + const Index64 startOffset, + const unsigned arrayIndex, + const Index stride, + const FilterT& filter, + const bool /*inCoreOnly*/) +{ + convertPointDataGridAttribute(attribute, tree, pointOffsets, startOffset, arrayIndex, stride, filter); +} + + //////////////////////////////////////// @@ -649,8 +662,7 @@ convertPointDataGridGroup( Group& group, const std::vector& pointOffsets, const Index64 startOffset, const AttributeSet::Descriptor::GroupIndex index, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) { using LeafManagerT= typename tree::LeafManager; @@ -662,7 +674,7 @@ convertPointDataGridGroup( Group& group, LeafManagerT leafManager(tree); ConvertPointDataGridGroupOp convert( group, pointOffsets, startOffset, index, - filter, inCoreOnly); + filter); tbb::parallel_for(leafManager.leafRange(), convert); // must call this after modifying point groups in parallel @@ -670,6 +682,22 @@ convertPointDataGridGroup( Group& group, group.finalize(); } + +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridGroup() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +void +convertPointDataGridGroup( Group& group, + const PointDataTreeT& tree, + const std::vector& pointOffsets, + const Index64 startOffset, + const AttributeSet::Descriptor::GroupIndex index, + const FilterT& filter, + const bool /*inCoreOnly*/) +{ + convertPointDataGridGroup(group, tree, pointOffsets, startOffset, index, filter); +} + + template inline float computeVoxelSize( const PositionWrapper& positions, diff --git a/openvdb/openvdb/points/impl/PointCountImpl.h b/openvdb/openvdb/points/impl/PointCountImpl.h index 29c998b9c8..9f913a5ce0 100644 --- a/openvdb/openvdb/points/impl/PointCountImpl.h +++ b/openvdb/openvdb/points/impl/PointCountImpl.h @@ -17,16 +17,14 @@ namespace points { template Index64 pointCount(const PointDataTreeT& tree, const FilterT& filter, - const bool inCoreOnly, const bool threaded) { using LeafManagerT = tree::LeafManager; using LeafRangeT = typename LeafManagerT::LeafRange; auto countLambda = - [&filter, &inCoreOnly] (const LeafRangeT& range, Index64 sum) -> Index64 { + [&filter] (const LeafRangeT& range, Index64 sum) -> Index64 { for (const auto& leaf : range) { - if (inCoreOnly && leaf.buffer().isOutOfCore()) continue; auto state = filter.state(leaf); if (state == index::ALL) { sum += leaf.pointCount(); @@ -48,11 +46,21 @@ Index64 pointCount(const PointDataTreeT& tree, } +template +OPENVDB_DEPRECATED_MESSAGE("Use pointCount() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +Index64 pointCount(const PointDataTreeT& tree, + const FilterT& filter, + const bool /*inCoreOnly*/, + const bool threaded) +{ + return pointCount(tree, filter, threaded); +} + + template Index64 pointOffsets( std::vector& pointOffsets, const PointDataTreeT& tree, const FilterT& filter, - const bool inCoreOnly, const bool threaded) { using LeafT = typename PointDataTreeT::LeafNodeType; @@ -67,8 +75,7 @@ Index64 pointOffsets( std::vector& pointOffsets, LeafManagerT leafManager(tree); leafManager.foreach( - [&pointOffsets, &filter, &inCoreOnly](const LeafT& leaf, size_t pos) { - if (inCoreOnly && leaf.buffer().isOutOfCore()) return; + [&pointOffsets, &filter](const LeafT& leaf, size_t pos) { auto state = filter.state(leaf); if (state == index::ALL) { pointOffsets[pos] = leaf.pointCount(); @@ -90,6 +97,18 @@ Index64 pointOffsets( std::vector& pointOffsets, } +template +OPENVDB_DEPRECATED_MESSAGE("Use pointOffsets() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +Index64 pointOffsets( std::vector& pointOffsets, + const PointDataTreeT& tree, + const FilterT& filter, + const bool /*inCoreOnly*/, + const bool threaded) +{ + return pointOffsets(pointOffsets, tree, filter, threaded); +} + + template typename GridT::Ptr pointCountGrid( const PointDataGridT& points, diff --git a/openvdb/openvdb/points/impl/PointMoveImpl.h b/openvdb/openvdb/points/impl/PointMoveImpl.h index 7d86a66d1f..87f2b35a4a 100644 --- a/openvdb/openvdb/points/impl/PointMoveImpl.h +++ b/openvdb/openvdb/points/impl/PointMoveImpl.h @@ -319,7 +319,6 @@ struct GlobalMovePointsOp // extract target array and ensure data is out-of-core and non-uniform auto& targetArray = leaf.attributeArray(mAttributeIndex); - targetArray.loadData(); targetArray.expand(); // perform the copy @@ -344,7 +343,6 @@ struct GlobalMovePointsOp const LeafT& sourceLeaf = mSourceLeafManager.leaf(sourceLeafIndex); const auto& sourceArray = sourceLeaf.constAttributeArray(mAttributeIndex); - sourceArray.loadData(); targetArray.copyValuesUnsafe(sourceArray, copyIterator); @@ -427,12 +425,10 @@ struct LocalMovePointsOp const Index sourceLeafOffset(mSourceIndices[idx]); LeafT& sourceLeaf = mSourceLeafManager.leaf(sourceLeafOffset); const auto& sourceArray = sourceLeaf.constAttributeArray(mAttributeIndex); - sourceArray.loadData(); // extract target array and ensure data is out-of-core and non-uniform auto& targetArray = leaf.attributeArray(mAttributeIndex); - targetArray.loadData(); targetArray.expand(); // perform the copy diff --git a/openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h b/openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h index 3578b44e83..88ee47f078 100644 --- a/openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h +++ b/openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h @@ -948,7 +948,7 @@ class GridToRasterize // generate leaf offsets (if necessary) if (mLeafOffsets.empty()) { openvdb::points::pointOffsets(mLeafOffsets, mGrid->constTree(), resolvedFilter, - /*inCoreOnly=*/false, mSettings.threaded); + mSettings.threaded); } // set streaming arbitrary attribute array flags diff --git a/openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h b/openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h index f2ce326efd..3b2a66e1c9 100644 --- a/openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h +++ b/openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h @@ -310,10 +310,7 @@ struct WeightPosSumsTransfer auto& leaf = this->mManager.leaf(idx); { - // @todo add API to get the array from the group handle. The handle - // calls loadData but not expand. auto& array = leaf.attributeArray(this->mIndices.mEllipsesGroupIndex.first); - array.loadData(); // so we can call setUnsafe/getUnsafe array.expand(); } diff --git a/openvdb/openvdb/tools/Count.h b/openvdb/openvdb/tools/Count.h index 03dc719b86..c1a3a9f9f9 100644 --- a/openvdb/openvdb/tools/Count.h +++ b/openvdb/openvdb/tools/Count.h @@ -60,19 +60,13 @@ Index64 countActiveTiles(const TreeT& tree, bool threaded = true); /// @brief Return the total amount of memory in bytes occupied by this tree. -/// @details This method returns the total in-core memory usage which can be -/// different to the maximum possible memory usage for trees which have not -/// been fully deserialized (via delay-loading). Thus, this is the current -/// true memory consumption. template Index64 memUsage(const TreeT& tree, bool threaded = true); -/// @brief Return the deserialized memory usage of this tree. This is not -/// necessarily equal to the current memory usage (returned by tools::memUsage) -/// if delay-loading is enabled. See File::open. template -Index64 memUsageIfLoaded(const TreeT& tree, bool threaded = true); +OPENVDB_DEPRECATED_MESSAGE("Use tools::memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +Index64 memUsageIfLoaded(const TreeT& tree, bool threaded = true) { return tools::memUsage(tree, threaded); } /// @brief Return the minimum and maximum active values in this tree. @@ -299,9 +293,9 @@ struct MemUsageOp using RootT = typename TreeType::RootNodeType; using LeafT = typename TreeType::LeafNodeType; - MemUsageOp(const bool inCoreOnly) : mInCoreOnly(inCoreOnly) {} - MemUsageOp(const MemUsageOp& other) : mCount(0), mInCoreOnly(other.mInCoreOnly) {} - MemUsageOp(const MemUsageOp& other, tbb::split) : MemUsageOp(other) {} + MemUsageOp() = default; + MemUsageOp(const MemUsageOp& other) = default; + MemUsageOp(const MemUsageOp&, tbb::split) : mCount(0) {} // accumulate size of the root node in bytes bool operator()(const RootT& root, size_t) @@ -323,8 +317,7 @@ struct MemUsageOp // accumulate size of leaf node in bytes bool operator()(const LeafT& leaf, size_t) { - if (mInCoreOnly) mCount += leaf.memUsage(); - else mCount += leaf.memUsageIfLoaded(); + mCount += leaf.memUsage(); return false; } @@ -334,7 +327,6 @@ struct MemUsageOp } openvdb::Index64 mCount{0}; - const bool mInCoreOnly; }; // struct MemUsageOp /// @brief A DynamicNodeManager operator to find the minimum and maximum active values in this tree. @@ -492,25 +484,12 @@ Index64 countActiveTiles(const TreeT& tree, bool threaded) template Index64 memUsage(const TreeT& tree, bool threaded) { - count_internal::MemUsageOp op(true); + count_internal::MemUsageOp op; tree::DynamicNodeManager nodeManager(tree); nodeManager.reduceTopDown(op, threaded); return op.mCount + sizeof(tree); } -template -Index64 memUsageIfLoaded(const TreeT& tree, bool threaded) -{ - /// @note For numeric (non-point) grids this really doesn't need to - /// traverse the tree and could instead be computed from the node counts. - /// We do so anyway as it ties this method into the tree data structure - /// which makes sure that changes to the tree/nodes are reflected/kept in - /// sync here. - count_internal::MemUsageOp op(false); - tree::DynamicNodeManager nodeManager(tree); - nodeManager.reduceTopDown(op, threaded); - return op.mCount + sizeof(tree); -} template math::MinMax minMax(const TreeT& tree, bool threaded) diff --git a/openvdb/openvdb/tools/Merge.h b/openvdb/openvdb/tools/Merge.h index a7afd4d055..54fbf927b4 100644 --- a/openvdb/openvdb/tools/Merge.h +++ b/openvdb/openvdb/tools/Merge.h @@ -581,16 +581,14 @@ struct UnallocatedBuffer static void allocateAndFill(BufferT& buffer, const ValueT& background) { if (buffer.empty()) { - if (!buffer.isOutOfCore()) { - buffer.allocate(); - buffer.fill(background); - } + buffer.allocate(); + buffer.fill(background); } } static bool isPartiallyConstructed(const BufferT& buffer) { - return !buffer.isOutOfCore() && buffer.empty(); + return buffer.empty(); } }; // struct AllocateAndFillBuffer diff --git a/openvdb/openvdb/tools/PointIndexGrid.h b/openvdb/openvdb/tools/PointIndexGrid.h index 3db213ea38..245b93ed5d 100644 --- a/openvdb/openvdb/tools/PointIndexGrid.h +++ b/openvdb/openvdb/tools/PointIndexGrid.h @@ -1491,7 +1491,8 @@ struct PointIndexLeafNode : public tree::LeafNode Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } //////////////////////////////////////// @@ -1798,13 +1799,6 @@ PointIndexLeafNode::memUsage() const return BaseLeaf::memUsage() + Index64((sizeof(T)*mIndices.capacity()) + sizeof(mIndices)); } -template -inline Index64 -PointIndexLeafNode::memUsageIfLoaded() const -{ - return BaseLeaf::memUsageIfLoaded() + Index64((sizeof(T)*mIndices.capacity()) + sizeof(mIndices)); -} - } // namespace tools diff --git a/openvdb/openvdb/tools/VolumeToMesh.h b/openvdb/openvdb/tools/VolumeToMesh.h index 509fc1187b..b3c268072a 100644 --- a/openvdb/openvdb/tools/VolumeToMesh.h +++ b/openvdb/openvdb/tools/VolumeToMesh.h @@ -616,8 +616,8 @@ template<> inline bool isInsideValue(bool value, bool /*isovalue*/) { return value; } -/// @brief Minor wrapper around the Leaf API to avoid atomic access with -/// delayed loading. +/// @brief Minor wrapper around the Leaf API. This was originally added to avoid +// atomic access with delayed loading, however delayed loading is no longer supported. template ::value> struct LeafBufferAccessor diff --git a/openvdb/openvdb/tree/LeafBuffer.h b/openvdb/openvdb/tree/LeafBuffer.h index 671781a129..80da87c721 100644 --- a/openvdb/openvdb/tree/LeafBuffer.h +++ b/openvdb/openvdb/tree/LeafBuffer.h @@ -37,27 +37,20 @@ class LeafBuffer static const Index SIZE = 1 << 3 * Log2Dim; /// Default constructor - inline LeafBuffer(): mData(new ValueType[SIZE]) - { - } + inline LeafBuffer(): mData(new ValueType[SIZE]) {} /// Construct a buffer populated with the specified value. explicit inline LeafBuffer(const ValueType&); /// Copy constructor inline LeafBuffer(const LeafBuffer&); /// Construct a buffer but don't allocate memory for the full array of values. - LeafBuffer(PartialCreate, const ValueType&): mData(nullptr) - { - } + LeafBuffer(PartialCreate, const ValueType&): mData(nullptr) {} /// Destructor inline ~LeafBuffer(); - /// Return @c true if this buffer's values have not yet been read from disk. - bool isOutOfCore() const - { - return false; - } + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool isOutOfCore() const { return false; } /// Return @c true if memory for this buffer has not yet been allocated. - bool empty() const { return !mData || this->isOutOfCore(); } + bool empty() const { return !mData; } /// Allocate memory for this buffer if it has not already been allocated. bool allocate() { if (mData == nullptr) mData = new ValueType[SIZE]; return true; } @@ -86,7 +79,8 @@ class LeafBuffer /// Return the memory footprint of this buffer in bytes. inline Index memUsage() const; - inline Index memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + inline Index memUsageIfLoaded() const { return memUsage(); } /// Return the number of values contained in this buffer. static Index size() { return SIZE; } @@ -112,24 +106,10 @@ class LeafBuffer bool deallocate(); - inline void setOutOfCore(bool b) - { - (void) b; - } - // To facilitate inlining in the common case in which the buffer is in-core, - // the loading logic is split into a separate function, doLoad(). - inline void loadValues() const - { - } - inline void doLoad() const; - inline bool detachFromFile(); - - using FlagsType = std::atomic; - - ValueType* mData; - FlagsType mOutOfCore; // interpreted as bool; extra bits reserved for future use - tbb::spin_mutex mMutex; // 1 byte - //int8_t mReserved[3]; // padding for alignment + ValueType* mData = nullptr; + // Deprecated members kept for ABI compatibility + std::atomic mDeprecatedAtomic{0}; + tbb::spin_mutex mDeprecatedSpinMutex; friend class ::TestLeaf; // Allow the parent LeafNode to access this buffer's data pointer. @@ -177,7 +157,6 @@ inline void LeafBuffer::setValue(Index i, const ValueType& val) { OPENVDB_ASSERT(i < SIZE); - this->loadValues(); if (mData) mData[i] = val; } @@ -198,11 +177,11 @@ LeafBuffer::operator=(const LeafBuffer& other) return *this; } + template inline void LeafBuffer::fill(const ValueType& val) { - this->detachFromFile(); if (mData != nullptr) { ValueType* target = mData; Index n = SIZE; @@ -215,8 +194,6 @@ template inline bool LeafBuffer::operator==(const LeafBuffer& other) const { - this->loadValues(); - other.loadValues(); const ValueType *target = mData, *source = other.mData; if (!target && !source) return true; if (!target || !source) return false; @@ -244,21 +221,10 @@ LeafBuffer::memUsage() const } -template -inline Index -LeafBuffer::memUsageIfLoaded() const -{ - size_t n = sizeof(*this); - n += SIZE * sizeof(ValueType); - return static_cast(n); -} - - template inline const typename LeafBuffer::ValueType* LeafBuffer::data() const { - this->loadValues(); if (mData == nullptr) { LeafBuffer* self = const_cast(this); if (mData == nullptr) self->mData = new ValueType[SIZE]; @@ -270,7 +236,6 @@ template inline typename LeafBuffer::ValueType* LeafBuffer::data() { - this->loadValues(); if (mData == nullptr) { if (mData == nullptr) mData = new ValueType[SIZE]; } @@ -284,7 +249,6 @@ LeafBuffer::at(Index i) const { static const ValueType sZero = zeroVal(); OPENVDB_ASSERT(i < SIZE); - this->loadValues(); // We can't use the ternary operator here, otherwise Visual C++ returns // a reference to a temporary. if (mData) return mData[i]; else return sZero; @@ -295,7 +259,6 @@ template inline bool LeafBuffer::deallocate() { - if (mData != nullptr) { delete[] mData; mData = nullptr; @@ -305,21 +268,6 @@ LeafBuffer::deallocate() } -template -inline void -LeafBuffer::doLoad() const -{ -} - - -template -inline bool -LeafBuffer::detachFromFile() -{ - return false; -} - - //////////////////////////////////////// @@ -364,7 +312,8 @@ class LeafBuffer void swap(LeafBuffer& other) { if (&other != this) std::swap(mData, other.mData); } Index memUsage() const { return sizeof(*this); } - Index memUsageIfLoaded() const { return sizeof(*this); } + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index memUsageIfLoaded() const { return memUsage(); } static Index size() { return SIZE; } /// @brief Return a pointer to the C-style array of words encoding the bits. diff --git a/openvdb/openvdb/tree/LeafNode.h b/openvdb/openvdb/tree/LeafNode.h index 6894d17391..8318f1dd5f 100644 --- a/openvdb/openvdb/tree/LeafNode.h +++ b/openvdb/openvdb/tree/LeafNode.h @@ -152,13 +152,14 @@ class LeafNode /// Return @c true if this node contains only active voxels. bool isDense() const { return mValueMask.isOn(); } /// Return @c true if memory for this node's buffer has been allocated. - bool isAllocated() const { return !mBuffer.isOutOfCore() && !mBuffer.empty(); } + bool isAllocated() const { return !mBuffer.empty(); } /// Allocate memory for this node's buffer if it has not already been allocated. bool allocate() { return mBuffer.allocate(); } /// Return the memory in bytes occupied by this node. Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } /// Expand the given bounding box so that it includes this leaf node's active voxels. /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all @@ -466,7 +467,6 @@ class LeafNode template void modifyValue(Index offset, const ModifyOp& op) { - mBuffer.loadValues(); if (!mBuffer.empty()) { // in-place modify value ValueType& val = const_cast(mBuffer[offset]); @@ -487,7 +487,6 @@ class LeafNode template void modifyValueAndActiveState(const Coord& xyz, const ModifyOp& op) { - mBuffer.loadValues(); if (!mBuffer.empty()) { const Index offset = this->coordToOffset(xyz); bool state = mValueMask.isOn(offset); @@ -1262,8 +1261,6 @@ template inline void LeafNode::copyToDense(const CoordBBox& bbox, DenseT& dense) const { - mBuffer.loadValues(); - using DenseValueType = typename DenseT::ValueType; const size_t xStride = dense.xStride(), yStride = dense.yStride(), zStride = dense.zStride(); @@ -1390,11 +1387,9 @@ LeafNode::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo // This node lies completely outside the clipping region. skipCompressedValues(seekable, is, fromHalf); mValueMask.setOff(); - mBuffer.setOutOfCore(false); } else { mBuffer.allocate(); io::readCompressedValues(is, mBuffer.mData, SIZE, mValueMask, fromHalf); - mBuffer.setOutOfCore(false); // Get this tree's background value. T background = zeroVal(); @@ -1417,9 +1412,6 @@ LeafNode::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo } } } - - // increment the leaf number - if (meta) meta->setLeaf(meta->leaf() + 1); } @@ -1430,8 +1422,6 @@ LeafNode::writeBuffers(std::ostream& os, bool toHalf) const // Write out the value mask. mValueMask.save(os); - mBuffer.loadValues(); - io::writeCompressedValues(os, mBuffer.mData, SIZE, mValueMask, /*childMask=*/NodeMaskType(), toHalf); } @@ -1460,16 +1450,6 @@ LeafNode::memUsage() const } -template -inline Index64 -LeafNode::memUsageIfLoaded() const -{ - // Use sizeof(*this) to capture alignment-related padding - // (but note that sizeof(*this) includes sizeof(mBuffer)). - return sizeof(*this) + mBuffer.memUsageIfLoaded() - sizeof(mBuffer); -} - - template inline void LeafNode::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const diff --git a/openvdb/openvdb/tree/LeafNodeBool.h b/openvdb/openvdb/tree/LeafNodeBool.h index 80dd31cb87..8caccdbbc6 100644 --- a/openvdb/openvdb/tree/LeafNodeBool.h +++ b/openvdb/openvdb/tree/LeafNodeBool.h @@ -145,7 +145,8 @@ class LeafNode /// Return the memory in bytes occupied by this node. Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } /// Expand the given bounding box so that it includes this leaf node's active voxels. /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all @@ -898,15 +899,6 @@ LeafNode::memUsage() const } -template -inline Index64 -LeafNode::memUsageIfLoaded() const -{ - // Use sizeof(*this) to capture alignment-related padding - return sizeof(*this); -} - - template inline void LeafNode::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const diff --git a/openvdb/openvdb/tree/LeafNodeMask.h b/openvdb/openvdb/tree/LeafNodeMask.h index 1a5e9a2bfc..41474874bd 100644 --- a/openvdb/openvdb/tree/LeafNodeMask.h +++ b/openvdb/openvdb/tree/LeafNodeMask.h @@ -144,7 +144,8 @@ class LeafNode /// Return the memory in bytes occupied by this node. Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } /// Expand the given bounding box so that it includes this leaf node's active voxels. /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all @@ -884,15 +885,6 @@ LeafNode::memUsage() const } -template -inline Index64 -LeafNode::memUsageIfLoaded() const -{ - // Use sizeof(*this) to capture alignment-related padding - return sizeof(*this); -} - - template inline void LeafNode::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const diff --git a/openvdb/openvdb/tree/Tree.h b/openvdb/openvdb/tree/Tree.h index e0e99881ba..a3371c71d7 100644 --- a/openvdb/openvdb/tree/Tree.h +++ b/openvdb/openvdb/tree/Tree.h @@ -95,13 +95,7 @@ class OPENVDB_API TreeBase virtual void getIndexRange(CoordBBox& bbox) const = 0; - /// @brief Replace with background tiles any nodes whose voxel buffers - /// have not yet been allocated. - /// @details Typically, unallocated nodes are leaf nodes whose voxel buffers - /// are not yet resident in memory because delayed loading is in effect. - /// @sa readNonresidentBuffers, io::File::open virtual void clipUnallocatedNodes() = 0; - /// Return the total number of unallocated leaf nodes residing in this tree. #if OPENVDB_ABI_VERSION_NUMBER >= 12 virtual Index64 unallocatedLeafCount() const = 0; #else @@ -167,11 +161,8 @@ class OPENVDB_API TreeBase virtual void readBuffers(std::istream&, bool saveFloatAsHalf = false) = 0; /// Read all of this tree's data buffers that intersect the given bounding box. virtual void readBuffers(std::istream&, const CoordBBox&, bool saveFloatAsHalf = false) = 0; - /// @brief Read all of this tree's data buffers that are not yet resident in memory - /// (because delayed loading is in effect). - /// @details If this tree was read from a memory-mapped file, this operation - /// disconnects the tree from the file. - /// @sa clipUnallocatedNodes, io::File::open, io::MappedFile + + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void readNonresidentBuffers() const = 0; /// Write out all the data buffers for this tree. virtual void writeBuffers(std::ostream&, bool saveFloatAsHalf = false) const = 0; @@ -339,12 +330,10 @@ class Tree: public TreeBase void readBuffers(std::istream&, bool saveFloatAsHalf = false) override; /// Read all of this tree's data buffers that intersect the given bounding box. void readBuffers(std::istream&, const CoordBBox&, bool saveFloatAsHalf = false) override; - /// @brief Read all of this tree's data buffers that are not yet resident in memory - /// (because delayed loading is in effect). - /// @details If this tree was read from a memory-mapped file, this operation - /// disconnects the tree from the file. - /// @sa clipUnallocatedNodes, io::File::open, io::MappedFile - void readNonresidentBuffers() const override; + + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void readNonresidentBuffers() const override { } + /// Write out all data buffers for this tree. void writeBuffers(std::ostream&, bool saveFloatAsHalf = false) const override; @@ -494,9 +483,6 @@ class Tree: public TreeBase void clip(const CoordBBox&); /// @brief Replace with background tiles any nodes whose voxel buffers /// have not yet been allocated. - /// @details Typically, unallocated nodes are leaf nodes whose voxel buffers - /// are not yet resident in memory because delayed loading is in effect. - /// @sa readNonresidentBuffers, io::File::open void clipUnallocatedNodes() override; /// Return the total number of unallocated leaf nodes residing in this tree. @@ -1315,17 +1301,6 @@ Tree::readBuffers(std::istream &is, const CoordBBox& bbox, bool sa } -template -inline void -Tree::readNonresidentBuffers() const -{ - for (LeafCIter it = this->cbeginLeaf(); it; ++it) { - // Retrieving the value of a leaf voxel forces loading of the leaf node's voxel buffer. - it->getValue(Index(0)); - } -} - - template inline void Tree::writeBuffers(std::ostream &os, bool saveFloatAsHalf) const diff --git a/openvdb/openvdb/unittest/TestCount.cc b/openvdb/openvdb/unittest/TestCount.cc index 0509b71359..201ab6efa6 100644 --- a/openvdb/openvdb/unittest/TestCount.cc +++ b/openvdb/openvdb/unittest/TestCount.cc @@ -223,8 +223,7 @@ TEST_F(TestCount, testMemUsage) internalNodeMemUsage += sizeof(Coord); for (auto leafIter = internal2Iter->cbeginChildOn(); leafIter; ++leafIter) { - EXPECT_EQ(leafIter->memUsage(), leafIter->memUsageIfLoaded()); - expectedMaxMem += leafIter->memUsageIfLoaded(); + expectedMaxMem += leafIter->memUsage(); ++leafCount; } } @@ -232,11 +231,9 @@ TEST_F(TestCount, testMemUsage) expectedMaxMem += internalNodeMemUsage; - Index64 inCoreMemUsage = tools::memUsage(grid->tree()); - Index64 memUsageIfLoaded = tools::memUsageIfLoaded(grid->tree()); + Index64 memUsage = tools::memUsage(grid->tree()); - EXPECT_EQ(expectedMaxMem, inCoreMemUsage); - EXPECT_EQ(expectedMaxMem, memUsageIfLoaded); + EXPECT_EQ(expectedMaxMem, memUsage); } diff --git a/openvdb/openvdb/unittest/TestMeshToVolume.cc b/openvdb/openvdb/unittest/TestMeshToVolume.cc index 5c4ff2ffdf..587c092cf2 100644 --- a/openvdb/openvdb/unittest/TestMeshToVolume.cc +++ b/openvdb/openvdb/unittest/TestMeshToVolume.cc @@ -211,8 +211,8 @@ TEST_F(TestMeshToVolume, testInterrupt) // Should have returned _something_ EXPECT_TRUE(grid); - // Expect to interrupt in under a second + // Expect to interrupt in under two seconds const auto duration = std::chrono::duration_cast(end - start); - EXPECT_LT(duration.count(), 1000); + EXPECT_LT(duration.count(), 2000); } diff --git a/openvdb/openvdb/unittest/TestPointCount.cc b/openvdb/openvdb/unittest/TestPointCount.cc index f02661456c..0d2332674b 100644 --- a/openvdb/openvdb/unittest/TestPointCount.cc +++ b/openvdb/openvdb/unittest/TestPointCount.cc @@ -313,17 +313,14 @@ TEST_F(TestPointCount, testGroup) GroupFilter groupFilter("test", attributeSet); - bool inCoreOnly; - inCoreOnly = false; - - EXPECT_EQ(pointCount(inputTree, NullFilter(), inCoreOnly), Index64(4)); - EXPECT_EQ(pointCount(inputTree, ActiveFilter(), inCoreOnly), Index64(3)); - EXPECT_EQ(pointCount(inputTree, InactiveFilter(), inCoreOnly), Index64(1)); - EXPECT_EQ(pointCount(inputTree, groupFilter, inCoreOnly), Index64(2)); + EXPECT_EQ(pointCount(inputTree, NullFilter()), Index64(4)); + EXPECT_EQ(pointCount(inputTree, ActiveFilter()), Index64(3)); + EXPECT_EQ(pointCount(inputTree, InactiveFilter()), Index64(1)); + EXPECT_EQ(pointCount(inputTree, groupFilter), Index64(2)); EXPECT_EQ(pointCount(inputTree, BinaryFilter( - groupFilter, ActiveFilter()), inCoreOnly), Index64(1)); + groupFilter, ActiveFilter())), Index64(1)); EXPECT_EQ(pointCount(inputTree, BinaryFilter( - groupFilter, InactiveFilter()), inCoreOnly), Index64(1)); + groupFilter, InactiveFilter())), Index64(1)); } std::remove(filename.c_str()); diff --git a/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc b/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc index 572b66dd2b..7d8997888f 100644 --- a/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc +++ b/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc @@ -2333,7 +2333,6 @@ TEST_F(TestPointRasterizeFrustum, testStreaming) Vec3SGrid::Ptr velocity1, velocity2, velocity3; const size_t mb = 1024*1024; - const size_t tinyMemory = static_cast(0.1*mb); { // memory test 1 - retain caches and streaming disabled Rasterizer rasterizer(settings); diff --git a/openvdb_ax/openvdb_ax/compiler/PointExecutable.cc b/openvdb_ax/openvdb_ax/compiler/PointExecutable.cc index 3598fb3adb..7b704e8524 100644 --- a/openvdb_ax/openvdb_ax/compiler/PointExecutable.cc +++ b/openvdb_ax/openvdb_ax/compiler/PointExecutable.cc @@ -362,7 +362,6 @@ struct PointFunctionArguments // @todo if the array is shared we should probably make it unique? if (mData.mUseBufferKernel) { - const_cast(array).loadData(); const char* data = array.constDataAsByteArray(); void* ptr = static_cast(const_cast(data)); mHandlesOrBuffers.emplace_back(ptr); @@ -387,14 +386,13 @@ struct PointFunctionArguments array.expand(); if (mData.mUseBufferKernel) { - array.loadData(); const char* data = array.constDataAsByteArray(); void* ptr = static_cast(const_cast(data)); mHandlesOrBuffers.emplace_back(ptr); const codegen::Codec* codec = codegen::getCodec(ast::tokens::tokenFromTypeString(array.valueType()), array.codecType()); if (codec) flag |= codec->flag(); - OPENVDB_ASSERT(array.isDataLoaded() && !array.isUniform()); + OPENVDB_ASSERT(!array.isUniform()); } else { typename WriteHandle::UniquePtr handle(new WriteHandle(leaf, Index(pos))); diff --git a/openvdb_cmd/vdb_ax/main.cc b/openvdb_cmd/vdb_ax/main.cc index 9543d9fe6c..b62c4559a7 100644 --- a/openvdb_cmd/vdb_ax/main.cc +++ b/openvdb_cmd/vdb_ax/main.cc @@ -791,9 +791,7 @@ main(int argc, char *argv[]) if (opts.mMode.get() == openvdb::ax::VDB_AX_MODE::Execute) { // read vdb file data for - axlog("[INFO] Reading VDB data" - << (openvdb::io::Archive::isDelayedLoadingEnabled() ? - " (delay-load)" : "") << '\n'); + axlog("[INFO] Reading VDB data" << '\n'); for (const auto& filename : opts.mInputVDBFiles.get()) { openvdb::io::File file(filename); try { diff --git a/openvdb_cmd/vdb_print/main.cc b/openvdb_cmd/vdb_print/main.cc index e093838768..3a329d4089 100644 --- a/openvdb_cmd/vdb_print/main.cc +++ b/openvdb_cmd/vdb_print/main.cc @@ -209,14 +209,9 @@ printShortListing(const StringVec& filenames, bool metadata) // Print the grid's size, in bytes - // no support for memUsageIfLoaded until ABI >= 10 for points::PointDataGrid types using ListT = openvdb::GridTypes; grid->apply([&](const auto& typed){ - // @todo combine these methods to avoid iterating across the tree twice - const openvdb::Index64 incore = openvdb::tools::memUsage(typed.tree()); - const openvdb::Index64 total = openvdb::tools::memUsageIfLoaded(typed.tree()); - - std::cout << " " << std::right << std::setw(6) << bytesAsString(incore) << " (In Core)"; + const openvdb::Index64 total = openvdb::tools::memUsage(typed.tree()); std::cout << " " << std::right << std::setw(6) << bytesAsString(total) << " (Total)"; }); diff --git a/openvdb_cmd/vdb_tool/include/Tool.h b/openvdb_cmd/vdb_tool/include/Tool.h index b6d5e3aaef..10ba883187 100644 --- a/openvdb_cmd/vdb_tool/include/Tool.h +++ b/openvdb_cmd/vdb_tool/include/Tool.h @@ -127,7 +127,7 @@ class Tool using FilterT = std::unique_ptr>; struct Points;// defined below struct Header;// defined below - + mutable util::CpuTimer mTimer; std::string mCmdName;// name of this command-line tool std::list mGeom;// list of geometries owned by this tool diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index 9608005294..b1fa29ba0a 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -563,10 +563,10 @@ GR_PrimVDBPoints::updatePosBuffer(RE_Render* r, int numPoints = 0; if (useGroup) { GroupFilter filter(groupName, iter->attributeSet()); - numPoints = static_cast(pointCount(tree, filter, /*inCoreOnly=*/true)); + numPoints = static_cast(pointCount(tree, filter)); } else { NullFilter filter; - numPoints = static_cast(pointCount(tree, filter, /*inCoreOnly=*/true)); + numPoints = static_cast(pointCount(tree, filter)); } if (numPoints == 0) return; @@ -596,13 +596,13 @@ GR_PrimVDBPoints::updatePosBuffer(RE_Render* r, MultiGroupFilter filter(includeGroups, excludeGroups, iter->attributeSet()); std::vector offsets; - pointOffsets(offsets, grid.tree(), filter, /*inCoreOnly=*/true); + pointOffsets(offsets, grid.tree(), filter); UT_UniquePtr pdata(new UT_Vector3F[numPoints]); PositionAttribute positionAttribute(pdata.get(), mCentroid); convertPointDataGridPosition(positionAttribute, grid, offsets, - /*startOffset=*/ 0, filter, /*inCoreOnly=*/true); + /*startOffset=*/ 0, filter); posGeo->setArray(r, pdata.get()); posGeo->setCacheVersion(version); @@ -647,20 +647,12 @@ GR_PrimVDBPoints::updateWireBuffer(RE_Render *r, using LeafNode = TreeType::LeafNodeType; const TreeType& tree = grid.tree(); + size_t leafCount = tree.leafCount(); if (tree.leafCount() == 0) return; - // count up total points ignoring any leaf nodes that are out of core - - size_t outOfCoreLeaves = 0; - for (auto iter = tree.cbeginLeaf(); iter; ++iter) { - if (iter->buffer().isOutOfCore()) outOfCoreLeaves++; - } - - if (outOfCoreLeaves == 0) return; - // Initialize the number of points for the wireframe box per leaf. - int numPoints = static_cast(outOfCoreLeaves*8*3); + int numPoints = static_cast(leafCount*8*3); myWire->setNumPoints(numPoints); // fetch wireframe position, if its cache version matches, no upload is required. @@ -680,9 +672,6 @@ GR_PrimVDBPoints::updateWireBuffer(RE_Render *r, for (auto iter = tree.cbeginLeaf(); iter; ++iter) { const LeafNode& leaf = *iter; - // skip in-core leaf nodes (for use when delay loading VDBs) - if (!leaf.buffer().isOutOfCore()) continue; - coords.push_back(leaf.origin()); } @@ -826,12 +815,12 @@ GR_PrimVDBPoints::updateVec3Buffer(RE_Render* r, MultiGroupFilter filter(includeGroups, excludeGroups, iter->attributeSet()); std::vector offsets; - pointOffsets(offsets, grid.tree(), filter, /*inCoreOnly=*/true); + pointOffsets(offsets, grid.tree(), filter); VectorAttribute typedAttribute(data.get()); convertPointDataGridAttribute(typedAttribute, grid.tree(), offsets, /*startOffset=*/ 0, static_cast(index), /*stride=*/1, - filter, /*inCoreOnly=*/true); + filter); } bufferGeo->setArray(r, data.get()); diff --git a/openvdb_houdini/openvdb_houdini/PointUtils.cc b/openvdb_houdini/openvdb_houdini/PointUtils.cc index e8070f0265..fdae57ccf1 100644 --- a/openvdb_houdini/openvdb_houdini/PointUtils.cc +++ b/openvdb_houdini/openvdb_houdini/PointUtils.cc @@ -1073,8 +1073,7 @@ convertPointDataGridToHoudini( const PointDataGrid& grid, const std::vector& attributes, const std::vector& includeGroups, - const std::vector& excludeGroups, - const bool inCoreOnly) + const std::vector& excludeGroups) { using namespace openvdb::math; @@ -1096,14 +1095,14 @@ convertPointDataGridToHoudini( // obtain cumulative point offsets and total points std::vector offsets; MultiGroupFilter filter(includeGroups, excludeGroups, leafIter->attributeSet()); - const Index64 total = pointOffsets(offsets, tree, filter, inCoreOnly); + const Index64 total = pointOffsets(offsets, tree, filter); // a block's global offset is needed to transform its point offsets to global offsets const Index64 startOffset = detail.appendPointBlock(total); HoudiniWriteAttribute positionAttribute(*detail.getP()); convertPointDataGridPosition(positionAttribute, grid, offsets, startOffset, - filter, inCoreOnly); + filter); // add other point attributes to the hdk detail const AttributeSet::Descriptor::NameToPosMap& nameToPosMap = descriptor.map(); @@ -1186,87 +1185,87 @@ convertPointDataGridToHoudini( if (valueType == "string") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "bool") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "int8") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "int16") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "int32") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "int64") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "float") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "double") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "vec3i") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "vec3s") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "vec3d") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "quats") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "quatd") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "mat3s") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "mat3d") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "mat4s") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "mat4d") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else { throw std::runtime_error("Unknown Attribute Type for Conversion: " + valueType); @@ -1288,7 +1287,7 @@ convertPointDataGridToHoudini( attributeSet.groupIndex(name); HoudiniGroup group(*pointGroup, startOffset, total); - convertPointDataGridGroup(group, tree, offsets, startOffset, index, filter, inCoreOnly); + convertPointDataGridGroup(group, tree, offsets, startOffset, index, filter); } } @@ -1615,32 +1614,7 @@ collectPointInfo(const PointDataGrid& grid, const PointDataTree& tree = grid.constTree(); - // iterate through all leaf nodes to find out if all are out-of-core - bool allOutOfCore = true; - for (auto iter = tree.cbeginLeaf(); iter; ++iter) { - if (!iter->buffer().isOutOfCore()) { - allOutOfCore = false; - break; - } - } - - openvdb::Index64 totalPointCount = 0; - - // it is more technically correct to rely on the voxel count as this may be - // out of sync with the attribute size, however for faster node preview when - // the voxel buffers are all out-of-core, count up the sizes of the first - // attribute array instead - - if (allOutOfCore) { - for (auto iter = tree.cbeginLeaf(); iter; ++iter) { - if (iter->attributeSet().size() > 0) { - totalPointCount += iter->constAttributeArray(0).size(); - } - } - } - else { - totalPointCount = openvdb::points::pointCount(tree); - } + openvdb::Index64 totalPointCount = openvdb::points::pointCount(tree); std::ostringstream os; os << openvdb::util::formattedInt(totalPointCount); @@ -1674,16 +1648,8 @@ collectPointInfo(const PointDataGrid& grid, os << it.first << "("; - // for faster node preview when all the voxel buffers are out-of-core, - // don't load the group arrays to display the group sizes, just print - // "out-of-core" instead @todo - put the group sizes into the grid - // metadata on write for this use case - - if (allOutOfCore) os << "out-of-core"; - else { - const openvdb::points::GroupFilter filter(it.first, attributeSet); - os << openvdb::util::formattedInt(pointCount(tree, filter)); - } + const openvdb::points::GroupFilter filter(it.first, attributeSet); + os << openvdb::util::formattedInt(pointCount(tree, filter)); os << ")"; } diff --git a/openvdb_houdini/openvdb_houdini/PointUtils.h b/openvdb_houdini/openvdb_houdini/PointUtils.h index 57dbf6306e..89fe54d941 100644 --- a/openvdb_houdini/openvdb_houdini/PointUtils.h +++ b/openvdb_houdini/openvdb_houdini/PointUtils.h @@ -109,7 +109,6 @@ convertHoudiniToPointDataGrid( /// (empty vector defaults to all) /// @param excludeGroups a vector of VDB Points groups to be excluded /// (empty vector defaults to none) -/// @param inCoreOnly true if out-of-core leaf nodes are to be ignored OPENVDB_HOUDINI_API void convertPointDataGridToHoudini( @@ -117,8 +116,7 @@ convertPointDataGridToHoudini( const openvdb::points::PointDataGrid& grid, const std::vector& attributes = {}, const std::vector& includeGroups = {}, - const std::vector& excludeGroups = {}, - const bool inCoreOnly = false); + const std::vector& excludeGroups = {}); /// @brief Populate VDB Points grid metadata from Houdini detail attributes diff --git a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc index e5abf786d3..665882b9ae 100644 --- a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc +++ b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc @@ -693,23 +693,10 @@ SOP_OpenVDB_Points_Convert::Cache::cookVDBSop(OP_Context& context) // all attributes should be converted const std::vector emptyNameVector; - // if all point data is being converted, sequentially pre-fetch any out-of-core - // data for faster performance when using delayed-loading - - const bool allData = emptyNameVector.empty() && - includeGroups.empty() && - excludeGroups.empty(); - for (const PointDataGrid::ConstPtr &grid : pointGrids) { GU_Detail geo; - // if all the data is being loaded, prefetch it for faster load performance - - if (allData) { - prefetch(grid->tree()); - } - // perform conversion hvdb::convertPointDataGridToHoudini( diff --git a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc index 2dcaf0aa26..d1a11f007c 100644 --- a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc +++ b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc @@ -516,59 +516,24 @@ struct GridsToRasterize struct Grid { - private: - static bool isTreeOutOfCore(const TreeType& tree) - { - using LeafManagerT = openvdb::tree::LeafManager; - using LeafRangeT = typename LeafManagerT::LeafRange; - LeafManagerT leafManager(tree); - return tbb::parallel_reduce(leafManager.leafRange(), true, - [] (const LeafRangeT& range, bool result) -> bool { - for (const auto& leaf : range) { - if (!leaf.buffer().isOutOfCore()) return false; - } - return result; - }, - [] (bool n, bool m) -> bool { return n || m; }); - } - public: explicit Grid(const ConstPtr& grid) - : mGrid(grid) - , mOutOfCore(isTreeOutOfCore(grid->constTree())) { } - - inline bool isOutOfCore() const { return mOutOfCore; } + : mGrid(grid) { } template void addToRasterizer(RasterizerT& rasterizer) { - if (mOutOfCore) { - auto newGrid = mGrid->deepCopy(); - rasterizer.addPoints(newGrid, /*stream=*/true); - } - else { - rasterizer.addPoints(mGrid); - } + rasterizer.addPoints(mGrid); } private: ConstPtr mGrid; - const bool mOutOfCore; }; // struct Grid void push_back(ConstPtr& grid) { mGrids.emplace_back(grid); } size_t size() const { return mGrids.size(); } bool empty() const { return mGrids.empty(); } - // return true if any of the grids is out-of-core - bool streaming() const - { - for (auto& grid : mGrids) { - if (grid.isOutOfCore()) return true; - } - return false; - } - template void addGridToRasterizer(RasterizerT& rasterizer, size_t index) { @@ -885,8 +850,7 @@ Rasterizing can be performed into cartesian and frustum grids (otherwise referre grids). For frustum grids, approximations are used by default to accelerate the rasterization.\n\ This can be disabled to achieve a more accurate result.\n\ \n\ -This node supports streaming of VDB points and attributes if one or more delayed-load VDB is\n\ -provided as an input.\n\ +This node supports streaming of VDB points and attributes\n\ \n\ @related\n\ - [OpenVDB Rasterize Points|Node:sop/DW_OpenVDBRasterizePoints]\n\ @@ -1257,8 +1221,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) iterations = 1; } - bool streaming = pointGrids.streaming(); - for (size_t i = 0; i < iterations; i++) { if (!mergeVDBPoints) { @@ -1274,12 +1236,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) if (std::find(vectorAttribNames.begin(), vectorAttribNames.end(), velocityAttribute) != vectorAttribNames.end()) { velocity = rasterizer.rasterizeAttribute(velocityAttribute, vectorMode, reduceMemory, scale, rasterGroups); - - // need to deep-copy input grids again if caches are being discarded - if (streaming && reduceMemory) { - if (mergeVDBPoints) pointGrids.addGridsToRasterizer(rasterizer); - else pointGrids.addGridToRasterizer(rasterizer, i); - } } // rasterize density @@ -1287,12 +1243,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) if (createDensity) { auto density = rasterizer.rasterizeDensity(densityAttribute, densityMode, reduceMemory, densityScale, rasterGroups); outputGrids.push_back(density); - - // need to deep-copy input grids again if caches are being discarded - if (streaming && reduceMemory) { - if (mergeVDBPoints) pointGrids.addGridsToRasterizer(rasterizer); - else pointGrids.addGridToRasterizer(rasterizer, i); - } } // rasterize mask @@ -1307,12 +1257,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) for (const auto& name : scalarAttribNames) { auto scalar = rasterizer.rasterizeAttribute(name, scalarMode, reduceMemory, scale, rasterGroups); outputGrids.push_back(scalar); - - // need to deep-copy input grids again if caches are being discarded - if (streaming && reduceMemory) { - if (mergeVDBPoints) pointGrids.addGridsToRasterizer(rasterizer); - else pointGrids.addGridToRasterizer(rasterizer, i); - } } // rasterize vector attributes @@ -1327,12 +1271,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) else { auto vector = rasterizer.rasterizeAttribute(name, vectorMode, reduceMemory, scale, rasterGroups); outputGrids.push_back(vector); - - // need to deep-copy input grids again if caches are being discarded - if (streaming && reduceMemory) { - if (mergeVDBPoints) pointGrids.addGridsToRasterizer(rasterizer); - else pointGrids.addGridToRasterizer(rasterizer, i); - } } } } diff --git a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Read.cc b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Read.cc index 7b11986b40..887a0a6e11 100644 --- a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Read.cc +++ b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Read.cc @@ -204,40 +204,20 @@ newSopOperator(OP_OperatorTable* table) .setCallbackFunc(&reloadCB) .setTooltip("Reread the VDB file.")); - parms.add(hutil::ParmFactory(PRM_SEPARATOR, "sep1", "Sep")); - - // Delayed loading - parms.add(hutil::ParmFactory(PRM_TOGGLE, "delayload", "Delay Loading") - .setDefault(PRMoneDefaults) - .setTooltip( - "Don't allocate memory for or read voxel values until the values" - " are actually accessed.\n\n" - "Delayed loading can significantly lower memory usage, but\n" - "note that viewport visualization of a volume usually requires\n" - "the entire volume to be loaded into memory.")); - - // Localization file size slider - parms.add(hutil::ParmFactory(PRM_FLT_J, "copylimit", "Copy If Smaller Than") + // Obsolete parameters + hutil::ParmList obsoleteParms; + obsoleteParms.add(hutil::ParmFactory(PRM_SEPARATOR, "sep1", "Sep")); + obsoleteParms.add(hutil::ParmFactory(PRM_TOGGLE, "delayload", "Delay Loading") + .setDefault(PRMoneDefaults)); + obsoleteParms.add(hutil::ParmFactory(PRM_FLT_J, "copylimit", "Copy If Smaller Than") .setTypeExtended(PRM_TYPE_JOIN_PAIR) - .setDefault(0.5f) - .setRange(PRM_RANGE_RESTRICTED, 0, PRM_RANGE_UI, 10) - .setTooltip( - "When delayed loading is enabled, a file must not be modified on disk before\n" - "it has been fully read. For safety, files smaller than the given size (in GB)\n" - "will be copied to a private, temporary location (either $OPENVDB_TEMP_DIR,\n" - "$TMPDIR or a system default temp directory).") - .setDocumentation( - "When delayed loading is enabled, a file must not be modified on disk before" - " it has been fully read. For safety, files smaller than the given size (in GB)" - " will be copied to a private, temporary location (either `$OPENVDB_TEMP_DIR`," - " `$TMPDIR` or a system default temp directory).")); - - parms.add(hutil::ParmFactory(PRM_LABEL, "copylimitlabel", "GB") - .setDocumentation(nullptr)); + .setDefault(0.5f)); + obsoleteParms.add(hutil::ParmFactory(PRM_LABEL, "copylimitlabel", "GB")); // Register this operator. hvdb::OpenVDBOpFactory("VDB Read", SOP_OpenVDB_Read::factory, parms, *table) .setNativeName("") + .setObsoleteParms(obsoleteParms) .addOptionalInput("Optional Bounding Geometry") .setDocumentation("\ #icon: COMMON/openvdb\n\ @@ -248,13 +228,7 @@ newSopOperator(OP_OperatorTable* table) @overview\n\ \n\ This node reads VDB volumes from a `.vdb` file.\n\ -It is usually preferable to use Houdini's native [File|Node:sop/file] node,\n\ -however unlike the native node, this node allows one to take advantage of\n\ -delayed loading, meaning that only those portions of a volume that are\n\ -actually accessed in a scene get loaded into memory.\n\ -Delayed loading can significantly reduce memory usage when working\n\ -with large volumes (but note that viewport visualization of a volume\n\ -usually requires the entire volume to be loaded into memory).\n\ +It is usually preferable to use Houdini's native [File|Node:sop/file] node.\n\ \n\ @related\n\ - [OpenVDB Write|Node:sop/DW_OpenVDBWrite]\n\ @@ -297,10 +271,6 @@ SOP_OpenVDB_Read::updateParmsFlags() changed |= enableParm("group", bool(evalInt("enable_grouping", 0, t))); - const bool delayedLoad = evalInt("delayload", 0, t); - changed |= enableParm("copylimit", delayedLoad); - changed |= enableParm("copylimitlabel", delayedLoad); - return changed; } @@ -344,10 +314,6 @@ SOP_OpenVDB_Read::cookVDBSop(OP_Context& context) //} } - const bool delayedLoad = evalInt("delayload", 0, t); - const openvdb::Index64 copyMaxBytes = - openvdb::Index64(1.0e9 * evalFloat("copylimit", 0, t)); - openvdb::BBoxd clipBBox; bool clip = evalInt("clip", 0, t); if (clip) { @@ -370,8 +336,7 @@ SOP_OpenVDB_Read::cookVDBSop(OP_Context& context) openvdb::MetaMap::Ptr fileMetadata; try { // Open the VDB file, but don't read any grids yet. - file.setCopyMaxBytes(copyMaxBytes); - file.open(delayedLoad); + file.open(); // Read the file-level metadata. fileMetadata = file.getMetadata(); diff --git a/openvdb_houdini/openvdb_houdini/VRAY_OpenVDB_Points.cc b/openvdb_houdini/openvdb_houdini/VRAY_OpenVDB_Points.cc index 89c808f406..a14b7b514a 100644 --- a/openvdb_houdini/openvdb_houdini/VRAY_OpenVDB_Points.cc +++ b/openvdb_houdini/openvdb_houdini/VRAY_OpenVDB_Points.cc @@ -351,7 +351,7 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) { struct Local { - static GridVecPtr loadGrids(const std::string& filename, const bool stream) + static GridVecPtr loadGrids(const std::string& filename) { GridVecPtr grids; @@ -367,11 +367,6 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) if (baseGrid->isType()) { auto grid = StaticPtrCast(file.readGrid(*iter)); assert(grid); - if (stream) { - // enable streaming mode to auto-collapse attributes - // on read for improved memory efficiency - points::setStreamingMode(grid->tree(), /*on=*/true); - } grids.push_back(grid); } } @@ -389,8 +384,6 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) import("file", mFilename); - int streamData; - import("streamdata", &streamData, 1); import("attrmask", mAttrStr); float fps; @@ -430,7 +423,7 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) } } - mGridPtrs = Local::loadGrids(mFilename.toStdString(), streamData ? true : false); + mGridPtrs = Local::loadGrids(mFilename.toStdString()); // extract which groups to include and exclude UT_StringHolder groupStr; @@ -447,11 +440,6 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) static_cast(vdbBox.max().y()), static_cast(vdbBox.max().z())); - // if streaming the data, re-open the file now that the bounding box has been computed - if (streamData) { - mGridPtrs = Local::loadGrids(mFilename.toStdString(), true); - } - return 1; } From 0f0143f464c6cf1d212850ebcfb724647ae4f536 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 11 Feb 2026 18:01:51 -0800 Subject: [PATCH 05/70] Delete wireframe rendering, this is only useful for delayed-loading Signed-off-by: Dan Bailey --- .../openvdb_houdini/GR_PrimVDBPoints.cc | 183 +----------------- 1 file changed, 1 insertion(+), 182 deletions(-) diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index b1fa29ba0a..4f93d05033 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -48,7 +48,6 @@ static RE_ShaderHandle theMarkerDecorShader("decor/GL32/point_marker.prog"); static RE_ShaderHandle theNormalDecorShader("decor/GL32/point_normal.prog"); static RE_ShaderHandle theVelocityDecorShader("decor/GL32/user_point_vector3.prog"); -static RE_ShaderHandle theLineShader("basic/GL32/wire_color.prog"); static RE_ShaderHandle thePixelShader("particle/GL32/pixel.prog"); static RE_ShaderHandle thePointShader("particle/GL32/point.prog"); @@ -145,10 +144,6 @@ class GR_PrimVDBPoints : public GR_Primitive const openvdb::points::PointDataGrid& grid, const RE_CacheVersion& version); - void updateWireBuffer(RE_Render* r, - const openvdb::points::PointDataGrid& grid, - const RE_CacheVersion& version); - bool updateVec3Buffer(RE_Render* r, const openvdb::points::PointDataGrid& grid, const std::string& attributeName, @@ -164,7 +159,6 @@ class GR_PrimVDBPoints : public GR_Primitive private: UT_UniquePtr myGeo; - UT_UniquePtr myWire; bool mDefaultPointColor = true; openvdb::Vec3f mCentroid{0, 0, 0}; openvdb::BBoxd mBbox; @@ -354,84 +348,6 @@ GR_PrimVDBPoints::acceptPrimitive(GT_PrimitiveType, return GR_NOT_PROCESSED; } -namespace gr_primitive_internal -{ - -struct FillGPUBuffersLeafBoxes -{ - FillGPUBuffersLeafBoxes(UT_Vector3H* buffer, - const std::vector& coords, - const openvdb::math::Transform& transform, - const openvdb::Vec3f& positionOffset) - : mBuffer(buffer) - , mCoords(coords) - , mTransform(transform) - , mPositionOffset(positionOffset) { } - - void operator()(const tbb::blocked_range& range) const - { - std::vector corners; - corners.reserve(8); - - for (size_t n = range.begin(), N = range.end(); n != N; ++n) { - const openvdb::Coord& origin = mCoords[n]; - - // define 8 corners - - corners.clear(); - - const openvdb::Vec3f pos000 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 0.0, 0.0)) - mPositionOffset; - corners.emplace_back(pos000.x(), pos000.y(), pos000.z()); - const openvdb::Vec3f pos001 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 0.0, 8.0)) - mPositionOffset; - corners.emplace_back(pos001.x(), pos001.y(), pos001.z()); - const openvdb::Vec3f pos010 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 8.0, 0.0)) - mPositionOffset; - corners.emplace_back(pos010.x(), pos010.y(), pos010.z()); - const openvdb::Vec3f pos011 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 8.0, 8.0)) - mPositionOffset; - corners.emplace_back(pos011.x(), pos011.y(), pos011.z()); - const openvdb::Vec3f pos100 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 0.0, 0.0)) - mPositionOffset; - corners.emplace_back(pos100.x(), pos100.y(), pos100.z()); - const openvdb::Vec3f pos101 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 0.0, 8.0)) - mPositionOffset; - corners.emplace_back(pos101.x(), pos101.y(), pos101.z()); - const openvdb::Vec3f pos110 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 8.0, 0.0)) - mPositionOffset; - corners.emplace_back(pos110.x(), pos110.y(), pos110.z()); - const openvdb::Vec3f pos111 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 8.0, 8.0)) - mPositionOffset; - corners.emplace_back(pos111.x(), pos111.y(), pos111.z()); - - openvdb::Index64 offset = n*8*3; - - // Z axis - - mBuffer[offset++] = corners[0]; mBuffer[offset++] = corners[1]; - mBuffer[offset++] = corners[2]; mBuffer[offset++] = corners[3]; - mBuffer[offset++] = corners[4]; mBuffer[offset++] = corners[5]; - mBuffer[offset++] = corners[6]; mBuffer[offset++] = corners[7]; - - // Y axis - - mBuffer[offset++] = corners[0]; mBuffer[offset++] = corners[2]; - mBuffer[offset++] = corners[1]; mBuffer[offset++] = corners[3]; - mBuffer[offset++] = corners[4]; mBuffer[offset++] = corners[6]; - mBuffer[offset++] = corners[5]; mBuffer[offset++] = corners[7]; - - // X axis - - mBuffer[offset++] = corners[0]; mBuffer[offset++] = corners[4]; - mBuffer[offset++] = corners[1]; mBuffer[offset++] = corners[5]; - mBuffer[offset++] = corners[2]; mBuffer[offset++] = corners[6]; - mBuffer[offset++] = corners[3]; mBuffer[offset++] = corners[7]; - } - } - - ////////// - - UT_Vector3H* mBuffer; - const std::vector& mCoords; - const openvdb::math::Transform& mTransform; - const openvdb::Vec3f mPositionOffset; -}; // class FillGPUBuffersLeafBoxes - -} // namespace gr_primitive_internal - void GR_PrimVDBPoints::computeCentroid(const openvdb::points::PointDataGrid& grid) { @@ -631,78 +547,6 @@ GR_PrimVDBPoints::updatePosBuffer(RE_Render* r, myGeo->connectAllPrims(r, RE_GEO_WIRE_IDX, primType, nullptr, true); } -void -GR_PrimVDBPoints::updateWireBuffer(RE_Render *r, - const openvdb::points::PointDataGrid& grid, - const RE_CacheVersion& version) -{ - const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3); - - // Initialize the geometry with the proper name for the GL cache - if (!myWire) myWire.reset(new RE_Geometry); - myWire->cacheBuffers(getCacheName()); - - using GridType = openvdb::points::PointDataGrid; - using TreeType = GridType::TreeType; - using LeafNode = TreeType::LeafNodeType; - - const TreeType& tree = grid.tree(); - size_t leafCount = tree.leafCount(); - if (tree.leafCount() == 0) return; - - // Initialize the number of points for the wireframe box per leaf. - - int numPoints = static_cast(leafCount*8*3); - myWire->setNumPoints(numPoints); - - // fetch wireframe position, if its cache version matches, no upload is required. - - RE_VertexArray* posWire = myWire->findCachedAttrib(r, "P", RE_GPU_FLOAT16, 3, RE_ARRAY_POINT, true); - - if (posWire->getCacheVersion() != version) - { - using gr_primitive_internal::FillGPUBuffersLeafBoxes; - - // fill the wire data - - UT_UniquePtr data(new UT_Vector3H[numPoints]); - - std::vector coords; - - for (auto iter = tree.cbeginLeaf(); iter; ++iter) { - const LeafNode& leaf = *iter; - - coords.push_back(leaf.origin()); - } - - FillGPUBuffersLeafBoxes fill(data.get(), coords, grid.transform(), mCentroid); - const tbb::blocked_range range(0, coords.size()); - tbb::parallel_for(range, fill); - - posWire->setArray(r, data.get()); - posWire->setCacheVersion(version); - } - - if (gl3) - { - // Extra constant inputs for the GL3 default shaders we are using. - - fpreal32 uv[2] = { 0.0, 0.0 }; - fpreal32 alpha = 1.0; - fpreal32 pnt = 0.0; - UT_Matrix4F instance; - instance.identity(); - - myWire->createConstAttribute(r, "uv", RE_GPU_FLOAT32, 2, uv); - myWire->createConstAttribute(r, "Alpha", RE_GPU_FLOAT32, 1, &alpha); - myWire->createConstAttribute(r, "pointSelection", RE_GPU_FLOAT32, 1,&pnt); - myWire->createConstAttribute(r, "instmat", RE_GPU_MATRIX4, 1, - instance.data()); - } - - myWire->connectAllPrims(r, RE_GEO_WIRE_IDX, RE_PRIM_LINES, nullptr, true); -} - void GR_PrimVDBPoints::update(RE_RenderContext r, const GT_PrimitiveHandle &primh, @@ -710,7 +554,6 @@ GR_PrimVDBPoints::update(RE_RenderContext r, { // patch the point shaders at run-time to add an offset (does nothing if already patched) - patchShaderVertexOffset(r, theLineShader); patchShaderVertexOffset(r, thePixelShader); patchShaderVertexOffset(r, thePointShader); @@ -738,7 +581,6 @@ GR_PrimVDBPoints::update(RE_RenderContext r, computeCentroid(pointDataGrid); computeBbox(pointDataGrid); updatePosBuffer(r, pointDataGrid, p.geo_version); - updateWireBuffer(r, pointDataGrid, p.geo_version); mDefaultPointColor = !updateVec3Buffer(r, pointDataGrid, "Cd", "Cd", p.geo_version); } @@ -856,7 +698,7 @@ GR_PrimVDBPoints::removeBuffer(const std::string& name) void GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_DrawParms dp) { - if (!myGeo && !myWire) return; + if (!myGeo) return; const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3); @@ -900,29 +742,6 @@ GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_D r->popShader(); } - - // draw leaf bboxes - - if (myWire && myWire->getNumPoints() > 0) { - - // bind the shader - - r->pushShader(); - r->bindShader(theLineShader); - - // bind the position offset - - UT_Vector3F positionOffset(mCentroid.x(), mCentroid.y(), mCentroid.z()); - theLineShader->bindVector(r, "offset", positionOffset); - - fpreal32 constcol[3] = { 0.6f, 0.6f, 0.6f }; - myWire->createConstAttribute(r, "Cd", RE_GPU_FLOAT32, 3, constcol); - - r->pushLineWidth(commonOpts.wireWidth()); - myWire->draw(r, RE_GEO_WIRE_IDX); - r->popLineWidth(); - r->popShader(); - } } From 759482d158666bef5c8993b8c88873cb865a92d7 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 11 Feb 2026 18:58:39 -0800 Subject: [PATCH 06/70] Fix the patching of marker decor shaders for recent Houdini releases Signed-off-by: Dan Bailey --- openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index 4f93d05033..6ffbed0fa8 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -246,10 +246,15 @@ bool patchShader(RE_Render* r, RE_ShaderHandle& shader, RE_ShaderType type, shader->getShaderSource(r, source, type); const int version = shader->getCodeVersion(); + // normalize whitespace (collapse tabs and runs of spaces to single space) + + source.substitute("\t", " "); + while (source.substitute(" ", " ")) {} + // patch the shader to replace the strings for (const auto& stringPair : stringReplacements) { - source.substitute(stringPair.first.c_str(), stringPair.second.c_str(), /*all=*/true); + source.substitute(stringPair.first.c_str(), stringPair.second.c_str()); } // patch the shader to insert the strings @@ -306,9 +311,6 @@ void patchShaderNoRedeclarations(RE_Render* r, RE_ShaderHandle& shader) { static const std::vector stringReplacements { - StringPair("\t", " "), - StringPair(" ", " "), - StringPair(" ", " "), StringPair("uniform vec2 glH_DepthProject;", "//uniform vec2 glH_DepthProject;"), StringPair("uniform vec2 glH_ScreenSize", "//uniform vec2 glH_ScreenSize") }; From eac8cc0208785b6615b277847d8a0be70bb2d1e3 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 12 Feb 2026 15:27:29 -0800 Subject: [PATCH 07/70] Remove support for Houdini < 20.0 Signed-off-by: Dan Bailey --- .../openvdb_houdini/GR_PrimVDBPoints.cc | 26 ++++--------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index 6ffbed0fa8..a86869e993 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -7,8 +7,6 @@ /// /// @brief GR Render Hook and Primitive for VDB PointDataGrid -#include - #include #include #include @@ -39,10 +37,6 @@ #include #include -#if UT_VERSION_INT < 0x14000000 // Below 20.0, there is no RE_RenderContext -#define RE_RenderContext RE_Render * -#endif - //////////////////////////////////////// static RE_ShaderHandle theMarkerDecorShader("decor/GL32/point_marker.prog"); @@ -119,11 +113,8 @@ class GR_PrimVDBPoints : public GR_Primitive /// return true if the primitive is in or overlaps the view frustum. /// always returning true will effectively disable frustum culling. - bool inViewFrustum(const UT_Matrix4D &objviewproj -#if (UT_VERSION_INT >= 0x1105014e) // 17.5.334 or later - , const UT_BoundingBoxD *bbox -#endif - ) override; + bool inViewFrustum(const UT_Matrix4D &objviewproj, + const UT_BoundingBoxD *bbox) override; /// Called whenever the primitive is required to render, which may be more /// than one time per viewport redraw (beauty, shadow passes, wireframe-over) @@ -589,20 +580,13 @@ GR_PrimVDBPoints::update(RE_RenderContext r, } bool -GR_PrimVDBPoints::inViewFrustum(const UT_Matrix4D& objviewproj -#if (UT_VERSION_INT >= 0x1105014e) // 17.5.334 or later - , const UT_BoundingBoxD *passed_bbox -#endif - ) +GR_PrimVDBPoints::inViewFrustum(const UT_Matrix4D& objviewproj, + const UT_BoundingBoxD *passed_bbox) { const UT_BoundingBoxD bbox( mBbox.min().x(), mBbox.min().y(), mBbox.min().z(), mBbox.max().x(), mBbox.max().y(), mBbox.max().z()); -#if (UT_VERSION_INT >= 0x1105014e) // 17.5.334 or later - return GR_Utils::inViewFrustum(passed_bbox ? *passed_bbox :bbox, + return GR_Utils::inViewFrustum(passed_bbox ? *passed_bbox : bbox, objviewproj); -#else - return GR_Utils::inViewFrustum(bbox, objviewproj); -#endif } bool From 94fafd52a585e9a7de4481a5f27decdc0b71340a Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 12 Feb 2026 15:29:19 -0800 Subject: [PATCH 08/70] Add Vulkan support Signed-off-by: Dan Bailey --- .../openvdb_houdini/CMakeLists.txt | 5 + .../openvdb_houdini/GR_PrimVDBPoints.cc | 517 +++++++++++++++++- .../glsl/openvdb/VK/points.frag | 28 + .../glsl/openvdb/VK/points.prog | 6 + .../glsl/openvdb/VK/points.vert | 28 + .../glsl/openvdb/VK/velocity.frag | 26 + .../glsl/openvdb/VK/velocity.prog | 8 + .../glsl/openvdb/VK/velocity.tcs | 46 ++ .../glsl/openvdb/VK/velocity.tes | 23 + .../glsl/openvdb/VK/velocity.vert | 30 + 10 files changed, 713 insertions(+), 4 deletions(-) create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.frag create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.prog create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.vert create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.frag create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.prog create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tcs create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tes create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.vert diff --git a/openvdb_houdini/openvdb_houdini/CMakeLists.txt b/openvdb_houdini/openvdb_houdini/CMakeLists.txt index 50d9ed4cab..1a8ae34dd0 100644 --- a/openvdb_houdini/openvdb_houdini/CMakeLists.txt +++ b/openvdb_houdini/openvdb_houdini/CMakeLists.txt @@ -123,6 +123,8 @@ set(OPENVDB_HOUDINI_ICON_INSTALL_PREFIX ${OPENVDB_HOUDINI_INSTALL_PREFIX}/config CACHE PATH "Install path for the OpenVDB Houdini node icons") set(OPENVDB_HOUDINI_PYTHON_INSTALL_PREFIX ${OPENVDB_HOUDINI_INSTALL_PREFIX}/python2.7libs CACHE PATH "Install path for the OpenVDB Houdini startup script") +set(OPENVDB_HOUDINI_GLSL_INSTALL_PREFIX ${OPENVDB_HOUDINI_INSTALL_PREFIX}/glsl + CACHE PATH "Install path for the OpenVDB Houdini GLSL shaders") ######################################################################### @@ -395,6 +397,9 @@ install(FILES install(DIRECTORY help/ DESTINATION ${OPENVDB_HOUDINI_HELP_INSTALL_PREFIX}) +install(DIRECTORY glsl/ + DESTINATION ${OPENVDB_HOUDINI_GLSL_INSTALL_PREFIX}) + if(OPENVDB_INSTALL_HOUDINI_PYTHONRC) install(FILES pythonrc.py diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index a86869e993..c4f45b7272 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -28,6 +28,18 @@ #include #include #include +#include + +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#include +#include +#include +#include +#include +#include +#include +#include +#endif #include #include @@ -45,6 +57,11 @@ static RE_ShaderHandle theVelocityDecorShader("decor/GL32/user_point_vector3.pro static RE_ShaderHandle thePixelShader("particle/GL32/pixel.prog"); static RE_ShaderHandle thePointShader("particle/GL32/point.prog"); +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +static RV_ShaderProgram* theVkPointShader = nullptr; +static RV_ShaderProgram* theVkVelocityShader = nullptr; +#endif + /// @note An additional scale for velocity trails to accurately match /// the visualization of velocity for Houdini points #define VELOCITY_DECOR_SCALE -0.041f; @@ -148,11 +165,35 @@ class GR_PrimVDBPoints : public GR_Primitive void removeBuffer(const std::string& name); +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support + void updatePosBufferVk(RV_Render* r, + const openvdb::points::PointDataGrid& grid, + const RE_CacheVersion& version); + + bool updateVec3BufferVk(RV_Render* r, + const openvdb::points::PointDataGrid& grid, + const std::string& attributeName, + const std::string& bufferName, + const RE_CacheVersion& version); +#endif + private: UT_UniquePtr myGeo; bool mDefaultPointColor = true; openvdb::Vec3f mCentroid{0, 0, 0}; openvdb::BBoxd mBbox; +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support + UT_UniquePtr myGeoVk; + UT_UniquePtr myVkObjectSet; + UT_UniquePtr myVkObjectBlock; + UT_UniquePtr myVkDrawingSet; + UT_UniquePtr myVkGeoBlock; + // velocity shader uniform state (separate to avoid clobbering point shader) + UT_UniquePtr myVkVelObjectSet; + UT_UniquePtr myVkVelObjectBlock; + bool mHasNormals = false; + bool mHasVelocity = false; +#endif }; @@ -439,12 +480,49 @@ struct VectorAttribute UT_Vector3H* mBuffer; }; // struct VectorAttribute +struct VectorAttribute4H +{ + using ValueType = Vec3f; + + struct Handle + { + Handle(VectorAttribute4H& attribute) + : mBuffer(attribute.mBuffer) { } + + template + void set(openvdb::Index offset, + openvdb::Index /*stride*/, + const openvdb::math::Vec3& value) + { + mBuffer[offset] = UT_Vector4H( + fpreal16(float(value.x())), + fpreal16(float(value.y())), + fpreal16(float(value.z())), + fpreal16(0)); + } + + private: + UT_Vector4H* mBuffer; + }; // struct Handle + + VectorAttribute4H(UT_Vector4H* buffer) + : mBuffer(buffer) { } + + void expand() { } + void compact() { } + +private: + UT_Vector4H* mBuffer; +}; // struct VectorAttribute4H + void GR_PrimVDBPoints::updatePosBuffer(RE_Render* r, const openvdb::points::PointDataGrid& grid, const RE_CacheVersion& version) { - const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3); + const GR_RenderVersion renderVersion = getRenderVersion(); + const bool gl3 = (renderVersion == GR_RENDER_GL3 + || renderVersion == GR_RENDER_GL4); // Initialize the geometry with the proper name for the GL cache if (!myGeo) myGeo.reset(new RE_Geometry); @@ -545,6 +623,36 @@ GR_PrimVDBPoints::update(RE_RenderContext r, const GT_PrimitiveHandle &primh, const GR_UpdateParms &p) { +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support + if (r.isVulkan()) + { + if (p.reason & (GR_GEO_CHANGED | GR_GEO_TOPOLOGY_CHANGED)) + { + const GT_PrimVDB& gt_primVDB = + static_cast(*primh); + + const openvdb::GridBase* grid = + const_cast(gt_primVDB).getGrid(); + + const openvdb::points::PointDataGrid& pointDataGrid = + static_cast(*grid); + + computeCentroid(pointDataGrid); + computeBbox(pointDataGrid); + + RV_Render* rv = r.vkRender(); + updatePosBufferVk(rv, pointDataGrid, p.geo_version); + mDefaultPointColor = !updateVec3BufferVk( + rv, pointDataGrid, "Cd", "Cd", p.geo_version); + updateVec3BufferVk( + rv, pointDataGrid, "N", "N", p.geo_version); + updateVec3BufferVk( + rv, pointDataGrid, "v", "V", p.geo_version); + } + return; + } +#endif + // patch the point shaders at run-time to add an offset (does nothing if already patched) patchShaderVertexOffset(r, thePixelShader); @@ -681,14 +789,294 @@ GR_PrimVDBPoints::removeBuffer(const std::string& name) myGeo->clearAttribute(name.c_str()); } + +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support + +void +GR_PrimVDBPoints::updatePosBufferVk(RV_Render* r, + const openvdb::points::PointDataGrid& grid, + const RE_CacheVersion& version) +{ + using GridType = openvdb::points::PointDataGrid; + using TreeType = GridType::TreeType; + using AttributeSet = openvdb::points::AttributeSet; + + const TreeType& tree = grid.tree(); + auto iter = tree.cbeginLeaf(); + if (!iter) return; + + const AttributeSet::Descriptor& descriptor = + iter->attributeSet().descriptor(); + + // check if group viewport is in use + + const openvdb::StringMetadata::ConstPtr s = + grid.getMetadata( + openvdb_houdini::META_GROUP_VIEWPORT); + + const std::string groupName = s ? s->value() : ""; + const bool useGroup = + !groupName.empty() && descriptor.hasGroup(groupName); + + // count up total points + + int numPoints = 0; + if (useGroup) { + GroupFilter filter(groupName, iter->attributeSet()); + numPoints = static_cast(pointCount(tree, filter)); + } else { + NullFilter filter; + numPoints = static_cast(pointCount(tree, filter)); + } + + if (numPoints == 0) return; + + const size_t positionIndex = descriptor.find("P"); + if (positionIndex == AttributeSet::INVALID_POS) return; + + // check for decoration attributes + + mHasNormals = (descriptor.find("N") != AttributeSet::INVALID_POS); + mHasVelocity = (descriptor.find("v") != AttributeSet::INVALID_POS); + + // (re)create the VK geometry + + myGeoVk.reset(new RV_Geometry); + myGeoVk->setName("vdb_points"); + myGeoVk->setNumPoints(numPoints); + myGeoVk->createAttribute("P", RV_GPU_FLOAT32, 3); + myGeoVk->createAttribute("Cd", RV_GPU_FLOAT16, 4); + if (mHasNormals) + myGeoVk->createAttribute("N", RV_GPU_FLOAT16, 4); + if (mHasVelocity) + myGeoVk->createAttribute("V", RV_GPU_FLOAT16, 4); + myGeoVk->createConstant("pointSelection", RV_GPU_UINT32, 1); + myGeoVk->connectAllPrims(0, RV_PRIM_POINTS); + if (mHasVelocity) + myGeoVk->connectAllPrims(1, RV_PRIM_PATCHES, /*patch_size=*/1); + myGeoVk->populateBuffers(r); + + // set pointSelection to zero (no selection) + + myGeoVk->setAttributeConstValue("pointSelection", 0.0); + + // fill the position buffer (world-space, no offset subtraction) + + { + std::vector includeGroups, excludeGroups; + if (useGroup) includeGroups.emplace_back(groupName); + + MultiGroupFilter filter( + includeGroups, excludeGroups, iter->attributeSet()); + + std::vector offsets; + pointOffsets(offsets, grid.tree(), filter); + + UT_UniquePtr pdata(new UT_Vector3F[numPoints]); + + // use zero offset so positions are stored in world space + PositionAttribute positionAttribute( + pdata.get(), Vec3f(0, 0, 0)); + convertPointDataGridPosition( + positionAttribute, grid, offsets, + /*startOffset=*/ 0, filter); + + const exint byteSize = + static_cast(numPoints) * sizeof(UT_Vector3F); + myGeoVk->getAttribute("P")->uploadData(r, pdata.get(), byteSize); + } + + // set a default Cd (will be overwritten if Cd attribute exists) + + myGeoVk->setAttributeConstVecValue( + "Cd", UT_Vector4F(0.6f, 0.6f, 0.5f, 1.0f)); + +} + + +bool +GR_PrimVDBPoints::updateVec3BufferVk(RV_Render* r, + const openvdb::points::PointDataGrid& grid, + const std::string& attributeName, + const std::string& bufferName, + const RE_CacheVersion& version) +{ + if (!myGeoVk) return false; + + using GridType = openvdb::points::PointDataGrid; + using TreeType = GridType::TreeType; + using AttributeSet = openvdb::points::AttributeSet; + + const TreeType& tree = grid.tree(); + auto iter = tree.cbeginLeaf(); + if (!iter) return false; + + const int numPoints = static_cast(myGeoVk->getNumPoints()); + if (numPoints == 0) return false; + + const AttributeSet::Descriptor& descriptor = + iter->attributeSet().descriptor(); + const size_t index = descriptor.find(attributeName); + + if (index == AttributeSet::INVALID_POS) return false; + + const openvdb::Name& type = descriptor.type(index).first; + if (type != "vec3s") return false; + + // check if group viewport is in use + + const openvdb::StringMetadata::ConstPtr s = + grid.getMetadata( + openvdb_houdini::META_GROUP_VIEWPORT); + + const std::string groupName = s ? s->value() : ""; + const bool useGroup = + !groupName.empty() && descriptor.hasGroup(groupName); + + std::vector includeGroups, excludeGroups; + if (useGroup) includeGroups.emplace_back(groupName); + + MultiGroupFilter filter( + includeGroups, excludeGroups, iter->attributeSet()); + + std::vector offsets; + pointOffsets(offsets, grid.tree(), filter); + + // read vec3s attribute data directly into 4-component float16 + // (VK_FORMAT_R16G16B16A16_SFLOAT is mandatory; 3-component is not) + + UT_UniquePtr data(new UT_Vector4H[numPoints]); + + VectorAttribute4H typedAttribute(data.get()); + convertPointDataGridAttribute(typedAttribute, grid.tree(), offsets, + /*startOffset=*/ 0, static_cast(index), + /*stride=*/1, filter); + + const exint byteSize = + static_cast(numPoints) * sizeof(UT_Vector4H); + RV_VKBuffer* buf = myGeoVk->getAttribute(bufferName.c_str()); + if (buf) { + buf->uploadData(r, data.get(), byteSize); + } + + return true; +} + +#endif // UT_VERSION_INT >= 0x15000000 + + void GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_DrawParms dp) { - if (!myGeo) return; +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support + if (r.isVulkan()) + { + if (!myGeoVk) return; + + RV_Render* rv = r.vkRender(); + GR_Uniforms* uniforms = r.uniforms(); + + // initialize the VK point shader on first use + + if (!theVkPointShader) { + theVkPointShader = RV_ShaderProgram::loadShaderProgram( + rv->instance(), "openvdb/VK/points.prog"); + if (!theVkPointShader) return; + } - const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3); + // set the shader + + rv->setShader(theVkPointShader); + + // bind global uniform blocks (pass info, object transforms) + + if (uniforms) { + bool globalBound = uniforms->bindRVGlobalBlock(rv, theVkPointShader); + + // bind set 1 (glH_Object) for per-object transform uniforms + if (theVkPointShader->hasSet(1)) { + const auto* objBinding = + theVkPointShader->getBinding(1, 0); + if (objBinding) { + if (!myVkObjectBlock) { + myVkObjectBlock.reset( + RV_ShaderBlock::create( + rv->instance(), *objBinding)); + } + if (!myVkObjectSet || + !theVkPointShader->isSetCompatible( + *myVkObjectSet)) { + myVkObjectSet = + theVkPointShader->createSet( + rv->instance(), 1); + } + uniforms->assignRVBlock( + rv, myVkObjectBlock.get(), theVkPointShader); + // override DecorationScale with the user's + // point size from display options + const GR_CommonDispOption& commonOpts = + dp.opts->common(); + myVkObjectBlock->bindFloat( + "DecorationScale", + static_cast(commonOpts.pointSize())); + myVkObjectBlock->uploadBuffer(rv); + myVkObjectSet->attachBufferBlock( + rv->instance(), "Object", + myVkObjectBlock.get()); + rv->bindSet( + myVkObjectSet.get(), theVkPointShader); + } + } + + // bind set 2 (auto-injected Geometry uniform block) + if (theVkPointShader->hasSet(2)) { + const auto* geoBinding = + theVkPointShader->getBinding(2, 2); + if (geoBinding) { + if (!myVkGeoBlock) { + myVkGeoBlock.reset( + RV_ShaderBlock::create( + rv->instance(), *geoBinding)); + } + if (!myVkDrawingSet || + !theVkPointShader->isSetCompatible( + *myVkDrawingSet)) { + myVkDrawingSet = + theVkPointShader->createSet( + rv->instance(), 2); + } + myVkGeoBlock->uploadBuffer(rv); + myVkDrawingSet->attachBufferBlock( + rv->instance(), "Geometry", + myVkGeoBlock.get()); + rv->bindSet( + myVkDrawingSet.get(), theVkPointShader); + } + } + + } + + // draw - must be inside beginRendering/endRendering block + + bool beganRendering = false; + if (!rv->isRendering()) { + beganRendering = rv->beginRendering(); + } - if (!gl3) return; + rv->draw(myGeoVk.get(), 0); + + if (beganRendering) { + rv->endRendering(); + } + return; + } +#endif + + if (!myGeo) return; + + const GR_RenderVersion renderVersion = getRenderVersion(); + if (renderVersion != GR_RENDER_GL3 && renderVersion != GR_RENDER_GL4) + return; const GR_CommonDispOption& commonOpts = dp.opts->common(); @@ -734,6 +1122,127 @@ GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_D void GR_PrimVDBPoints::renderDecoration(RE_RenderContext r, GR_Decoration decor, const GR_DecorationParms& p) { +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support + if (r.isVulkan()) + { + if (!myGeoVk) return; + + if (decor == GR_POINT_MARKER) + { + drawDecorationForGeo(r, myGeoVk.get(), decor, p.opts, + GR_DECOR_RENDER_FLAG_NONE, + /*overlay=*/false, /*override_vis=*/false, + /*instance_group=*/-1, GR_SELECT_NONE, + GR_DecorationRender::PRIM_POINT); + } + else if (decor == GR_POINT_NORMAL && mHasNormals) + { + drawDecorationForGeo(r, myGeoVk.get(), decor, p.opts, + GR_DECOR_RENDER_FLAG_NONE, + /*overlay=*/false, /*override_vis=*/false, + /*instance_group=*/-1, GR_SELECT_NONE, + GR_DecorationRender::PRIM_POINT); + } + else if (decor == GR_POINT_VELOCITY && mHasVelocity) + { + // GPU-based velocity trail rendering using tessellation. + // The VK decoration system requires GR_GeoRenderVK for its + // built-in tessellation shaders, which custom GR_Primitive + // subclasses don't have. Instead, we use a custom velocity + // shader that reads P and V as vertex inputs and generates + // isolines via tessellation on the GPU. + + RV_Render* rv = r.vkRender(); + GR_Uniforms* uniforms = r.uniforms(); + + // load the velocity tessellation shader on first use + + if (!theVkVelocityShader) { + theVkVelocityShader = + RV_ShaderProgram::loadShaderProgram( + rv->instance(), + "openvdb/VK/velocity.prog"); + if (!theVkVelocityShader) return; + } + + rv->setShader(theVkVelocityShader); + + // compute velocity scale and trail color + + const GR_CommonDispOption& commonOpts = + p.opts->common(); + const float velocityScale = + static_cast(commonOpts.vectorScale()) + * -0.041f; + + UT_Color trailColor = + commonOpts.getColor(GR_POINT_TRAIL_COLOR); + float cr, cg, cb; + trailColor.getRGB(&cr, &cg, &cb); + + // bind uniform blocks + + if (uniforms) { + uniforms->bindRVGlobalBlock( + rv, theVkVelocityShader); + + // bind set 1 (glH_Object) with velocity overrides + if (theVkVelocityShader->hasSet(1)) { + const auto* objBinding = + theVkVelocityShader->getBinding(1, 0); + if (objBinding) { + if (!myVkVelObjectBlock) { + myVkVelObjectBlock.reset( + RV_ShaderBlock::create( + rv->instance(), + *objBinding)); + } + if (!myVkVelObjectSet || + !theVkVelocityShader->isSetCompatible( + *myVkVelObjectSet)) { + myVkVelObjectSet = + theVkVelocityShader->createSet( + rv->instance(), 1); + } + // fill with viewport transforms + uniforms->assignRVBlock(rv, + myVkVelObjectBlock.get(), + theVkVelocityShader); + // override decoration scale and wire color + myVkVelObjectBlock->bindFloat( + "DecorationScale", velocityScale); + myVkVelObjectBlock->bindVector( + "WireColor", + UT_Vector4F(cr, cg, cb, 1.0f)); + myVkVelObjectBlock->uploadBuffer(rv); + myVkVelObjectSet->attachBufferBlock( + rv->instance(), "Object", + myVkVelObjectBlock.get()); + rv->bindSet(myVkVelObjectSet.get(), + theVkVelocityShader); + } + } + } + + // draw using PATCHES connection group (index 1) + + bool beganRendering = false; + if (!rv->isRendering()) { + beganRendering = rv->beginRendering(); + } + + rv->draw(myGeoVk.get(), 1); + + if (beganRendering) { + rv->endRendering(); + } + } + return; + } +#endif + + if (!myGeo) return; + // just render native GR_Primitive decorations if position not available const RE_VertexArray* const position = myGeo->getAttribute("P"); diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.frag b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.frag new file mode 100644 index 0000000000..22229d6310 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.frag @@ -0,0 +1,28 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB Points fragment shader for Vulkan. +// Outputs the interpolated color from the vertex shader. +// Uses gl_PointCoord to discard fragments outside a circular disc +// with smooth anti-aliased edges. + +layout(location = 0) in vec4 pnt_color; + +layout(location = 0) out vec4 color_out; + +void main() +{ + if (pnt_color.a == 0.0) + discard; + + // circular point sprite with anti-aliased edge + vec2 coord = gl_PointCoord * 2.0 - 1.0; + float r2 = dot(coord, coord); + if (r2 > 1.0) + discard; + + // smooth edge over the outermost ~1 pixel + float alpha = 1.0 - smoothstep(0.8, 1.0, r2); + + color_out = vec4(pnt_color.rgb, pnt_color.a * alpha); +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.prog b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.prog new file mode 100644 index 0000000000..ee21815c92 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.prog @@ -0,0 +1,6 @@ +#name VDB Points +#version 450 +#output color 0 + +points.vert +points.frag diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.vert b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.vert new file mode 100644 index 0000000000..5257e48d01 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.vert @@ -0,0 +1,28 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// Minimal VDB Points vertex shader for Vulkan. +// Transforms position P by the standard Houdini view/projection matrices, +// passes Cd (color) to the fragment shader. +// +// Uses direct vertex inputs (not ATTRIB macros) because the geometry is +// populated with createAttribute() which creates VBOs, not SSBOs. + +layout(location = 0) in vec3 P; +layout(location = 1) in vec3 Cd; + +layout(location = 0) out vec4 pnt_color; + +layout(set=0, binding=0) +#using glH_PassInfo + +layout(set=1, binding=0) +#using glH_Object + +void main() +{ + vec4 pos = glH_Object.ObjView * vec4(P, 1.0); + gl_Position = glH_PassInfo.Projection * pos; + gl_PointSize = glH_Object.DecorationScale; + pnt_color = vec4(Cd, 1.0); +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.frag b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.frag new file mode 100644 index 0000000000..16d45e8293 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.frag @@ -0,0 +1,26 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB velocity decoration fragment shader. +// Applies a fade effect along the velocity trail using the u parameter +// from the tessellation evaluation shader. Trail color comes from +// glH_Object.WireColor which is set by the C++ code to the trail color. + +layout(location = 0) in float fsU; + +layout(location = 0) out vec4 color_out; + +layout(set=1, binding=0) +#using glH_Object + +void main() +{ + vec4 col = glH_Object.WireColor; + + // cubic fade from opaque (tip) to transparent (tail) + float a = 1.0 - fsU; + a = 1.0 - a * a * a; + a *= col.a; + + color_out = vec4(col.rgb, a); +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.prog b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.prog new file mode 100644 index 0000000000..3470f6fba5 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.prog @@ -0,0 +1,8 @@ +#name VDB Velocity Lines +#version 450 +#output color 0 + +velocity.vert +velocity.tcs +velocity.tes +velocity.frag diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tcs b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tcs new file mode 100644 index 0000000000..3bf7e2bfbb --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tcs @@ -0,0 +1,46 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB velocity decoration tessellation control shader. +// Takes a single control point (patch size 1) and computes the +// start and end positions of the velocity vector in clip space. +// Tessellation generates one isoline segment (2 vertices). + +layout(vertices = 1) out; + +layout(location = 0) in parms +{ + vec3 velocity; +} tsIn[]; + +layout(location = 0) out patch nparms +{ + vec4 startPos; + vec4 endPos; +} tsOut; + +layout(set=0, binding=0) +#using glH_PassInfo + +layout(set=1, binding=0) +#using glH_Object + +void main() +{ + vec3 vel = tsIn[0].velocity * glH_Object.DecorationScale; + + // start position in clip space + tsOut.startPos = glH_PassInfo.Projection * gl_in[0].gl_Position; + + // end position in clip space (add scaled velocity in view space) + tsOut.endPos = glH_PassInfo.Projection * + (gl_in[0].gl_Position + vec4(vel, 0.0)); + + // one isoline segment = 2 vertices + gl_TessLevelInner[0] = 0.0; + gl_TessLevelInner[1] = 0.0; + gl_TessLevelOuter[0] = 1.0; + gl_TessLevelOuter[1] = 1.0; + gl_TessLevelOuter[2] = 1.0; + gl_TessLevelOuter[3] = 1.0; +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tes b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tes new file mode 100644 index 0000000000..0eea5f48e4 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tes @@ -0,0 +1,23 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB velocity decoration tessellation evaluation shader. +// Generates isoline vertices by interpolating between start and end +// positions. The u parameter drives a fade effect in the fragment shader. + +layout(isolines) in; + +layout(location = 0) in patch nparms +{ + vec4 startPos; + vec4 endPos; +} tsIn; + +layout(location = 0) out float fsU; + +void main() +{ + float t = gl_TessCoord.x; + fsU = 1.0 - t; + gl_Position = mix(tsIn.startPos, tsIn.endPos, t); +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.vert b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.vert new file mode 100644 index 0000000000..bc2f0a0a16 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.vert @@ -0,0 +1,30 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB velocity decoration vertex shader for Vulkan. +// Reads P (position) and V (velocity) as vertex inputs. +// Transforms position to view space. Velocity is passed through +// in view space for the tessellation stage to generate isolines. + +layout(location = 0) in vec3 P; +layout(location = 1) in vec3 V; + +layout(location = 0) out parms +{ + vec3 velocity; +} vsOut; + +layout(set=0, binding=0) +#using glH_PassInfo + +layout(set=1, binding=0) +#using glH_Object + +void main() +{ + // position in view space (projection applied in TCS) + gl_Position = glH_Object.ObjView * vec4(P, 1.0); + + // velocity direction in view space + vsOut.velocity = mat3(glH_Object.ObjView) * V; +} From d6a78e79a44ac5315b979210d131e01fd6ef38a2 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 12 Feb 2026 17:59:36 -0800 Subject: [PATCH 09/70] Add USE_VULKAN macro for Mac OSX Signed-off-by: Dan Bailey --- .../openvdb_houdini/GR_PrimVDBPoints.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index c4f45b7272..7869ab94a7 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -30,7 +30,7 @@ #include #include -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support #include #include #include @@ -57,7 +57,7 @@ static RE_ShaderHandle theVelocityDecorShader("decor/GL32/user_point_vector3.pro static RE_ShaderHandle thePixelShader("particle/GL32/pixel.prog"); static RE_ShaderHandle thePointShader("particle/GL32/point.prog"); -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support static RV_ShaderProgram* theVkPointShader = nullptr; static RV_ShaderProgram* theVkVelocityShader = nullptr; #endif @@ -165,7 +165,7 @@ class GR_PrimVDBPoints : public GR_Primitive void removeBuffer(const std::string& name); -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support void updatePosBufferVk(RV_Render* r, const openvdb::points::PointDataGrid& grid, const RE_CacheVersion& version); @@ -182,7 +182,7 @@ class GR_PrimVDBPoints : public GR_Primitive bool mDefaultPointColor = true; openvdb::Vec3f mCentroid{0, 0, 0}; openvdb::BBoxd mBbox; -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support UT_UniquePtr myGeoVk; UT_UniquePtr myVkObjectSet; UT_UniquePtr myVkObjectBlock; @@ -623,7 +623,7 @@ GR_PrimVDBPoints::update(RE_RenderContext r, const GT_PrimitiveHandle &primh, const GR_UpdateParms &p) { -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support if (r.isVulkan()) { if (p.reason & (GR_GEO_CHANGED | GR_GEO_TOPOLOGY_CHANGED)) @@ -790,7 +790,7 @@ GR_PrimVDBPoints::removeBuffer(const std::string& name) } -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support void GR_PrimVDBPoints::updatePosBufferVk(RV_Render* r, @@ -962,13 +962,13 @@ GR_PrimVDBPoints::updateVec3BufferVk(RV_Render* r, return true; } -#endif // UT_VERSION_INT >= 0x15000000 +#endif // UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) void GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_DrawParms dp) { -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support if (r.isVulkan()) { if (!myGeoVk) return; @@ -1122,7 +1122,7 @@ GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_D void GR_PrimVDBPoints::renderDecoration(RE_RenderContext r, GR_Decoration decor, const GR_DecorationParms& p) { -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support if (r.isVulkan()) { if (!myGeoVk) return; From c48b99f759a8542838da29ce723ab61cec41ee1f Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 12 Feb 2026 17:14:07 -0800 Subject: [PATCH 10/70] Remove unused class members and methods in future ABI=14 Signed-off-by: Dan Bailey --- .github/workflows/weekly.yml | 1 + openvdb/openvdb/Grid.h | 4 ++++ openvdb/openvdb/tree/LeafBuffer.h | 6 +++++- openvdb/openvdb/tree/Tree.h | 4 ++++ openvdb/openvdb/unittest/TestGrid.cc | 2 ++ 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index 614cb37de9..ee35ee2315 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -154,6 +154,7 @@ jobs: - { name: 'asan', image: '2026', build: 'asan', components: 'core,test', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DNANOVDB_USE_OPENVDB=ON -DOPENVDB_AX_STATIC=OFF -DOPENVDB_CORE_STATIC=OFF -DUSE_BLOSC=OFF' } # We never called blosc_destroy(), so disable blosc to silence these errors - { name: 'ubsan', image: '2026', build: 'ubsan', components: 'core,test', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DCMAKE_CXX_FLAGS="-Wno-deprecated-declarations" ' } - { name: 'c++20', image: '2026', build: 'Release', components: 'core,test', cmake: '-DCMAKE_CXX_STANDARD=20' } + - { name: 'abi14', image: '2026', build: 'Release', components: 'core,test', cmake: '-DOPENVDB_USE_FUTURE_ABI_14=ON -DOPENVDB_ENABLE_ASSERTS=ON' } - { name: 'conf', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON' } fail-fast: false steps: diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index ecb18f9b66..b364e9efc0 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -458,8 +458,10 @@ class OPENVDB_API GridBase: public MetaMap /// Read all of this grid's data buffers that intersect the given index-space bounding box. virtual void readBuffers(std::istream&, const CoordBBox&) = 0; +#if OPENVDB_ABI_VERSION_NUMBER < 14 OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void readNonresidentBuffers() const = 0; +#endif /// Write out all data buffers for this grid. virtual void writeBuffers(std::ostream&) const = 0; @@ -938,8 +940,10 @@ class Grid: public GridBase /// Read all of this grid's data buffers that intersect the given index-space bounding box. void readBuffers(std::istream&, const CoordBBox&) override; +#if OPENVDB_ABI_VERSION_NUMBER < 14 OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") void readNonresidentBuffers() const override { } +#endif /// Write out all data buffers for this grid. void writeBuffers(std::ostream&) const override; diff --git a/openvdb/openvdb/tree/LeafBuffer.h b/openvdb/openvdb/tree/LeafBuffer.h index 80da87c721..bb6222cf84 100644 --- a/openvdb/openvdb/tree/LeafBuffer.h +++ b/openvdb/openvdb/tree/LeafBuffer.h @@ -8,9 +8,11 @@ #include // for io::readCompressedValues(), etc #include #include +#if OPENVDB_ABI_VERSION_NUMBER < 14 #include -#include // for std::swap #include +#endif +#include // for std::swap #include // for offsetof() #include #include @@ -107,9 +109,11 @@ class LeafBuffer bool deallocate(); ValueType* mData = nullptr; +#if OPENVDB_ABI_VERSION_NUMBER < 14 // Deprecated members kept for ABI compatibility std::atomic mDeprecatedAtomic{0}; tbb::spin_mutex mDeprecatedSpinMutex; +#endif friend class ::TestLeaf; // Allow the parent LeafNode to access this buffer's data pointer. diff --git a/openvdb/openvdb/tree/Tree.h b/openvdb/openvdb/tree/Tree.h index a3371c71d7..9277c57b0f 100644 --- a/openvdb/openvdb/tree/Tree.h +++ b/openvdb/openvdb/tree/Tree.h @@ -162,8 +162,10 @@ class OPENVDB_API TreeBase /// Read all of this tree's data buffers that intersect the given bounding box. virtual void readBuffers(std::istream&, const CoordBBox&, bool saveFloatAsHalf = false) = 0; +#if OPENVDB_ABI_VERSION_NUMBER < 14 OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void readNonresidentBuffers() const = 0; +#endif /// Write out all the data buffers for this tree. virtual void writeBuffers(std::ostream&, bool saveFloatAsHalf = false) const = 0; @@ -331,8 +333,10 @@ class Tree: public TreeBase /// Read all of this tree's data buffers that intersect the given bounding box. void readBuffers(std::istream&, const CoordBBox&, bool saveFloatAsHalf = false) override; +#if OPENVDB_ABI_VERSION_NUMBER < 14 OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") void readNonresidentBuffers() const override { } +#endif /// Write out all data buffers for this tree. void writeBuffers(std::ostream&, bool saveFloatAsHalf = false) const override; diff --git a/openvdb/openvdb/unittest/TestGrid.cc b/openvdb/openvdb/unittest/TestGrid.cc index 147405c996..724109ce53 100644 --- a/openvdb/openvdb/unittest/TestGrid.cc +++ b/openvdb/openvdb/unittest/TestGrid.cc @@ -65,7 +65,9 @@ class ProxyTree: public openvdb::TreeBase void readBuffers(std::istream& is, const openvdb::CoordBBox&, bool /*saveFloatAsHalf*/=false) override { is.seekg(0); } +#if OPENVDB_ABI_VERSION_NUMBER < 14 void readNonresidentBuffers() const override {} +#endif void readBuffers(std::istream& is, bool /*saveFloatAsHalf*/=false) override { is.seekg(0); } void writeBuffers(std::ostream& os, bool /*saveFloatAsHalf*/=false) const override { os.seekp(0, std::ios::beg); } From 6e55069e92c470e686ebc560b03caefea4d42cb8 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 17 Feb 2026 11:27:01 -0800 Subject: [PATCH 11/70] Remove TreeBase readTopology() and writeTopology() Signed-off-by: Dan Bailey --- openvdb/openvdb/tree/Tree.h | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/openvdb/openvdb/tree/Tree.h b/openvdb/openvdb/tree/Tree.h index 9277c57b0f..07ed98e5db 100644 --- a/openvdb/openvdb/tree/Tree.h +++ b/openvdb/openvdb/tree/Tree.h @@ -151,11 +151,11 @@ class OPENVDB_API TreeBase /// @brief Read the tree topology from a stream. /// /// This will read the tree structure and tile values, but not voxel data. - virtual void readTopology(std::istream&, bool saveFloatAsHalf = false); + virtual void readTopology(std::istream&, bool saveFloatAsHalf = false) = 0; /// @brief Write the tree topology to a stream. /// /// This will write the tree structure and tile values, but not voxel data. - virtual void writeTopology(std::ostream&, bool saveFloatAsHalf = false) const; + virtual void writeTopology(std::ostream&, bool saveFloatAsHalf = false) const = 0; /// Read all data buffers for this tree. virtual void readBuffers(std::istream&, bool saveFloatAsHalf = false) = 0; @@ -1130,23 +1130,6 @@ struct Tree5 { //////////////////////////////////////// -inline void -TreeBase::readTopology(std::istream& is, bool /*saveFloatAsHalf*/) -{ - int32_t bufferCount; - is.read(reinterpret_cast(&bufferCount), sizeof(int32_t)); - if (bufferCount != 1) OPENVDB_LOG_WARN("multi-buffer trees are no longer supported"); -} - - -inline void -TreeBase::writeTopology(std::ostream& os, bool /*saveFloatAsHalf*/) const -{ - int32_t bufferCount = 1; - os.write(reinterpret_cast(&bufferCount), sizeof(int32_t)); -} - - inline void TreeBase::print(std::ostream& os, int /*verboseLevel*/) const { @@ -1273,7 +1256,9 @@ void Tree::readTopology(std::istream& is, bool saveFloatAsHalf) { this->clearAllAccessors(); - TreeBase::readTopology(is, saveFloatAsHalf); + int32_t bufferCount; + is.read(reinterpret_cast(&bufferCount), sizeof(int32_t)); + if (bufferCount != 1) OPENVDB_LOG_WARN("multi-buffer trees are no longer supported"); mRoot.readTopology(is, saveFloatAsHalf); } @@ -1282,7 +1267,8 @@ template void Tree::writeTopology(std::ostream& os, bool saveFloatAsHalf) const { - TreeBase::writeTopology(os, saveFloatAsHalf); + int32_t bufferCount = 1; + os.write(reinterpret_cast(&bufferCount), sizeof(int32_t)); mRoot.writeTopology(os, saveFloatAsHalf); } From bdaceec3ef7fe0d22c5573917f32678610ec8cd0 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 28 Nov 2025 16:06:59 -0800 Subject: [PATCH 12/70] Update all existing unit tests to use Grid I/O methods instead of Leaf I/O methods Signed-off-by: Dan Bailey --- openvdb/openvdb/unittest/TestLeafBool.cc | 50 ++++++-- openvdb/openvdb/unittest/TestLeafIO.cc | 109 +++++++++++++---- openvdb/openvdb/unittest/TestLeafMask.cc | 50 ++++++-- openvdb/openvdb/unittest/TestPointDataLeaf.cc | 113 ++++++++++-------- openvdb/openvdb/unittest/TestTree.cc | 47 ++++---- pendingchanges/removeleafiotests.txt | 3 + 6 files changed, 252 insertions(+), 120 deletions(-) create mode 100644 pendingchanges/removeleafiotests.txt diff --git a/openvdb/openvdb/unittest/TestLeafBool.cc b/openvdb/openvdb/unittest/TestLeafBool.cc index 6acae26ea2..79f9a00a83 100644 --- a/openvdb/openvdb/unittest/TestLeafBool.cc +++ b/openvdb/openvdb/unittest/TestLeafBool.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include "util.h" // for unittest_util::makeSphere() #include #include @@ -293,26 +294,49 @@ TEST_F(TestLeafBool, testIO) leaf.setValueOn(openvdb::Coord(0, 1, 0)); leaf.setValueOn(openvdb::Coord(1, 0, 0)); - std::ostringstream ostr(std::ios_base::binary); + // read and write topology to disk - leaf.writeBuffers(ostr); + { + // create a grid with the leaf for topology testing + typedef openvdb::Grid, 5>>>> BoolGrid; + BoolGrid::Ptr grid = BoolGrid::create(); + grid->setName("bool_leaf"); + grid->tree().addLeaf(new LeafType(leaf)); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); + + // write to file + { + openvdb::io::File file("leaf_bool.vdb"); + file.write(grids); + file.close(); + } - leaf.setValueOff(openvdb::Coord(0, 1, 0)); - leaf.setValueOn(openvdb::Coord(0, 1, 1)); + // read grid from file + BoolGrid::Ptr gridFromDisk; + { + openvdb::io::File file("leaf_bool.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("bool_leaf"); + file.close(); - std::istringstream istr(ostr.str(), std::ios_base::binary); - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(istr); + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } - leaf.readBuffers(istr); + LeafType* leaf2 = gridFromDisk->tree().probeLeaf(origin); + EXPECT_TRUE(leaf2); - EXPECT_EQ(origin, leaf.origin()); + // check topology and values match - EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(0, 1, 0))); - EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(1, 0, 0))); + EXPECT_EQ(origin, leaf2->origin()); + EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(0, 1, 0))); + EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(1, 0, 0))); + EXPECT_TRUE(leaf2->onVoxelCount() == 2); - EXPECT_TRUE(leaf.onVoxelCount() == 2); + remove("leaf_bool.vdb"); + } } diff --git a/openvdb/openvdb/unittest/TestLeafIO.cc b/openvdb/openvdb/unittest/TestLeafIO.cc index 3b35259f53..a67cd9fdf6 100644 --- a/openvdb/openvdb/unittest/TestLeafIO.cc +++ b/openvdb/openvdb/unittest/TestLeafIO.cc @@ -2,8 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 #include +#include #include #include +#include #include #include // for toupper() @@ -21,35 +23,65 @@ template void TestLeafIO::testBuffer() { - openvdb::tree::LeafNode leaf(openvdb::Coord(0, 0, 0)); + using LeafT = openvdb::tree::LeafNode; + LeafT leaf(openvdb::Coord(0, 0, 0)); + const openvdb::Coord origin = leaf.origin(); leaf.setValueOn(openvdb::Coord(0, 1, 0), T(1)); leaf.setValueOn(openvdb::Coord(1, 0, 0), T(1)); - std::ostringstream ostr(std::ios_base::binary); + // read and write topology to disk - leaf.writeBuffers(ostr); + { + // create a grid with the leaf for topology testing + typedef openvdb::Grid, 5>>>> GridType; + if (!GridType::isRegistered()) GridType::registerGrid(); - leaf.setValueOn(openvdb::Coord(0, 1, 0), T(0)); - leaf.setValueOn(openvdb::Coord(0, 1, 1), T(1)); + typename GridType::Ptr grid = GridType::create(); + grid->setName("leaf_io"); + grid->tree().addLeaf(new LeafT(leaf)); - std::istringstream istr(ostr.str(), std::ios_base::binary); + openvdb::GridCPtrVec grids; + grids.push_back(grid); - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(istr); + // write to file + { + openvdb::io::File file("leaf_io.vdb"); + file.write(grids); + file.close(); + } - leaf.readBuffers(istr); + // read grid from file + typename GridType::Ptr gridFromDisk; + { + openvdb::io::File file("leaf_io.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("leaf_io"); + file.close(); - EXPECT_NEAR(T(1), leaf.getValue(openvdb::Coord(0, 1, 0)), /*tolerance=*/0); - EXPECT_NEAR(T(1), leaf.getValue(openvdb::Coord(1, 0, 0)), /*tolerance=*/0); + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } - EXPECT_TRUE(leaf.onVoxelCount() == 2); + LeafT* leaf2 = gridFromDisk->tree().probeLeaf(origin); + EXPECT_TRUE(leaf2); + + // check topology and values match + + EXPECT_NEAR(T(1), leaf2->getValue(openvdb::Coord(0, 1, 0)), /*tolerance=*/0); + EXPECT_NEAR(T(1), leaf2->getValue(openvdb::Coord(1, 0, 0)), /*tolerance=*/0); + EXPECT_TRUE(leaf2->onVoxelCount() == 2); + + remove("leaf_io.vdb"); + } } class TestLeafIOTest: public ::testing::Test { +public: + void SetUp() override { openvdb::initialize(); } + void TearDown() override { openvdb::uninitialize(); } }; @@ -62,28 +94,53 @@ TEST_F(TestLeafIOTest, testBufferByte) { TestLeafIO::testBuffer() TEST_F(TestLeafIOTest, testBufferVec3R) { - openvdb::tree::LeafNode leaf(openvdb::Coord(0, 0, 0)); + using LeafT = openvdb::tree::LeafNode; + LeafT leaf(openvdb::Coord(0, 0, 0)); + const openvdb::Coord origin = leaf.origin(); leaf.setValueOn(openvdb::Coord(0, 1, 0), openvdb::Vec3R(1, 1, 1)); leaf.setValueOn(openvdb::Coord(1, 0, 0), openvdb::Vec3R(1, 1, 1)); - std::ostringstream ostr(std::ios_base::binary); + // read and write topology to disk + + { + // create a grid with the leaf for topology testing + typedef openvdb::Grid, 5>>>> GridType; + GridType::Ptr grid = GridType::create(); + grid->setName("leaf_vec3r"); + grid->tree().addLeaf(new LeafT(leaf)); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); - leaf.writeBuffers(ostr); + // write to file + { + openvdb::io::File file("leaf_vec3r.vdb"); + file.write(grids); + file.close(); + } - leaf.setValueOn(openvdb::Coord(0, 1, 0), openvdb::Vec3R(0, 0, 0)); - leaf.setValueOn(openvdb::Coord(0, 1, 1), openvdb::Vec3R(1, 1, 1)); + // read grid from file + GridType::Ptr gridFromDisk; + { + openvdb::io::File file("leaf_vec3r.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("leaf_vec3r"); + file.close(); - std::istringstream istr(ostr.str(), std::ios_base::binary); + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(istr); + LeafT* leaf2 = gridFromDisk->tree().probeLeaf(origin); + EXPECT_TRUE(leaf2); - leaf.readBuffers(istr); + // check topology and values match - EXPECT_TRUE(leaf.getValue(openvdb::Coord(0, 1, 0)) == openvdb::Vec3R(1, 1, 1)); - EXPECT_TRUE(leaf.getValue(openvdb::Coord(1, 0, 0)) == openvdb::Vec3R(1, 1, 1)); + EXPECT_TRUE(leaf2->getValue(openvdb::Coord(0, 1, 0)) == openvdb::Vec3R(1, 1, 1)); + EXPECT_TRUE(leaf2->getValue(openvdb::Coord(1, 0, 0)) == openvdb::Vec3R(1, 1, 1)); + EXPECT_TRUE(leaf2->onVoxelCount() == 2); - EXPECT_TRUE(leaf.onVoxelCount() == 2); + remove("leaf_vec3r.vdb"); + } } diff --git a/openvdb/openvdb/unittest/TestLeafMask.cc b/openvdb/openvdb/unittest/TestLeafMask.cc index 7597158f45..ada92c2e12 100644 --- a/openvdb/openvdb/unittest/TestLeafMask.cc +++ b/openvdb/openvdb/unittest/TestLeafMask.cc @@ -6,6 +6,7 @@ #include #include #include +#include #include "util.h" // for unittest_util::makeSphere() #include #include @@ -290,26 +291,49 @@ TEST_F(TestLeafMask, testIO) leaf.setValueOn(openvdb::Coord(0, 1, 0)); leaf.setValueOn(openvdb::Coord(1, 0, 0)); - std::ostringstream ostr(std::ios_base::binary); + // read and write topology to disk - leaf.writeBuffers(ostr); + { + // create a grid with the leaf for topology testing + typedef openvdb::Grid, 5>>>> MaskGrid; + MaskGrid::Ptr grid = MaskGrid::create(); + grid->setName("leaf_mask"); + grid->tree().addLeaf(new LeafType(leaf)); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); + + // write to file + { + openvdb::io::File file("leaf_mask.vdb"); + file.write(grids); + file.close(); + } - leaf.setValueOff(openvdb::Coord(0, 1, 0)); - leaf.setValueOn(openvdb::Coord(0, 1, 1)); + // read grid from file + MaskGrid::Ptr gridFromDisk; + { + openvdb::io::File file("leaf_mask.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("leaf_mask"); + file.close(); - std::istringstream istr(ostr.str(), std::ios_base::binary); - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(istr); + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } - leaf.readBuffers(istr); + LeafType* leaf2 = gridFromDisk->tree().probeLeaf(origin); + EXPECT_TRUE(leaf2); - EXPECT_EQ(origin, leaf.origin()); + // check topology and values match - EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(0, 1, 0))); - EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(1, 0, 0))); + EXPECT_EQ(origin, leaf2->origin()); + EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(0, 1, 0))); + EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(1, 0, 0))); + EXPECT_TRUE(leaf2->onVoxelCount() == 2); - EXPECT_TRUE(leaf.onVoxelCount() == 2); + remove("leaf_mask.vdb"); + } } diff --git a/openvdb/openvdb/unittest/TestPointDataLeaf.cc b/openvdb/openvdb/unittest/TestPointDataLeaf.cc index 651b3174d9..24fd1a679e 100644 --- a/openvdb/openvdb/unittest/TestPointDataLeaf.cc +++ b/openvdb/openvdb/unittest/TestPointDataLeaf.cc @@ -1058,76 +1058,93 @@ TEST_F(TestPointDataLeaf, testIO) // read and write topology to disk { - LeafType leaf2(openvdb::Coord(0, 0, 0)); + // create a grid with the leaf for topology testing + PointDataGrid::Ptr grid = PointDataGrid::create(); + grid->setName("points"); + grid->tree().addLeaf(new LeafType(leaf)); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); + + // write to file + { + io::File file("leaf_topology.vdb"); + file.write(grids); + file.close(); + } - std::ostringstream ostr(std::ios_base::binary); - leaf.writeTopology(ostr); + // read grid from file + PointDataGrid::Ptr gridFromDisk; + { + io::File file("leaf_topology.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); + file.close(); - std::istringstream istr(ostr.str(), std::ios_base::binary); - leaf2.readTopology(istr); + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } + + LeafType* leaf2 = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); + EXPECT_TRUE(leaf2); // check topology matches - EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount()); - EXPECT_TRUE(leaf2.isValueOn(4)); - EXPECT_TRUE(!leaf2.isValueOn(5)); + EXPECT_EQ(leaf.onVoxelCount(), leaf2->onVoxelCount()); + EXPECT_TRUE(leaf2->isValueOn(4)); + EXPECT_TRUE(!leaf2->isValueOn(5)); + + // check that values and attributes are correctly read - // check only topology (values and attributes still empty) + EXPECT_EQ(leaf2->getValue(4), ValueType(20)); + EXPECT_EQ(leaf2->attributeSet().size(), size_t(2)); - EXPECT_EQ(leaf2.getValue(4), ValueType(0)); - EXPECT_EQ(leaf2.attributeSet().size(), size_t(0)); + remove("leaf_topology.vdb"); } // read and write buffers to disk { - LeafType leaf2(openvdb::Coord(0, 0, 0)); - - io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata); - - std::ostringstream ostr(std::ios_base::binary); - io::setStreamMetadataPtr(ostr, streamMetadata); - io::setDataCompression(ostr, io::COMPRESS_BLOSC); - leaf.writeTopology(ostr); - for (Index b = 0; b < leaf.buffers(); b++) { - uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b); - streamMetadata->setPass(pass); - leaf.writeBuffers(ostr); - } - { // error checking - streamMetadata->setPass(1000); - leaf.writeBuffers(ostr); + // create a grid with the leaf for buffer testing + PointDataGrid::Ptr grid = PointDataGrid::create(); + grid->setName("points"); + grid->tree().addLeaf(new LeafType(leaf)); - io::StreamMetadata::Ptr meta; - io::setStreamMetadataPtr(ostr, meta); - EXPECT_THROW(leaf.writeBuffers(ostr), openvdb::IoError); - } + openvdb::GridCPtrVec grids; + grids.push_back(grid); - std::istringstream istr(ostr.str(), std::ios_base::binary); - io::setStreamMetadataPtr(istr, streamMetadata); - io::setDataCompression(istr, io::COMPRESS_BLOSC); + // write to file + { + io::File file("leaf_buffers.vdb"); + file.write(grids); + file.close(); + } - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - io::setCurrentVersion(istr); + // read grid from file + PointDataGrid::Ptr gridFromDisk; + { + io::File file("leaf_buffers.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); + file.close(); - leaf2.readTopology(istr); - for (Index b = 0; b < leaf.buffers(); b++) { - uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b); - streamMetadata->setPass(pass); - leaf2.readBuffers(istr); + gridFromDisk = openvdb::gridPtrCast(baseGrid); } + LeafType* leaf2 = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); + EXPECT_TRUE(leaf2); + // check topology matches - EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount()); - EXPECT_TRUE(leaf2.isValueOn(4)); - EXPECT_TRUE(!leaf2.isValueOn(5)); + EXPECT_EQ(leaf.onVoxelCount(), leaf2->onVoxelCount()); + EXPECT_TRUE(leaf2->isValueOn(4)); + EXPECT_TRUE(!leaf2->isValueOn(5)); + + // check values and attributes are correctly read - // check only topology (values and attributes still empty) + EXPECT_EQ(leaf2->getValue(4), ValueType(20)); + EXPECT_EQ(leaf2->attributeSet().size(), size_t(2)); - EXPECT_EQ(leaf2.getValue(4), ValueType(20)); - EXPECT_EQ(leaf2.attributeSet().size(), size_t(2)); + remove("leaf_buffers.vdb"); } { // test multi-buffer IO diff --git a/openvdb/openvdb/unittest/TestTree.cc b/openvdb/openvdb/unittest/TestTree.cc index a5ac2afade..a132b2fd57 100644 --- a/openvdb/openvdb/unittest/TestTree.cc +++ b/openvdb/openvdb/unittest/TestTree.cc @@ -731,37 +731,44 @@ TEST_F(TestTree, testIterators) TEST_F(TestTree, testIO) { - const char* filename = "testIO.dbg"; + using TreeType = openvdb::tree::Tree; + using GridType = openvdb::Grid; + + const char* filename = "testIO.vdb"; openvdb::SharedPtr scopedFile(filename, ::remove); { ValueType background=5.0f; - RootNodeType root_node(background); - root_node.setValueOn(openvdb::Coord(5,10,20),0.234f); - root_node.setValueOn(openvdb::Coord(50000,20000,30000),4.5678f); + GridType::Ptr grid = GridType::create(background); + grid->setName("test_grid"); + grid->tree().setValueOn(openvdb::Coord(5,10,20),0.234f); + grid->tree().setValueOn(openvdb::Coord(50000,20000,30000),4.5678f); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); - std::ofstream os(filename, std::ios_base::binary); - root_node.writeTopology(os); - root_node.writeBuffers(os); - EXPECT_TRUE(!os.fail()); + openvdb::io::File file(filename); + file.write(grids); + file.close(); } { ValueType background=2.0f; - RootNodeType root_node(background); - ASSERT_DOUBLES_EXACTLY_EQUAL(background, root_node.getValue(openvdb::Coord(5,10,20))); + GridType::Ptr grid = GridType::create(background); + ASSERT_DOUBLES_EXACTLY_EQUAL(background, grid->tree().getValue(openvdb::Coord(5,10,20))); + { - std::ifstream is(filename, std::ios_base::binary); - // Since the test file doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(is); - root_node.readTopology(is); - root_node.readBuffers(is); - EXPECT_TRUE(!is.fail()); + openvdb::io::File file(filename); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("test_grid"); + file.close(); + + grid = openvdb::gridPtrCast(baseGrid); + EXPECT_TRUE(grid.get() != nullptr); } - ASSERT_DOUBLES_EXACTLY_EQUAL(0.234f, root_node.getValue(openvdb::Coord(5,10,20))); - ASSERT_DOUBLES_EXACTLY_EQUAL(5.0f, root_node.getValue(openvdb::Coord(5,11,20))); + ASSERT_DOUBLES_EXACTLY_EQUAL(0.234f, grid->tree().getValue(openvdb::Coord(5,10,20))); + ASSERT_DOUBLES_EXACTLY_EQUAL(5.0f, grid->tree().getValue(openvdb::Coord(5,11,20))); ValueType sum=0.0f; - for (RootNodeType::ChildOnIter root_iter = root_node.beginChildOn(); + for (RootNodeType::ChildOnIter root_iter = grid->tree().root().beginChildOn(); root_iter.test(); ++root_iter) { for (InternalNodeType2::ChildOnIter internal_iter2 = root_iter->beginChildOn(); diff --git a/pendingchanges/removeleafiotests.txt b/pendingchanges/removeleafiotests.txt new file mode 100644 index 0000000000..481a910de8 --- /dev/null +++ b/pendingchanges/removeleafiotests.txt @@ -0,0 +1,3 @@ +OpenVDB: + Improvements: + - Remove direct testing of Tree I/O methods in favor of indirect testing using the Grid I/O. From 47cf7af05288ea3805e8d93b50867f410d5b41c7 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 10 Mar 2026 22:03:49 -0700 Subject: [PATCH 13/70] Add TreeIO methods for regression testing Signed-off-by: Dan Bailey --- openvdb/openvdb/unittest/TestLeafBool.cc | 29 ++++++ openvdb/openvdb/unittest/TestLeafIO.cc | 63 +++++++++++++ openvdb/openvdb/unittest/TestLeafMask.cc | 29 ++++++ openvdb/openvdb/unittest/TestPointDataLeaf.cc | 91 +++++++++++++++++++ openvdb/openvdb/unittest/TestTree.cc | 52 +++++++++++ 5 files changed, 264 insertions(+) diff --git a/openvdb/openvdb/unittest/TestLeafBool.cc b/openvdb/openvdb/unittest/TestLeafBool.cc index 79f9a00a83..3bb4522001 100644 --- a/openvdb/openvdb/unittest/TestLeafBool.cc +++ b/openvdb/openvdb/unittest/TestLeafBool.cc @@ -340,6 +340,35 @@ TEST_F(TestLeafBool, testIO) } +TEST_F(TestLeafBool, testTreeIO) +{ + LeafType leaf(openvdb::Coord(1, 3, 5)); + const openvdb::Coord origin = leaf.origin(); + + leaf.setValueOn(openvdb::Coord(0, 1, 0)); + leaf.setValueOn(openvdb::Coord(1, 0, 0)); + + std::ostringstream ostr(std::ios_base::binary); + + leaf.writeBuffers(ostr); + + leaf.setValueOff(openvdb::Coord(0, 1, 0)); + leaf.setValueOn(openvdb::Coord(0, 1, 1)); + + std::istringstream istr(ostr.str(), std::ios_base::binary); + openvdb::io::setCurrentVersion(istr); + + leaf.readBuffers(istr); + + EXPECT_EQ(origin, leaf.origin()); + + EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(0, 1, 0))); + EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(1, 0, 0))); + + EXPECT_TRUE(leaf.onVoxelCount() == 2); +} + + TEST_F(TestLeafBool, testConstructors) { using openvdb::Coord; diff --git a/openvdb/openvdb/unittest/TestLeafIO.cc b/openvdb/openvdb/unittest/TestLeafIO.cc index a67cd9fdf6..7e4cf04575 100644 --- a/openvdb/openvdb/unittest/TestLeafIO.cc +++ b/openvdb/openvdb/unittest/TestLeafIO.cc @@ -17,6 +17,7 @@ class TestLeafIO { public: static void testBuffer(); + static void testTreeIO(); }; template @@ -77,6 +78,35 @@ TestLeafIO::testBuffer() } +template +void +TestLeafIO::testTreeIO() +{ + using LeafT = openvdb::tree::LeafNode; + LeafT leaf(openvdb::Coord(0, 0, 0)); + + leaf.setValueOn(openvdb::Coord(0, 1, 0), T(1)); + leaf.setValueOn(openvdb::Coord(1, 0, 0), T(1)); + + std::ostringstream ostr(std::ios_base::binary); + + leaf.writeBuffers(ostr); + + leaf.setValueOn(openvdb::Coord(0, 1, 0), T(0)); + leaf.setValueOn(openvdb::Coord(0, 1, 1), T(1)); + + std::istringstream istr(ostr.str(), std::ios_base::binary); + openvdb::io::setCurrentVersion(istr); + + leaf.readBuffers(istr); + + EXPECT_NEAR(T(1), leaf.getValue(openvdb::Coord(0, 1, 0)), /*tolerance=*/0); + EXPECT_NEAR(T(1), leaf.getValue(openvdb::Coord(1, 0, 0)), /*tolerance=*/0); + + EXPECT_TRUE(leaf.onVoxelCount() == 2); +} + + class TestLeafIOTest: public ::testing::Test { public: @@ -144,3 +174,36 @@ TEST_F(TestLeafIOTest, testBufferVec3R) remove("leaf_vec3r.vdb"); } } + +TEST_F(TestLeafIOTest, testTreeIOInt) { TestLeafIO::testTreeIO(); } +TEST_F(TestLeafIOTest, testTreeIOFloat) { TestLeafIO::testTreeIO(); } +TEST_F(TestLeafIOTest, testTreeIODouble) { TestLeafIO::testTreeIO(); } +TEST_F(TestLeafIOTest, testTreeIOBool) { TestLeafIO::testTreeIO(); } +TEST_F(TestLeafIOTest, testTreeIOByte) { TestLeafIO::testTreeIO(); } + + +TEST_F(TestLeafIOTest, testTreeIOVec3R) +{ + using LeafT = openvdb::tree::LeafNode; + LeafT leaf(openvdb::Coord(0, 0, 0)); + + leaf.setValueOn(openvdb::Coord(0, 1, 0), openvdb::Vec3R(1, 1, 1)); + leaf.setValueOn(openvdb::Coord(1, 0, 0), openvdb::Vec3R(1, 1, 1)); + + std::ostringstream ostr(std::ios_base::binary); + + leaf.writeBuffers(ostr); + + leaf.setValueOn(openvdb::Coord(0, 1, 0), openvdb::Vec3R(0, 0, 0)); + leaf.setValueOn(openvdb::Coord(0, 1, 1), openvdb::Vec3R(1, 1, 1)); + + std::istringstream istr(ostr.str(), std::ios_base::binary); + openvdb::io::setCurrentVersion(istr); + + leaf.readBuffers(istr); + + EXPECT_TRUE(leaf.getValue(openvdb::Coord(0, 1, 0)) == openvdb::Vec3R(1, 1, 1)); + EXPECT_TRUE(leaf.getValue(openvdb::Coord(1, 0, 0)) == openvdb::Vec3R(1, 1, 1)); + + EXPECT_TRUE(leaf.onVoxelCount() == 2); +} diff --git a/openvdb/openvdb/unittest/TestLeafMask.cc b/openvdb/openvdb/unittest/TestLeafMask.cc index ada92c2e12..6545b3461a 100644 --- a/openvdb/openvdb/unittest/TestLeafMask.cc +++ b/openvdb/openvdb/unittest/TestLeafMask.cc @@ -337,6 +337,35 @@ TEST_F(TestLeafMask, testIO) } +TEST_F(TestLeafMask, testTreeIO) +{ + LeafType leaf(openvdb::Coord(1, 3, 5)); + const openvdb::Coord origin = leaf.origin(); + + leaf.setValueOn(openvdb::Coord(0, 1, 0)); + leaf.setValueOn(openvdb::Coord(1, 0, 0)); + + std::ostringstream ostr(std::ios_base::binary); + + leaf.writeBuffers(ostr); + + leaf.setValueOff(openvdb::Coord(0, 1, 0)); + leaf.setValueOn(openvdb::Coord(0, 1, 1)); + + std::istringstream istr(ostr.str(), std::ios_base::binary); + openvdb::io::setCurrentVersion(istr); + + leaf.readBuffers(istr); + + EXPECT_EQ(origin, leaf.origin()); + + EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(0, 1, 0))); + EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(1, 0, 0))); + + EXPECT_TRUE(leaf.onVoxelCount() == 2); +} + + TEST_F(TestLeafMask, testTopologyCopy) { using openvdb::Coord; diff --git a/openvdb/openvdb/unittest/TestPointDataLeaf.cc b/openvdb/openvdb/unittest/TestPointDataLeaf.cc index 24fd1a679e..c5956e2991 100644 --- a/openvdb/openvdb/unittest/TestPointDataLeaf.cc +++ b/openvdb/openvdb/unittest/TestPointDataLeaf.cc @@ -1276,6 +1276,97 @@ TEST_F(TestPointDataLeaf, testIO) } +TEST_F(TestPointDataLeaf, testTreeIO) +{ + using AttributeVec3s = TypedAttributeArray; + using AttributeF = TypedAttributeArray; + + using Descriptor = AttributeSet::Descriptor; + + Descriptor::Ptr descrA = Descriptor::create(AttributeVec3s::attributeType()); + + const size_t size = LeafType::NUM_VOXELS; + + LeafType leaf(openvdb::Coord(0, 0, 0)); + leaf.initializeAttributes(descrA, /*arrayLength=*/size/2); + + descrA = descrA->duplicateAppend("density", AttributeF::attributeType()); + leaf.appendAttribute(leaf.attributeSet().descriptor(), descrA, descrA->find("density")); + + leaf.setOffsetOn(1, 10); + leaf.setOffsetOn(4, 20); + leaf.setOffsetOn(7, 5); + + TypedAttributeArray& attr = + TypedAttributeArray::cast(leaf.attributeArray("density")); + + attr.set(0, 5.0f); + attr.set(50, 2.0f); + attr.set(51, 8.1f); + + { + LeafType leaf2(openvdb::Coord(0, 0, 0)); + + std::ostringstream ostr(std::ios_base::binary); + leaf.writeTopology(ostr); + + std::istringstream istr(ostr.str(), std::ios_base::binary); + leaf2.readTopology(istr); + + EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount()); + EXPECT_TRUE(leaf2.isValueOn(4)); + EXPECT_TRUE(!leaf2.isValueOn(5)); + + EXPECT_EQ(leaf2.getValue(4), ValueType(0)); + EXPECT_EQ(leaf2.attributeSet().size(), size_t(0)); + } + + { + LeafType leaf2(openvdb::Coord(0, 0, 0)); + + io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata); + + std::ostringstream ostr(std::ios_base::binary); + io::setStreamMetadataPtr(ostr, streamMetadata); + io::setDataCompression(ostr, io::COMPRESS_BLOSC); + leaf.writeTopology(ostr); + for (Index b = 0; b < leaf.buffers(); b++) { + uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b); + streamMetadata->setPass(pass); + leaf.writeBuffers(ostr); + } + { + streamMetadata->setPass(1000); + leaf.writeBuffers(ostr); + + io::StreamMetadata::Ptr meta; + io::setStreamMetadataPtr(ostr, meta); + EXPECT_THROW(leaf.writeBuffers(ostr), openvdb::IoError); + } + + std::istringstream istr(ostr.str(), std::ios_base::binary); + io::setStreamMetadataPtr(istr, streamMetadata); + io::setDataCompression(istr, io::COMPRESS_BLOSC); + + io::setCurrentVersion(istr); + + leaf2.readTopology(istr); + for (Index b = 0; b < leaf.buffers(); b++) { + uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b); + streamMetadata->setPass(pass); + leaf2.readBuffers(istr); + } + + EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount()); + EXPECT_TRUE(leaf2.isValueOn(4)); + EXPECT_TRUE(!leaf2.isValueOn(5)); + + EXPECT_EQ(leaf2.getValue(4), ValueType(20)); + EXPECT_EQ(leaf2.attributeSet().size(), size_t(2)); + } +} + + TEST_F(TestPointDataLeaf, testSwap) { using AttributeVec3s = TypedAttributeArray; diff --git a/openvdb/openvdb/unittest/TestTree.cc b/openvdb/openvdb/unittest/TestTree.cc index a132b2fd57..bc08d087c6 100644 --- a/openvdb/openvdb/unittest/TestTree.cc +++ b/openvdb/openvdb/unittest/TestTree.cc @@ -790,6 +790,58 @@ TEST_F(TestTree, testIO) } +TEST_F(TestTree, testTreeIO) +{ + const char* filename = "testTreeIO.dbg"; + openvdb::SharedPtr scopedFile(filename, ::remove); + { + ValueType background=5.0f; + RootNodeType root_node(background); + root_node.setValueOn(openvdb::Coord(5,10,20),0.234f); + root_node.setValueOn(openvdb::Coord(50000,20000,30000),4.5678f); + + std::ofstream os(filename, std::ios_base::binary); + root_node.writeTopology(os); + root_node.writeBuffers(os); + EXPECT_TRUE(!os.fail()); + } + { + ValueType background=2.0f; + RootNodeType root_node(background); + ASSERT_DOUBLES_EXACTLY_EQUAL(background, root_node.getValue(openvdb::Coord(5,10,20))); + { + std::ifstream is(filename, std::ios_base::binary); + openvdb::io::setCurrentVersion(is); + root_node.readTopology(is); + root_node.readBuffers(is); + EXPECT_TRUE(!is.fail()); + } + + ASSERT_DOUBLES_EXACTLY_EQUAL(0.234f, root_node.getValue(openvdb::Coord(5,10,20))); + ASSERT_DOUBLES_EXACTLY_EQUAL(5.0f, root_node.getValue(openvdb::Coord(5,11,20))); + ValueType sum=0.0f; + for (RootNodeType::ChildOnIter root_iter = root_node.beginChildOn(); + root_iter.test(); ++root_iter) + { + for (InternalNodeType2::ChildOnIter internal_iter2 = root_iter->beginChildOn(); + internal_iter2.test(); ++internal_iter2) + { + for (InternalNodeType1::ChildOnIter internal_iter1 = + internal_iter2->beginChildOn(); internal_iter1.test(); ++internal_iter1) + { + for (LeafNodeType::ValueOnIter block_iter = + internal_iter1->beginValueOn(); block_iter.test(); ++block_iter) + { + sum += *block_iter; + } + } + } + } + ASSERT_DOUBLES_EXACTLY_EQUAL(sum, (0.234f + 4.5678f)); + } +} + + TEST_F(TestTree, testNegativeIndexing) { ValueType background=5.0f; From 6d3f320c2a5236826630b1f0ffe6ba733aaad84c Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 15 Apr 2026 21:59:20 -0700 Subject: [PATCH 14/70] Clean up some custom initialization logic in the unit tests Signed-off-by: Dan Bailey --- openvdb/openvdb/unittest/TestFile.cc | 184 +------------------ openvdb/openvdb/unittest/TestMultiResGrid.cc | 3 +- openvdb/openvdb/unittest/TestStream.cc | 20 +- 3 files changed, 4 insertions(+), 203 deletions(-) diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index 123408c524..d4e976fedc 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -44,7 +44,7 @@ class TestFile: public ::testing::Test { public: - void SetUp() override {} + void SetUp() override { openvdb::initialize(); } void TearDown() override { openvdb::uninitialize(); } void testHeader(); @@ -126,8 +126,6 @@ TestFile::testWriteGrid() tree.setValue(Coord(0, 0, 0), 5); // Add some metadata. - Metadata::clearRegistry(); - StringMetadata::registerType(); const std::string meta0Val, meta1Val("Hello, world."); Metadata::Ptr stringMetadata = Metadata::createMetadata(typeNameAsString()); EXPECT_TRUE(stringMetadata); @@ -157,22 +155,6 @@ TestFile::testWriteGrid() io::setCurrentVersion(istr); GridBase::Ptr gd2_grid; - EXPECT_THROW(gd2.read(istr), openvdb::LookupError); - - // Register the grid and the transform and the blocks. - GridBase::clearRegistry(); - GridType::registerGrid(); - - // Register transform maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - istr.seekg(0, std::ios_base::beg); EXPECT_NO_THROW(gd2_grid = gd2.read(istr)); @@ -207,10 +189,6 @@ TestFile::testWriteGrid() EXPECT_EQ( grid->baseTree().treeDepth(), gd2_grid->baseTree().treeDepth()); - //EXPECT_EQ(0.1, gd2_grid->getTransform()->getVoxelSizeX()); - //EXPECT_EQ(0.1, gd2_grid->getTransform()->getVoxelSizeY()); - //EXPECT_EQ(0.1, gd2_grid->getTransform()->getVoxelSizeZ()); - // Read in the data blocks. gd2.seekToBlocks(istr); gd2_grid->readBuffers(istr); @@ -220,10 +198,6 @@ TestFile::testWriteGrid() EXPECT_EQ(5, tree2->getValue(Coord(0, 0, 0))); EXPECT_EQ(1, tree2->getValue(Coord(1000, 1000, 16000))); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); remove("something.vdb2"); } @@ -276,20 +250,6 @@ TestFile::testWriteMultipleGrids() EXPECT_TRUE(gd2.getBlockPos() != 0); EXPECT_TRUE(gd2.getEndPos() != 0); - // register the grid - GridBase::clearRegistry(); - GridType::registerGrid(); - - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Read in the first grid descriptor. GridDescriptor gd_in; std::istringstream istr(ostr.str(), std::ios_base::binary); @@ -377,10 +337,6 @@ TestFile::testWriteMultipleGrids() EXPECT_EQ(10, grid2_in->getValue(Coord(0, 0, 0))); EXPECT_EQ(2, grid2_in->getValue(Coord(100000, 100000, 16000))); - // Clear registries. - GridBase::clearRegistry(); - - math::MapRegistry::clear(); remove("something.vdb2"); } TEST_F(TestFile, testWriteMultipleGrids) { testWriteMultipleGrids(); } @@ -394,12 +350,6 @@ TEST_F(TestFile, testWriteFloatAsHalf) using TreeType = Vec3STree; using GridType = Grid; - // Register all grid types. - initialize(); - // Ensure that the registry is cleared on exit. - struct Local { static void uninitialize(char*) { openvdb::uninitialize(); } }; - SharedPtr onExit(nullptr, Local::uninitialize); - // Create two test grids. GridType::Ptr grid1 = createGrid(/*bg=*/Vec3s(1, 1, 1)); TreeType& tree1 = grid1->tree(); @@ -460,9 +410,6 @@ TEST_F(TestFile, testWriteInstancedGrids) { using namespace openvdb; - // Register data types. - openvdb::initialize(); - // Remove something.vdb2 when done. We must declare this here before the // other grid smart_ptr's because we re-use them in the test several times. // We will not be able to remove something.vdb2 on Windows if the pointers @@ -680,19 +627,6 @@ TestFile::testReadGridDescriptors() file.writeGrid(gd, grid, ostr, /*seekable=*/true); file.writeGrid(gd2, grid2, ostr, /*seekable=*/true); - // Register the grid and the transform and the blocks. - GridBase::clearRegistry(); - GridType::registerGrid(); - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Read in the grid descriptors. File file2("something.vdb2"); std::istringstream istr(ostr.str(), std::ios_base::binary); @@ -716,10 +650,6 @@ TestFile::testReadGridDescriptors() EXPECT_EQ(gd2.getBlockPos(), file2gd.getBlockPos()); EXPECT_EQ(gd2.getEndPos(), file2gd.getEndPos()); - // Clear registries. - GridBase::clearRegistry(); - math::MapRegistry::clear(); - remove("something.vdb2"); } TEST_F(TestFile, testReadGridDescriptors) { testReadGridDescriptors(); } @@ -732,9 +662,6 @@ TEST_F(TestFile, testGridNaming) using TreeType = Int32Tree; - // Register data types. - openvdb::initialize(); - logging::LevelScope suppressLogging{logging::Level::Fatal}; // Create several grids that share a single tree. @@ -947,19 +874,6 @@ TestFile::testEmptyGridIO() EXPECT_EQ(gd.getEndPos(), gd.getBlockPos()); EXPECT_EQ(gd2.getEndPos(), gd2.getBlockPos()); - // Register the grid and the transform and the blocks. - GridBase::clearRegistry(); - GridType::registerGrid(); - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Read in the grid descriptors. File file2(filename); std::istringstream istr(ostr.str(), std::ios_base::binary); @@ -1002,10 +916,6 @@ TestFile::testEmptyGridIO() EXPECT_EQ(gd2.getGridPos(), file2gd.getGridPos()); EXPECT_EQ(gd2.getBlockPos(), file2gd.getBlockPos()); EXPECT_EQ(gd2.getEndPos(), file2gd.getEndPos()); - - // Clear registries. - GridBase::clearRegistry(); - math::MapRegistry::clear(); } TEST_F(TestFile, testEmptyGridIO) { testEmptyGridIO(); } @@ -1055,23 +965,6 @@ void TestFile::testOpen() EXPECT_TRUE(meta.metaValue("author") == "Einstein"); EXPECT_EQ(2009, meta.metaValue("year")); - // Register grid and transform. - GridBase::clearRegistry(); - IntGrid::registerGrid(); - FloatGrid::registerGrid(); - Metadata::clearRegistry(); - StringMetadata::registerType(); - Int32Metadata::registerType(); - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1120,11 +1013,6 @@ void TestFile::testOpen() EXPECT_THROW(vdbfile2.open(), openvdb::IoError); EXPECT_THROW(vdbfile2.inputStream(), openvdb::IoError); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); - // Test closing the file. vdbfile.close(); EXPECT_TRUE(vdbfile.isOpen() == false); @@ -1166,11 +1054,6 @@ TEST_F(TestFile, testGetMetadata) meta.insertMeta("author", StringMetadata("Einstein")); meta.insertMeta("year", Int32Metadata(2009)); - // Adjust registry before writing. - Metadata::clearRegistry(); - StringMetadata::registerType(); - Int32Metadata::registerType(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1187,9 +1070,6 @@ TEST_F(TestFile, testGetMetadata) EXPECT_TRUE(meta2->metaValue("author") == "Einstein"); EXPECT_EQ(2009, meta2->metaValue("year")); - // Clear registry. - Metadata::clearRegistry(); - remove("something.vdb2"); } @@ -1234,9 +1114,6 @@ TEST_F(TestFile, testReadAll) grids.push_back(grid1); grids.push_back(grid2); - // Register grid and transform. - openvdb::initialize(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1276,11 +1153,6 @@ TEST_F(TestFile, testReadAll) EXPECT_NEAR(10, temperature->getValue(Coord(0, 0, 0)), /*tolerance=*/0); EXPECT_NEAR(11, temperature->getValue(Coord(0, 100, 0)), /*tolerance=*/0); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); - vdbfile2.close(); remove("something.vdb2"); @@ -1295,11 +1167,6 @@ TEST_F(TestFile, testWriteOpenFile) meta->insertMeta("author", StringMetadata("Einstein")); meta->insertMeta("year", Int32Metadata(2009)); - // Register metadata - Metadata::clearRegistry(); - StringMetadata::registerType(); - Int32Metadata::registerType(); - // Write the metadata out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(GridPtrVec(), *meta); @@ -1330,9 +1197,6 @@ TEST_F(TestFile, testWriteOpenFile) EXPECT_NO_THROW(vdbfile2.write(*grids)); - // Clear registries. - Metadata::clearRegistry(); - remove("something.vdb2"); } @@ -1341,8 +1205,6 @@ TEST_F(TestFile, testReadGridMetadata) { using namespace openvdb; - openvdb::initialize(); - const char* filename = "testReadGridMetadata.vdb2"; SharedPtr scopedFile(filename, ::remove); @@ -1512,9 +1374,6 @@ TEST_F(TestFile, testReadGrid) grids.push_back(grid); grids.push_back(grid2); - // Register grid and transform. - openvdb::initialize(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1549,11 +1408,6 @@ TEST_F(TestFile, testReadGrid) EXPECT_NEAR(5,typedDensity->getValue(Coord(0, 0, 0)), /*tolerance=*/0); EXPECT_NEAR(6,typedDensity->getValue(Coord(100, 0, 0)), /*tolerance=*/0); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); - vdbfile2.close(); remove("something.vdb2"); @@ -1604,9 +1458,6 @@ TEST_F(TestFile, testReadClippedGrid) { using namespace openvdb; - // Register types. - openvdb::initialize(); - // World-space clipping region const BBoxd clipBox(Vec3d(4.0, 4.0, -6.0), Vec3d(4.9, 4.9, 6.0)); @@ -1958,23 +1809,6 @@ TEST_F(TestFile, testHasGrid) grids.push_back(grid); grids.push_back(grid2); - // Register grid and transform. - GridBase::clearRegistry(); - IntGrid::registerGrid(); - FloatGrid::registerGrid(); - Metadata::clearRegistry(); - StringMetadata::registerType(); - Int32Metadata::registerType(); - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1990,11 +1824,6 @@ TEST_F(TestFile, testHasGrid) EXPECT_TRUE(!vdbfile2.hasGrid("Temperature")); EXPECT_TRUE(!vdbfile2.hasGrid("densitY")); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); - vdbfile2.close(); remove("something.vdb2"); @@ -2040,9 +1869,6 @@ TEST_F(TestFile, testNameIterator) grid->setName("level_set"); grids.push_back(grid); - // Register types. - openvdb::initialize(); - const char* filename = "testNameIterator.vdb2"; SharedPtr scopedFile(filename, ::remove); @@ -2090,9 +1916,6 @@ TEST_F(TestFile, testCompression) using IntGrid = openvdb::Int32Grid; - // Register types. - openvdb::initialize(); - // Create reference grids. IntGrid::Ptr intGrid = IntGrid::create(/*background=*/0); intGrid->fill(CoordBBox(Coord(0), Coord(49)), /*value=*/999, /*active=*/true); @@ -2302,9 +2125,6 @@ TEST_F(TestFile, testAsync) { using namespace openvdb; - // Register types. - openvdb::initialize(); - // Create a grid. FloatGrid::Ptr lsGrid = createLevelSet(); unittest_util::makeSphere(/*dim=*/Coord(100), /*ctr=*/Vec3f(50, 50, 50), /*r=*/20.0, @@ -2422,8 +2242,6 @@ TEST_F(TestFile, testAsync) // (see https://github.com/Blosc/c-blosc/pull/63). TEST_F(TestFile, testBlosc) { - openvdb::initialize(); - const unsigned char rawdata[] = { 0x93, 0xb0, 0x49, 0xaf, 0x62, 0xad, 0xe3, 0xaa, 0xe4, 0xa5, 0x43, 0x20, 0x24, 0x29, 0xc9, 0xaf, 0xee, 0xad, 0x0b, 0xac, 0x3d, 0xa8, 0x1f, 0x99, 0x53, 0x27, diff --git a/openvdb/openvdb/unittest/TestMultiResGrid.cc b/openvdb/openvdb/unittest/TestMultiResGrid.cc index d30cb0a28d..3930fbe047 100644 --- a/openvdb/openvdb/unittest/TestMultiResGrid.cc +++ b/openvdb/openvdb/unittest/TestMultiResGrid.cc @@ -208,6 +208,8 @@ TEST_F(TestMultiResGrid, testIO) { using namespace openvdb; + openvdb::initialize(); + const float radius = 1.0f; const Vec3f center(0.0f, 0.0f, 0.0f); const float voxelSize = 0.01f; @@ -243,7 +245,6 @@ TEST_F(TestMultiResGrid, testIO) outputFile.close(); // Read grids - openvdb::initialize(); openvdb::io::File file( filename ); file.open(); GridPtrVecPtr grids = file.getGrids(); diff --git a/openvdb/openvdb/unittest/TestStream.cc b/openvdb/openvdb/unittest/TestStream.cc index 8183317ea9..35850fab3d 100644 --- a/openvdb/openvdb/unittest/TestStream.cc +++ b/openvdb/openvdb/unittest/TestStream.cc @@ -36,25 +36,7 @@ class TestStream: public ::testing::Test void TestStream::SetUp() { - openvdb::uninitialize(); - - openvdb::Int32Grid::registerGrid(); - openvdb::FloatGrid::registerGrid(); - - openvdb::StringMetadata::registerType(); - openvdb::Int32Metadata::registerType(); - openvdb::Int64Metadata::registerType(); - openvdb::Vec3IMetadata::registerType(); - - // Register maps - openvdb::math::MapRegistry::clear(); - openvdb::math::AffineMap::registerMap(); - openvdb::math::ScaleMap::registerMap(); - openvdb::math::UniformScaleMap::registerMap(); - openvdb::math::TranslationMap::registerMap(); - openvdb::math::ScaleTranslateMap::registerMap(); - openvdb::math::UniformScaleTranslateMap::registerMap(); - openvdb::math::NonlinearFrustumMap::registerMap(); + openvdb::initialize(); } From 77162630c1f85146c7635f6c3c71130c6107e1ea Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 15 Apr 2026 22:42:42 -0700 Subject: [PATCH 15/70] Change testWriteHalf() to use a Stream when testing write as half Signed-off-by: Dan Bailey --- openvdb/openvdb/unittest/TestTree.cc | 74 +++++++++++++++++++++------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/openvdb/openvdb/unittest/TestTree.cc b/openvdb/openvdb/unittest/TestTree.cc index bc08d087c6..90a3de5e45 100644 --- a/openvdb/openvdb/unittest/TestTree.cc +++ b/openvdb/openvdb/unittest/TestTree.cc @@ -7,6 +7,7 @@ #include // for tools::setValueOnMin(), et al. #include #include // for io::RealToHalf +#include #include // for Abs() #include #include @@ -104,6 +105,10 @@ TEST_F(TestTree, testChangeBackground) TEST_F(TestTree, testHalf) { + // explicitly register these grid types as they are not registered by default + openvdb::Grid::registerGrid(); + openvdb::Grid::registerGrid(); + testWriteHalf(); testWriteHalf(); testWriteHalf(); @@ -125,27 +130,54 @@ TestTree::testWriteHalf() using GridType = openvdb::Grid; using ValueT = typename TreeType::ValueType; ValueT background(5); - GridType grid(background); + typename GridType::Ptr grid = GridType::create(background); + grid->setName("density"); unittest_util::makeSphere(openvdb::Coord(64, 64, 64), openvdb::Vec3f(35, 30, 40), - /*radius=*/10, grid, + /*radius=*/10, *grid, /*dx=*/1.0f, unittest_util::SPHERE_DENSE); - EXPECT_TRUE(!grid.tree().empty()); + EXPECT_TRUE(!grid->tree().empty()); + + // find stream and grid header position for save as float + + size_t headerSize = 0; + { + std::ostringstream outFull(std::ios_base::binary); + openvdb::io::Stream streamFull(outFull); + streamFull.write({}); + openvdb::io::GridDescriptor gd(grid->getName(), grid->type()); + gd.writeHeader(outFull); + headerSize = outFull.str().size(); + } + + // find stream and grid header position for save as half + + size_t halfHeaderSize = 0; + { + std::ostringstream outHalf(std::ios_base::binary); + openvdb::io::Stream streamHalf(outHalf); + streamHalf.write({}); + openvdb::io::GridDescriptor gdHalf(grid->getName(), grid->type(), /*half=*/true); + gdHalf.writeHeader(outHalf); + halfHeaderSize = outHalf.str().size(); + } // Write grid blocks in both float and half formats. std::ostringstream outFull(std::ios_base::binary); - grid.setSaveFloatAsHalf(false); - grid.writeBuffers(outFull); + openvdb::io::Stream streamFull(outFull); + grid->setSaveFloatAsHalf(false); + streamFull.write({grid}); outFull.flush(); - const size_t fullBytes = outFull.str().size(); + const size_t fullBytes = outFull.str().size() - headerSize; if (fullBytes == 0) FAIL() << "wrote empty full float buffers"; std::ostringstream outHalf(std::ios_base::binary); - grid.setSaveFloatAsHalf(true); - grid.writeBuffers(outHalf); + openvdb::io::Stream streamHalf(outHalf); + grid->setSaveFloatAsHalf(true); + streamHalf.write({grid}); outHalf.flush(); - const size_t halfBytes = outHalf.str().size(); + const size_t halfBytes = outHalf.str().size() - halfHeaderSize; if (halfBytes == 0) FAIL() << "wrote empty half float buffers"; if (openvdb::io::RealToHalf::isReal) { @@ -166,22 +198,26 @@ TestTree::testWriteHalf() // then write it out again in half float format. Verify that the resulting file // is identical to the original half float file. { - openvdb::Grid gridCopy(grid); - gridCopy.setSaveFloatAsHalf(true); std::istringstream is(outHalf.str(), std::ios_base::binary); + openvdb::io::Stream streamHalf2(is); - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(is); - - gridCopy.readBuffers(is); + openvdb::GridPtrVecPtr grids = streamHalf2.getGrids(); + openvdb::GridBase::Ptr gridCopy = (*grids)[0]; + gridCopy->setSaveFloatAsHalf(true); std::ostringstream outDiff(std::ios_base::binary); - gridCopy.writeBuffers(outDiff); + openvdb::io::Stream streamDiff(outDiff); + streamDiff.write({gridCopy}); outDiff.flush(); - if (outHalf.str() != outDiff.str()) { - FAIL() << "half-from-full and half-from-half buffers differ"; + // Compare the two buffers, skipping the header region + { + const std::string s1 = outHalf.str().substr(halfHeaderSize); + const std::string s2 = outDiff.str().substr(halfHeaderSize); + if (s1 != s2) + { + FAIL() << "half-from-full and half-from-half buffers differ"; + } } } } From 4042079c4de3e9f49b54052c32e178e8f8fd12cd Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 15 Apr 2026 23:02:04 -0700 Subject: [PATCH 16/70] Refactor GridDescriptor reading Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 6 +++-- openvdb/openvdb/io/GridDescriptor.cc | 27 ++++++++++++------- openvdb/openvdb/io/GridDescriptor.h | 10 +++++++ openvdb/openvdb/io/Stream.cc | 3 ++- openvdb/openvdb/unittest/TestFile.cc | 17 +++++++----- .../openvdb/unittest/TestGridDescriptor.cc | 7 +++-- 6 files changed, 47 insertions(+), 23 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 4cbda52eb6..00265f9839 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -281,7 +281,8 @@ File::open() const int32_t gridCount = readGridCount(inputStream()); for (int32_t i = 0; i < gridCount; ++i) { GridDescriptor gd; - gd.read(inputStream()); + gd.readHeader(inputStream()); + gd.readStreamPos(inputStream()); GridBase::Ptr grid = createGrid(gd); Archive::readGrid(grid, gd, inputStream()); @@ -584,7 +585,8 @@ File::readGridDescriptors(std::istream& is) for (int32_t i = 0, N = readGridCount(is); i < N; ++i) { // Read the grid descriptor. GridDescriptor gd; - gd.read(is); + gd.readHeader(is); + gd.readStreamPos(is); // Add the descriptor to the dictionary. gridDescriptors().insert(std::make_pair(gd.gridName(), gd)); diff --git a/openvdb/openvdb/io/GridDescriptor.cc b/openvdb/openvdb/io/GridDescriptor.cc index 961f622008..9a59d22d1c 100644 --- a/openvdb/openvdb/io/GridDescriptor.cc +++ b/openvdb/openvdb/io/GridDescriptor.cc @@ -69,8 +69,8 @@ GridDescriptor::writeStreamPos(std::ostream &os) const os.write(reinterpret_cast(&mEndPos), sizeof(int64_t)); } -GridBase::Ptr -GridDescriptor::read(std::istream &is) +void +GridDescriptor::readHeader(std::istream &is) { checkFormatVersion(is); @@ -86,21 +86,28 @@ GridDescriptor::read(std::istream &is) } mInstanceParentName = readString(is); +} + +void +GridDescriptor::readStreamPos(std::istream &is) +{ + is.read(reinterpret_cast(&mGridPos), sizeof(int64_t)); + is.read(reinterpret_cast(&mBlockPos), sizeof(int64_t)); + is.read(reinterpret_cast(&mEndPos), sizeof(int64_t)); +} + +GridBase::Ptr +GridDescriptor::read(std::istream &is) +{ + readHeader(is); + readStreamPos(is); - // Create the grid of the type if it has been registered. if (!GridBase::isRegistered(mGridType)) { OPENVDB_THROW(LookupError, "Cannot read grid." << " Grid type " << mGridType << " is not registered."); } - // else GridBase::Ptr grid = GridBase::createGrid(mGridType); if (grid) grid->setSaveFloatAsHalf(mSaveFloatAsHalf); - - // Read in the offsets. - is.read(reinterpret_cast(&mGridPos), sizeof(int64_t)); - is.read(reinterpret_cast(&mBlockPos), sizeof(int64_t)); - is.read(reinterpret_cast(&mEndPos), sizeof(int64_t)); - return grid; } diff --git a/openvdb/openvdb/io/GridDescriptor.h b/openvdb/openvdb/io/GridDescriptor.h index 9ef03262e2..1aaf5f10a4 100644 --- a/openvdb/openvdb/io/GridDescriptor.h +++ b/openvdb/openvdb/io/GridDescriptor.h @@ -61,8 +61,18 @@ class OPENVDB_API GridDescriptor /// written out separately. void writeStreamPos(std::ostream&) const; + /// @brief Read this descriptor's header information (all data except for + /// stream offsets) from the given stream. + void readHeader(std::istream&); + + /// @brief Read stream positions (grid, block, and end offsets) from the + /// given stream. + void readStreamPos(std::istream&); + /// @brief Read a grid descriptor from the given stream. /// @return an empty grid of the type specified by the grid descriptor. + /// @deprecated Use readHeader() followed by readStreamPos() instead. + OPENVDB_DEPRECATED_MESSAGE("Use readHeader() followed by readStreamPos() instead.") GridBase::Ptr read(std::istream&); /// @brief Append the number @a n to the given name (separated by an ASCII diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index be3a003515..80326af693 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -72,7 +72,8 @@ Stream::Stream(std::istream& is): mImpl(new Impl) Archive::NamedGridMap namedGrids; for (int32_t i = 0; i < gridCount; ++i) { GridDescriptor gd; - gd.read(is); + gd.readHeader(is); + gd.readStreamPos(is); descriptors.push_back(gd); GridBase::Ptr grid = readGrid(gd, is); mImpl->mGrids->push_back(grid); diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index d4e976fedc..9120bed409 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -154,9 +154,12 @@ TestFile::testWriteGrid() // it doesn't have a header), set the file format version number explicitly. io::setCurrentVersion(istr); - GridBase::Ptr gd2_grid; istr.seekg(0, std::ios_base::beg); - EXPECT_NO_THROW(gd2_grid = gd2.read(istr)); + gd2.readHeader(istr); + gd2.readStreamPos(istr); + + // manually create the grid + GridBase::Ptr gd2_grid = GridBase::createGrid(gd2.gridType()); EXPECT_EQ(gd.gridName(), gd2.gridName()); EXPECT_EQ(GridType::gridType(), gd2_grid->type()); @@ -255,8 +258,9 @@ TestFile::testWriteMultipleGrids() std::istringstream istr(ostr.str(), std::ios_base::binary); io::setCurrentVersion(istr); - GridBase::Ptr gd_in_grid; - EXPECT_NO_THROW(gd_in_grid = gd_in.read(istr)); + gd_in.readHeader(istr); + gd_in.readStreamPos(istr); + GridBase::Ptr gd_in_grid = GridBase::createGrid(gd_in.gridType()); // Ensure read in the right values. EXPECT_EQ(gd.gridName(), gd_in.gridName()); @@ -300,8 +304,9 @@ TestFile::testWriteMultipleGrids() gd_in.seekToEnd(istr); GridDescriptor gd2_in; - GridBase::Ptr gd2_in_grid; - EXPECT_NO_THROW(gd2_in_grid = gd2_in.read(istr)); + gd2_in.readHeader(istr); + gd2_in.readStreamPos(istr); + GridBase::Ptr gd2_in_grid = GridBase::createGrid(gd2_in.gridType()); // Ensure that we read in the right values. EXPECT_EQ(gd2.gridName(), gd2_in.gridName()); diff --git a/openvdb/openvdb/unittest/TestGridDescriptor.cc b/openvdb/openvdb/unittest/TestGridDescriptor.cc index c45e8d0433..a39fde278e 100644 --- a/openvdb/openvdb/unittest/TestGridDescriptor.cc +++ b/openvdb/openvdb/unittest/TestGridDescriptor.cc @@ -42,16 +42,15 @@ TEST_F(TestGridDescriptor, testIO) GridDescriptor gd2; - EXPECT_THROW(gd2.read(istr), openvdb::LookupError); - // Register the grid. GridBase::clearRegistry(); GridType::registerGrid(); // seek back and read again. istr.seekg(0, std::ios_base::beg); - GridBase::Ptr grid; - EXPECT_NO_THROW(grid = gd2.read(istr)); + gd2.readHeader(istr); + gd2.readStreamPos(istr); + GridBase::Ptr grid = GridBase::createGrid(gd2.gridType()); EXPECT_EQ(gd.gridName(), gd2.gridName()); EXPECT_EQ(gd.uniqueName(), gd2.uniqueName()); From b2691d0797774237ee25bc59a6fe5e0988ff7f3e Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 19 Mar 2026 22:27:20 -0700 Subject: [PATCH 17/70] Move body of readGridDescriptors into sole call site and delete method Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 41 +++++++++++++------------------------- openvdb/openvdb/io/File.h | 3 --- 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 00265f9839..33a3ccd044 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -296,8 +296,20 @@ File::open() Archive::connectInstance(it->second, mImpl->mNamedGrids); } } else { - // Read in just the grid descriptors. - readGridDescriptors(inputStream()); + gridDescriptors().clear(); + + for (int32_t i = 0, N = readGridCount(inputStream()); i < N; ++i) { + // Read the grid descriptor. + GridDescriptor gd; + gd.readHeader(inputStream()); + gd.readStreamPos(inputStream()); + + // Add the descriptor to the dictionary. + gridDescriptors().insert(std::make_pair(gd.gridName(), gd)); + + // Skip forward to the next descriptor. + gd.seekToEnd(inputStream()); + } } mImpl->mIsOpen = true; @@ -571,31 +583,6 @@ File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta) const } -//////////////////////////////////////// - - -void -File::readGridDescriptors(std::istream& is) -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - gridDescriptors().clear(); - - for (int32_t i = 0, N = readGridCount(is); i < N; ++i) { - // Read the grid descriptor. - GridDescriptor gd; - gd.readHeader(is); - gd.readStreamPos(is); - - // Add the descriptor to the dictionary. - gridDescriptors().insert(std::make_pair(gd.gridName(), gd)); - - // Skip forward to the next descriptor. - gd.seekToEnd(is); - } -} - //////////////////////////////////////// diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index be8ab89968..3b64058c0b 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -146,9 +146,6 @@ class OPENVDB_API File: public Archive NameIterator endName() const; private: - /// Read in all grid descriptors that are stored in the given stream. - void readGridDescriptors(std::istream&); - /// @brief Return an iterator to the descriptor for the grid with the given name. /// If the name is non-unique, return an iterator to the first matching descriptor. NameMapCIter findDescriptor(const Name&) const; From f5495e29dee65444f4cc0e3061cbaa5b8b260d1c Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 19 Mar 2026 22:27:50 -0700 Subject: [PATCH 18/70] Remove File::readGridByName() Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 9 +-------- openvdb/openvdb/io/File.h | 4 ---- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 33a3ccd044..d399c09586 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -491,19 +491,12 @@ File::readGridMetadata(const Name& name) GridBase::Ptr File::readGrid(const Name& name) { - return readGridByName(name, BBoxd()); + return readGrid(name, BBoxd()); } GridBase::Ptr File::readGrid(const Name& name, const BBoxd& bbox) -{ - return readGridByName(name, bbox); -} - - -GridBase::Ptr -File::readGridByName(const Name& name, const BBoxd& bbox) { if (!isOpen()) { OPENVDB_THROW(IoError, filename() << " is not open for reading."); diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index 3b64058c0b..f9e11b342c 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -153,10 +153,6 @@ class OPENVDB_API File: public Archive /// Return a newly created, empty grid of the type specified by the given grid descriptor. GridBase::Ptr createGrid(const GridDescriptor&) const; - /// @brief Read a grid, including its data blocks, but only where it - /// intersects the given world-space bounding box. - GridBase::Ptr readGridByName(const Name&, const BBoxd&); - /// Read in and return the partially-populated grid specified by the given grid descriptor. GridBase::ConstPtr readGridPartial(const GridDescriptor&, bool readTopology) const; From 2fc931221c0dfa6652b1fa0165deeac050f50a1f Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 19 Mar 2026 22:28:33 -0700 Subject: [PATCH 19/70] Simplify readGridPartial() and remove unused topology read option Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 48 ++++++++++++++------------------------ openvdb/openvdb/io/File.h | 6 +---- 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index d399c09586..ff5aa6345b 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -442,7 +442,7 @@ File::readAllGridMetadata() // Read just the metadata and transforms for all grids. for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) { const GridDescriptor& gd = i->second; - GridBase::ConstPtr grid = readGridPartial(gd, /*readTopology=*/false); + GridBase::ConstPtr grid = readGridPartial(gd); // Return copies of the grids, but with empty trees. // (As of 0.98.0, at least, it would suffice to just const cast // the grid pointers returned by readGridPartial(), but shallow @@ -479,7 +479,7 @@ File::readGridMetadata(const Name& name) // Seek to and read in the grid from the file. const GridDescriptor& gd = it->second; - ret = readGridPartial(gd, /*readTopology=*/false); + ret = readGridPartial(gd); } return ret->copyGridWithNewTree(); } @@ -644,18 +644,30 @@ File::createGrid(const GridDescriptor& gd) const GridBase::ConstPtr -File::readGridPartial(const GridDescriptor& gd, bool readTopology) const +File::readGridPartial(const GridDescriptor& gd) const { // This method should not be called for files that don't contain grid offsets. OPENVDB_ASSERT(inputHasGridOffsets()); + std::istream& is = inputStream(); + GridBase::Ptr grid = createGrid(gd); // Seek to grid. - gd.seekToGrid(inputStream()); + gd.seekToGrid(is); + + // This code needs to stay in sync with io::Archive::readGrid(), in terms of + // the order of operations. + readGridCompression(is); + grid->readMeta(is); + + // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists + if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { + grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); + } - // Read the grid partially. - readGridPartial(grid, inputStream(), gd.isInstance(), readTopology); + // Read the transform. + grid->readTransform(is); // Promote to a const grid. GridBase::ConstPtr constGrid = grid; @@ -685,30 +697,6 @@ File::readGrid(const GridDescriptor& gd, const CoordBBox& bbox) const } -void -File::readGridPartial(GridBase::Ptr grid, std::istream& is, - bool isInstance, bool readTopology) const -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - // This code needs to stay in sync with io::Archive::readGrid(), in terms of - // the order of operations. - readGridCompression(is); - grid->readMeta(is); - - // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists - if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { - grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } - - grid->readTransform(is); - if (!isInstance && readTopology) { - grid->readTopology(is); - } -} - - //////////////////////////////////////// diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index f9e11b342c..49ce98eb6d 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -154,7 +154,7 @@ class OPENVDB_API File: public Archive GridBase::Ptr createGrid(const GridDescriptor&) const; /// Read in and return the partially-populated grid specified by the given grid descriptor. - GridBase::ConstPtr readGridPartial(const GridDescriptor&, bool readTopology) const; + GridBase::ConstPtr readGridPartial(const GridDescriptor&) const; /// Read in and return the grid specified by the given grid descriptor. GridBase::Ptr readGrid(const GridDescriptor&) const; @@ -165,10 +165,6 @@ class OPENVDB_API File: public Archive /// that intersects the given index-space bounding box. GridBase::Ptr readGrid(const GridDescriptor&, const CoordBBox&) const; - /// @brief Partially populate the given grid by reading its metadata and transform and, - /// if the grid is not an instance, its tree structure, but not the tree's leaf nodes. - void readGridPartial(GridBase::Ptr, std::istream&, bool isInstance, bool readTopology) const; - /// @brief Retrieve a grid from @c mNamedGrids. Return a null pointer /// if @c mNamedGrids was not populated (because this file is random-access). /// @throw KeyError if no grid with the given name exists in this file. From 5eb40b118171389041eb752c5d5b4102f6f8d276 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 19 Mar 2026 23:15:22 -0700 Subject: [PATCH 20/70] Eliminate File::Impl Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 286 ++++++++++++--------------- openvdb/openvdb/io/File.h | 20 +- openvdb/openvdb/unittest/TestFile.cc | 56 ++++-- 3 files changed, 183 insertions(+), 179 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index ff5aa6345b..0424c89da4 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -25,83 +25,24 @@ OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { namespace io { -// Implementation details of the File class -struct File::Impl -{ - struct NoBBox {}; - - // Common implementation of the various File::readGrid() overloads, - // with and without bounding box clipping - template - static GridBase::Ptr readGrid(const File& file, const GridDescriptor& gd, const BoxType& bbox) - { - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(file.inputHasGridOffsets()); - - GridBase::Ptr grid = file.createGrid(gd); - gd.seekToGrid(file.inputStream()); - unarchive(file, grid, gd, bbox); - return grid; - } - - static void unarchive(const File& file, GridBase::Ptr& grid, - const GridDescriptor& gd, NoBBox) - { - file.Archive::readGrid(grid, gd, file.inputStream()); - } - - static void unarchive(const File& file, GridBase::Ptr& grid, - const GridDescriptor& gd, const CoordBBox& indexBBox) - { - file.Archive::readGrid(grid, gd, file.inputStream(), indexBBox); - } - - static void unarchive(const File& file, GridBase::Ptr& grid, - const GridDescriptor& gd, const BBoxd& worldBBox) - { - file.Archive::readGrid(grid, gd, file.inputStream(), worldBBox); - } - std::string mFilename; - // The file-level metadata - MetaMap::Ptr mMeta; - // The file stream that is open for reading - std::unique_ptr mInStream; - // File-level stream metadata (file format, compression, etc.) - StreamMetadata::Ptr mStreamMetadata; - // Flag indicating if we have read in the global information (header, - // metadata, and grid descriptors) for this VDB file - bool mIsOpen; - // Grid descriptors for all grids stored in the file, indexed by grid name - NameMap mGridDescriptors; - // All grids, indexed by unique name (used only when mHasGridOffsets is false) - Archive::NamedGridMap mNamedGrids; - // All grids stored in the file (used only when mHasGridOffsets is false) - GridPtrVecPtr mGrids; -}; // class File::Impl - - -//////////////////////////////////////// - - -File::File(const std::string& filename): mImpl(new Impl) +File::File(const std::string& filename) + : Archive() + , mFilename(filename) { - mImpl->mFilename = filename; - mImpl->mIsOpen = false; setInputHasGridOffsets(true); } -File::~File() -{ -} - - File::File(const File& other) : Archive(other) - , mImpl(new Impl) + , mFilename(other.mFilename) + , mMeta(other.mMeta) + , mIsOpen(false) + , mGridDescriptors(other.mGridDescriptors) + , mNamedGrids(other.mNamedGrids) + , mGrids(other.mGrids) { - *this = other; } @@ -110,13 +51,12 @@ File::operator=(const File& other) { if (&other != this) { Archive::operator=(other); - const Impl& otherImpl = *other.mImpl; - mImpl->mFilename = otherImpl.mFilename; - mImpl->mMeta = otherImpl.mMeta; - mImpl->mIsOpen = false; // don't want two file objects reading from the same stream - mImpl->mGridDescriptors = otherImpl.mGridDescriptors; - mImpl->mNamedGrids = otherImpl.mNamedGrids; - mImpl->mGrids = otherImpl.mGrids; + mFilename = other.mFilename; + mMeta = other.mMeta; + mIsOpen = false; // don't want two file objects reading from the same stream + mGridDescriptors = other.mGridDescriptors; + mNamedGrids = other.mNamedGrids; + mGrids = other.mGrids; } return *this; } @@ -135,43 +75,43 @@ File::copy() const const std::string& File::filename() const { - return mImpl->mFilename; + return mFilename; } MetaMap::Ptr File::fileMetadata() { - return mImpl->mMeta; + return mMeta; } MetaMap::ConstPtr File::fileMetadata() const { - return mImpl->mMeta; + return mMeta; } const File::NameMap& File::gridDescriptors() const { - return mImpl->mGridDescriptors; + return mGridDescriptors; } File::NameMap& File::gridDescriptors() { - return mImpl->mGridDescriptors; + return mGridDescriptors; } std::istream& File::inputStream() const { - if (!mImpl->mInStream) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mInStream) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } - return *mImpl->mInStream; + return *mInStream; } @@ -187,11 +127,11 @@ File::getSize() const Index64 result = std::numeric_limits::max(); - std::string mesg = "could not get size of file " + filename(); + std::string mesg = "could not get size of file " + mFilename; #ifdef _WIN32 // Get the file size by seeking to the end of the file. - std::ifstream fstrm(filename()); + std::ifstream fstrm(mFilename); if (fstrm) { fstrm.seekg(0, fstrm.end); result = static_cast(fstrm.tellg()); @@ -201,7 +141,7 @@ File::getSize() const #else // Get the file size using the stat() system call. struct stat info; - if (0 != ::stat(filename().c_str(), &info)) { + if (0 != ::stat(mFilename.c_str(), &info)) { std::string s = getErrorString(); if (!s.empty()) mesg += " (" + s + ")"; OPENVDB_THROW(IoError, mesg); @@ -223,25 +163,25 @@ File::getSize() const bool File::isOpen() const { - return mImpl->mIsOpen; + return mIsOpen; } bool File::open() { - if (isOpen()) { - OPENVDB_THROW(IoError, filename() << " is already open"); + if (mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is already open"); } - mImpl->mInStream.reset(); + mInStream.reset(); // Open the file using standard I/O (delayed loading has been removed) std::unique_ptr newStream; newStream.reset(new std::ifstream( - filename().c_str(), std::ios_base::in | std::ios_base::binary)); + mFilename.c_str(), std::ios_base::in | std::ios_base::binary)); if (newStream->fail()) { - OPENVDB_THROW(IoError, "could not open file " << filename()); + OPENVDB_THROW(IoError, "could not open file " << mFilename); } // Read in the file header. @@ -251,31 +191,31 @@ File::open() } catch (IoError& e) { if (e.what() && std::string("not a VDB file") == e.what()) { // Rethrow, adding the filename. - OPENVDB_THROW(IoError, filename() << " is not a VDB file"); + OPENVDB_THROW(IoError, mFilename << " is not a VDB file"); } throw; } - mImpl->mInStream.swap(newStream); + mInStream.swap(newStream); // Tag the input stream with the file format and library version numbers // and other metadata. - mImpl->mStreamMetadata.reset(new StreamMetadata); - mImpl->mStreamMetadata->setSeekable(true); - io::setStreamMetadataPtr(inputStream(), mImpl->mStreamMetadata, /*transfer=*/false); + mStreamMetadata.reset(new StreamMetadata); + mStreamMetadata->setSeekable(true); + io::setStreamMetadataPtr(inputStream(), mStreamMetadata, /*transfer=*/false); Archive::setFormatVersion(inputStream()); Archive::setLibraryVersion(inputStream()); Archive::setDataCompression(inputStream()); // Read in the VDB metadata. - mImpl->mMeta = MetaMap::Ptr(new MetaMap); - mImpl->mMeta->readMeta(inputStream()); + mMeta = MetaMap::Ptr(new MetaMap); + mMeta->readMeta(inputStream()); if (!inputHasGridOffsets()) { - OPENVDB_LOG_DEBUG_RUNTIME("file " << filename() << " does not support partial reading"); + OPENVDB_LOG_DEBUG_RUNTIME("file " << mFilename << " does not support partial reading"); - mImpl->mGrids.reset(new GridPtrVec); - mImpl->mNamedGrids.clear(); + mGrids.reset(new GridPtrVec); + mNamedGrids.clear(); // Stream in the entire contents of the file and append all grids to mGrids. const int32_t gridCount = readGridCount(inputStream()); @@ -287,16 +227,16 @@ File::open() GridBase::Ptr grid = createGrid(gd); Archive::readGrid(grid, gd, inputStream()); - gridDescriptors().insert(std::make_pair(gd.gridName(), gd)); - mImpl->mGrids->push_back(grid); - mImpl->mNamedGrids[gd.uniqueName()] = grid; + mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); + mGrids->push_back(grid); + mNamedGrids[gd.uniqueName()] = grid; } // Connect instances (grids that share trees with other grids). - for (NameMapCIter it = gridDescriptors().begin(); it != gridDescriptors().end(); ++it) { - Archive::connectInstance(it->second, mImpl->mNamedGrids); + for (NameMapCIter it = mGridDescriptors.begin(); it != mGridDescriptors.end(); ++it) { + Archive::connectInstance(it->second, mNamedGrids); } } else { - gridDescriptors().clear(); + mGridDescriptors.clear(); for (int32_t i = 0, N = readGridCount(inputStream()); i < N; ++i) { // Read the grid descriptor. @@ -305,14 +245,14 @@ File::open() gd.readStreamPos(inputStream()); // Add the descriptor to the dictionary. - gridDescriptors().insert(std::make_pair(gd.gridName(), gd)); + mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); // Skip forward to the next descriptor. gd.seekToEnd(inputStream()); } } - mImpl->mIsOpen = true; + mIsOpen = true; return newFile; // true if file is not identical to opened file } @@ -321,14 +261,14 @@ void File::close() { // Reset all data. - mImpl->mMeta.reset(); - mImpl->mGridDescriptors.clear(); - mImpl->mGrids.reset(); - mImpl->mNamedGrids.clear(); - mImpl->mInStream.reset(); - mImpl->mStreamMetadata.reset(); - - mImpl->mIsOpen = false; + mMeta.reset(); + mGridDescriptors.clear(); + mGrids.reset(); + mNamedGrids.clear(); + mInStream.reset(); + mStreamMetadata.reset(); + + mIsOpen = false; setInputHasGridOffsets(true); } @@ -339,44 +279,44 @@ File::close() bool File::hasGrid(const Name& name) const { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } - return (findDescriptor(name) != gridDescriptors().end()); + return (findDescriptor(name) != mGridDescriptors.end()); } MetaMap::Ptr File::getMetadata() const { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } // Return a deep copy of the file-level metadata, which was read // when the file was opened. - return MetaMap::Ptr(new MetaMap(*mImpl->mMeta)); + return MetaMap::Ptr(new MetaMap(*mMeta)); } GridPtrVecPtr File::getGrids() const { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } GridPtrVecPtr ret; if (!inputHasGridOffsets()) { // If the input file doesn't have grid offsets, then all of the grids // have already been streamed in and stored in mGrids. - ret = mImpl->mGrids; + ret = mGrids; } else { ret.reset(new GridPtrVec); Archive::NamedGridMap namedGrids; // Read all grids represented by the GridDescriptors. - for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) { + for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { const GridDescriptor& gd = i->second; GridBase::Ptr grid = readGrid(gd); ret->push_back(grid); @@ -384,7 +324,7 @@ File::getGrids() const } // Connect instances (grids that share trees with other grids). - for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) { + for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { Archive::connectInstance(i->second, namedGrids); } } @@ -404,11 +344,11 @@ File::retrieveCachedGrid(const Name& name) const // Search by unique name. Archive::NamedGridMap::const_iterator it = - mImpl->mNamedGrids.find(GridDescriptor::stringAsUniqueName(name)); + mNamedGrids.find(GridDescriptor::stringAsUniqueName(name)); // If not found, search by grid name. - if (it == mImpl->mNamedGrids.end()) it = mImpl->mNamedGrids.find(name); - if (it == mImpl->mNamedGrids.end()) { - OPENVDB_THROW(KeyError, filename() << " has no grid named \"" << name << "\""); + if (it == mNamedGrids.end()) it = mNamedGrids.find(name); + if (it == mNamedGrids.end()) { + OPENVDB_THROW(KeyError, mFilename << " has no grid named \"" << name << "\""); } return it->second; } @@ -420,8 +360,8 @@ File::retrieveCachedGrid(const Name& name) const GridPtrVecPtr File::readAllGridMetadata() { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } if (fileVersion() < OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) { @@ -434,13 +374,13 @@ File::readAllGridMetadata() if (!inputHasGridOffsets()) { // If the input file doesn't have grid offsets, then all of the grids // have already been streamed in and stored in mGrids. - for (size_t i = 0, N = mImpl->mGrids->size(); i < N; ++i) { + for (size_t i = 0, N = mGrids->size(); i < N; ++i) { // Return copies of the grids, but with empty trees. - ret->push_back((*mImpl->mGrids)[i]->copyGridWithNewTree()); + ret->push_back((*mGrids)[i]->copyGridWithNewTree()); } } else { // Read just the metadata and transforms for all grids. - for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) { + for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { const GridDescriptor& gd = i->second; GridBase::ConstPtr grid = readGridPartial(gd); // Return copies of the grids, but with empty trees. @@ -457,8 +397,8 @@ File::readAllGridMetadata() GridBase::Ptr File::readGridMetadata(const Name& name) { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading."); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading."); } if (fileVersion() < OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) { @@ -473,8 +413,8 @@ File::readGridMetadata(const Name& name) ret = readGrid(name); } else { NameMapCIter it = findDescriptor(name); - if (it == gridDescriptors().end()) { - OPENVDB_THROW(KeyError, filename() << " has no grid named \"" << name << "\""); + if (it == mGridDescriptors.end()) { + OPENVDB_THROW(KeyError, mFilename << " has no grid named \"" << name << "\""); } // Seek to and read in the grid from the file. @@ -498,8 +438,8 @@ File::readGrid(const Name& name) GridBase::Ptr File::readGrid(const Name& name, const BBoxd& bbox) { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading."); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading."); } const bool clip = bbox.isSorted(); @@ -517,8 +457,8 @@ File::readGrid(const Name& name, const BBoxd& bbox) } NameMapCIter it = findDescriptor(name); - if (it == gridDescriptors().end()) { - OPENVDB_THROW(KeyError, filename() << " has no grid named \"" << name << "\""); + if (it == mGridDescriptors.end()) { + OPENVDB_THROW(KeyError, mFilename << " has no grid named \"" << name << "\""); } // Seek to and read in the grid from the file. @@ -529,11 +469,11 @@ File::readGrid(const Name& name, const BBoxd& bbox) /// @todo Refactor to share code with Archive::connectInstance()? NameMapCIter parentIt = findDescriptor(GridDescriptor::nameAsString(gd.instanceParentName())); - if (parentIt == gridDescriptors().end()) { + if (parentIt == mGridDescriptors.end()) { OPENVDB_THROW(KeyError, "missing instance parent \"" << GridDescriptor::nameAsString(gd.instanceParentName()) << "\" for grid " << GridDescriptor::nameAsString(gd.uniqueName()) - << " in file " << filename()); + << " in file " << mFilename); } GridBase::Ptr parent; @@ -555,18 +495,18 @@ File::readGrid(const Name& name, const BBoxd& bbox) void File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta) const { - if (isOpen()) { + if (mIsOpen) { OPENVDB_THROW(IoError, - filename() << " cannot be written because it is open for reading"); + mFilename << " cannot be written because it is open for reading"); } // Create a file stream and write it out. std::ofstream file; - file.open(filename().c_str(), + file.open(mFilename.c_str(), std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); if (file.fail()) { - OPENVDB_THROW(IoError, "could not open " << filename() << " for writing"); + OPENVDB_THROW(IoError, "could not open " << mFilename << " for writing"); } // Write out the vdb. @@ -586,20 +526,20 @@ File::findDescriptor(const Name& name) const const Name uniqueName = GridDescriptor::stringAsUniqueName(name); // Find all descriptors with the given grid name. - std::pair range = gridDescriptors().equal_range(name); + std::pair range = mGridDescriptors.equal_range(name); if (range.first == range.second) { // If no descriptors were found with the given grid name, the name might have // a suffix ("name[N]"). In that case, remove the "[N]" suffix and search again. - range = gridDescriptors().equal_range(GridDescriptor::stripSuffix(uniqueName)); + range = mGridDescriptors.equal_range(GridDescriptor::stripSuffix(uniqueName)); } const size_t count = size_t(std::distance(range.first, range.second)); if (count > 1 && name == uniqueName) { - OPENVDB_LOG_WARN(filename() << " has more than one grid named \"" << name << "\""); + OPENVDB_LOG_WARN(mFilename << " has more than one grid named \"" << name << "\""); } - NameMapCIter ret = gridDescriptors().end(); + NameMapCIter ret = mGridDescriptors.end(); if (count > 0) { if (name == uniqueName) { @@ -632,7 +572,7 @@ File::createGrid(const GridDescriptor& gd) const if (!GridBase::isRegistered(gd.gridType())) { OPENVDB_THROW(KeyError, "Cannot read grid " << GridDescriptor::nameAsString(gd.uniqueName()) - << " from " << filename() << ": grid type " + << " from " << mFilename << ": grid type " << gd.gridType() << " is not registered"); } @@ -679,21 +619,39 @@ File::readGridPartial(const GridDescriptor& gd) const GridBase::Ptr File::readGrid(const GridDescriptor& gd) const { - return Impl::readGrid(*this, gd, Impl::NoBBox()); + // This method should not be called for files that don't contain grid offsets. + OPENVDB_ASSERT(inputHasGridOffsets()); + + GridBase::Ptr grid = createGrid(gd); + gd.seekToGrid(inputStream()); + Archive::readGrid(grid, gd, inputStream()); + return grid; } GridBase::Ptr File::readGrid(const GridDescriptor& gd, const BBoxd& bbox) const { - return Impl::readGrid(*this, gd, bbox); + // This method should not be called for files that don't contain grid offsets. + OPENVDB_ASSERT(inputHasGridOffsets()); + + GridBase::Ptr grid = createGrid(gd); + gd.seekToGrid(inputStream()); + Archive::readGrid(grid, gd, inputStream(), bbox); + return grid; } GridBase::Ptr File::readGrid(const GridDescriptor& gd, const CoordBBox& bbox) const { - return Impl::readGrid(*this, gd, bbox); + // This method should not be called for files that don't contain grid offsets. + OPENVDB_ASSERT(inputHasGridOffsets()); + + GridBase::Ptr grid = createGrid(gd); + gd.seekToGrid(inputStream()); + Archive::readGrid(grid, gd, inputStream(), bbox); + return grid; } @@ -703,17 +661,17 @@ File::readGrid(const GridDescriptor& gd, const CoordBBox& bbox) const File::NameIterator File::beginName() const { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } - return File::NameIterator(gridDescriptors().begin()); + return File::NameIterator(mGridDescriptors.begin()); } File::NameIterator File::endName() const { - return File::NameIterator(gridDescriptors().end()); + return File::NameIterator(mGridDescriptors.end()); } } // namespace io diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index 49ce98eb6d..a572b3f841 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -34,7 +34,7 @@ class OPENVDB_API File: public Archive using NameMapCIter = NameMap::const_iterator; explicit File(const std::string& filename); - ~File() override; + ~File() override { } /// @brief Copy constructor /// @details The copy will be closed and will not reference the same @@ -183,8 +183,22 @@ class OPENVDB_API File: public Archive friend class ::TestFile; friend class ::TestStream; - struct Impl; - std::unique_ptr mImpl; + std::string mFilename; + // The file-level metadata + MetaMap::Ptr mMeta; + // The file stream that is open for reading + std::unique_ptr mInStream; + // File-level stream metadata (file format, compression, etc.) + StreamMetadata::Ptr mStreamMetadata; + // Flag indicating if we have read in the global information (header, + // metadata, and grid descriptors) for this VDB file + bool mIsOpen = false; + // Grid descriptors for all grids stored in the file, indexed by grid name + NameMap mGridDescriptors; + // All grids, indexed by unique name (used only when mHasGridOffsets is false) + Archive::NamedGridMap mNamedGrids; + // All grids stored in the file (used only when mHasGridOffsets is false) + GridPtrVecPtr mGrids; }; diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index 9120bed409..189fed9589 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -636,11 +636,27 @@ TestFile::testReadGridDescriptors() File file2("something.vdb2"); std::istringstream istr(ostr.str(), std::ios_base::binary); io::setCurrentVersion(istr); - file2.readGridDescriptors(istr); + // file2.readGridDescriptors(istr); + //////////////////////////// + file2.mGridDescriptors.clear(); + + for (int32_t i = 0, N = file2.readGridCount(istr); i < N; ++i) { + // Read the grid descriptor. + GridDescriptor gd; + gd.readHeader(istr); + gd.readStreamPos(istr); + + // Add the descriptor to the dictionary. + file2.mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); + + // Skip forward to the next descriptor. + gd.seekToEnd(istr); + } + //////////////////////////// // Compare with the initial grid descriptors. File::NameMapCIter it = file2.findDescriptor("temperature"); - EXPECT_TRUE(it != file2.gridDescriptors().end()); + EXPECT_TRUE(it != file2.mGridDescriptors.end()); GridDescriptor file2gd = it->second; EXPECT_EQ(gd.gridName(), file2gd.gridName()); EXPECT_EQ(gd.getGridPos(), file2gd.getGridPos()); @@ -648,7 +664,7 @@ TestFile::testReadGridDescriptors() EXPECT_EQ(gd.getEndPos(), file2gd.getEndPos()); it = file2.findDescriptor("density"); - EXPECT_TRUE(it != file2.gridDescriptors().end()); + EXPECT_TRUE(it != file2.mGridDescriptors.end()); file2gd = it->second; EXPECT_EQ(gd2.gridName(), file2gd.gridName()); EXPECT_EQ(gd2.getGridPos(), file2gd.getGridPos()); @@ -883,11 +899,27 @@ TestFile::testEmptyGridIO() File file2(filename); std::istringstream istr(ostr.str(), std::ios_base::binary); io::setCurrentVersion(istr); - file2.readGridDescriptors(istr); + // file2.readGridDescriptors(istr); + //////////////////////////// + file2.mGridDescriptors.clear(); + + for (int32_t i = 0, N = file2.readGridCount(istr); i < N; ++i) { + // Read the grid descriptor. + GridDescriptor gd; + gd.readHeader(istr); + gd.readStreamPos(istr); + + // Add the descriptor to the dictionary. + file2.mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); + + // Skip forward to the next descriptor. + gd.seekToEnd(istr); + } + //////////////////////////// // Compare with the initial grid descriptors. File::NameMapCIter it = file2.findDescriptor("temperature"); - EXPECT_TRUE(it != file2.gridDescriptors().end()); + EXPECT_TRUE(it != file2.mGridDescriptors.end()); GridDescriptor file2gd = it->second; file2gd.seekToGrid(istr); GridBase::Ptr gd_grid = GridBase::createGrid(file2gd.gridType()); @@ -905,7 +937,7 @@ TestFile::testEmptyGridIO() EXPECT_EQ(gd.getEndPos(), file2gd.getEndPos()); it = file2.findDescriptor("density"); - EXPECT_TRUE(it != file2.gridDescriptors().end()); + EXPECT_TRUE(it != file2.mGridDescriptors.end()); file2gd = it->second; file2gd.seekToGrid(istr); gd_grid = GridBase::createGrid(file2gd.gridType()); @@ -1000,16 +1032,16 @@ void TestFile::testOpen() EXPECT_EQ(2009, vdbfile.getMetadata()->metaValue("year")); // Ensure we got the grid descriptors. - EXPECT_EQ(1, int(vdbfile.gridDescriptors().count("density"))); - EXPECT_EQ(1, int(vdbfile.gridDescriptors().count("temperature"))); + EXPECT_EQ(1, int(vdbfile.mGridDescriptors.count("density"))); + EXPECT_EQ(1, int(vdbfile.mGridDescriptors.count("temperature"))); io::File::NameMapCIter it = vdbfile.findDescriptor("density"); - EXPECT_TRUE(it != vdbfile.gridDescriptors().end()); + EXPECT_TRUE(it != vdbfile.mGridDescriptors.end()); io::GridDescriptor gd = it->second; EXPECT_EQ(IntTree::treeType(), gd.gridType()); it = vdbfile.findDescriptor("temperature"); - EXPECT_TRUE(it != vdbfile.gridDescriptors().end()); + EXPECT_TRUE(it != vdbfile.mGridDescriptors.end()); gd = it->second; EXPECT_EQ(FloatTree::treeType(), gd.gridType()); @@ -1021,8 +1053,8 @@ void TestFile::testOpen() // Test closing the file. vdbfile.close(); EXPECT_TRUE(vdbfile.isOpen() == false); - EXPECT_TRUE(vdbfile.fileMetadata().get() == nullptr); - EXPECT_EQ(0, int(vdbfile.gridDescriptors().size())); + EXPECT_TRUE(vdbfile.mMeta.get() == nullptr); + EXPECT_EQ(0, int(vdbfile.mGridDescriptors.size())); EXPECT_THROW(vdbfile.inputStream(), openvdb::IoError); remove("something.vdb2"); From c388ae6431f39c52b69a92070b1197ba74047b1b Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 20 Mar 2026 11:25:38 -0700 Subject: [PATCH 21/70] Eliminate Stream::Impl Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Stream.cc | 69 +++++++++++------------------------- openvdb/openvdb/io/Stream.h | 10 +++--- 2 files changed, 26 insertions(+), 53 deletions(-) diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 80326af693..9d52a8a1c4 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -19,32 +19,8 @@ OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { namespace io { -struct Stream::Impl -{ - Impl(): mOutputStream{nullptr} {} - Impl(const Impl& other) { *this = other; } - Impl& operator=(const Impl& other) - { - if (&other != this) { - mMeta = other.mMeta; ///< @todo deep copy? - mGrids = other.mGrids; ///< @todo deep copy? - mOutputStream = other.mOutputStream; - mFile.reset(); - } - return *this; - } - - MetaMap::Ptr mMeta; - GridPtrVecPtr mGrids; - std::ostream* mOutputStream; - std::unique_ptr mFile; -}; - -//////////////////////////////////////// - - -Stream::Stream(std::istream& is): mImpl(new Impl) +Stream::Stream(std::istream& is) { if (!is) return; @@ -59,14 +35,14 @@ Stream::Stream(std::istream& is): mImpl(new Impl) io::setDataCompression(is, compression()); // Read in the VDB metadata. - mImpl->mMeta.reset(new MetaMap); - mImpl->mMeta->readMeta(is); + mMeta.reset(new MetaMap); + mMeta->readMeta(is); // Read in the number of grids. const int32_t gridCount = readGridCount(is); // Read in all grids and insert them into mGrids. - mImpl->mGrids.reset(new GridPtrVec); + mGrids.reset(new GridPtrVec); std::vector descriptors; descriptors.reserve(gridCount); Archive::NamedGridMap namedGrids; @@ -76,7 +52,7 @@ Stream::Stream(std::istream& is): mImpl(new Impl) gd.readStreamPos(is); descriptors.push_back(gd); GridBase::Ptr grid = readGrid(gd, is); - mImpl->mGrids->push_back(grid); + mGrids->push_back(grid); namedGrids[gd.uniqueName()] = grid; } @@ -87,23 +63,18 @@ Stream::Stream(std::istream& is): mImpl(new Impl) } -Stream::Stream(): mImpl(new Impl) -{ -} - - -Stream::Stream(std::ostream& os): mImpl(new Impl) -{ - mImpl->mOutputStream = &os; -} - - -Stream::~Stream() +Stream::Stream(std::ostream& os) + : Archive() + , mOutputStream(&os) { } -Stream::Stream(const Stream& other): Archive(other), mImpl(new Impl(*other.mImpl)) +Stream::Stream(const Stream& other) + : Archive(other) + , mMeta(other.mMeta) + , mGrids(other.mGrids) + , mOutputStream(other.mOutputStream) { } @@ -112,7 +83,9 @@ Stream& Stream::operator=(const Stream& other) { if (&other != this) { - mImpl.reset(new Impl(*other.mImpl)); + mMeta = other.mMeta; + mGrids = other.mGrids; + mOutputStream = other.mOutputStream; } return *this; } @@ -150,10 +123,10 @@ Stream::readGrid(const GridDescriptor& gd, std::istream& is) const void Stream::write(const GridCPtrVec& grids, const MetaMap& metadata) const { - if (mImpl->mOutputStream == nullptr) { + if (mOutputStream == nullptr) { OPENVDB_THROW(ValueError, "no output stream was specified"); } - this->writeGrids(*mImpl->mOutputStream, grids, metadata); + this->writeGrids(*mOutputStream, grids, metadata); } @@ -171,10 +144,10 @@ MetaMap::Ptr Stream::getMetadata() const { MetaMap::Ptr result; - if (mImpl->mMeta) { + if (mMeta) { // Return a deep copy of the file-level metadata // that was read when this object was constructed. - result.reset(new MetaMap(*mImpl->mMeta)); + result.reset(new MetaMap(*mMeta)); } return result; } @@ -183,7 +156,7 @@ Stream::getMetadata() const GridPtrVecPtr Stream::getGrids() { - return mImpl->mGrids; + return mGrids; } } // namespace io diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index 480f5bf074..d6644953b8 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -29,14 +29,14 @@ class OPENVDB_API Stream: public Archive Stream(std::istream& is, bool /*delayLoad*/) : Stream(is) { } /// Construct an archive for stream output. - Stream(); + Stream() = default; /// Construct an archive for output to the given stream. explicit Stream(std::ostream&); Stream(const Stream&); Stream& operator=(const Stream&); - ~Stream() override; + ~Stream() override { } /// @brief Return a copy of this archive. Archive::Ptr copy() const override; @@ -64,9 +64,9 @@ class OPENVDB_API Stream: public Archive void writeGrids(std::ostream&, const GridCPtrVec&, const MetaMap&) const; - - struct Impl; - std::unique_ptr mImpl; + MetaMap::Ptr mMeta; + GridPtrVecPtr mGrids; + std::ostream* mOutputStream = nullptr; }; From aaa82b537e2d7fff7d7aa3df1001b03eecaf1ba0 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 20 Mar 2026 10:43:50 -0700 Subject: [PATCH 22/70] Refactor Archive::readGrid() to always take a BBox Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Archive.cc | 57 ++++++++--------------------------- openvdb/openvdb/io/Archive.h | 5 --- openvdb/openvdb/io/File.cc | 39 +++--------------------- openvdb/openvdb/io/File.h | 5 --- openvdb/openvdb/io/Stream.cc | 2 +- 5 files changed, 17 insertions(+), 91 deletions(-) diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 7ddd4f48b8..3d298390c4 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -886,23 +886,13 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co //////////////////////////////////////// -namespace { - -struct NoBBox {}; - -template void -doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const BoxType& bbox) +Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, + std::istream& is, const BBoxd& worldBBox) { - struct Local { - static void readBuffers(GridBase& g, std::istream& istrm, NoBBox) { g.readBuffers(istrm); } - static void readBuffers(GridBase& g, std::istream& istrm, const CoordBBox& indexBBox) { - g.readBuffers(istrm, indexBBox); - } - static void readBuffers(GridBase& g, std::istream& istrm, const BBoxd& worldBBox) { - g.readBuffers(istrm, g.constTransform().worldToIndexNodeCentered(worldBBox)); - } - }; + // Read the compression settings for this grid and tag the stream with them + // so that downstream functions can reference them. + readGridCompression(is); // Restore the file-level stream metadata on exit. struct OnExit { @@ -942,39 +932,16 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const grid->readTransform(is); if (!gd.isInstance()) { grid->readTopology(is); - Local::readBuffers(*grid, is, bbox); + const bool clip = worldBBox.isSorted(); + if (clip) { + const auto indexBBox = grid->constTransform().worldToIndexNodeCentered(worldBBox); + grid->readBuffers(is, indexBBox); + } else { + grid->readBuffers(is); + } } } -} // unnamed namespace - - -void -Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is) -{ - // Read the compression settings for this grid and tag the stream with them - // so that downstream functions can reference them. - readGridCompression(is); - - doReadGrid(grid, gd, is, NoBBox()); -} - -void -Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, - std::istream& is, const BBoxd& worldBBox) -{ - readGridCompression(is); - doReadGrid(grid, gd, is, worldBBox); -} - -void -Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, - std::istream& is, const CoordBBox& indexBBox) -{ - readGridCompression(is); - doReadGrid(grid, gd, is, indexBBox); -} - //////////////////////////////////////// diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index 87f31b14eb..32b011ec26 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -127,14 +127,9 @@ class OPENVDB_API Archive /// Read in and return the number of grids on the input stream. static int32_t readGridCount(std::istream&); - /// Populate the given grid from the input stream. - static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&); /// @brief Populate the given grid from the input stream, but only where it /// intersects the given world-space bounding box. static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&, const BBoxd&); - /// @brief Populate the given grid from the input stream, but only where it - /// intersects the given index-space bounding box. - static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&, const CoordBBox&); using NamedGridMap = std::map; diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 0424c89da4..2273cdc34f 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -225,7 +225,7 @@ File::open() gd.readStreamPos(inputStream()); GridBase::Ptr grid = createGrid(gd); - Archive::readGrid(grid, gd, inputStream()); + Archive::readGrid(grid, gd, inputStream(), BBoxd()); mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); mGrids->push_back(grid); @@ -318,7 +318,7 @@ File::getGrids() const // Read all grids represented by the GridDescriptors. for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { const GridDescriptor& gd = i->second; - GridBase::Ptr grid = readGrid(gd); + GridBase::Ptr grid = readGrid(gd, BBoxd()); ret->push_back(grid); namedGrids[gd.uniqueName()] = grid; } @@ -463,7 +463,7 @@ File::readGrid(const Name& name, const BBoxd& bbox) // Seek to and read in the grid from the file. const GridDescriptor& gd = it->second; - grid = (clip ? readGrid(gd, bbox) : readGrid(gd)); + grid = readGrid(gd, bbox); if (gd.isInstance()) { /// @todo Refactor to share code with Archive::connectInstance()? @@ -477,12 +477,7 @@ File::readGrid(const Name& name, const BBoxd& bbox) } GridBase::Ptr parent; - if (clip) { - const CoordBBox indexBBox = grid->constTransform().worldToIndexNodeCentered(bbox); - parent = readGrid(parentIt->second, indexBBox); - } else { - parent = readGrid(parentIt->second); - } + parent = readGrid(parentIt->second, bbox); if (parent) grid->setTree(parent->baseTreePtr()); } return grid; @@ -616,19 +611,6 @@ File::readGridPartial(const GridDescriptor& gd) const } -GridBase::Ptr -File::readGrid(const GridDescriptor& gd) const -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - GridBase::Ptr grid = createGrid(gd); - gd.seekToGrid(inputStream()); - Archive::readGrid(grid, gd, inputStream()); - return grid; -} - - GridBase::Ptr File::readGrid(const GridDescriptor& gd, const BBoxd& bbox) const { @@ -642,19 +624,6 @@ File::readGrid(const GridDescriptor& gd, const BBoxd& bbox) const } -GridBase::Ptr -File::readGrid(const GridDescriptor& gd, const CoordBBox& bbox) const -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - GridBase::Ptr grid = createGrid(gd); - gd.seekToGrid(inputStream()); - Archive::readGrid(grid, gd, inputStream(), bbox); - return grid; -} - - //////////////////////////////////////// diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index a572b3f841..b837e2d5e4 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -156,14 +156,9 @@ class OPENVDB_API File: public Archive /// Read in and return the partially-populated grid specified by the given grid descriptor. GridBase::ConstPtr readGridPartial(const GridDescriptor&) const; - /// Read in and return the grid specified by the given grid descriptor. - GridBase::Ptr readGrid(const GridDescriptor&) const; /// Read in and return the region of the grid specified by the given grid descriptor /// that intersects the given world-space bounding box. GridBase::Ptr readGrid(const GridDescriptor&, const BBoxd&) const; - /// Read in and return the region of the grid specified by the given grid descriptor - /// that intersects the given index-space bounding box. - GridBase::Ptr readGrid(const GridDescriptor&, const CoordBBox&) const; /// @brief Retrieve a grid from @c mNamedGrids. Return a null pointer /// if @c mNamedGrids was not populated (because this file is random-access). diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 9d52a8a1c4..b37c393688 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -114,7 +114,7 @@ Stream::readGrid(const GridDescriptor& gd, std::istream& is) const grid = GridBase::createGrid(gd.gridType()); if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); - Archive::readGrid(grid, gd, is); + Archive::readGrid(grid, gd, is, BBoxd()); } return grid; } From 3b2ea9c636f05b9767915efc3ebe1e6c39be9a72 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 20 Mar 2026 11:26:25 -0700 Subject: [PATCH 23/70] Archive::readGrid() now creates the grid, remove File::readGridPartial() Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Archive.cc | 19 +++++-- openvdb/openvdb/io/Archive.h | 7 +-- openvdb/openvdb/io/File.cc | 93 +++++++---------------------------- openvdb/openvdb/io/File.h | 10 ---- openvdb/openvdb/io/Stream.cc | 21 +------- openvdb/openvdb/io/Stream.h | 5 -- 6 files changed, 38 insertions(+), 117 deletions(-) diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 3d298390c4..e0f37f5535 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -886,9 +886,8 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co //////////////////////////////////////// -void -Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, - std::istream& is, const BBoxd& worldBBox) +GridBase::Ptr +Archive::readGrid(const GridDescriptor& gd, std::istream& is, const BBoxd& worldBBox, bool partial/* = false*/) { // Read the compression settings for this grid and tag the stream with them // so that downstream functions can reference them. @@ -903,6 +902,16 @@ Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, }; OnExit restore(is); + // Create the grid. + if (!GridBase::isRegistered(gd.gridType())) { + OPENVDB_THROW(KeyError, "Cannot read grid " + << GridDescriptor::nameAsString(gd.uniqueName()) + << ": grid type " << gd.gridType() << " is not registered"); + } + + GridBase::Ptr grid = GridBase::createGrid(gd.gridType()); + if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); + // Stream metadata varies per grid, and it needs to persist // in case delayed load is in effect. io::StreamMetadata::Ptr streamMetadata; @@ -930,7 +939,7 @@ Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, io::setGridClass(is, gridClass); grid->readTransform(is); - if (!gd.isInstance()) { + if (!partial && !gd.isInstance()) { grid->readTopology(is); const bool clip = worldBBox.isSorted(); if (clip) { @@ -940,6 +949,8 @@ Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, grid->readBuffers(is); } } + + return grid; } diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index 32b011ec26..e61a0c8735 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -127,9 +127,10 @@ class OPENVDB_API Archive /// Read in and return the number of grids on the input stream. static int32_t readGridCount(std::istream&); - /// @brief Populate the given grid from the input stream, but only where it - /// intersects the given world-space bounding box. - static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&, const BBoxd&); + /// @brief Read in and create the grid represented by the given grid descriptor using the + /// given input stream, but only where it intersects the given world-space bounding box. + /// @param partial If true, only read the metadata and transform of the grid. + static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, const BBoxd&, bool partial = false); using NamedGridMap = std::map; diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 2273cdc34f..1e0e6acd35 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -224,8 +224,7 @@ File::open() gd.readHeader(inputStream()); gd.readStreamPos(inputStream()); - GridBase::Ptr grid = createGrid(gd); - Archive::readGrid(grid, gd, inputStream(), BBoxd()); + GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), BBoxd()); mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); mGrids->push_back(grid); @@ -318,7 +317,9 @@ File::getGrids() const // Read all grids represented by the GridDescriptors. for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { const GridDescriptor& gd = i->second; - GridBase::Ptr grid = readGrid(gd, BBoxd()); + // Seek to the grid in the file. + gd.seekToGrid(inputStream()); + GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), BBoxd()); ret->push_back(grid); namedGrids[gd.uniqueName()] = grid; } @@ -382,10 +383,12 @@ File::readAllGridMetadata() // Read just the metadata and transforms for all grids. for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { const GridDescriptor& gd = i->second; - GridBase::ConstPtr grid = readGridPartial(gd); + // Seek to the grid in the file. + gd.seekToGrid(inputStream()); + GridBase::ConstPtr grid = Archive::readGrid(gd, inputStream(), BBoxd(), /*partial=*/true); // Return copies of the grids, but with empty trees. // (As of 0.98.0, at least, it would suffice to just const cast - // the grid pointers returned by readGridPartial(), but shallow + // the grid pointers returned by readGrid(partial=true), but shallow // copying the grids helps to ensure future compatibility.) ret->push_back(grid->copyGridWithNewTree()); } @@ -419,7 +422,8 @@ File::readGridMetadata(const Name& name) // Seek to and read in the grid from the file. const GridDescriptor& gd = it->second; - ret = readGridPartial(gd); + gd.seekToGrid(inputStream()); + ret = Archive::readGrid(gd, inputStream(), BBoxd(), /*partial=*/true); } return ret->copyGridWithNewTree(); } @@ -463,7 +467,11 @@ File::readGrid(const Name& name, const BBoxd& bbox) // Seek to and read in the grid from the file. const GridDescriptor& gd = it->second; - grid = readGrid(gd, bbox); + // This method should not be called for files that don't contain grid offsets. + OPENVDB_ASSERT(inputHasGridOffsets()); + // Seek to the grid in the file. + gd.seekToGrid(inputStream()); + grid = Archive::readGrid(gd, inputStream(), bbox); if (gd.isInstance()) { /// @todo Refactor to share code with Archive::connectInstance()? @@ -477,7 +485,9 @@ File::readGrid(const Name& name, const BBoxd& bbox) } GridBase::Ptr parent; - parent = readGrid(parentIt->second, bbox); + OPENVDB_ASSERT(inputHasGridOffsets()); + parentIt->second.seekToGrid(inputStream()); + parent = Archive::readGrid(parentIt->second, inputStream(), bbox); if (parent) grid->setTree(parent->baseTreePtr()); } return grid; @@ -560,73 +570,6 @@ File::findDescriptor(const Name& name) const //////////////////////////////////////// -GridBase::Ptr -File::createGrid(const GridDescriptor& gd) const -{ - // Create the grid. - if (!GridBase::isRegistered(gd.gridType())) { - OPENVDB_THROW(KeyError, "Cannot read grid " - << GridDescriptor::nameAsString(gd.uniqueName()) - << " from " << mFilename << ": grid type " - << gd.gridType() << " is not registered"); - } - - GridBase::Ptr grid = GridBase::createGrid(gd.gridType()); - if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); - - return grid; -} - - -GridBase::ConstPtr -File::readGridPartial(const GridDescriptor& gd) const -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - std::istream& is = inputStream(); - - GridBase::Ptr grid = createGrid(gd); - - // Seek to grid. - gd.seekToGrid(is); - - // This code needs to stay in sync with io::Archive::readGrid(), in terms of - // the order of operations. - readGridCompression(is); - grid->readMeta(is); - - // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists - if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { - grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } - - // Read the transform. - grid->readTransform(is); - - // Promote to a const grid. - GridBase::ConstPtr constGrid = grid; - - return constGrid; -} - - -GridBase::Ptr -File::readGrid(const GridDescriptor& gd, const BBoxd& bbox) const -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - GridBase::Ptr grid = createGrid(gd); - gd.seekToGrid(inputStream()); - Archive::readGrid(grid, gd, inputStream(), bbox); - return grid; -} - - -//////////////////////////////////////// - - File::NameIterator File::beginName() const { diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index b837e2d5e4..8222263cf1 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -150,16 +150,6 @@ class OPENVDB_API File: public Archive /// If the name is non-unique, return an iterator to the first matching descriptor. NameMapCIter findDescriptor(const Name&) const; - /// Return a newly created, empty grid of the type specified by the given grid descriptor. - GridBase::Ptr createGrid(const GridDescriptor&) const; - - /// Read in and return the partially-populated grid specified by the given grid descriptor. - GridBase::ConstPtr readGridPartial(const GridDescriptor&) const; - - /// Read in and return the region of the grid specified by the given grid descriptor - /// that intersects the given world-space bounding box. - GridBase::Ptr readGrid(const GridDescriptor&, const BBoxd&) const; - /// @brief Retrieve a grid from @c mNamedGrids. Return a null pointer /// if @c mNamedGrids was not populated (because this file is random-access). /// @throw KeyError if no grid with the given name exists in this file. diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index b37c393688..9021fd7706 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -51,7 +51,7 @@ Stream::Stream(std::istream& is) gd.readHeader(is); gd.readStreamPos(is); descriptors.push_back(gd); - GridBase::Ptr grid = readGrid(gd, is); + GridBase::Ptr grid = Archive::readGrid(gd, is, BBoxd()); mGrids->push_back(grid); namedGrids[gd.uniqueName()] = grid; } @@ -101,25 +101,6 @@ Stream::copy() const //////////////////////////////////////// -GridBase::Ptr -Stream::readGrid(const GridDescriptor& gd, std::istream& is) const -{ - GridBase::Ptr grid; - - if (!GridBase::isRegistered(gd.gridType())) { - OPENVDB_THROW(TypeError, "can't read grid \"" - << GridDescriptor::nameAsString(gd.uniqueName()) << - "\" from input stream because grid type " << gd.gridType() << " is unknown"); - } else { - grid = GridBase::createGrid(gd.gridType()); - if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); - - Archive::readGrid(grid, gd, is, BBoxd()); - } - return grid; -} - - void Stream::write(const GridCPtrVec& grids, const MetaMap& metadata) const { diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index d6644953b8..4bdc1662ee 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -57,11 +57,6 @@ class OPENVDB_API Stream: public Archive void write(const GridPtrContainerT&, const MetaMap& = MetaMap()) const; private: - /// Create a new grid of the type specified by the given descriptor, - /// then populate the grid from the given input stream. - /// @return the newly created grid. - GridBase::Ptr readGrid(const GridDescriptor&, std::istream&) const; - void writeGrids(std::ostream&, const GridCPtrVec&, const MetaMap&) const; MetaMap::Ptr mMeta; From f32548c6c2f001b6f5b2a29b0796bcf6a39dc764 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 16 Apr 2026 13:29:50 -0700 Subject: [PATCH 24/70] Update TestFile to use Archive::readGrid Signed-off-by: Dan Bailey --- openvdb/openvdb/unittest/TestFile.cc | 55 +++------------------------- 1 file changed, 6 insertions(+), 49 deletions(-) diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index 189fed9589..47a26cbaf9 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -158,21 +158,7 @@ TestFile::testWriteGrid() gd2.readHeader(istr); gd2.readStreamPos(istr); - // manually create the grid - GridBase::Ptr gd2_grid = GridBase::createGrid(gd2.gridType()); - - EXPECT_EQ(gd.gridName(), gd2.gridName()); - EXPECT_EQ(GridType::gridType(), gd2_grid->type()); - EXPECT_EQ(gd.getGridPos(), gd2.getGridPos()); - EXPECT_EQ(gd.getBlockPos(), gd2.getBlockPos()); - EXPECT_EQ(gd.getEndPos(), gd2.getEndPos()); - - // Position the stream to beginning of the grid storage and read the grid. - gd2.seekToGrid(istr); - Archive::readGridCompression(istr); - gd2_grid->readMeta(istr); - gd2_grid->readTransform(istr); - gd2_grid->readTopology(istr); + GridBase::Ptr gd2_grid = Archive::readGrid(gd2, istr, BBoxd()); // Delay load metadata should not exist. ASSERT_FALSE(bool((*gd2_grid)["file_delayed_load"])); @@ -192,9 +178,6 @@ TestFile::testWriteGrid() EXPECT_EQ( grid->baseTree().treeDepth(), gd2_grid->baseTree().treeDepth()); - // Read in the data blocks. - gd2.seekToBlocks(istr); - gd2_grid->readBuffers(istr); TreeType::Ptr tree2 = DynamicPtrCast(gd2_grid->baseTreePtr()); EXPECT_TRUE(tree2.get() != nullptr); EXPECT_EQ(10, tree2->getValue(Coord(10, 1, 2))); @@ -260,7 +243,8 @@ TestFile::testWriteMultipleGrids() gd_in.readHeader(istr); gd_in.readStreamPos(istr); - GridBase::Ptr gd_in_grid = GridBase::createGrid(gd_in.gridType()); + + GridBase::Ptr gd_in_grid = Archive::readGrid(gd_in, istr, BBoxd()); // Ensure read in the right values. EXPECT_EQ(gd.gridName(), gd_in.gridName()); @@ -269,13 +253,6 @@ TestFile::testWriteMultipleGrids() EXPECT_EQ(gd.getBlockPos(), gd_in.getBlockPos()); EXPECT_EQ(gd.getEndPos(), gd_in.getEndPos()); - // Position the stream to beginning of the grid storage and read the grid. - gd_in.seekToGrid(istr); - Archive::readGridCompression(istr); - gd_in_grid->readMeta(istr); - gd_in_grid->readTransform(istr); - gd_in_grid->readTopology(istr); - // Ensure that we have the same topology and transform. EXPECT_EQ( grid->baseTree().leafCount(), gd_in_grid->baseTree().leafCount()); @@ -289,8 +266,6 @@ TestFile::testWriteMultipleGrids() // EXPECT_EQ(0.1, gd_in_grid->getTransform()->getVoxelSizeZ()); // Read in the data blocks. - gd_in.seekToBlocks(istr); - gd_in_grid->readBuffers(istr); TreeType::Ptr grid_in = DynamicPtrCast(gd_in_grid->baseTreePtr()); EXPECT_TRUE(grid_in.get() != nullptr); EXPECT_EQ(10, grid_in->getValue(Coord(10, 1, 2))); @@ -306,7 +281,7 @@ TestFile::testWriteMultipleGrids() GridDescriptor gd2_in; gd2_in.readHeader(istr); gd2_in.readStreamPos(istr); - GridBase::Ptr gd2_in_grid = GridBase::createGrid(gd2_in.gridType()); + GridBase::Ptr gd2_in_grid = Archive::readGrid(gd2_in, istr, BBoxd()); // Ensure that we read in the right values. EXPECT_EQ(gd2.gridName(), gd2_in.gridName()); @@ -315,13 +290,6 @@ TestFile::testWriteMultipleGrids() EXPECT_EQ(gd2.getBlockPos(), gd2_in.getBlockPos()); EXPECT_EQ(gd2.getEndPos(), gd2_in.getEndPos()); - // Position the stream to beginning of the grid storage and read the grid. - gd2_in.seekToGrid(istr); - Archive::readGridCompression(istr); - gd2_in_grid->readMeta(istr); - gd2_in_grid->readTransform(istr); - gd2_in_grid->readTopology(istr); - // Ensure that we have the same topology and transform. EXPECT_EQ( grid2->baseTree().leafCount(), gd2_in_grid->baseTree().leafCount()); @@ -333,9 +301,6 @@ TestFile::testWriteMultipleGrids() // EXPECT_EQ(0.2, gd2_in_grid->getTransform()->getVoxelSizeY()); // EXPECT_EQ(0.2, gd2_in_grid->getTransform()->getVoxelSizeZ()); - // Read in the data blocks. - gd2_in.seekToBlocks(istr); - gd2_in_grid->readBuffers(istr); TreeType::Ptr grid2_in = DynamicPtrCast(gd2_in_grid->baseTreePtr()); EXPECT_TRUE(grid2_in.get() != nullptr); EXPECT_EQ(50, grid2_in->getValue(Coord(1000, 1000, 1000))); @@ -922,11 +887,7 @@ TestFile::testEmptyGridIO() EXPECT_TRUE(it != file2.mGridDescriptors.end()); GridDescriptor file2gd = it->second; file2gd.seekToGrid(istr); - GridBase::Ptr gd_grid = GridBase::createGrid(file2gd.gridType()); - Archive::readGridCompression(istr); - gd_grid->readMeta(istr); - gd_grid->readTransform(istr); - gd_grid->readTopology(istr); + GridBase::Ptr gd_grid = Archive::readGrid(file2gd, istr, BBoxd()); EXPECT_EQ(gd.gridName(), file2gd.gridName()); EXPECT_TRUE(gd_grid.get() != nullptr); EXPECT_EQ(0, int(gd_grid->baseTree().leafCount())); @@ -940,11 +901,7 @@ TestFile::testEmptyGridIO() EXPECT_TRUE(it != file2.mGridDescriptors.end()); file2gd = it->second; file2gd.seekToGrid(istr); - gd_grid = GridBase::createGrid(file2gd.gridType()); - Archive::readGridCompression(istr); - gd_grid->readMeta(istr); - gd_grid->readTransform(istr); - gd_grid->readTopology(istr); + gd_grid = Archive::readGrid(file2gd, istr, BBoxd()); EXPECT_EQ(gd2.gridName(), file2gd.gridName()); EXPECT_TRUE(gd_grid.get() != nullptr); EXPECT_EQ(0, int(gd_grid->baseTree().leafCount())); From 518d4bda27fc0ae5013aeae62309aa2d256fd5fc Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 23 Mar 2026 13:50:14 -0700 Subject: [PATCH 25/70] Add io/Codec.h with ReadOptions and WriteOptions Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 2 ++ openvdb/openvdb/io/Codec.h | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 openvdb/openvdb/io/Codec.h diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 91fc5fcfd7..6a613f0326 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -366,6 +366,7 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES set(OPENVDB_LIBRARY_IO_INCLUDE_FILES io/Archive.h + io/Codec.h io/Compression.h io/File.h io/GridDescriptor.h @@ -734,6 +735,7 @@ endif() install(FILES ${OPENVDB_LIBRARY_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/openvdb/version.h DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb) install(FILES ${OPENVDB_LIBRARY_IO_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/io) +install(FILES ${OPENVDB_LIBRARY_CODECS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/codecs) install(FILES ${OPENVDB_LIBRARY_MATH_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/math) install(FILES ${OPENVDB_LIBRARY_POINTS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/points) install(FILES ${OPENVDB_LIBRARY_POINTS_IMPL_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/points/impl) diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h new file mode 100644 index 0000000000..a96b076b41 --- /dev/null +++ b/openvdb/openvdb/io/Codec.h @@ -0,0 +1,51 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED + +#include + +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace io { + + +enum class ReadMode { + Original, // Read data as-is (default) + TopologyOnly // Read topology only, no buffer data +}; + +/// Global read configuration that applies to all grids +/// Contains common options shared across all codec types +struct OPENVDB_API ReadOptions +{ + using Ptr = std::shared_ptr; + + virtual ~ReadOptions() = default; + + // BBox Clip Data + BBoxd clipBBox = BBoxd(); + + // Grid Conversion Options + ReadMode readMode = ReadMode::Original; +}; // struct ReadOptions + +/// Global write configuration that applies to all grids +/// Contains common options shared across all codec types +struct OPENVDB_API WriteOptions +{ + using Ptr = std::shared_ptr; + + virtual ~WriteOptions() = default; +}; // struct WriteOptions + + +} // namespace io +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED From 7405a77417f6edd3df06f83a92a525aeaff8167b Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 23 Mar 2026 13:51:26 -0700 Subject: [PATCH 26/70] Pass writeOptions as an argument to all the I/O write routines Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Archive.cc | 14 +++++++------- openvdb/openvdb/io/Archive.h | 19 +++++++++++++------ openvdb/openvdb/io/File.cc | 4 ++-- openvdb/openvdb/io/File.h | 18 +++++++++++------- openvdb/openvdb/io/Stream.cc | 10 ++++++---- openvdb/openvdb/io/Stream.h | 14 +++++++++----- 6 files changed, 48 insertions(+), 31 deletions(-) diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index e0f37f5535..cf89a5aba1 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -959,15 +959,15 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const BBoxd& world void Archive::write(std::ostream& os, const GridPtrVec& grids, bool seekable, - const MetaMap& metadata) const + const MetaMap& metadata, const io::WriteOptions& writeOptions) const { - this->write(os, GridCPtrVec(grids.begin(), grids.end()), seekable, metadata); + this->write(os, GridCPtrVec(grids.begin(), grids.end()), seekable, metadata, writeOptions); } void Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, - const MetaMap& metadata) const + const MetaMap& metadata, const io::WriteOptions& writeOptions) const { // Set stream flags so that downstream functions can reference them. io::StreamMetadata::Ptr streamMetadata = io::getStreamMetadataPtr(os); @@ -1039,7 +1039,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, // Get the name of the other grid. gd.setInstanceParentName(mapIter->second.uniqueName()); // Write out this grid's descriptor and metadata, but not its tree. - writeGridInstance(gd, grid, os, seekable); + writeGridInstance(gd, grid, os, seekable, writeOptions); OPENVDB_LOG_DEBUG_RUNTIME("io::Archive::write(): " << GridDescriptor::nameAsString(gd.uniqueName()) @@ -1048,7 +1048,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, << GridDescriptor::nameAsString(gd.instanceParentName())); } else { // Write out the grid descriptor and its associated grid. - writeGrid(gd, grid, os, seekable); + writeGrid(gd, grid, os, seekable, writeOptions); // Record the grid's tree pointer so that the tree doesn't get written // more than once. treeMap[treePtr] = gd; @@ -1064,7 +1064,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, void Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, - std::ostream& os, bool seekable) const + std::ostream& os, bool seekable, const io::WriteOptions& writeOptions) const { // Restore file-level stream metadata on exit. struct OnExit { @@ -1143,7 +1143,7 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, void Archive::writeGridInstance(GridDescriptor& gd, GridBase::ConstPtr grid, - std::ostream& os, bool seekable) const + std::ostream& os, bool seekable, const io::WriteOptions& writeOptions) const { // Write out the Descriptor's header information (grid name, type // and instance parent name). diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index e61a0c8735..8065a3b0b5 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -5,11 +5,14 @@ #define OPENVDB_IO_ARCHIVE_HAS_BEEN_INCLUDED #include -#include "Compression.h" // for COMPRESS_ZIP, etc. #include #include #include #include // for VersionId + +#include "Codec.h" +#include "Compression.h" // for COMPRESS_ZIP, etc. + #include #include #include @@ -89,7 +92,8 @@ class OPENVDB_API Archive void setGridStatsMetadataEnabled(bool b) { mEnableGridStats = b; } /// @brief Write the grids in the given container to this archive's output stream. - virtual void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const {} + virtual void write(const GridCPtrVec&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const {} /// @brief Return @c false (delayed loading has been removed). static bool isDelayedLoadingEnabled() { return false; } @@ -141,13 +145,14 @@ class OPENVDB_API Archive /// Write the given grid descriptor and grid to an output stream /// and update the GridDescriptor offsets. /// @param seekable if true, the output stream supports seek operations - void writeGrid(GridDescriptor&, GridBase::ConstPtr, std::ostream&, bool seekable) const; + void writeGrid(GridDescriptor&, GridBase::ConstPtr, std::ostream&, bool seekable, + const io::WriteOptions& writeOptions = io::WriteOptions{}) const; /// Write the given grid descriptor and grid metadata to an output stream /// and update the GridDescriptor offsets, but don't write the grid's tree, /// since it is shared with another grid. /// @param seekable if true, the output stream supports seek operations void writeGridInstance(GridDescriptor&, GridBase::ConstPtr, - std::ostream&, bool seekable) const; + std::ostream&, bool seekable, const io::WriteOptions& writeOptions = io::WriteOptions{}) const; /// @brief Read the magic number, version numbers, UUID, etc. from the given input stream. /// @return @c true if the input UUID differs from the previously-read UUID. @@ -159,8 +164,10 @@ class OPENVDB_API Archive //@{ /// Write the given grids to an output stream. - void write(std::ostream&, const GridPtrVec&, bool seekable, const MetaMap& = MetaMap()) const; - void write(std::ostream&, const GridCPtrVec&, bool seekable, const MetaMap& = MetaMap()) const; + void write(std::ostream&, const GridPtrVec&, bool seekable, const MetaMap&, + const io::WriteOptions& writeOptions = io::WriteOptions{}) const; + void write(std::ostream&, const GridCPtrVec&, bool seekable, const MetaMap&, + const io::WriteOptions& writeOptions = io::WriteOptions{}) const; //@} private: diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 1e0e6acd35..551716ee73 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -498,7 +498,7 @@ File::readGrid(const Name& name, const BBoxd& bbox) void -File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta) const +File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta, const io::WriteOptions& writeOptions) const { if (mIsOpen) { OPENVDB_THROW(IoError, @@ -515,7 +515,7 @@ File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta) const } // Write out the vdb. - Archive::write(file, grids, /*seekable=*/true, meta); + Archive::write(file, grids, /*seekable=*/true, meta, writeOptions); file.close(); } diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index 8222263cf1..8752e45f5c 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -110,12 +110,14 @@ class OPENVDB_API File: public Archive /// @brief Write the grids in the given container to the file whose name /// was given in the constructor. - void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const override; + void write(const GridCPtrVec&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const override; /// @brief Write the grids in the given container to the file whose name /// was given in the constructor. template - void write(const GridPtrContainerT&, const MetaMap& = MetaMap()) const; + void write(const GridPtrContainerT&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const; /// A const iterator that iterates over all names in the file. This is only /// valid once the file has been opened. @@ -155,7 +157,7 @@ class OPENVDB_API File: public Archive /// @throw KeyError if no grid with the given name exists in this file. GridBase::Ptr retrieveCachedGrid(const Name&) const; - void writeGrids(const GridCPtrVec&, const MetaMap&) const; + void writeGrids(const GridCPtrVec&, const MetaMap&, const io::WriteOptions&) const; MetaMap::Ptr fileMetadata(); MetaMap::ConstPtr fileMetadata() const; @@ -191,19 +193,21 @@ class OPENVDB_API File: public Archive inline void -File::write(const GridCPtrVec& grids, const MetaMap& meta) const +File::write(const GridCPtrVec& grids, const MetaMap& meta, + const io::WriteOptions& writeOptions) const { - this->writeGrids(grids, meta); + this->writeGrids(grids, meta, writeOptions); } template inline void -File::write(const GridPtrContainerT& container, const MetaMap& meta) const +File::write(const GridPtrContainerT& container, const MetaMap& meta, + const io::WriteOptions& writeOptions) const { GridCPtrVec grids; std::copy(container.begin(), container.end(), std::back_inserter(grids)); - this->writeGrids(grids, meta); + this->writeGrids(grids, meta, writeOptions); } } // namespace io diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 9021fd7706..b65e32703a 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -102,19 +102,21 @@ Stream::copy() const void -Stream::write(const GridCPtrVec& grids, const MetaMap& metadata) const +Stream::write(const GridCPtrVec& grids, const MetaMap& metadata, + const io::WriteOptions& writeOptions) const { if (mOutputStream == nullptr) { OPENVDB_THROW(ValueError, "no output stream was specified"); } - this->writeGrids(*mOutputStream, grids, metadata); + this->writeGrids(*mOutputStream, grids, metadata, writeOptions); } void -Stream::writeGrids(std::ostream& os, const GridCPtrVec& grids, const MetaMap& metadata) const +Stream::writeGrids(std::ostream& os, const GridCPtrVec& grids, const MetaMap& metadata, + const io::WriteOptions& writeOptions) const { - Archive::write(os, grids, /*seekable=*/false, metadata); + Archive::write(os, grids, /*seekable=*/false, metadata, writeOptions); } diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index 4bdc1662ee..418a06b142 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -49,15 +49,18 @@ class OPENVDB_API Stream: public Archive /// @brief Write the grids in the given container to this archive's output stream. /// @throw ValueError if this archive was constructed without specifying an output stream. - void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const override; + void write(const GridCPtrVec&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const override; /// @brief Write the grids in the given container to this archive's output stream. /// @throw ValueError if this archive was constructed without specifying an output stream. template - void write(const GridPtrContainerT&, const MetaMap& = MetaMap()) const; + void write(const GridPtrContainerT&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const; private: - void writeGrids(std::ostream&, const GridCPtrVec&, const MetaMap&) const; + void writeGrids(std::ostream&, const GridCPtrVec&, const MetaMap&, + const io::WriteOptions&) const; MetaMap::Ptr mMeta; GridPtrVecPtr mGrids; @@ -70,11 +73,12 @@ class OPENVDB_API Stream: public Archive template inline void -Stream::write(const GridPtrContainerT& container, const MetaMap& metadata) const +Stream::write(const GridPtrContainerT& container, const MetaMap& metadata, + const io::WriteOptions& writeOptions) const { GridCPtrVec grids; std::copy(container.begin(), container.end(), std::back_inserter(grids)); - this->write(grids, metadata); + this->write(grids, metadata, writeOptions); } } // namespace io From de3a3e0813e868b5402fce87dbf4bf2e31ab5a17 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 23 Mar 2026 17:44:57 -0700 Subject: [PATCH 27/70] Pass readOptions as an argument to all the I/O read routines Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Archive.cc | 9 +++++---- openvdb/openvdb/io/Archive.h | 6 +++--- openvdb/openvdb/io/File.cc | 30 +++++++++++++++++----------- openvdb/openvdb/io/File.h | 6 +++--- openvdb/openvdb/io/Stream.cc | 2 +- openvdb/openvdb/unittest/TestFile.cc | 10 +++++----- 6 files changed, 35 insertions(+), 28 deletions(-) diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index cf89a5aba1..4eb9d0fe74 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -887,7 +887,7 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co GridBase::Ptr -Archive::readGrid(const GridDescriptor& gd, std::istream& is, const BBoxd& worldBBox, bool partial/* = false*/) +Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOptions& readOptions) { // Read the compression settings for this grid and tag the stream with them // so that downstream functions can reference them. @@ -939,8 +939,9 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const BBoxd& world io::setGridClass(is, gridClass); grid->readTransform(is); - if (!partial && !gd.isInstance()) { + if (readOptions.readMode != io::ReadMode::TopologyOnly && !gd.isInstance()) { grid->readTopology(is); + const auto& worldBBox = readOptions.clipBBox; const bool clip = worldBBox.isSorted(); if (clip) { const auto indexBBox = grid->constTransform().worldToIndexNodeCentered(worldBBox); @@ -1064,7 +1065,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, void Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, - std::ostream& os, bool seekable, const io::WriteOptions& writeOptions) const + std::ostream& os, bool seekable, const io::WriteOptions&) const { // Restore file-level stream metadata on exit. struct OnExit { @@ -1143,7 +1144,7 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, void Archive::writeGridInstance(GridDescriptor& gd, GridBase::ConstPtr grid, - std::ostream& os, bool seekable, const io::WriteOptions& writeOptions) const + std::ostream& os, bool seekable, const io::WriteOptions&) const { // Write out the Descriptor's header information (grid name, type // and instance parent name). diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index 8065a3b0b5..a3b0893ea6 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -132,9 +132,9 @@ class OPENVDB_API Archive static int32_t readGridCount(std::istream&); /// @brief Read in and create the grid represented by the given grid descriptor using the - /// given input stream, but only where it intersects the given world-space bounding box. - /// @param partial If true, only read the metadata and transform of the grid. - static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, const BBoxd&, bool partial = false); + /// given input stream, using the provided options if given. + static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, + const io::ReadOptions& readOptions = io::ReadOptions{}); using NamedGridMap = std::map; diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 551716ee73..bd4ebdd4f5 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -224,7 +224,7 @@ File::open() gd.readHeader(inputStream()); gd.readStreamPos(inputStream()); - GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), BBoxd()); + GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), io::ReadOptions{}); mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); mGrids->push_back(grid); @@ -298,7 +298,7 @@ File::getMetadata() const GridPtrVecPtr -File::getGrids() const +File::getGrids(const io::ReadOptions& readOptions) const { if (!mIsOpen) { OPENVDB_THROW(IoError, mFilename << " is not open for reading"); @@ -319,7 +319,7 @@ File::getGrids() const const GridDescriptor& gd = i->second; // Seek to the grid in the file. gd.seekToGrid(inputStream()); - GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), BBoxd()); + GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), readOptions); ret->push_back(grid); namedGrids[gd.uniqueName()] = grid; } @@ -385,7 +385,9 @@ File::readAllGridMetadata() const GridDescriptor& gd = i->second; // Seek to the grid in the file. gd.seekToGrid(inputStream()); - GridBase::ConstPtr grid = Archive::readGrid(gd, inputStream(), BBoxd(), /*partial=*/true); + io::ReadOptions readOptions; + readOptions.readMode = io::ReadMode::TopologyOnly; + GridBase::ConstPtr grid = Archive::readGrid(gd, inputStream(), readOptions); // Return copies of the grids, but with empty trees. // (As of 0.98.0, at least, it would suffice to just const cast // the grid pointers returned by readGrid(partial=true), but shallow @@ -423,7 +425,9 @@ File::readGridMetadata(const Name& name) // Seek to and read in the grid from the file. const GridDescriptor& gd = it->second; gd.seekToGrid(inputStream()); - ret = Archive::readGrid(gd, inputStream(), BBoxd(), /*partial=*/true); + io::ReadOptions readOptions; + readOptions.readMode = io::ReadMode::TopologyOnly; + ret = Archive::readGrid(gd, inputStream(), readOptions); } return ret->copyGridWithNewTree(); } @@ -433,26 +437,28 @@ File::readGridMetadata(const Name& name) GridBase::Ptr -File::readGrid(const Name& name) +File::readGrid(const Name& name, const BBoxd& bbox) { - return readGrid(name, BBoxd()); + io::ReadOptions readOptions; + readOptions.clipBBox = bbox; + return readGrid(name, readOptions); } GridBase::Ptr -File::readGrid(const Name& name, const BBoxd& bbox) +File::readGrid(const Name& name, const io::ReadOptions& readOptions) { if (!mIsOpen) { OPENVDB_THROW(IoError, mFilename << " is not open for reading."); } - const bool clip = bbox.isSorted(); - // If a grid with the given name was already read and cached // (along with the entire contents of the file, because the file // doesn't support random access), retrieve and return it. GridBase::Ptr grid = retrieveCachedGrid(name); if (grid) { + const auto& bbox = readOptions.clipBBox; + const bool clip = bbox.isSorted(); if (clip) { grid = grid->deepCopyGrid(); grid->clipGrid(bbox); @@ -471,7 +477,7 @@ File::readGrid(const Name& name, const BBoxd& bbox) OPENVDB_ASSERT(inputHasGridOffsets()); // Seek to the grid in the file. gd.seekToGrid(inputStream()); - grid = Archive::readGrid(gd, inputStream(), bbox); + grid = Archive::readGrid(gd, inputStream(), readOptions); if (gd.isInstance()) { /// @todo Refactor to share code with Archive::connectInstance()? @@ -487,7 +493,7 @@ File::readGrid(const Name& name, const BBoxd& bbox) GridBase::Ptr parent; OPENVDB_ASSERT(inputHasGridOffsets()); parentIt->second.seekToGrid(inputStream()); - parent = Archive::readGrid(parentIt->second, inputStream(), bbox); + parent = Archive::readGrid(parentIt->second, inputStream(), readOptions); if (parent) grid->setTree(parent->baseTreePtr()); } return grid; diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index 8752e45f5c..920b86a50f 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -87,7 +87,7 @@ class OPENVDB_API File: public Archive MetaMap::Ptr getMetadata() const; /// Read the entire contents of the file and return a list of grid pointers. - GridPtrVecPtr getGrids() const; + GridPtrVecPtr getGrids(const io::ReadOptions& readOptions = io::ReadOptions{}) const; /// @brief Read just the grid metadata and transforms from the file and return a list /// of pointers to grids that are empty except for their metadata and transforms. @@ -100,8 +100,8 @@ class OPENVDB_API File: public Archive /// @throw KeyError if no grid with the given name exists in this file. GridBase::Ptr readGridMetadata(const Name&); - /// Read an entire grid, including all of its data blocks. - GridBase::Ptr readGrid(const Name&); + /// Read an entire grid, including all of its data blocks, using the provided options if given. + GridBase::Ptr readGrid(const Name&, const io::ReadOptions& readOptions = io::ReadOptions{}); /// @brief Read a grid, including its data blocks, but only where it /// intersects the given world-space bounding box. GridBase::Ptr readGrid(const Name&, const BBoxd&); diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index b65e32703a..938cc321b0 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -51,7 +51,7 @@ Stream::Stream(std::istream& is) gd.readHeader(is); gd.readStreamPos(is); descriptors.push_back(gd); - GridBase::Ptr grid = Archive::readGrid(gd, is, BBoxd()); + GridBase::Ptr grid = Archive::readGrid(gd, is, io::ReadOptions{}); mGrids->push_back(grid); namedGrids[gd.uniqueName()] = grid; } diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index 47a26cbaf9..fad51e9567 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -158,7 +158,7 @@ TestFile::testWriteGrid() gd2.readHeader(istr); gd2.readStreamPos(istr); - GridBase::Ptr gd2_grid = Archive::readGrid(gd2, istr, BBoxd()); + GridBase::Ptr gd2_grid = Archive::readGrid(gd2, istr); // Delay load metadata should not exist. ASSERT_FALSE(bool((*gd2_grid)["file_delayed_load"])); @@ -244,7 +244,7 @@ TestFile::testWriteMultipleGrids() gd_in.readHeader(istr); gd_in.readStreamPos(istr); - GridBase::Ptr gd_in_grid = Archive::readGrid(gd_in, istr, BBoxd()); + GridBase::Ptr gd_in_grid = Archive::readGrid(gd_in, istr); // Ensure read in the right values. EXPECT_EQ(gd.gridName(), gd_in.gridName()); @@ -281,7 +281,7 @@ TestFile::testWriteMultipleGrids() GridDescriptor gd2_in; gd2_in.readHeader(istr); gd2_in.readStreamPos(istr); - GridBase::Ptr gd2_in_grid = Archive::readGrid(gd2_in, istr, BBoxd()); + GridBase::Ptr gd2_in_grid = Archive::readGrid(gd2_in, istr); // Ensure that we read in the right values. EXPECT_EQ(gd2.gridName(), gd2_in.gridName()); @@ -887,7 +887,7 @@ TestFile::testEmptyGridIO() EXPECT_TRUE(it != file2.mGridDescriptors.end()); GridDescriptor file2gd = it->second; file2gd.seekToGrid(istr); - GridBase::Ptr gd_grid = Archive::readGrid(file2gd, istr, BBoxd()); + GridBase::Ptr gd_grid = Archive::readGrid(file2gd, istr); EXPECT_EQ(gd.gridName(), file2gd.gridName()); EXPECT_TRUE(gd_grid.get() != nullptr); EXPECT_EQ(0, int(gd_grid->baseTree().leafCount())); @@ -901,7 +901,7 @@ TestFile::testEmptyGridIO() EXPECT_TRUE(it != file2.mGridDescriptors.end()); file2gd = it->second; file2gd.seekToGrid(istr); - gd_grid = Archive::readGrid(file2gd, istr, BBoxd()); + gd_grid = Archive::readGrid(file2gd, istr); EXPECT_EQ(gd2.gridName(), file2gd.gridName()); EXPECT_TRUE(gd_grid.get() != nullptr); EXPECT_EQ(0, int(gd_grid->baseTree().leafCount())); From a6eaed5b390c5abc1e2e38617dd4cbb2fc9e2e54 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 27 Mar 2026 17:25:41 -0700 Subject: [PATCH 28/70] Make ReadOptions and WriteOptions non-virtual and remove Ptr typdefs Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Codec.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index a96b076b41..bd257056a1 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -23,10 +23,6 @@ enum class ReadMode { /// Contains common options shared across all codec types struct OPENVDB_API ReadOptions { - using Ptr = std::shared_ptr; - - virtual ~ReadOptions() = default; - // BBox Clip Data BBoxd clipBBox = BBoxd(); @@ -38,9 +34,6 @@ struct OPENVDB_API ReadOptions /// Contains common options shared across all codec types struct OPENVDB_API WriteOptions { - using Ptr = std::shared_ptr; - - virtual ~WriteOptions() = default; }; // struct WriteOptions From 2ee82e57ccd7a0c7c7de2211793f9818f6f13182 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 23 Mar 2026 18:57:12 -0700 Subject: [PATCH 29/70] Add BoolCodec, Codec Registry and Archive integration Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 7 + openvdb/openvdb/codecs/BoolCodec.h | 154 +++++++++++ openvdb/openvdb/codecs/TopologyCodec.h | 326 ++++++++++++++++++++++++ openvdb/openvdb/io/Archive.cc | 77 ++++-- openvdb/openvdb/io/Archive.h | 3 + openvdb/openvdb/io/Codec.cc | 106 ++++++++ openvdb/openvdb/io/Codec.h | 92 ++++++- openvdb/openvdb/openvdb.cc | 4 + openvdb/openvdb/unittest/CMakeLists.txt | 1 + openvdb/openvdb/unittest/TestCodec.cc | 185 ++++++++++++++ 10 files changed, 937 insertions(+), 18 deletions(-) create mode 100644 openvdb/openvdb/codecs/BoolCodec.h create mode 100644 openvdb/openvdb/codecs/TopologyCodec.h create mode 100644 openvdb/openvdb/io/Codec.cc create mode 100644 openvdb/openvdb/unittest/TestCodec.cc diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 6a613f0326..f0fd0fed5a 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -329,6 +329,7 @@ configure_file(version.h.in openvdb/version.h) set(OPENVDB_LIBRARY_SOURCE_FILES Grid.cc io/Archive.cc + io/Codec.cc io/Compression.cc io/File.cc io/GridDescriptor.cc @@ -364,6 +365,11 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES TypeList.h ) +set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES + codecs/BoolCodec.h + codecs/TopologyCodec.h +) + set(OPENVDB_LIBRARY_IO_INCLUDE_FILES io/Archive.h io/Codec.h @@ -559,6 +565,7 @@ if(USE_EXPLICIT_INSTANTIATION) # inexpensive to generate in this case. set(OPENVDB_LIBRARY_INSTANTIATION_HEADERS ${OPENVDB_LIBRARY_TOOLS_INCLUDE_FILES} + ${OPENVDB_LIBRARY_CODECS_INCLUDE_FILES} ) # A new source file is generated by CMake for every header diff --git a/openvdb/openvdb/codecs/BoolCodec.h b/openvdb/openvdb/codecs/BoolCodec.h new file mode 100644 index 0000000000..2aafb86aaa --- /dev/null +++ b/openvdb/openvdb/codecs/BoolCodec.h @@ -0,0 +1,154 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_BOOLCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_BOOLCODEC_HAS_BEEN_INCLUDED + +#include + +#include + +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct ReadBoolBuffersOp +{ + using TreeT = typename GridT::TreeType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + ReadBoolBuffersOp(std::istream& _is, bool _saveFloatAsHalf, + const ValueT& _background, const CoordBBox* _clipBBox) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) + , clipBBox(_clipBBox) { } + + void operator()(RootT& root, size_t) + { + // Clip root-level tiles and prune children that were clipped. + if (clipBBox) { + root.clip(*clipBBox); + } + } + + template + void operator()(NodeT& node, size_t) + { + // Clip internal node tiles and prune children that were clipped. + if (clipBBox) { + node.clip(*clipBBox, background); + } + } + + void operator()(LeafT& leaf, size_t) + { + // Read in the value mask. + leaf.getValueMask().load(is); + // Read in the origin. + Coord origin; + is.read(reinterpret_cast(&origin), sizeof(Coord::ValueType) * 3); + leaf.setOrigin(origin); + + // Read in the mask for the voxel values. + typename LeafT::Buffer::NodeMaskType nodeMask; + nodeMask.load(is); + typename LeafT::Buffer temp(nodeMask); + leaf.swap(temp); + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; + const CoordBBox* clipBBox = nullptr; +}; // struct ReadBoolBuffersOp + +template +struct WriteBoolBuffersOp +{ + using TreeT = typename GridT::TreeType; + using LeafT = typename TreeT::LeafNodeType; + + WriteBoolBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + // Write out the value mask. + leaf.getValueMask().save(os); + + // Write out the origin. + os.write(reinterpret_cast(&leaf.origin()), sizeof(Coord::ValueType) * 3); + + // Write out the voxel values. + leaf.buffer().storage().save(os); + } + + std::ostream& os; + const bool saveFloatAsHalf; +}; // struct WriteBoolBuffersOp + +} // namespace internal + +template +struct BoolCodec : public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~BoolCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options) final + { + GridT& grid = static_cast(*data.grid); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in BoolCodec"); + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + std::unique_ptr clipIndexBBox; + if (options.clipBBox.isSorted()) { + clipIndexBBox = std::make_unique(grid.constTransform().worldToIndexNodeCentered(options.clipBBox)); + } + + internal::ReadBoolBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), clipIndexBBox.get()); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + const GridT& grid = static_cast(gridBase); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in BoolCodec"); + } + + internal::WriteBoolBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); + } +}; // struct BoolCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_BOOLCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h new file mode 100644 index 0000000000..0f411598c0 --- /dev/null +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -0,0 +1,326 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_TOPOLOGYCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_TOPOLOGYCODEC_HAS_BEEN_INCLUDED + +#include +#include +#include +#include +#include +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct WriteTopologyOp +{ + using ValueT = typename TreeT::ValueType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + + WriteTopologyOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + void operator()(const RootT& root, size_t) + { + int32_t bufferCount = 1; + os.write(reinterpret_cast(&bufferCount), sizeof(int32_t)); + + background = &root.background(); + if (!saveFloatAsHalf) { + os.write(reinterpret_cast(background), sizeof(ValueT)); + } else { + ValueT truncatedVal = io::truncateRealToHalf(*background); + os.write(reinterpret_cast(&truncatedVal), sizeof(ValueT)); + } + io::setGridBackgroundValuePtr(os, background); + + const Index numTiles = root.tileCount(), numChildren = root.childCount(); + os.write(reinterpret_cast(&numTiles), sizeof(Index)); + os.write(reinterpret_cast(&numChildren), sizeof(Index)); + + // Write tiles. + for (auto iter = root.cbeginValueAll(); iter; ++iter) { + const auto& ijk = iter.getCoord(); + os.write(reinterpret_cast(ijk.asPointer()), 3 * sizeof(Int32)); + ValueT value; + bool active = root.getTileValueUnsafe(ijk, value); + os.write(reinterpret_cast(&value), sizeof(ValueT)); + os.write(reinterpret_cast(&active), sizeof(bool)); + } + + rootChildLevel = RootT::LEVEL-1; + } + + template + void operator()(const NodeT& node, size_t) + { + // Write origin for RootNode children only + if (rootChildLevel == NodeT::LEVEL) { + const auto& ijk = node.origin(); + os.write(reinterpret_cast(ijk.asPointer()), 3 * sizeof(Int32)); + } + + const auto& childMask = node.getChildMask(); + const auto& valueMask = node.getValueMask(); + + childMask.save(os); + valueMask.save(os); + + { + // Copy all of this node's values into an array. + std::unique_ptr valuePtr(new ValueT[NodeT::NUM_VALUES]); + ValueT* values = valuePtr.get(); + const ValueT zero = zeroVal(); + for (Index i = 0; i < NodeT::NUM_VALUES; ++i) { + values[i] = (node.isChildMaskOff(i) ? node.getValueUnsafe(i) : zero); + } + // Compress (optionally) and write out the contents of the array. + io::writeCompressedValues(os, values, NodeT::NUM_VALUES, valueMask, childMask, saveFloatAsHalf); + } + } + + void operator()(const LeafT& leaf, size_t) + { + leaf.getValueMask().save(os); + } + + std::ostream& os; + const bool saveFloatAsHalf; + const ValueT* background = nullptr; + Index rootChildLevel = std::numeric_limits::max(); +}; // struct WriteTopologyOp + + +template +struct ReadTopologyOp +{ + using ValueT = typename TreeT::ValueType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using StorageValueT = typename StorageTreeT::ValueType; + + ReadTopologyOp(std::istream& _is, bool _saveFloatAsHalf) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + void operator()(RootT& root) + { + using ChildT = typename RootT::ChildNodeType; + + int32_t bufferCount; + is.read(reinterpret_cast(&bufferCount), sizeof(int32_t)); + if (bufferCount != 1) OPENVDB_LOG_WARN("multi-buffer trees are no longer supported"); + + // Delete the existing tree. + root.clear(); + + // Read a RootNode that was stored in the current format. + + if constexpr (std::is_same_v) { + is.read(reinterpret_cast(&background), sizeof(ValueT)); + } else { + StorageValueT _background; + is.read(reinterpret_cast(&_background), sizeof(StorageValueT)); + background = static_cast(_background); + } + + root.setBackground(background, false); + io::setGridBackgroundValuePtr(is, &root.background()); + + Index numTiles = 0, numChildren = 0; + is.read(reinterpret_cast(&numTiles), sizeof(Index)); + is.read(reinterpret_cast(&numChildren), sizeof(Index)); + + Int32 vec[3]; + StorageValueT value; + bool active; + + // Read tiles. + for (Index n = 0; n < numTiles; ++n) { + is.read(reinterpret_cast(vec), 3 * sizeof(Int32)); + is.read(reinterpret_cast(&value), sizeof(StorageValueT)); + is.read(reinterpret_cast(&active), sizeof(bool)); + Coord origin(vec); + if constexpr (std::is_same_v) { + root.addTile(origin, value, active); + } else { + root.addTile(origin, static_cast(value), active); + } + } + + // Read child nodes. + for (Index n = 0; n < numChildren; ++n) { + is.read(reinterpret_cast(vec), 3 * sizeof(Int32)); + Coord origin(vec); + ChildT* child = new ChildT(PartialCreate(), origin, background); + (*this)(*child); + root.addChild(child); + } + } + + template + void operator()(NodeT& node) + { + using ChildT = typename NodeT::ChildNodeType; + using NodeMaskT = typename NodeT::NodeMaskType; + + NodeMaskT childMask, valueMask; + childMask.load(is); + valueMask.load(is); + node.setValueMaskUnsafe(valueMask); + + const Index numValues = NodeT::NUM_VALUES; + { + // Read in (and uncompress, if necessary) all of this node's values + // into a contiguous array. + std::unique_ptr valuePtr(new StorageValueT[numValues]); + StorageValueT* values = valuePtr.get(); + io::readCompressedValues(is, values, numValues, valueMask, saveFloatAsHalf); + + // Copy values from the array into this node's table. + if constexpr (std::is_same_v) { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), values[iter.pos()]); + } + } else { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); + } + } + } + + // Read in all child nodes and insert them into the table at their proper locations. + for (auto iter = childMask.beginOn(); iter; ++iter) { + Coord origin = node.offsetToGlobalCoord(iter.pos()); + auto* child = new ChildT(PartialCreate(), origin, background); + (*this)(*child); + node.setChildUnsafe(iter.pos(), child); + } + } + + void operator()(LeafT& leaf) + { + typename LeafT::NodeMaskType valueMask; + valueMask.load(is); + leaf.setValueMask(valueMask); + } + + std::istream& is; + bool saveFloatAsHalf; + ValueT background; +}; // struct ReadTopologyOp + +template +struct SetTilesToBackgroundOp +{ + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + explicit SetTilesToBackgroundOp(const ValueT& background) + : mBackground(background) { } + + bool operator()(RootT& root, size_t) const + { + for (auto it = root.beginValueAll(); it; ++it) { + it.setValue(mBackground); + } + return true; + } + + template + bool operator()(NodeT& node, size_t) const + { + for (Index i = 0; i < NodeT::NUM_VALUES; ++i) { + if (node.isChildMaskOff(i)) { + node.setValueOnlyUnsafe(i, mBackground); + } + } + return true; + } + + bool operator()(LeafT&, size_t) const { return true; } + +private: + const ValueT mBackground; +}; // struct SetTilesToBackgroundOp + +template +void setTilesToBackground(TreeT& tree) +{ + const typename TreeT::ValueType& background = tree.root().background(); + SetTilesToBackgroundOp op(background); + tree::DynamicNodeManager nodeManager(tree); + nodeManager.foreachTopDown(op); +} + +// Free-standing function for read case (supports type conversion via StorageGridT) +template +void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::ReadOptions& options) +{ + io::checkFormatVersion(is); + + GridT& grid = static_cast(gridBase); + grid.tree().clearAllAccessors(); + + internal::ReadTopologyOp readTopologyOp(is, grid.saveFloatAsHalf()); + readTopologyOp(grid.tree().root()); + + if (options.readMode == io::ReadMode::TopologyOnly) { + internal::setTilesToBackground(grid.tree()); + return; + } +} + +// Free-standing function for write case (no StorageGridT needed) +template +void topologyCodecWriteTopology(const GridBase& gridBase, std::ostream& os) +{ + const GridT& grid = static_cast(gridBase); + using TreeType = typename GridT::TreeType; + + internal::WriteTopologyOp writeTopologyOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeTopologyOp); +} + +} // namespace internal + +template +struct TopologyCodec : public io::Codec +{ + using Ptr = std::unique_ptr>; + + ~TopologyCodec() noexcept = default; + + io::CodecData::Ptr createData() override + { + auto data = std::make_unique(); + data->grid = GridT::create(); + return data; + } + + void readTopology(std::istream& is, io::CodecData& data, const io::ReadOptions& options) final + { + internal::topologyCodecReadTopology(*data.grid, is, options); + } + + void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + internal::topologyCodecWriteTopology(gridBase, os); + } +}; // struct TopologyCodec + + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_TOPOLOGYCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 4eb9d0fe74..712d3d2b99 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -855,6 +855,17 @@ Archive::readGridCount(std::istream& is) //////////////////////////////////////// +io::Codec* +Archive::findCodec(const std::string& gridType, const io::ReadOptions&) +{ + // Determine the I/O codec to use to read this grid + return io::CodecRegistry::get(gridType); +} + + +//////////////////////////////////////// + + void Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) const { @@ -902,15 +913,26 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti }; OnExit restore(is); + // Find the codec for the grid type and options. + io::Codec* codec = findCodec(gd.gridType(), readOptions); + + GridBase::Ptr grid; + io::CodecData::Ptr codecData; + // Create the grid. - if (!GridBase::isRegistered(gd.gridType())) { - OPENVDB_THROW(KeyError, "Cannot read grid " - << GridDescriptor::nameAsString(gd.uniqueName()) - << ": grid type " << gd.gridType() << " is not registered"); - } + if (codec) { + codecData = codec->createData(); + if (codecData->grid) grid = codecData->grid; + } else { + if (!GridBase::isRegistered(gd.gridType())) { + OPENVDB_THROW(KeyError, "Cannot read grid " + << GridDescriptor::nameAsString(gd.uniqueName()) + << ": grid type " << gd.gridType() << " is not registered"); + } - GridBase::Ptr grid = GridBase::createGrid(gd.gridType()); - if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); + grid = GridBase::createGrid(gd.gridType()); + } + grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); // Stream metadata varies per grid, and it needs to persist // in case delayed load is in effect. @@ -940,14 +962,24 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti grid->readTransform(is); if (readOptions.readMode != io::ReadMode::TopologyOnly && !gd.isInstance()) { - grid->readTopology(is); - const auto& worldBBox = readOptions.clipBBox; - const bool clip = worldBBox.isSorted(); - if (clip) { - const auto indexBBox = grid->constTransform().worldToIndexNodeCentered(worldBBox); - grid->readBuffers(is, indexBBox); + // read topology + if (codec) { + codec->readTopology(is, *codecData, readOptions); + } else { + grid->readTopology(is); + } + // read buffers + if (codec) { + codec->readBuffers(is, *codecData, readOptions); } else { - grid->readBuffers(is); + const auto& worldBBox = readOptions.clipBBox; + const bool clip = worldBBox.isSorted(); + if (clip) { + const auto indexBBox = grid->constTransform().worldToIndexNodeCentered(worldBBox); + grid->readBuffers(is, indexBBox); + } else { + grid->readBuffers(is); + } } } @@ -1065,7 +1097,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, void Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, - std::ostream& os, bool seekable, const io::WriteOptions&) const + std::ostream& os, bool seekable, const io::WriteOptions& writeOptions) const { // Restore file-level stream metadata on exit. struct OnExit { @@ -1076,6 +1108,9 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, }; OnExit restore(os); + // Find the codec for the grid type and options. + io::Codec* codec = findCodec(gd.gridType()); + // Stream metadata varies per grid, so make a copy of the file-level stream metadata. io::StreamMetadata::Ptr streamMetadata; if (io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(os)) { @@ -1119,13 +1154,21 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, grid->writeTransform(os); // Save the grid's structure. - grid->writeTopology(os); + if (codec) { + codec->writeTopology(os, *grid, writeOptions); + } else { + grid->writeTopology(os); + } // Now we know the grid block storage position. if (seekable) gd.setBlockPos(os.tellp()); // Save out the data blocks of the grid. - grid->writeBuffers(os); + if (codec) { + codec->writeBuffers(os, *grid, writeOptions); + } else { + grid->writeBuffers(os); + } // Now we know the end position of this grid. if (seekable) gd.setEndPos(os.tellp()); diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index a3b0893ea6..dad4cb8816 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -131,6 +131,9 @@ class OPENVDB_API Archive /// Read in and return the number of grids on the input stream. static int32_t readGridCount(std::istream&); + /// @brief Find the codec for the given grid type and options. + static io::Codec* findCodec(const std::string& gridType, const io::ReadOptions& options = io::ReadOptions{}); + /// @brief Read in and create the grid represented by the given grid descriptor using the /// given input stream, using the provided options if given. static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc new file mode 100644 index 0000000000..6c4661bdae --- /dev/null +++ b/openvdb/openvdb/io/Codec.cc @@ -0,0 +1,106 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#include "Codec.h" + +#include + +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace io { + +namespace { + +using CodecFactoryMap = std::map; +using CodecFactoryMapCIter = CodecFactoryMap::const_iterator; + +struct LockedCodecRegistry { + LockedCodecRegistry() {} + ~LockedCodecRegistry() {} + std::mutex mMutex; + CodecFactoryMap mMap; +}; + +// Global function for accessing the registry +static LockedCodecRegistry* +getCodecRegistry() +{ + static LockedCodecRegistry registry; + return ®istry; +} + +} // unnamed namespace + + +namespace internal { + +void initialize() +{ + CodecRegistry::registerCodec>(); +} + +void uninitialize() +{ + CodecRegistry::clear(); +} + +} // namespace internal + + +//////////////////////////////////////// + + +bool +CodecRegistry::isRegistered(const std::string& name) +{ + LockedCodecRegistry* registry = getCodecRegistry(); + std::lock_guard lock(registry->mMutex); + + return (registry->mMap.find(name) != registry->mMap.end()); +} + + +void +CodecRegistry::registerCodecByName(const std::string& name, Codec::Ptr&& codec) +{ + LockedCodecRegistry* registry = getCodecRegistry(); + std::lock_guard lock(registry->mMutex); + + if (registry->mMap.find(name) != registry->mMap.end()) { + OPENVDB_THROW(KeyError, + "Cannot register codec " << name << ". Codec is already registered"); + } + + registry->mMap[name] = std::move(codec); +} + + +Codec* +CodecRegistry::get(const std::string& name) +{ + LockedCodecRegistry* registry = getCodecRegistry(); + std::lock_guard lock(registry->mMutex); + + CodecFactoryMapCIter iter = registry->mMap.find(name); + + return (iter != registry->mMap.end()) ? iter->second.get() : nullptr; +} + + +void +CodecRegistry::clear() +{ + LockedCodecRegistry* registry = getCodecRegistry(); + std::lock_guard lock(registry->mMutex); + + registry->mMap.clear(); +} + + +} // namespace io +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index bd257056a1..4923ee7f83 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -4,21 +4,53 @@ #ifndef OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED #define OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED +#include +#include #include +#include -#include +#include namespace openvdb { OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { + +// Forward declaration +class GridBase; + namespace io { +namespace internal { + +OPENVDB_API void initialize(); +OPENVDB_API void uninitialize(); + +} // namespace internal + +//////////////////////////////////////// enum class ReadMode { Original, // Read data as-is (default) TopologyOnly // Read topology only, no buffer data }; +/// Base class for per-grid-type codec-specific options +/// Derive from this to add codec-specific read configuration +struct OPENVDB_API ReadTypedOptions +{ + using Ptr = std::shared_ptr; + + virtual ~ReadTypedOptions() = default; + + /// Type-safe casting helper for derived ReadTypedOptions types + /// Uses OPENVDB_ASSERT to verify type safety when assertions are enabled + template + static T& cast(const Ptr& data) { + OPENVDB_ASSERT(dynamic_cast(data.get()) != nullptr); + return *static_cast(data.get()); + } +}; // struct ReadTypedOptions + /// Global read configuration that applies to all grids /// Contains common options shared across all codec types struct OPENVDB_API ReadOptions @@ -28,6 +60,9 @@ struct OPENVDB_API ReadOptions // Grid Conversion Options ReadMode readMode = ReadMode::Original; + + // Per-grid-type codec-specific options, keyed by grid type string + std::unordered_map typeData; }; // struct ReadOptions /// Global write configuration that applies to all grids @@ -36,6 +71,61 @@ struct OPENVDB_API WriteOptions { }; // struct WriteOptions +/// Per-codec mutable data created by Codec::createData() +/// Contains the grid and any codec-specific state +struct OPENVDB_API CodecData +{ + using Ptr = std::unique_ptr; + + virtual ~CodecData() = default; + + SharedPtr grid; +}; // struct CodecData + +struct OPENVDB_API Codec +{ + using Ptr = std::unique_ptr; + + virtual ~Codec() = default; + + /// Create codec data including a new grid of the appropriate type + /// @return A new CodecData object containing the grid and any codec-specific state + virtual CodecData::Ptr createData() = 0; + + /// Read the grid topology + virtual void readTopology(std::istream&, CodecData&, const ReadOptions&) { } + + /// Read all data buffers for this grid + virtual void readBuffers(std::istream&, CodecData&, const ReadOptions&) { } + + /// Write the grid topology + virtual void writeTopology(std::ostream&, const GridBase&, const WriteOptions&) { } + + /// Write all data buffers for this grid + virtual void writeBuffers(std::ostream&, const GridBase&, const WriteOptions&) { } +}; // struct Codec + +struct OPENVDB_API CodecRegistry +{ + /// Return @c true if the given codec name is registered. + static bool isRegistered(const std::string& name); + + /// Register a codec by name with a factory function. + static void registerCodecByName(const std::string& name, Codec::Ptr&& codec); + + /// Register a codec by type. + template + static void registerCodec() + { + registerCodecByName(CodecT::name(), std::make_unique()); + } + + /// Return a pointer to the codec with the given name, or nullptr if not found. + static Codec* get(const std::string& name); + + /// Clear the codec registry. + static void clear(); +}; // struct CodecRegistry } // namespace io } // namespace OPENVDB_VERSION_NAME diff --git a/openvdb/openvdb/openvdb.cc b/openvdb/openvdb/openvdb.cc index f830054849..ff163071de 100644 --- a/openvdb/openvdb/openvdb.cc +++ b/openvdb/openvdb/openvdb.cc @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "openvdb.h" +#include "io/Codec.h" #include "points/PointDataGrid.h" #include "tools/PointIndexGrid.h" #include "util/logging.h" @@ -70,6 +71,8 @@ initialize() logging::initialize(); + io::internal::initialize(); + // Register metadata. Metadata::clearRegistry(); MetaTypes::foreach(); @@ -130,6 +133,7 @@ __pragma(warning(default:1711)) Metadata::clearRegistry(); GridBase::clearRegistry(); math::MapRegistry::clear(); + io::internal::uninitialize(); points::internal::uninitialize(); #ifdef OPENVDB_USE_BLOSC diff --git a/openvdb/openvdb/unittest/CMakeLists.txt b/openvdb/openvdb/unittest/CMakeLists.txt index 1a0168b6c7..18046cd8e0 100644 --- a/openvdb/openvdb/unittest/CMakeLists.txt +++ b/openvdb/openvdb/unittest/CMakeLists.txt @@ -82,6 +82,7 @@ else() TestAttributeSet.cc TestBBox.cc TestClip.cc + TestCodec.cc TestConjGradient.cc TestCoord.cc TestCount.cc diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc new file mode 100644 index 0000000000..65996a3a56 --- /dev/null +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -0,0 +1,185 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include + +class TestCodec: public ::testing::Test +{ +public: + void SetUp() override { openvdb::initialize(); } + void TearDown() override { openvdb::uninitialize(); } +}; + +struct MockCodec : public openvdb::io::Codec +{ + static std::string name() { return "mock"; } + + openvdb::io::CodecData::Ptr createData() final { return nullptr; } +}; + +TEST_F(TestCodec, testCodecRegistry) +{ + using namespace openvdb::io; + + // Start clean + CodecRegistry::clear(); + + // Test isRegistered on empty registry + EXPECT_FALSE(CodecRegistry::isRegistered("mock")); + + // Test registerCodecByName + EXPECT_NO_THROW( + CodecRegistry::registerCodecByName("mock", std::make_unique()) + ); + + EXPECT_TRUE(CodecRegistry::isRegistered("mock")); + EXPECT_FALSE(CodecRegistry::isRegistered("nonexistent")); + + // Test duplicate registration throws KeyError + EXPECT_THROW( + CodecRegistry::registerCodecByName("mock", std::make_unique()), + openvdb::KeyError + ); + + // Test registerCodec template form also throws on duplicate + EXPECT_THROW( + CodecRegistry::registerCodec(), + openvdb::KeyError + ); + + // Test get + EXPECT_NE(CodecRegistry::get("mock"), nullptr); + EXPECT_EQ(CodecRegistry::get("nonexistent"), nullptr); + + // Test clear + CodecRegistry::clear(); + EXPECT_FALSE(CodecRegistry::isRegistered("mock")); + EXPECT_NO_THROW(CodecRegistry::clear()); // Clear on empty registry + + // Test registerCodec template form on fresh registry + EXPECT_NO_THROW(CodecRegistry::registerCodec()); + EXPECT_TRUE(CodecRegistry::isRegistered("mock")); + + // Test io::initialize and io::uninitialize + CodecRegistry::clear(); + EXPECT_FALSE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType())); + + EXPECT_NO_THROW(internal::initialize()); + EXPECT_TRUE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType())); + + EXPECT_NO_THROW(internal::uninitialize()); + EXPECT_FALSE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType())); +} + + +template +void testIOImpl( + const std::string& gridName, + const typename GridT::ValueType& bgValue, + const typename GridT::ValueType& fillValue) +{ + using namespace openvdb; + using namespace openvdb::io; + + typename GridT::Ptr srcGrid = GridT::create(bgValue); + srcGrid->setName(gridName); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), fillValue, true); + + std::stringstream ss("test"); + if (CodecRegistry::isRegistered(GridT::gridType())) { + ss << "_codec"; + } else { + ss << "_tree"; + } + ss << "_" << GridT::gridType() << ".vdb"; + const std::string path = ss.str(); + { + io::File f(path); + f.write(GridPtrVec{srcGrid}); + } + + typename GridT::Ptr readGrid; + { + io::File f(path); + f.open(); + readGrid = gridPtrCast(f.readGrid(gridName)); + f.close(); + } + ASSERT_TRUE(readGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(readGrid->tree())); + { + auto readAcc = readGrid->getConstAccessor(); + for (typename GridT::ValueOnCIter it = srcGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, readAcc.getValue(it.getCoord())); + } + } + + // clip read + const BBoxd clipBBox(Vec3d(0.0), Vec3d(3.5)); + auto srcClipped = tools::clip(*srcGrid, clipBBox); + + typename GridT::Ptr readClipped; + { + io::File f(path); + f.open(); + readClipped = gridPtrCast(f.readGrid(gridName, clipBBox)); + f.close(); + } + ASSERT_TRUE(readClipped); + EXPECT_TRUE(srcClipped->tree().hasSameTopology(readClipped->tree())); + { + auto readAcc = readClipped->getConstAccessor(); + for (typename GridT::ValueOnCIter it = srcClipped->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, readAcc.getValue(it.getCoord())); + } + } + + // topology-only read + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + typename GridT::Ptr readTopo; + { + io::File f(path); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid(gridName, topoOpts)); + readTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(readTopo); + EXPECT_EQ(readTopo->activeVoxelCount(), Index64(0)); + EXPECT_TRUE(readTopo->tree().leafCount() == 0); + EXPECT_EQ(readTopo->getName(), gridName); + + // Cleanup + std::remove(path.c_str()); +} + +template +void testCodecIOImpl( + const std::string& gridName, + const typename GridT::ValueType& bgValue, + const typename GridT::ValueType& fillValue) +{ + // initialize to register all the codecs + openvdb::io::CodecRegistry::clear(); + openvdb::io::internal::initialize(); + // ensure the codec is registered + ASSERT_TRUE(openvdb::io::CodecRegistry::isRegistered(GridT::gridType())); + // test the io implementation (codec) + testIOImpl(gridName, bgValue, fillValue); + // clear the codec registry (now read/write falls back to Tree I/O) + openvdb::io::CodecRegistry::clear(); + // ensure the codec is not registered + ASSERT_FALSE(openvdb::io::CodecRegistry::isRegistered(GridT::gridType())); + // test the io implementation (tree I/O) + testIOImpl(gridName, bgValue, fillValue); +} + +TEST_F(TestCodec, testBoolCodecIO) { testCodecIOImpl("bool_grid", false, true); } From fa84a408b2b97136a53dca50f3f9c64295c9ec63 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 31 Mar 2026 21:49:53 -0700 Subject: [PATCH 30/70] Add ReadDiagnostics to Codecs Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/BoolCodec.h | 2 +- openvdb/openvdb/codecs/TopologyCodec.h | 22 ++++++--- openvdb/openvdb/io/Archive.cc | 43 +++++++++++++++-- openvdb/openvdb/io/Archive.h | 17 +++++++ openvdb/openvdb/io/Codec.h | 49 ++++++++++++++++++- openvdb/openvdb/io/File.cc | 7 +-- openvdb/openvdb/unittest/TestCodec.cc | 65 ++++++++++++++++++++++++++ 7 files changed, 189 insertions(+), 16 deletions(-) diff --git a/openvdb/openvdb/codecs/BoolCodec.h b/openvdb/openvdb/codecs/BoolCodec.h index 2aafb86aaa..68d7997a89 100644 --- a/openvdb/openvdb/codecs/BoolCodec.h +++ b/openvdb/openvdb/codecs/BoolCodec.h @@ -110,7 +110,7 @@ struct BoolCodec : public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options) final + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { GridT& grid = static_cast(*data.grid); diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 0f411598c0..7e2fc952b8 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -107,9 +107,12 @@ struct ReadTopologyOp using LeafT = typename TreeT::LeafNodeType; using StorageValueT = typename StorageTreeT::ValueType; - ReadTopologyOp(std::istream& _is, bool _saveFloatAsHalf) + ReadTopologyOp(std::istream& _is, bool _saveFloatAsHalf, io::ReadDiagnostics& _diagnostics, + const std::string& _gridName) : is(_is) - , saveFloatAsHalf(_saveFloatAsHalf) { } + , saveFloatAsHalf(_saveFloatAsHalf) + , diagnostics(_diagnostics) + , gridName(_gridName) { } void operator()(RootT& root) { @@ -117,7 +120,9 @@ struct ReadTopologyOp int32_t bufferCount; is.read(reinterpret_cast(&bufferCount), sizeof(int32_t)); - if (bufferCount != 1) OPENVDB_LOG_WARN("multi-buffer trees are no longer supported"); + if (bufferCount != 1) { + diagnostics.addWarning(gridName, "multi-buffer trees are no longer supported"); + } // Delete the existing tree. root.clear(); @@ -216,6 +221,8 @@ struct ReadTopologyOp std::istream& is; bool saveFloatAsHalf; ValueT background; + io::ReadDiagnostics& diagnostics; + std::string gridName; }; // struct ReadTopologyOp template @@ -264,14 +271,14 @@ void setTilesToBackground(TreeT& tree) // Free-standing function for read case (supports type conversion via StorageGridT) template -void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::ReadOptions& options) +void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) { io::checkFormatVersion(is); GridT& grid = static_cast(gridBase); grid.tree().clearAllAccessors(); - internal::ReadTopologyOp readTopologyOp(is, grid.saveFloatAsHalf()); + internal::ReadTopologyOp readTopologyOp(is, grid.saveFloatAsHalf(), diagnostics, grid.getName()); readTopologyOp(grid.tree().root()); if (options.readMode == io::ReadMode::TopologyOnly) { @@ -307,9 +314,10 @@ struct TopologyCodec : public io::Codec return data; } - void readTopology(std::istream& is, io::CodecData& data, const io::ReadOptions& options) final + void readTopology(std::istream& is, io::CodecData& data, const io::ReadOptions& options, + io::ReadDiagnostics& diagnostics) final { - internal::topologyCodecReadTopology(*data.grid, is, options); + internal::topologyCodecReadTopology(*data.grid, is, options, diagnostics); } void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 712d3d2b99..92e99f7c83 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -384,6 +384,35 @@ Archive::copy() const } +void +Archive::enableReadDiagnostics() +{ + mReadDiagnostics.enable(); +} + + +void +Archive::disableReadDiagnostics() +{ + mReadDiagnostics.disable(); + mReadDiagnostics.clear(); +} + + +const ReadDiagnostics& +Archive::readDiagnostics() const +{ + return mReadDiagnostics; +} + + +void +Archive::clearReadDiagnostics() +{ + mReadDiagnostics.clear(); +} + + //////////////////////////////////////// @@ -898,7 +927,7 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co GridBase::Ptr -Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOptions& readOptions) +Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOptions& readOptions, ReadDiagnostics& diagnostics) { // Read the compression settings for this grid and tag the stream with them // so that downstream functions can reference them. @@ -964,13 +993,13 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti if (readOptions.readMode != io::ReadMode::TopologyOnly && !gd.isInstance()) { // read topology if (codec) { - codec->readTopology(is, *codecData, readOptions); + codec->readTopology(is, *codecData, readOptions, diagnostics); } else { grid->readTopology(is); } // read buffers if (codec) { - codec->readBuffers(is, *codecData, readOptions); + codec->readBuffers(is, *codecData, readOptions, diagnostics); } else { const auto& worldBBox = readOptions.clipBBox; const bool clip = worldBBox.isSorted(); @@ -987,6 +1016,14 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti } +GridBase::Ptr +Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOptions& readOptions) +{ + ReadDiagnostics nullDiagnostics; + return readGrid(gd, is, readOptions, nullDiagnostics); +} + + //////////////////////////////////////// diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index dad4cb8816..e0414f166a 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -98,6 +98,18 @@ class OPENVDB_API Archive /// @brief Return @c false (delayed loading has been removed). static bool isDelayedLoadingEnabled() { return false; } + /// @brief Enable collection of read diagnostics (warnings, etc.) during I/O operations. + void enableReadDiagnostics(); + + /// @brief Disable collection of read diagnostics. + void disableReadDiagnostics(); + + /// @brief Return a const reference to the diagnostics collector. + const ReadDiagnostics& readDiagnostics() const; + + /// @brief Clear the diagnostics list while keeping collection active. + void clearReadDiagnostics(); + protected: /// @brief Return @c true if the input stream contains grid offsets /// that allow for random access or partial reading. @@ -136,6 +148,8 @@ class OPENVDB_API Archive /// @brief Read in and create the grid represented by the given grid descriptor using the /// given input stream, using the provided options if given. + static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, + const io::ReadOptions& readOptions, ReadDiagnostics& diagnostics); static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, const io::ReadOptions& readOptions = io::ReadOptions{}); @@ -173,6 +187,9 @@ class OPENVDB_API Archive const io::WriteOptions& writeOptions = io::WriteOptions{}) const; //@} + /// Diagnostics collector for read operations (always valid; enabled/disabled via flag) + mutable ReadDiagnostics mReadDiagnostics; + private: friend class ::TestFile; diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 4923ee7f83..304a5b7976 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include @@ -71,6 +73,49 @@ struct OPENVDB_API WriteOptions { }; // struct WriteOptions +enum class DiagnosticSeverity { Warning }; + +struct ReadDiagnostic { + DiagnosticSeverity severity; + std::string context; + std::string message; +}; + +struct OPENVDB_API ReadDiagnostics { + ReadDiagnostics() = default; + ReadDiagnostics(const ReadDiagnostics& other) { + std::lock_guard lock(other.mMutex); + mEnabled = other.mEnabled; + mDiagnostics = other.mDiagnostics; + } + ReadDiagnostics& operator=(const ReadDiagnostics& other) { + if (this != &other) { + std::scoped_lock lock(mMutex, other.mMutex); + mEnabled = other.mEnabled; + mDiagnostics = other.mDiagnostics; + } + return *this; + } + void addWarning(const std::string& context, const std::string& message) { + if (!mEnabled) return; + std::lock_guard lock(mMutex); + mDiagnostics.push_back({DiagnosticSeverity::Warning, context, message}); + } + void clear() { + std::lock_guard lock(mMutex); + mDiagnostics.clear(); + } + void enable() { mEnabled = true; } + void disable() { mEnabled = false; } + bool enabled() const { return mEnabled; } + const std::vector& diagnostics() const { return mDiagnostics; } + bool empty() const { return mDiagnostics.empty(); } +private: + bool mEnabled = false; + std::vector mDiagnostics; + mutable std::mutex mMutex; +}; // struct ReadDiagnostics + /// Per-codec mutable data created by Codec::createData() /// Contains the grid and any codec-specific state struct OPENVDB_API CodecData @@ -93,10 +138,10 @@ struct OPENVDB_API Codec virtual CodecData::Ptr createData() = 0; /// Read the grid topology - virtual void readTopology(std::istream&, CodecData&, const ReadOptions&) { } + virtual void readTopology(std::istream&, CodecData&, const ReadOptions&, ReadDiagnostics&) { } /// Read all data buffers for this grid - virtual void readBuffers(std::istream&, CodecData&, const ReadOptions&) { } + virtual void readBuffers(std::istream&, CodecData&, const ReadOptions&, ReadDiagnostics&) { } /// Write the grid topology virtual void writeTopology(std::ostream&, const GridBase&, const WriteOptions&) { } diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index bd4ebdd4f5..d1e3856d9d 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -319,7 +319,7 @@ File::getGrids(const io::ReadOptions& readOptions) const const GridDescriptor& gd = i->second; // Seek to the grid in the file. gd.seekToGrid(inputStream()); - GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), readOptions); + GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), readOptions, mReadDiagnostics); ret->push_back(grid); namedGrids[gd.uniqueName()] = grid; } @@ -477,7 +477,7 @@ File::readGrid(const Name& name, const io::ReadOptions& readOptions) OPENVDB_ASSERT(inputHasGridOffsets()); // Seek to the grid in the file. gd.seekToGrid(inputStream()); - grid = Archive::readGrid(gd, inputStream(), readOptions); + grid = Archive::readGrid(gd, inputStream(), readOptions, mReadDiagnostics); if (gd.isInstance()) { /// @todo Refactor to share code with Archive::connectInstance()? @@ -493,7 +493,7 @@ File::readGrid(const Name& name, const io::ReadOptions& readOptions) GridBase::Ptr parent; OPENVDB_ASSERT(inputHasGridOffsets()); parentIt->second.seekToGrid(inputStream()); - parent = Archive::readGrid(parentIt->second, inputStream(), readOptions); + parent = Archive::readGrid(parentIt->second, inputStream(), readOptions, mReadDiagnostics); if (parent) grid->setTree(parent->baseTreePtr()); } return grid; @@ -592,6 +592,7 @@ File::endName() const return File::NameIterator(mGridDescriptors.end()); } + } // namespace io } // namespace OPENVDB_VERSION_NAME } // namespace openvdb diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index 65996a3a56..384c52eaa2 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -77,6 +77,71 @@ TEST_F(TestCodec, testCodecRegistry) } +TEST_F(TestCodec, testReadDiagnostics) +{ + using namespace openvdb; + using namespace openvdb::io; + + // ReadDiagnostics struct: disabled by default, addWarning is a no-op until enabled + { + ReadDiagnostics diags; + EXPECT_FALSE(diags.enabled()); + diags.addWarning("grid_a", "something went wrong"); + EXPECT_TRUE(diags.diagnostics().empty()); + + diags.enable(); + diags.addWarning("grid_a", "something went wrong"); + ASSERT_EQ(diags.diagnostics().size(), size_t(1)); + EXPECT_EQ(diags.diagnostics()[0].severity, DiagnosticSeverity::Warning); + + diags.clear(); + EXPECT_TRUE(diags.diagnostics().empty()); + } + + CodecRegistry::clear(); + openvdb::io::internal::initialize(); + + // Archive API and getGrids() with diagnostics + + BoolGrid::Ptr srcGrid = BoolGrid::create(false); + srcGrid->setName("bool_grid"); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), true, true); + + const std::string codecPath = "testReadDiagnostics.vdb"; + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + // Disabled by default; enabling produces no warnings on a clean read + { + io::File f(codecPath); + f.open(); + EXPECT_FALSE(f.readDiagnostics().enabled()); + f.enableReadDiagnostics(); + EXPECT_TRUE(f.readDiagnostics().enabled()); + f.readGrid("bool_grid"); + EXPECT_TRUE(f.readDiagnostics().diagnostics().empty()); + f.close(); + } + + // clearReadDiagnostics() resets entries but keeps diagnostics enabled + { + io::File f(codecPath); + f.open(); + f.enableReadDiagnostics(); + GridPtrVecPtr grids = f.getGrids(); + ASSERT_TRUE(grids && !grids->empty()); + f.clearReadDiagnostics(); + EXPECT_TRUE(f.readDiagnostics().enabled()); + EXPECT_TRUE(f.readDiagnostics().diagnostics().empty()); + f.close(); + } + + std::remove(codecPath.c_str()); +} + + template void testIOImpl( const std::string& gridName, From 906eef63afaf2f0c41c451a3f6ed4a562b909615 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 23 Apr 2026 11:28:15 -0700 Subject: [PATCH 31/70] Add Scalar and ValueMask codecs Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 2 + openvdb/openvdb/codecs/ScalarCodec.h | 185 +++++++++++++++++++++++ openvdb/openvdb/codecs/ScalarLeafCodec.h | 104 +++++++++++++ openvdb/openvdb/codecs/TopologyCodec.h | 7 +- openvdb/openvdb/codecs/ValueMaskCodec.h | 146 ++++++++++++++++++ openvdb/openvdb/io/Codec.cc | 9 ++ openvdb/openvdb/io/Codec.h | 5 + openvdb/openvdb/unittest/TestCodec.cc | 9 ++ openvdb/openvdb/unittest/TestGridIO.cc | 8 +- openvdb/openvdb/unittest/TestTree.cc | 5 + 10 files changed, 477 insertions(+), 3 deletions(-) create mode 100644 openvdb/openvdb/codecs/ScalarCodec.h create mode 100644 openvdb/openvdb/codecs/ScalarLeafCodec.h create mode 100644 openvdb/openvdb/codecs/ValueMaskCodec.h diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index f0fd0fed5a..9d8d819d49 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -367,7 +367,9 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES codecs/BoolCodec.h + codecs/ScalarCodec.h codecs/TopologyCodec.h + codecs/ValueMaskCodec.h ) set(OPENVDB_LIBRARY_IO_INCLUDE_FILES diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h new file mode 100644 index 0000000000..c9f8dbe49e --- /dev/null +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -0,0 +1,185 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_SCALARCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_SCALARCODEC_HAS_BEEN_INCLUDED + +#include +#include +#include +#include +#include + +#include "ScalarLeafCodec.h" +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct WriteBuffersOp +{ + using LeafT = typename TreeT::LeafNodeType; + + WriteBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + writeScalarLeafBuffers(leaf, os, saveFloatAsHalf); + } + + std::ostream& os; + const bool saveFloatAsHalf; +}; // struct WriteBuffersOp + + +template +struct ReadBuffersOp +{ + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + using StorageLeafT = typename StorageTreeT::LeafNodeType; + + ReadBuffersOp(std::istream& _is, bool _saveFloatAsHalf, const ValueT& _background, + const CoordBBox* _clipBBox) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) + , clipBBox(_clipBBox) { } + + void operator()(RootT& root, size_t) + { + if (clipBBox) { + root.clip(*clipBBox); + } + } + + template + void operator()(NodeT& node, size_t) + { + if (clipBBox) { + node.clip(*clipBBox, background); + } + } + + void operator()(LeafT& leaf, size_t) + { + readScalarLeafBuffers(leaf, is, saveFloatAsHalf, background, /*skip=*/false, clipBBox); + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; + const CoordBBox* clipBBox = nullptr; +}; // struct ReadBuffersOp + + +// Free-standing function for both standard and conversion codec cases +// Uses StorageGridT = GridT by default, but allows different storage type for conversions +template +void scalarCodecReadBuffers(GridT& grid, std::istream& is, const io::ReadOptions& options) +{ + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ScalarCodec"); + } + + using TreeT = typename GridT::TreeType; + using StorageTreeT = typename StorageGridT::TreeType; + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + std::unique_ptr clipIndexBBox; + if (options.clipBBox.isSorted()) { + clipIndexBBox = std::make_unique(grid.constTransform().worldToIndexNodeCentered(options.clipBBox)); + } + + // Works for both standard (TreeT == StorageTreeT) and conversion cases + ReadBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), + clipIndexBBox.get()); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); +} + +// Free-standing function for write case (no StorageGridT needed) +template +void scalarCodecWriteBuffers(const GridT& grid, std::ostream& os) +{ + using TreeType = typename GridT::TreeType; + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ScalarCodec"); + } + + WriteBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); +} + +} // namespace internal + +template +struct ScalarCodec : public TopologyCodec +{ + static_assert(GridT::TreeType::RootNodeType::template SameConfiguration< + typename StorageGridT::TreeType::RootNodeType>::value, + "GridT and StorageGridT must have the same configuration"); + + using Ptr = std::unique_ptr>; + + struct ScalarCodecData : public io::CodecData + { + std::vector leafCoords; + }; + + ~ScalarCodec() noexcept = default; + + io::CodecData::Ptr createData() final + { + auto data = std::make_unique(); + data->grid = GridT::create(); + return data; + } + + static inline std::string name() + { + if constexpr (std::is_same_v) { + return GridT::gridType(); + } else { + std::string buildType = typeNameAsString(); + return StorageGridT::gridType() + "_to_" + buildType; + } + } + + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + { + GridT& grid = static_cast(*data.grid); + internal::scalarCodecReadBuffers(grid, is, options); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + if constexpr (Mode == io::CodecMode::ReadOnly) return; + + const GridT& grid = static_cast(gridBase); + internal::scalarCodecWriteBuffers(grid, os); + } +}; // struct ScalarCodec + + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_SCALARCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/codecs/ScalarLeafCodec.h b/openvdb/openvdb/codecs/ScalarLeafCodec.h new file mode 100644 index 0000000000..f72efc56d9 --- /dev/null +++ b/openvdb/openvdb/codecs/ScalarLeafCodec.h @@ -0,0 +1,104 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_SCALARLEAFCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_SCALARLEAFCODEC_HAS_BEEN_INCLUDED + +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +void writeScalarLeafBuffers(const LeafT& leaf, std::ostream& os, bool saveFloatAsHalf) +{ + using NodeMaskT = typename LeafT::NodeMaskType; + + // Write out the value mask. + leaf.getValueMask().save(os); + + leaf.buffer().data(); // load values + + io::writeCompressedValues(os, leaf.buffer().data(), LeafT::SIZE, + leaf.getValueMask(), /*childMask=*/NodeMaskT(), saveFloatAsHalf); +} + + +template +void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, + const typename LeafT::ValueType& background, bool skip = false, + const math::CoordBBox* clipBBox = nullptr) +{ + using ValueT = typename LeafT::ValueType; + using NodeMaskT = typename LeafT::NodeMaskType; + using StorageBufferT = typename StorageLeafT::Buffer; + using StorageValueT = typename StorageLeafT::ValueType; + + constexpr Index SIZE = LeafT::SIZE; + + SharedPtr meta = io::getStreamMetadataPtr(is); + const bool seekable = meta && meta->seekable(); + + auto& valueMask = leaf.getValueMask(); + + // Load or seek the value mask + if (seekable) valueMask.seek(is); + else valueMask.load(is); + + if (skip) { + if (seekable) { + io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf); + } else { + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + } + // Clear the value mask so that the skipped leaf has no active + // voxels. Without this, the leaf retains its on-disk active + // topology even though no data was read into its buffer. + valueMask.setOff(); + return; + } + + if constexpr (std::is_same_v) { + // ValueMask leaf: value == active state, already captured in the value mask above. + // Seek/consume past the storage buffer without populating any separate leaf buffer. + if (seekable) { + io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf); + } else { + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + } + } else if constexpr (std::is_same_v) { + // Bool leaf: must read storage values regardless of seekability, then convert. + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + for (Index i = 0; i < SIZE; ++i) { + leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); + } + } else { + leaf.buffer().allocate(); + if constexpr (std::is_same_v) { + io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf); + } else { + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + for (Index i = 0; i < SIZE; ++i) { + leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); + } + } + } + + if (clipBBox) { + leaf.clip(*clipBBox, background); + } +} + +} // namespace internal +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_SCALARLEAFCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 7e2fc952b8..c99b39277b 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -300,10 +300,10 @@ void topologyCodecWriteTopology(const GridBase& gridBase, std::ostream& os) } // namespace internal -template +template struct TopologyCodec : public io::Codec { - using Ptr = std::unique_ptr>; + using Ptr = std::unique_ptr>; ~TopologyCodec() noexcept = default; @@ -322,6 +322,9 @@ struct TopologyCodec : public io::Codec void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final { + // disable implementation when read only + if constexpr (Mode == io::CodecMode::ReadOnly) return; + internal::topologyCodecWriteTopology(gridBase, os); } }; // struct TopologyCodec diff --git a/openvdb/openvdb/codecs/ValueMaskCodec.h b/openvdb/openvdb/codecs/ValueMaskCodec.h new file mode 100644 index 0000000000..17171cebf7 --- /dev/null +++ b/openvdb/openvdb/codecs/ValueMaskCodec.h @@ -0,0 +1,146 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_VALUEMASKCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_VALUEMASKCODEC_HAS_BEEN_INCLUDED + +#include + +#include +#include + +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct ReadValueMaskBuffersOp +{ + using TreeT = typename GridT::TreeType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + ReadValueMaskBuffersOp(std::istream& _is, bool _saveFloatAsHalf, + const ValueT& _background, const CoordBBox* _clipBBox) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) + , clipBBox(_clipBBox) { } + + void operator()(RootT& root, size_t) + { + // Clip root-level tiles and prune children that were clipped. + if (clipBBox) { + root.clip(*clipBBox); + } + } + + template + void operator()(NodeT& node, size_t) + { + // Clip internal node tiles and prune children that were clipped. + if (clipBBox) { + node.clip(*clipBBox, background); + } + } + + void operator()(LeafT& leaf, size_t) + { + // Read in the value mask. + leaf.getValueMask().load(is); + // Read in the origin. + Coord origin; + is.read(reinterpret_cast(&origin), sizeof(Coord::ValueType) * 3); + leaf.setOrigin(origin); + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; + const CoordBBox* clipBBox = nullptr; +}; // struct ReadValueMaskBuffersOp + +template +struct WriteValueMaskBuffersOp +{ + using TreeT = typename GridT::TreeType; + using LeafT = typename TreeT::LeafNodeType; + + WriteValueMaskBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + // Write out the value mask. + leaf.getValueMask().save(os); + + // Write out the origin. + os.write(reinterpret_cast(&leaf.origin()), sizeof(Coord::ValueType) * 3); + } + + std::ostream& os; + const bool saveFloatAsHalf; +}; // struct WriteValueMaskBuffersOp + +} // namespace internal + +template +struct ValueMaskCodec : public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~ValueMaskCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + { + GridT& grid = static_cast(*data.grid); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ValueMaskCodec"); + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + std::unique_ptr clipIndexBBox; + if (options.clipBBox.isSorted()) { + clipIndexBBox = std::make_unique(grid.constTransform().worldToIndexNodeCentered(options.clipBBox)); + } + + internal::ReadValueMaskBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), clipIndexBBox.get()); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + const GridT& grid = static_cast(gridBase); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ValueMaskCodec"); + } + + internal::WriteValueMaskBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); + } +}; // struct ValueMaskCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_VALUEMASKCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc index 6c4661bdae..f123f0ec0a 100644 --- a/openvdb/openvdb/io/Codec.cc +++ b/openvdb/openvdb/io/Codec.cc @@ -3,7 +3,9 @@ #include "Codec.h" +#include #include +#include #include @@ -37,9 +39,16 @@ getCodecRegistry() namespace internal { +template +struct RegisterCodec { inline void operator()() { CodecRegistry::registerCodec>(); } }; + void initialize() { + NumericGridTypes::foreach(); + Vec3GridTypes::foreach(); + CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); } void uninitialize() diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 304a5b7976..8f41b7d017 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -31,6 +31,11 @@ OPENVDB_API void uninitialize(); //////////////////////////////////////// +enum class CodecMode { + ReadWrite, // Both read and write enabled (default) + ReadOnly // Only read methods enabled +}; + enum class ReadMode { Original, // Read data as-is (default) TopologyOnly // Read topology only, no buffer data diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index 384c52eaa2..da68e4b321 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -247,4 +247,13 @@ void testCodecIOImpl( testIOImpl(gridName, bgValue, fillValue); } +TEST_F(TestCodec, testFloatCodecIO) { testCodecIOImpl("float_grid", 0.0f, 1.0f); } +TEST_F(TestCodec, testDoubleCodecIO) { testCodecIOImpl("double_grid", 0.0, 1.0); } +TEST_F(TestCodec, testInt32CodecIO) { testCodecIOImpl("int32_grid", 0, 1); } +TEST_F(TestCodec, testInt64CodecIO) { testCodecIOImpl("int64_grid", openvdb::Int64(0), openvdb::Int64(1)); } +TEST_F(TestCodec, testHalfCodecIO) { testCodecIOImpl("half_grid", openvdb::Half(0.0), openvdb::Half(1.5)); } +TEST_F(TestCodec, testVec3ICodecIO) { testCodecIOImpl("vec3i_grid", openvdb::Vec3i(0), openvdb::Vec3i(1, 2, 3)); } +TEST_F(TestCodec, testVec3SCodecIO) { testCodecIOImpl("vec3s_grid", openvdb::Vec3s(0.0f), openvdb::Vec3s(1.0f, 2.0f, 3.0f)); } +TEST_F(TestCodec, testVec3DCodecIO) { testCodecIOImpl("vec3d_grid", openvdb::Vec3d(0.0), openvdb::Vec3d(1.0, 2.0, 3.0)); } TEST_F(TestCodec, testBoolCodecIO) { testCodecIOImpl("bool_grid", false, true); } +TEST_F(TestCodec, testMaskCodecIO) { testCodecIOImpl("mask_grid", false, true); } diff --git a/openvdb/openvdb/unittest/TestGridIO.cc b/openvdb/openvdb/unittest/TestGridIO.cc index 184379a030..3f605b86dd 100644 --- a/openvdb/openvdb/unittest/TestGridIO.cc +++ b/openvdb/openvdb/unittest/TestGridIO.cc @@ -3,6 +3,7 @@ #include #include +#include #include #include // for remove() @@ -278,5 +279,10 @@ TEST_F(TestGridIO, testReadAllBool) { readAllTest(); } TEST_F(TestGridIO, testReadAllFloat) { readAllTest(); } TEST_F(TestGridIO, testReadAllHalf) { readAllTest(); } TEST_F(TestGridIO, testReadAllVec3S) { readAllTest(); } -TEST_F(TestGridIO, testReadAllFloat5432) { Float5432Grid::registerGrid(); readAllTest(); } +TEST_F(TestGridIO, testReadAllFloat5432) +{ + Float5432Grid::registerGrid(); + openvdb::io::CodecRegistry::registerCodec>(); + readAllTest(); +} TEST_F(TestGridIO, testCreateWriteReadHalf) { testCreateWriteReadHalf(); } \ No newline at end of file diff --git a/openvdb/openvdb/unittest/TestTree.cc b/openvdb/openvdb/unittest/TestTree.cc index 90a3de5e45..624906b199 100644 --- a/openvdb/openvdb/unittest/TestTree.cc +++ b/openvdb/openvdb/unittest/TestTree.cc @@ -3,6 +3,7 @@ #include #include +#include #include #include // for tools::setValueOnMin(), et al. #include @@ -109,6 +110,10 @@ TEST_F(TestTree, testHalf) openvdb::Grid::registerGrid(); openvdb::Grid::registerGrid(); + // Register scalar codecs for all vec2 tree types which are not registered by default + openvdb::io::CodecRegistry::registerCodec>>(); + openvdb::io::CodecRegistry::registerCodec>>(); + testWriteHalf(); testWriteHalf(); testWriteHalf(); From 42fbdf741c2864008316a366bd44d76e511cb003 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 20 Apr 2026 21:44:28 -0700 Subject: [PATCH 32/70] Add support for implicit conversion - float to half and scalar to bool/mask Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Archive.cc | 18 +++- openvdb/openvdb/io/Codec.cc | 15 +++ openvdb/openvdb/io/Codec.h | 3 + openvdb/openvdb/unittest/TestCodec.cc | 146 ++++++++++++++++++++++++++ 4 files changed, 180 insertions(+), 2 deletions(-) diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 92e99f7c83..d11e7a1688 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -885,10 +885,24 @@ Archive::readGridCount(std::istream& is) io::Codec* -Archive::findCodec(const std::string& gridType, const io::ReadOptions&) +Archive::findCodec(const std::string& gridType, const io::ReadOptions& options) { // Determine the I/O codec to use to read this grid - return io::CodecRegistry::get(gridType); + auto codec = io::CodecRegistry::get(gridType); + + // if readMode is Half, then search for a codec that converts + // from the storage grid type to the grid type + if (options.readMode == ReadMode::Half) { + codec = io::CodecRegistry::get(gridType + "_to_half"); + } + if (options.readMode == ReadMode::Bool) { + codec = io::CodecRegistry::get(gridType + "_to_bool"); + } + if (options.readMode == ReadMode::Mask) { + codec = io::CodecRegistry::get(gridType + "_to_mask"); + } + + return codec; } diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc index f123f0ec0a..cfb1489efc 100644 --- a/openvdb/openvdb/io/Codec.cc +++ b/openvdb/openvdb/io/Codec.cc @@ -42,6 +42,15 @@ namespace internal { template struct RegisterCodec { inline void operator()() { CodecRegistry::registerCodec>(); } }; +template +struct RegisterConvertCodec { + inline void operator()() + { + CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); + } +}; // struct RegisterConvertCodec + void initialize() { NumericGridTypes::foreach(); @@ -49,6 +58,12 @@ void initialize() CodecRegistry::registerCodec>(); CodecRegistry::registerCodec>(); + + // register the plugin that converts from scalar to mask/bool + NumericGridTypes::foreach(); + + // register the plugin that converts from float to half + CodecRegistry::registerCodec>(); } void uninitialize() diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 8f41b7d017..28b00c4a91 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -38,6 +38,9 @@ enum class CodecMode { enum class ReadMode { Original, // Read data as-is (default) + Half, // Read data as half precision + Bool, // Read data as boolean + Mask, // Read data as mask TopologyOnly // Read topology only, no buffer data }; diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index da68e4b321..d8981ca3f7 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -257,3 +258,148 @@ TEST_F(TestCodec, testVec3SCodecIO) { testCodecIOImpl("vec3s TEST_F(TestCodec, testVec3DCodecIO) { testCodecIOImpl("vec3d_grid", openvdb::Vec3d(0.0), openvdb::Vec3d(1.0, 2.0, 3.0)); } TEST_F(TestCodec, testBoolCodecIO) { testCodecIOImpl("bool_grid", false, true); } TEST_F(TestCodec, testMaskCodecIO) { testCodecIOImpl("mask_grid", false, true); } + +TEST_F(TestCodec, testFloatToHalfCodecConversion) +{ + using namespace openvdb; + using namespace openvdb::io; + + openvdb::initialize(); + CodecRegistry::clear(); + + // Verify the conversion codec name + const std::string expectedName = FloatGrid::gridType() + "_to_half"; + EXPECT_EQ((codecs::ScalarCodec::name()), + expectedName); + + // Verify the codec is registered after initialize() + io::internal::initialize(); + EXPECT_TRUE(CodecRegistry::isRegistered(expectedName)); + + // Write a FloatGrid with a known fill value (1.5f is exactly representable in half) + const std::string floatPath = "test_float_to_half.vdb"; + const std::string gridName = "float_grid"; + FloatGrid::Ptr srcGrid = FloatGrid::create(0.0f); + srcGrid->setName(gridName); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + { + io::File f(floatPath); + f.write(GridPtrVec{srcGrid}); + } + + // Read back with ReadMode::Half — triggers the float-to-half conversion codec + ReadOptions halfOpts; + halfOpts.readMode = ReadMode::Half; + + GridBase::Ptr base; + { + io::File f(floatPath); + f.open(); + base = f.readGrid(gridName, halfOpts); + f.close(); + } + ASSERT_TRUE(base); + + // The returned grid must be a HalfGrid + EXPECT_TRUE(base->isType()); + HalfGrid::Ptr halfGrid = gridPtrCast(base); + ASSERT_TRUE(halfGrid); + + // Topology must match the source FloatGrid + EXPECT_TRUE(srcGrid->tree().hasSameTopology(halfGrid->tree())); + + // All active voxel values must equal Half(1.5f) + for (HalfGrid::ValueOnCIter it = halfGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, Half(1.5f)); + } + + // Background value must equal Half(0.0f) + EXPECT_EQ(halfGrid->background(), Half(0.0f)); + + // Cleanup + std::remove(floatPath.c_str()); +} + +template +void testConvertCodecImpl() +{ + using namespace openvdb; + using namespace openvdb::io; + + openvdb::io::CodecRegistry::clear(); + openvdb::io::internal::initialize(); + + // Verify the conversion codec name + const std::string expectedName = + SrcGridT::gridType() + "_to_" + typeNameAsString(); + EXPECT_EQ((codecs::ScalarCodec::name()), + expectedName); + + // Verify the codec is registered after initialize() + EXPECT_TRUE(CodecRegistry::isRegistered(expectedName)); + + // Write a SrcGridT with a non-zero fill value + const std::string testPath = "test_" + expectedName + ".vdb"; + const std::string gridName = SrcGridT::gridType(); + typename SrcGridT::Ptr srcGrid = + SrcGridT::create(typename SrcGridT::ValueType(0)); + srcGrid->setName(gridName); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), + typename SrcGridT::ValueType(1), true); + + { + io::File f(testPath); + f.write(GridPtrVec{srcGrid}); + } + + // Read back with the specified ReadMode — triggers the conversion codec + ReadOptions opts; + opts.readMode = mode; + + GridBase::Ptr base; + { + io::File f(testPath); + f.open(); + base = f.readGrid(gridName, opts); + f.close(); + } + ASSERT_TRUE(base); + + // The returned grid must be a DstGridT + EXPECT_TRUE(base->isType()); + typename DstGridT::Ptr dstGrid = gridPtrCast(base); + ASSERT_TRUE(dstGrid); + + // Topology must match the source grid + EXPECT_TRUE(srcGrid->tree().hasSameTopology(dstGrid->tree())); + + // All active voxel values must be true + for (typename DstGridT::ValueOnCIter it = dstGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, typename DstGridT::ValueType(true)); + } + + // Background value must be false + EXPECT_EQ(dstGrid->background(), typename DstGridT::ValueType(false)); + + // Cleanup + std::remove(testPath.c_str()); +} + +TEST_F(TestCodec, testNumericToBoolCodecConversion) +{ + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); +} + +TEST_F(TestCodec, testNumericToMaskCodecConversion) +{ + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); +} From 94e20c0239f495843f04e3392e9426f6f1946e5a Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 17 Feb 2026 11:22:54 -0800 Subject: [PATCH 33/70] Update to Tree Iterator to work with Mask Grids Signed-off-by: Dan Bailey --- openvdb/openvdb/tree/Iterator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvdb/openvdb/tree/Iterator.h b/openvdb/openvdb/tree/Iterator.h index 1116c2b34f..e80ee1e6e5 100644 --- a/openvdb/openvdb/tree/Iterator.h +++ b/openvdb/openvdb/tree/Iterator.h @@ -83,7 +83,7 @@ class IteratorBase /// @details If this iterator is pointing to a child node, then the current item /// in the parent node's table is required to be inactive. In that case, /// this method has no effect. - void setValueOff() const { parent().mValueMask.setOff(this->pos()); } + void setValueOff() const { parent().setValueMask(this->pos(), false); } /// Return the coordinates of the item to which this iterator is pointing. Coord getCoord() const { return parent().offsetToGlobalCoord(this->pos()); } From c1eabdbbdf76b3e72a060de3550077aba49e28d6 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 6 May 2026 16:43:12 -0700 Subject: [PATCH 34/70] Add pending changes Signed-off-by: Dan Bailey --- pendingchanges/iorefactor.txt | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 pendingchanges/iorefactor.txt diff --git a/pendingchanges/iorefactor.txt b/pendingchanges/iorefactor.txt new file mode 100644 index 0000000000..331fd2ba22 --- /dev/null +++ b/pendingchanges/iorefactor.txt @@ -0,0 +1,8 @@ +OpenVDB: + API changes: + - GridDescriptor::read() has been deprecated, use GridDescriptor::readHeader() + followed by GridDescriptor::readStreamPos() instead. + + Improvements: + - Large refactor of I/O classes, many private and protected member functions have + been modified. No change in behavior for io::File and io::Stream. From ea040c95543afa9103e86a9c8216df6c1c74a48d Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 6 May 2026 21:43:24 -0700 Subject: [PATCH 35/70] Add pending changes Signed-off-by: Dan Bailey --- pendingchanges/treebaseio.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pendingchanges/treebaseio.txt diff --git a/pendingchanges/treebaseio.txt b/pendingchanges/treebaseio.txt new file mode 100644 index 0000000000..260862ee03 --- /dev/null +++ b/pendingchanges/treebaseio.txt @@ -0,0 +1,4 @@ + OpenVDB: + API changes: + - TreeBase::readTopology() and TreeBase::writeTopology() are now pure virtual. Derived + classes of TreeBase now need to read and write int32_t(1) to remain backwards-compatible. From c4f28dad40d7f53f2a78be767e8f112931010378 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 7 May 2026 11:39:23 -0700 Subject: [PATCH 36/70] Add pending changes Signed-off-by: Dan Bailey --- pendingchanges/iooptions.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 pendingchanges/iooptions.txt diff --git a/pendingchanges/iooptions.txt b/pendingchanges/iooptions.txt new file mode 100644 index 0000000000..ca67c833f5 --- /dev/null +++ b/pendingchanges/iooptions.txt @@ -0,0 +1,6 @@ +OpenVDB: + API changes: + - Added optional trailing WriteOptions parameter to io::File::write() and io::Stream::write(). + No changes in behavior. + - Added optional trailing ReadOptions parameter to io::File::getGrids() and io::File::readGrid(). + No changes in behavior. From 0e420878120e269524fc26c57f90091e93a1b80c Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 12 May 2026 10:51:09 -0700 Subject: [PATCH 37/70] Improve and extend docs Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Codec.h | 329 +++++++++++++++++++++++++++++++++---- 1 file changed, 296 insertions(+), 33 deletions(-) diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 304a5b7976..6800fa240c 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -24,6 +24,13 @@ namespace io { namespace internal { +// Global registration of codecs. These initialization functions are NOT intended +// to be called directly by the user. +// While they are thread-safe, they lack the early-exit logic of openvdb::initialize() - +// this can result in exceptions being thrown when attempting to register a codec that is +// already registered. +// They are also used extensively in unit tests. + OPENVDB_API void initialize(); OPENVDB_API void uninitialize(); @@ -31,21 +38,58 @@ OPENVDB_API void uninitialize(); //////////////////////////////////////// +/// @brief Controls how grid data is read. +/// +/// Passed via @c ReadOptions::readMode to influence which portions of a grid +/// are deserialized and whether any on-the-fly conversion is performed during read. +/// Codecs are expected to honour this setting in their @c readTopology() and +/// @c readBuffers() implementations. enum class ReadMode { - Original, // Read data as-is (default) - TopologyOnly // Read topology only, no buffer data + /// Deserialize both topology and all value buffers. This is the default + /// behaviour and produces a fully populated grid identical to the one + /// that was written. + Original, + /// Deserialize topology only; value buffers are skipped. The resulting + /// grid has a valid tree structure (active/inactive state, node + /// hierarchy) but leaf buffer data is left at its default (background) + /// value. Useful when only the active-voxel mask is needed and + /// avoiding the cost of reading large value buffers is desirable. + TopologyOnly }; -/// Base class for per-grid-type codec-specific options -/// Derive from this to add codec-specific read configuration +/// @brief Base class for per-grid-type, codec-specific read options. +/// +/// Codecs that require type-specific read configuration should derive from +/// this class and store an instance in @c ReadOptions::typeData, keyed by +/// the grid type string (e.g. @c "Vec3SGrid"). +/// +/// @par Example +/// @code +/// struct MyCodecReadOptions : public ReadTypedOptions { +/// bool someFlag = false; +/// }; +/// +/// ReadOptions options; +/// auto data = std::make_shared(); +/// data->someFlag = true; +/// options.typeData["Vec3SGrid"] = data; +/// @endcode struct OPENVDB_API ReadTypedOptions { using Ptr = std::shared_ptr; virtual ~ReadTypedOptions() = default; - /// Type-safe casting helper for derived ReadTypedOptions types - /// Uses OPENVDB_ASSERT to verify type safety when assertions are enabled + /// @brief Downcast a @c ReadTypedOptions::Ptr to a concrete derived type. + /// + /// Performs a @c static_cast to @c T after verifying via @c dynamic_cast + /// (when @c OPENVDB_ASSERT is enabled) that @p data actually holds an + /// instance of @c T. Calling this with a mismatched type is undefined + /// behaviour in release builds. + /// + /// @tparam T The concrete @c ReadTypedOptions subclass to cast to. + /// @param data A shared pointer to the base @c ReadTypedOptions object. + /// @return A reference to the underlying @c T instance. template static T& cast(const Ptr& data) { OPENVDB_ASSERT(dynamic_cast(data.get()) != nullptr); @@ -53,34 +97,73 @@ struct OPENVDB_API ReadTypedOptions } }; // struct ReadTypedOptions -/// Global read configuration that applies to all grids -/// Contains common options shared across all codec types +/// @brief Global read configuration passed to every codec during deserialization. +/// +/// An instance of this struct is threaded through all codec @c readTopology() and +/// @c readBuffers() calls so that site-wide policy (clipping, read mode) and any +/// type-specific overrides are available in a single place. +/// +/// @par Clipping +/// When @c clipBBox is non-empty (i.e. @c !clipBBox.empty()), codecs should +/// restrict the voxel data they load to the region that intersects that world-space +/// bounding box. An empty bbox (the default) means no clipping is applied. +/// +/// @par Read mode +/// @c readMode controls the granularity of deserialization. The default, +/// @c ReadMode::Original, loads both topology and value buffers. +/// @c ReadMode::TopologyOnly skips value buffers entirely, which can be +/// significantly faster when only the active-voxel mask is needed. +/// +/// @par Per-type options +/// @c typeData allows callers to attach codec-specific configuration for +/// individual grid types. Entries are keyed by the grid type string +/// (e.g. @c "Vec3SGrid") and hold a @c ReadTypedOptions-derived object. +/// Codecs retrieve their entry via @c ReadTypedOptions::cast(). struct OPENVDB_API ReadOptions { - // BBox Clip Data + /// World-space bounding box used to spatially clip the read. + /// An empty bbox (the default) disables clipping. BBoxd clipBBox = BBoxd(); - // Grid Conversion Options + /// Controls which portions of each grid are deserialized. + /// Defaults to @c ReadMode::Original (full topology + value buffers). ReadMode readMode = ReadMode::Original; - // Per-grid-type codec-specific options, keyed by grid type string + /// Optional per-grid-type codec configuration, keyed by grid type string. + /// Values are @c ReadTypedOptions subclass instances; use + /// @c ReadTypedOptions::cast() to retrieve the concrete type. std::unordered_map typeData; }; // struct ReadOptions -/// Global write configuration that applies to all grids -/// Contains common options shared across all codec types +/// @brief Global write configuration passed to every codec during serialization. +/// +/// Currently carries no fields, but is provided for forward compatibility: +/// future write-time options (e.g. compression hints, metadata policies) can +/// be added here without changing the codec interface. struct OPENVDB_API WriteOptions { }; // struct WriteOptions +/// @brief Severity level for a read diagnostic. +/// @note Currently only @c Warning is defined. enum class DiagnosticSeverity { Warning }; +/// @brief A single read diagnostic message. +/// @note @c context is typically the grid name or another identifier that +/// locates the source of the message. struct ReadDiagnostic { DiagnosticSeverity severity; std::string context; std::string message; }; +/// @brief Thread-safe collection of read diagnostics accumulated during a codec read. +/// +/// @details Diagnostics report situations where a requested read option could +/// not be honoured by the codec — for example, clipping that was not applied +/// natively and had to fall back to a post-process, or an option that the codec +/// ignores entirely. The object must be explicitly enabled before any messages +/// are recorded; it is disabled by default. struct OPENVDB_API ReadDiagnostics { ReadDiagnostics() = default; ReadDiagnostics(const ReadDiagnostics& other) { @@ -96,19 +179,28 @@ struct OPENVDB_API ReadDiagnostics { } return *this; } + /// @brief Append a warning diagnostic. No-op if the object is disabled. + /// @note This method is thread-safe. void addWarning(const std::string& context, const std::string& message) { if (!mEnabled) return; std::lock_guard lock(mMutex); mDiagnostics.push_back({DiagnosticSeverity::Warning, context, message}); } + /// @brief Remove all recorded diagnostics. + /// @note This method is thread-safe. void clear() { std::lock_guard lock(mMutex); mDiagnostics.clear(); } + /// @brief Enable diagnostic recording. void enable() { mEnabled = true; } + /// @brief Disable diagnostic recording. void disable() { mEnabled = false; } + /// @brief Return @c true if diagnostic recording is enabled. bool enabled() const { return mEnabled; } + /// @brief Return the list of recorded diagnostics. const std::vector& diagnostics() const { return mDiagnostics; } + /// @brief Return @c true if no diagnostics have been recorded. bool empty() const { return mDiagnostics.empty(); } private: bool mEnabled = false; @@ -116,59 +208,230 @@ struct OPENVDB_API ReadDiagnostics { mutable std::mutex mMutex; }; // struct ReadDiagnostics -/// Per-codec mutable data created by Codec::createData() -/// Contains the grid and any codec-specific state +/// @brief Mutable per-operation state created by @c Codec::createData() and +/// passed into every read and write call on a given codec. +/// +/// @details The main purpose of this class is to provide a mechanism for passing +/// additional state between the different methods in the codec. @c ReadOptions is +/// immutable and cannot carry per-operation state; the codec itself is stateless +/// by design (to allow sharing across threads) and therefore also cannot be used +/// for this purpose. +/// +/// The base class provides the single field that every codec requires: +/// a shared pointer to the @c GridBase being populated (on read) or inspected +/// (on write). Derived codecs that need additional transient state should +/// subclass @c CodecData and add those fields. +/// +/// A @c CodecData instance is created once per grid per read or write operation +/// by @c Codec::createData(), which also allocates the concrete @c GridBase +/// subclass and stores it in @c grid. +/// +/// @par Deriving from CodecData +/// @code +/// struct MyCodecData : public io::CodecData { +/// // Additional per-operation state goes here +/// SomeCache intermediateBuffer; +/// }; +/// @endcode +/// +/// Inside @c Codec::createData(), the derived type is allocated and returned: +/// @code +/// io::CodecData::Ptr createData() override { +/// auto data = std::make_unique(); +/// data->grid = MyGridType::create(); +/// return data; +/// } +/// @endcode +/// +/// In the read and write methods, downcast the reference back to the concrete +/// type to access the extra fields: +/// @code +/// void readBuffers(std::istream& is, io::CodecData& data, ...) override { +/// auto& myData = static_cast(data); +/// MyGridType& grid = static_cast(*myData.grid); +/// // Use myData.intermediateBuffer, grid, etc. +/// } +/// @endcode struct OPENVDB_API CodecData { using Ptr = std::unique_ptr; virtual ~CodecData() = default; + /// The grid being populated on read, or being serialized on write. SharedPtr grid; }; // struct CodecData +/// @brief Abstract base class for grid I/O codecs. +/// +/// @details A codec encapsulates the serialization and deserialization logic +/// for a specific grid type (or family of grid types). The I/O subsystem +/// selects a codec by name at runtime, calls @c createData() to allocate +/// per-operation state, and then dispatches to the appropriate read or write +/// methods. +/// +/// Codecs must be stateless: all mutable per-operation state is stored in a +/// @c CodecData object (see above) that is created fresh for each grid read +/// or write. This ensures that a single registered codec instance can safely +/// handle concurrent I/O operations. +/// +/// @par Implementing a new codec +/// +/// 1. **Declare the codec struct**, deriving publicly from @c Codec (or from a +/// convenience intermediate that already handles topology, if one exists for +/// the grid family in question): +/// @code +/// struct MyCodec : public io::Codec { ... }; +/// @endcode +/// +/// 2. **Provide a static @c name() method** that returns the unique string +/// identifier under which the codec will be registered. Any globally +/// unique ASCII string is acceptable: +/// @code +/// static std::string name() { return "mycodec"; } +/// @endcode +/// +/// 3. **Override @c createData()** (the only pure-virtual method). Allocate a +/// @c CodecData (or a derived subclass if extra per-operation state is +/// needed), create the appropriate @c GridBase subclass, assign it to +/// @c CodecData::grid, and return the object: +/// @code +/// io::CodecData::Ptr createData() override { +/// auto data = std::make_unique(); +/// data->grid = MyGridType::create(); +/// return data; +/// } +/// @endcode +/// +/// 4. **Override the read and/or write methods** as required. All four +/// methods have default no-op implementations, so override only those that +/// the codec actually uses. The I/O layer separates topology (the tree +/// structure and active/inactive voxel mask) from value buffers (the actual +/// voxel data), allowing callers to request topology-only reads via +/// @c ReadOptions::readMode. Implementations should honour that setting: +/// @code +/// void readTopology(std::istream& is, io::CodecData& data, +/// const io::ReadOptions& options, +/// io::ReadDiagnostics& diagnostics) override +/// { +/// MyGridType& grid = static_cast(*data.grid); +/// // Deserialize the tree structure into grid... +/// } +/// +/// void readBuffers(std::istream& is, io::CodecData& data, +/// const io::ReadOptions& options, +/// io::ReadDiagnostics& diagnostics) override +/// { +/// if (options.readMode == io::ReadMode::TopologyOnly) return; +/// MyGridType& grid = static_cast(*data.grid); +/// // Deserialize voxel values into grid... +/// } +/// @endcode +/// If a requested option (e.g. spatial clipping) cannot be honoured natively, +/// record a warning via @c diagnostics.addWarning() rather than silently ignoring it. +/// +/// 5. **Register the codec** once at start-up, typically from the library's +/// @c initialize() function, using @c CodecRegistry::registerCodec(): +/// @code +/// io::CodecRegistry::registerCodec(); +/// @endcode +/// +/// @note The codec struct itself is never copied; the registry takes ownership +/// of a single heap-allocated instance via @c Codec::Ptr +/// (@c std::unique_ptr). struct OPENVDB_API Codec { using Ptr = std::unique_ptr; virtual ~Codec() = default; - /// Create codec data including a new grid of the appropriate type - /// @return A new CodecData object containing the grid and any codec-specific state + /// @brief Allocate per-operation codec state, including the target grid. + /// + /// This is the only pure-virtual method. Implementations must create a + /// concrete @c GridBase subclass appropriate for the codec, store it in + /// @c CodecData::grid, and return the @c CodecData (or a derived subclass + /// carrying additional state). The returned object is passed by reference + /// to every subsequent read or write call for the same grid. + /// + /// @return A fully initialized @c CodecData whose @c grid field is non-null. virtual CodecData::Ptr createData() = 0; - /// Read the grid topology - virtual void readTopology(std::istream&, CodecData&, const ReadOptions&, ReadDiagnostics&) { } - - /// Read all data buffers for this grid - virtual void readBuffers(std::istream&, CodecData&, const ReadOptions&, ReadDiagnostics&) { } - - /// Write the grid topology - virtual void writeTopology(std::ostream&, const GridBase&, const WriteOptions&) { } - - /// Write all data buffers for this grid - virtual void writeBuffers(std::ostream&, const GridBase&, const WriteOptions&) { } + /// @brief Deserialize the grid topology (tree structure and active-voxel + /// mask) from @a is into the grid held by @a data. + virtual void readTopology(std::istream& /*is*/, CodecData& /*data*/, + const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) { } + + /// @brief Deserialize all voxel-value buffers from @a is into the grid + /// held by @a data. + /// + /// The default implementation is a no-op. Override to populate leaf-node + /// value buffers after topology has been established. When + /// @c options.readMode is @c ReadMode::TopologyOnly this method will not + /// be called by the I/O layer, so implementations may also guard against + /// that mode internally for safety. If @c options.clipBBox is non-empty, + /// restrict the loaded data to the region that intersects it; if the codec + /// cannot honour clipping natively, fall back to a post-process and record + /// a warning via @a diagnostics. + virtual void readBuffers(std::istream& /*is*/, CodecData& /*data*/, + const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) { } + + /// @brief Serialize the grid topology (tree structure and active-voxel + /// mask) from @a grid to @a os. + /// + /// The default implementation is a no-op. Override when the codec stores + /// topology as a distinct section that precedes the value buffers in the + /// stream. + virtual void writeTopology(std::ostream& /*os*/, const GridBase& /*grid*/, + const WriteOptions& /*options*/) { } + + /// @brief Serialize all voxel-value buffers from @a grid to @a os. + /// + /// The default implementation is a no-op. Override to write the leaf-node + /// value buffers that follow the topology section. + virtual void writeBuffers(std::ostream& /*os*/, const GridBase& /*grid*/, + const WriteOptions& /*options*/) { } }; // struct Codec +/// @brief A thread-safe, process-global registry that maps codec names to +/// @c Codec instances. +/// +/// Codecs are identified by a unique string name (typically provided by @c Codec::name()) +/// and stored as @c Codec::Ptr (i.e. @c std::unique_ptr). +/// +/// @par Typical usage +/// @code +/// // Register a custom codec once at start-up (e.g. from initialize()): +/// CodecRegistry::registerCodec(); +/// @endcode +/// +/// @note Attempting to register a codec whose name is already present throws +/// @c openvdb::KeyError. Call @c isRegistered() first when the caller +/// cannot guarantee uniqueness. struct OPENVDB_API CodecRegistry { - /// Return @c true if the given codec name is registered. + /// Return @c true if a codec with the given @a name has been registered. static bool isRegistered(const std::string& name); - /// Register a codec by name with a factory function. + /// Register a codec under the explicit string @a name, transferring + /// ownership of @a codec to the registry. + /// @throw KeyError if @a name is already registered. static void registerCodecByName(const std::string& name, Codec::Ptr&& codec); - /// Register a codec by type. + /// Convenience wrapper that registers @c CodecT using the name returned + /// by @c CodecT::name(), constructing the instance internally. + /// @throw KeyError if the codec is already registered. template static void registerCodec() { registerCodecByName(CodecT::name(), std::make_unique()); } - /// Return a pointer to the codec with the given name, or nullptr if not found. + /// Return a raw (non-owning) pointer to the codec registered under + /// @a name, or @c nullptr if no such codec exists. + /// The returned pointer remains valid for the lifetime of the registry. static Codec* get(const std::string& name); - /// Clear the codec registry. + /// Deregister all codecs and reset the registry to an empty state. static void clear(); }; // struct CodecRegistry From 2a54d6d86850ea0349a71d6fa955b6cb09aacd38 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 12 May 2026 13:50:14 -0700 Subject: [PATCH 38/70] Update docs Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Codec.h | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index bc26233a6d..4fbf3412d1 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -38,9 +38,18 @@ OPENVDB_API void uninitialize(); //////////////////////////////////////// +/// @brief Controls which operations a codec exposes. +/// +/// Passed to codec registration to restrict how the codec may be used at +/// runtime. A @c ReadOnly codec can still be used to read existing files but +/// will not be selected as the write codec for new grids. enum class CodecMode { - ReadWrite, // Both read and write enabled (default) - ReadOnly // Only read methods enabled + /// Both @c readTopology()/@c readBuffers() and + /// @c writeTopology()/@c writeBuffers() are enabled. This is the default. + ReadWrite, + /// Only @c readTopology() and @c readBuffers() are enabled; the codec + /// will not be offered as a write target. + ReadOnly }; /// @brief Controls how grid data is read. @@ -54,9 +63,20 @@ enum class ReadMode { /// behaviour and produces a fully populated grid identical to the one /// that was written. Original, - Half, // Read data as half precision - Bool, // Read data as boolean - Mask, // Read data as mask + /// Deserialize topology and value buffers, converting each voxel value + /// to half-precision floating point on the fly. Codecs that do not + /// support native half conversion should fall back to @c Original and + /// record a @c ReadDiagnostic warning. + Half, + /// Deserialize topology and value buffers, converting each voxel value + /// to @c bool (non-zero to @c true, zero to @c false). Produces a + /// @c BoolGrid whose active set matches the source grid's active set. + Bool, + /// Deserialize topology only and promote the result to a mask grid, + /// discarding all value data. Equivalent to reading @c TopologyOnly and + /// then constructing a @c MaskGrid from the active-voxel set, but may + /// be performed more efficiently inside the codec. + Mask, /// Deserialize topology only; value buffers are skipped. The resulting /// grid has a valid tree structure (active/inactive state, node /// hierarchy) but leaf buffer data is left at its default (background) @@ -117,8 +137,11 @@ struct OPENVDB_API ReadTypedOptions /// bounding box. An empty bbox (the default) means no clipping is applied. /// /// @par Read mode -/// @c readMode controls the granularity of deserialization. The default, -/// @c ReadMode::Original, loads both topology and value buffers. +/// @c readMode controls the granularity and on-the-fly conversion applied +/// during deserialization. The default, @c ReadMode::Original, loads both +/// topology and value buffers without modification. +/// @c ReadMode::Half, @c ReadMode::Bool, and @c ReadMode::Mask request +/// in-place type conversion as data is read. /// @c ReadMode::TopologyOnly skips value buffers entirely, which can be /// significantly faster when only the active-voxel mask is needed. /// From f4682d14678101d2fe5153fa94dbfbc5908a2757 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 13 May 2026 16:36:41 -0700 Subject: [PATCH 39/70] Add pending notes Signed-off-by: Dan Bailey --- pendingchanges/houdinivulkan.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pendingchanges/houdinivulkan.txt diff --git a/pendingchanges/houdinivulkan.txt b/pendingchanges/houdinivulkan.txt new file mode 100644 index 0000000000..b64d663f0a --- /dev/null +++ b/pendingchanges/houdinivulkan.txt @@ -0,0 +1,3 @@ +OpenVDB: + Houdini: + - Add new Vulkan viewport support for VDB Points primitives and remove support for delayed loading. From 27f6e19edd8142598fb830d9c7e75ddcba0b8bbf Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 13 May 2026 16:57:35 -0700 Subject: [PATCH 40/70] Add back support for 221 VDB files Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/ScalarLeafCodec.h | 15 +++++++++++++ openvdb/openvdb/codecs/TopologyCodec.h | 28 +++++++++++++++++++----- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/openvdb/openvdb/codecs/ScalarLeafCodec.h b/openvdb/openvdb/codecs/ScalarLeafCodec.h index f72efc56d9..df13bc8f7f 100644 --- a/openvdb/openvdb/codecs/ScalarLeafCodec.h +++ b/openvdb/openvdb/codecs/ScalarLeafCodec.h @@ -48,6 +48,21 @@ void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, if (seekable) valueMask.seek(is); else valueMask.load(is); + // Pre-node-mask-compression format stored the origin and buffer count + // inline after the value mask. + int8_t numBuffers = 1; + if (io::getFormatVersion(is) < OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) { + Coord origin; + is.read(reinterpret_cast(&origin), sizeof(Coord::ValueType) * 3); + leaf.setOrigin(origin); + + is.read(reinterpret_cast(&numBuffers), sizeof(int8_t)); + + if (numBuffers > 1) { + OPENVDB_THROW(IoError, "Old file format 221 (FLOAT_FRUSTUM_BBOX) with multiple buffers is not supported"); + } + } + if (skip) { if (seekable) { io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf); diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index c99b39277b..e2a09b8ddc 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -182,7 +182,9 @@ struct ReadTopologyOp valueMask.load(is); node.setValueMaskUnsafe(valueMask); - const Index numValues = NodeT::NUM_VALUES; + const bool oldVersion = + (io::getFormatVersion(is) < OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION); + const Index numValues = (oldVersion ? childMask.countOff() : NodeT::NUM_VALUES); { // Read in (and uncompress, if necessary) all of this node's values // into a contiguous array. @@ -191,13 +193,27 @@ struct ReadTopologyOp io::readCompressedValues(is, values, numValues, valueMask, saveFloatAsHalf); // Copy values from the array into this node's table. - if constexpr (std::is_same_v) { - for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), values[iter.pos()]); + if (oldVersion) { + Index n = 0; + if constexpr (std::is_same_v) { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), values[n++]); + } + } else { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[n++])); + } } + OPENVDB_ASSERT(n == numValues); } else { - for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); + if constexpr (std::is_same_v) { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), values[iter.pos()]); + } + } else { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); + } } } } From ff40f237791a56ee32226b4b3e8a9f17a9ea27df Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 14 May 2026 14:21:29 -0700 Subject: [PATCH 41/70] Address feedback Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 6 ++++++ openvdb/openvdb/codecs/BoolCodec.h | 2 +- openvdb/openvdb/codecs/ScalarCodec.h | 16 ++------------- openvdb/openvdb/codecs/TopologyCodec.h | 20 ++++--------------- openvdb/openvdb/codecs/ValueMaskCodec.h | 6 +++--- .../codecs/{ => impl}/ScalarLeafCodec.h | 18 ++++++++++------- openvdb/openvdb/io/Archive.cc | 18 +++++++---------- 7 files changed, 34 insertions(+), 52 deletions(-) rename openvdb/openvdb/codecs/{ => impl}/ScalarLeafCodec.h (86%) diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 9d8d819d49..3572583e20 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -372,6 +372,10 @@ set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES codecs/ValueMaskCodec.h ) +set(OPENVDB_LIBRARY_CODECS_IMPL_INCLUDE_FILES + codecs/impl/ScalarLeafCodec.h +) + set(OPENVDB_LIBRARY_IO_INCLUDE_FILES io/Archive.h io/Codec.h @@ -568,6 +572,7 @@ if(USE_EXPLICIT_INSTANTIATION) set(OPENVDB_LIBRARY_INSTANTIATION_HEADERS ${OPENVDB_LIBRARY_TOOLS_INCLUDE_FILES} ${OPENVDB_LIBRARY_CODECS_INCLUDE_FILES} + ${OPENVDB_LIBRARY_CODECS_IMPL_INCLUDE_FILES} ) # A new source file is generated by CMake for every header @@ -745,6 +750,7 @@ install(FILES ${OPENVDB_LIBRARY_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INC install(FILES ${CMAKE_CURRENT_BINARY_DIR}/openvdb/version.h DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb) install(FILES ${OPENVDB_LIBRARY_IO_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/io) install(FILES ${OPENVDB_LIBRARY_CODECS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/codecs) +install(FILES ${OPENVDB_LIBRARY_CODECS_IMPL_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/codecs/impl) install(FILES ${OPENVDB_LIBRARY_MATH_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/math) install(FILES ${OPENVDB_LIBRARY_POINTS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/points) install(FILES ${OPENVDB_LIBRARY_POINTS_IMPL_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/points/impl) diff --git a/openvdb/openvdb/codecs/BoolCodec.h b/openvdb/openvdb/codecs/BoolCodec.h index 68d7997a89..e8537fa662 100644 --- a/openvdb/openvdb/codecs/BoolCodec.h +++ b/openvdb/openvdb/codecs/BoolCodec.h @@ -102,7 +102,7 @@ struct WriteBoolBuffersOp } // namespace internal template -struct BoolCodec : public TopologyCodec +struct BoolCodec final: public TopologyCodec { using Ptr = std::unique_ptr>; diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h index c9f8dbe49e..114c575bcf 100644 --- a/openvdb/openvdb/codecs/ScalarCodec.h +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -10,7 +10,7 @@ #include #include -#include "ScalarLeafCodec.h" +#include "impl/ScalarLeafCodec.h" #include "TopologyCodec.h" namespace openvdb { @@ -130,7 +130,7 @@ void scalarCodecWriteBuffers(const GridT& grid, std::ostream& os) } // namespace internal template -struct ScalarCodec : public TopologyCodec +struct ScalarCodec final: public TopologyCodec { static_assert(GridT::TreeType::RootNodeType::template SameConfiguration< typename StorageGridT::TreeType::RootNodeType>::value, @@ -138,20 +138,8 @@ struct ScalarCodec : public TopologyCodec using Ptr = std::unique_ptr>; - struct ScalarCodecData : public io::CodecData - { - std::vector leafCoords; - }; - ~ScalarCodec() noexcept = default; - io::CodecData::Ptr createData() final - { - auto data = std::make_unique(); - data->grid = GridT::create(); - return data; - } - static inline std::string name() { if constexpr (std::is_same_v) { diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index e2a09b8ddc..e602f176ce 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -195,25 +195,13 @@ struct ReadTopologyOp // Copy values from the array into this node's table. if (oldVersion) { Index n = 0; - if constexpr (std::is_same_v) { - for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), values[n++]); - } - } else { - for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), static_cast(values[n++])); - } + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[n++])); } OPENVDB_ASSERT(n == numValues); } else { - if constexpr (std::is_same_v) { - for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), values[iter.pos()]); - } - } else { - for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); - } + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); } } } diff --git a/openvdb/openvdb/codecs/ValueMaskCodec.h b/openvdb/openvdb/codecs/ValueMaskCodec.h index 17171cebf7..27f6289cb9 100644 --- a/openvdb/openvdb/codecs/ValueMaskCodec.h +++ b/openvdb/openvdb/codecs/ValueMaskCodec.h @@ -55,7 +55,7 @@ struct ReadValueMaskBuffersOp leaf.getValueMask().load(is); // Read in the origin. Coord origin; - is.read(reinterpret_cast(&origin), sizeof(Coord::ValueType) * 3); + origin.read(is); leaf.setOrigin(origin); } @@ -84,7 +84,7 @@ struct WriteValueMaskBuffersOp leaf.getValueMask().save(os); // Write out the origin. - os.write(reinterpret_cast(&leaf.origin()), sizeof(Coord::ValueType) * 3); + leaf.origin().write(os); } std::ostream& os; @@ -94,7 +94,7 @@ struct WriteValueMaskBuffersOp } // namespace internal template -struct ValueMaskCodec : public TopologyCodec +struct ValueMaskCodec final: public TopologyCodec { using Ptr = std::unique_ptr>; diff --git a/openvdb/openvdb/codecs/ScalarLeafCodec.h b/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h similarity index 86% rename from openvdb/openvdb/codecs/ScalarLeafCodec.h rename to openvdb/openvdb/codecs/impl/ScalarLeafCodec.h index df13bc8f7f..ad9e088604 100644 --- a/openvdb/openvdb/codecs/ScalarLeafCodec.h +++ b/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h @@ -1,8 +1,8 @@ // Copyright Contributors to the OpenVDB Project // SPDX-License-Identifier: Apache-2.0 -#ifndef OPENVDB_IO_CODECS_SCALARLEAFCODEC_HAS_BEEN_INCLUDED -#define OPENVDB_IO_CODECS_SCALARLEAFCODEC_HAS_BEEN_INCLUDED +#ifndef OPENVDB_IO_CODECS_IMPL_SCALARLEAFCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_IMPL_SCALARLEAFCODEC_HAS_BEEN_INCLUDED #include @@ -88,10 +88,14 @@ void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, } } else if constexpr (std::is_same_v) { // Bool leaf: must read storage values regardless of seekability, then convert. - StorageBufferT storageTemp; - io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); - for (Index i = 0; i < SIZE; ++i) { - leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); + if constexpr (std::is_same_v) { + io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf); + } else { + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + for (Index i = 0; i < SIZE; ++i) { + leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); + } } } else { leaf.buffer().allocate(); @@ -116,4 +120,4 @@ void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, } // namespace OPENVDB_VERSION_NAME } // namespace openvdb -#endif // OPENVDB_IO_CODECS_SCALARLEAFCODEC_HAS_BEEN_INCLUDED +#endif // OPENVDB_IO_CODECS_IMPL_SCALARLEAFCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index b05d8eefb5..86dd2755cc 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -887,22 +887,18 @@ Archive::readGridCount(std::istream& is) io::Codec* Archive::findCodec(const std::string& gridType, const io::ReadOptions& options) { - // Determine the I/O codec to use to read this grid - auto codec = io::CodecRegistry::get(gridType); - // if readMode is Half, then search for a codec that converts // from the storage grid type to the grid type if (options.readMode == ReadMode::Half) { - codec = io::CodecRegistry::get(gridType + "_to_half"); - } - if (options.readMode == ReadMode::Bool) { - codec = io::CodecRegistry::get(gridType + "_to_bool"); - } - if (options.readMode == ReadMode::Mask) { - codec = io::CodecRegistry::get(gridType + "_to_mask"); + return io::CodecRegistry::get(gridType + "_to_half"); + } else if (options.readMode == ReadMode::Bool) { + return io::CodecRegistry::get(gridType + "_to_bool"); + } else if (options.readMode == ReadMode::Mask) { + return io::CodecRegistry::get(gridType + "_to_mask"); } - return codec; + // Determine the I/O codec to use to read this grid + return io::CodecRegistry::get(gridType); } From 96964f2c3aba8bab887283d493fa1f359a706681 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 13 Apr 2026 16:28:10 -0700 Subject: [PATCH 42/70] Add PointIndexCodec Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 1 + openvdb/openvdb/codecs/PointIndexCodec.h | 157 +++++++++++++++++++++ openvdb/openvdb/io/Codec.cc | 2 + openvdb/openvdb/unittest/CMakeLists.txt | 1 + openvdb/openvdb/unittest/TestPointCodec.cc | 147 +++++++++++++++++++ 5 files changed, 308 insertions(+) create mode 100644 openvdb/openvdb/codecs/PointIndexCodec.h create mode 100644 openvdb/openvdb/unittest/TestPointCodec.cc diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 3572583e20..e9437b86de 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -367,6 +367,7 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES codecs/BoolCodec.h + codecs/PointIndexCodec.h codecs/ScalarCodec.h codecs/TopologyCodec.h codecs/ValueMaskCodec.h diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h new file mode 100644 index 0000000000..d8d9f1ee32 --- /dev/null +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -0,0 +1,157 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED + +#include + +#include + +#include "ScalarLeafCodec.h" +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct ReadPointIndexBuffersOp +{ + using TreeT = typename GridT::TreeType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + ReadPointIndexBuffersOp(std::istream& _is, bool _saveFloatAsHalf, + const ValueT& _background) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) { } + + template + void operator()(NodeT&, size_t) { } + + void operator()(LeafT& leaf, size_t) + { + using BaseLeaf = typename LeafT::BaseLeaf; + + // Read the value mask and voxel data via base class + BaseLeaf& baseLeaf = static_cast(leaf); + readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background, /*skip=*/false, /*clipBBox=*/nullptr); + + // Read the number of indices. + Index64 numIndices = Index64(0); + is.read(reinterpret_cast(&numIndices), sizeof(Index64)); + + // Read the indices data. + leaf.indices().resize(size_t(numIndices)); + is.read(reinterpret_cast(leaf.indices().data()), numIndices * sizeof(ValueT)); + + // Reserved for future use. + Index64 auxDataBytes = Index64(0); + is.read(reinterpret_cast(&auxDataBytes), sizeof(Index64)); + if (auxDataBytes > 0) { + // For now, read and discard any auxiliary data. + std::unique_ptr auxData{new char[auxDataBytes]}; + is.read(auxData.get(), auxDataBytes); + } + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; +}; // struct ReadPointIndexBuffersOp + +template +struct WritePointIndexBuffersOp +{ + using TreeT = typename GridT::TreeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + WritePointIndexBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + using BaseLeaf = typename LeafT::BaseLeaf; + + // Write out the value mask and voxel values via base class + const BaseLeaf& baseLeaf = static_cast(leaf); + writeScalarLeafBuffers(baseLeaf, os, saveFloatAsHalf); + + // Write the number of indices. + Index64 numIndices = Index64(leaf.indices().size()); + os.write(reinterpret_cast(&numIndices), sizeof(Index64)); + + // Write the indices data. + os.write(reinterpret_cast(leaf.indices().data()), numIndices * sizeof(ValueT)); + + // Reserved for future use. + const Index64 auxDataBytes = Index64(0); + os.write(reinterpret_cast(&auxDataBytes), sizeof(Index64)); + } + + std::ostream& os; + const bool saveFloatAsHalf; +}; // struct WritePointIndexBuffersOp + +} // namespace internal + +template +struct PointIndexCodec : public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~PointIndexCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final + { + GridT& grid = static_cast(*data.grid); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in PointIndexCodec"); + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + if (options.clipBBox.isSorted()) { + diagnostics.addWarning(grid.getName(), "bounding box clipping is not supported for PointIndexGrids"); + } + + internal::ReadPointIndexBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background()); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + const GridT& grid = static_cast(gridBase); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in PointIndexCodec"); + } + + internal::WritePointIndexBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); + } +}; // struct PointIndexCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc index cfb1489efc..3e437fab17 100644 --- a/openvdb/openvdb/io/Codec.cc +++ b/openvdb/openvdb/io/Codec.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -58,6 +59,7 @@ void initialize() CodecRegistry::registerCodec>(); CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); // register the plugin that converts from scalar to mask/bool NumericGridTypes::foreach(); diff --git a/openvdb/openvdb/unittest/CMakeLists.txt b/openvdb/openvdb/unittest/CMakeLists.txt index 18046cd8e0..446d118027 100644 --- a/openvdb/openvdb/unittest/CMakeLists.txt +++ b/openvdb/openvdb/unittest/CMakeLists.txt @@ -146,6 +146,7 @@ else() TestParticlesToLevelSet.cc TestPointAdvect.cc TestPointAttribute.cc + TestPointCodec.cc TestPointConversion.cc TestPointCount.cc TestPointDataLeaf.cc diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc new file mode 100644 index 0000000000..2b8911419e --- /dev/null +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -0,0 +1,147 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include "util.h" // for unittest_util::genPoints + +namespace { + +class PointList { +public: + using PosType = openvdb::Vec3R; + PointList(const std::vector& points) : mPoints(&points) {} + size_t size() const { return mPoints->size(); } + void getPos(size_t n, PosType& xyz) const { xyz = (*mPoints)[n]; } +private: + std::vector const * const mPoints; +}; + +} // namespace + +class TestPointCodec: public ::testing::Test +{ +}; + +TEST_F(TestPointCodec, testPointIndexCodecIO) +{ + using namespace openvdb; + using namespace openvdb::io; + using PointIndexGrid = tools::PointIndexGrid; + + openvdb::initialize(); + CodecRegistry::clear(); + + // Generate points on a unit sphere and build a PointIndexGrid + std::vector points; + unittest_util::genPoints(100, points); + PointList pointList(points); + + const double voxelSize = 0.1; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointIndexGrid::Ptr srcGrid = + tools::createPointIndexGrid(pointList, *transform); + srcGrid->setName("point_index_grid"); + + const std::string rawPath = "testPointIndexCodec_raw.vdb"; + + // Phase 1: write/read without codec + { + io::File f(rawPath); + f.write(GridPtrVec{srcGrid}); + } + + PointIndexGrid::Ptr rawGrid; + { + io::File f(rawPath); + f.open(); + rawGrid = gridPtrCast(f.readGrid("point_index_grid")); + f.close(); + } + ASSERT_TRUE(rawGrid); + + PointIndexGrid::Ptr rawTopo; + { + io::File f(rawPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("point_index_grid")); + rawTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(rawTopo); + EXPECT_EQ(rawTopo->activeVoxelCount(), Index64(97)); + EXPECT_EQ(rawTopo->getName(), std::string("point_index_grid")); + + std::remove(rawPath.c_str()); + + const std::string codecPath = "testPointIndexCodec_codec.vdb"; + + // Phase 2: register codec, write/read with codec + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointIndexGrid::gridType())); + + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + PointIndexGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + codecGrid = gridPtrCast(f.readGrid("point_index_grid")); + f.close(); + } + ASSERT_TRUE(codecGrid); + + // Phase 3: full read comparison + EXPECT_TRUE(srcGrid->tree().hasSameTopology(srcGrid->tree())); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree())); + { + auto codecAcc = codecGrid->getConstAccessor(); + for (PointIndexGrid::ValueOnCIter it = srcGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, codecAcc.getValue(it.getCoord())); + } + } + + // Compare leaf indices arrays + { + auto srcLeafIt = srcGrid->tree().cbeginLeaf(); + auto codecLeafIt = codecGrid->tree().cbeginLeaf(); + for (; srcLeafIt; ++srcLeafIt, ++codecLeafIt) { + ASSERT_TRUE(codecLeafIt); + EXPECT_EQ(srcLeafIt->indices().size(), codecLeafIt->indices().size()); + for (size_t i = 0; i < srcLeafIt->indices().size(); ++i) { + EXPECT_EQ(srcLeafIt->indices()[i], codecLeafIt->indices()[i]); + } + } + EXPECT_TRUE(!codecLeafIt); + } + + // Phase 4: TopologyOnly read + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + PointIndexGrid::Ptr codecTopo; + { + io::File f(codecPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("point_index_grid", topoOpts)); + codecTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(codecTopo); + EXPECT_EQ(codecTopo->activeVoxelCount(), Index64(0)); + EXPECT_TRUE(codecTopo->tree().leafCount() == 0); + EXPECT_EQ(codecTopo->getName(), std::string("point_index_grid")); + + // Cleanup + CodecRegistry::clear(); + std::remove(codecPath.c_str()); +} From 000fb2473fe385ebfd40de143c4c4fea4fb01a99 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 17 Feb 2026 11:24:26 -0800 Subject: [PATCH 43/70] Move PointData I/O overloads to a new header Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 1 + openvdb/openvdb/points/PointDataGrid.h | 130 +--------------------- openvdb/openvdb/points/PointDataIO.h | 148 +++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 128 deletions(-) create mode 100644 openvdb/openvdb/points/PointDataIO.h diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index e9437b86de..97317b1fb7 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -426,6 +426,7 @@ set(OPENVDB_LIBRARY_POINTS_INCLUDE_FILES points/PointConversion.h points/PointCount.h points/PointDataGrid.h + points/PointDataIO.h points/PointDelete.h points/PointGroup.h points/PointMask.h diff --git a/openvdb/openvdb/points/PointDataGrid.h b/openvdb/openvdb/points/PointDataGrid.h index fcb10bdb2a..946b5764c3 100644 --- a/openvdb/openvdb/points/PointDataGrid.h +++ b/openvdb/openvdb/points/PointDataGrid.h @@ -31,140 +31,14 @@ #include // std::pair, std::make_pair #include +#include // io::readCompressedValues(), io::writeCompressedValues(), io::writeCompressedValuesSize() + class TestPointDataLeaf; namespace openvdb { OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { -namespace io -{ - -/// @brief openvdb::io::readCompressedValues specialized on PointDataIndex32 arrays to -/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit -template<> -inline void -readCompressedValues( std::istream& is, PointDataIndex32* destBuf, Index destCount, - const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/) -{ - using compression::bloscDecompress; - - const bool seek = destBuf == nullptr; - - const size_t destBytes = destCount*sizeof(PointDataIndex32); - const size_t maximumBytes = std::numeric_limits::max(); - if (destBytes >= maximumBytes) { - OPENVDB_THROW(openvdb::IoError, "Cannot read more than " << - maximumBytes << " bytes in voxel values.") - } - - uint16_t bytes16; - - const io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is); - - if (seek && meta) { - // buffer size temporarily stored in the StreamMetadata pass - // to avoid having to perform an expensive disk read for 2-bytes - bytes16 = static_cast(meta->pass()); - // seek over size of the compressed buffer - is.seekg(sizeof(uint16_t), std::ios_base::cur); - } - else { - // otherwise read from disk - is.read(reinterpret_cast(&bytes16), sizeof(uint16_t)); - } - - if (bytes16 == std::numeric_limits::max()) { - // read or seek uncompressed data - if (seek) { - is.seekg(destBytes, std::ios_base::cur); - } - else { - is.read(reinterpret_cast(destBuf), destBytes); - } - } - else { - // read or seek uncompressed data - if (seek) { - is.seekg(int(bytes16), std::ios_base::cur); - } - else { - // decompress into the destination buffer - std::unique_ptr bloscBuffer(new char[int(bytes16)]); - is.read(bloscBuffer.get(), bytes16); - std::unique_ptr buffer = bloscDecompress( bloscBuffer.get(), - destBytes, - /*resize=*/false); - std::memcpy(destBuf, buffer.get(), destBytes); - } - } -} - -/// @brief openvdb::io::writeCompressedValues specialized on PointDataIndex32 arrays to -/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit -template<> -inline void -writeCompressedValues( std::ostream& os, const PointDataIndex32* srcBuf, Index srcCount, - const util::NodeMask<3>& /*valueMask*/, - const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/) -{ - using compression::bloscCompress; - - const size_t srcBytes = srcCount*sizeof(PointDataIndex32); - const size_t maximumBytes = std::numeric_limits::max(); - if (srcBytes >= maximumBytes) { - OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << - maximumBytes << " bytes in voxel values.") - } - - const char* charBuffer = reinterpret_cast(srcBuf); - - size_t compressedBytes; - std::unique_ptr buffer = bloscCompress( charBuffer, srcBytes, - compressedBytes, /*resize=*/false); - - if (compressedBytes > 0) { - auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - os.write(reinterpret_cast(buffer.get()), compressedBytes); - } - else { - auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - os.write(reinterpret_cast(srcBuf), srcBytes); - } -} - -template -inline void -writeCompressedValuesSize(std::ostream& os, const T* srcBuf, Index srcCount) -{ - using compression::bloscCompressedSize; - - const size_t srcBytes = srcCount*sizeof(T); - const size_t maximumBytes = std::numeric_limits::max(); - if (srcBytes >= maximumBytes) { - OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << - maximumBytes << " bytes in voxel values.") - } - - const char* charBuffer = reinterpret_cast(srcBuf); - - // calculate voxel buffer size after compression - size_t compressedBytes = bloscCompressedSize(charBuffer, srcBytes); - - if (compressedBytes > 0) { - auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - } - else { - auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - } -} - -} // namespace io - // forward declaration namespace tree { diff --git a/openvdb/openvdb/points/PointDataIO.h b/openvdb/openvdb/points/PointDataIO.h new file mode 100644 index 0000000000..fddbaef6c4 --- /dev/null +++ b/openvdb/openvdb/points/PointDataIO.h @@ -0,0 +1,148 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED +#define OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED + + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { + + +//////////////////////////////////////// + + +namespace io +{ + +/// @brief openvdb::io::readCompressedValues specialized on PointDataIndex32 arrays to +/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit +template<> +inline void +readCompressedValues( std::istream& is, PointDataIndex32* destBuf, Index destCount, + const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/) +{ + using compression::bloscDecompress; + + const bool seek = destBuf == nullptr; + + const size_t destBytes = destCount*sizeof(PointDataIndex32); + const size_t maximumBytes = std::numeric_limits::max(); + if (destBytes >= maximumBytes) { + OPENVDB_THROW(openvdb::IoError, "Cannot read more than " << + maximumBytes << " bytes in voxel values.") + } + + uint16_t bytes16; + + const io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is); + + if (seek && meta) { + // buffer size temporarily stored in the StreamMetadata pass + // to avoid having to perform an expensive disk read for 2-bytes + bytes16 = static_cast(meta->pass()); + // seek over size of the compressed buffer + is.seekg(sizeof(uint16_t), std::ios_base::cur); + } + else { + // otherwise read from disk + is.read(reinterpret_cast(&bytes16), sizeof(uint16_t)); + } + + if (bytes16 == std::numeric_limits::max()) { + // read or seek uncompressed data + if (seek) { + is.seekg(destBytes, std::ios_base::cur); + } + else { + is.read(reinterpret_cast(destBuf), destBytes); + } + } + else { + // read or seek uncompressed data + if (seek) { + is.seekg(int(bytes16), std::ios_base::cur); + } + else { + // decompress into the destination buffer + std::unique_ptr bloscBuffer(new char[int(bytes16)]); + is.read(bloscBuffer.get(), bytes16); + std::unique_ptr buffer = bloscDecompress( bloscBuffer.get(), + destBytes, + /*resize=*/false); + std::memcpy(destBuf, buffer.get(), destBytes); + } + } +} + +/// @brief openvdb::io::writeCompressedValues specialized on PointDataIndex32 arrays to +/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit +template<> +inline void +writeCompressedValues( std::ostream& os, const PointDataIndex32* srcBuf, Index srcCount, + const util::NodeMask<3>& /*valueMask*/, + const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/) +{ + using compression::bloscCompress; + + const size_t srcBytes = srcCount*sizeof(PointDataIndex32); + const size_t maximumBytes = std::numeric_limits::max(); + if (srcBytes >= maximumBytes) { + OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << + maximumBytes << " bytes in voxel values.") + } + + const char* charBuffer = reinterpret_cast(srcBuf); + + size_t compressedBytes; + std::unique_ptr buffer = bloscCompress( charBuffer, srcBytes, + compressedBytes, /*resize=*/false); + + if (compressedBytes > 0) { + auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + os.write(reinterpret_cast(buffer.get()), compressedBytes); + } + else { + auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + os.write(reinterpret_cast(srcBuf), srcBytes); + } +} + +template +inline void +writeCompressedValuesSize(std::ostream& os, const T* srcBuf, Index srcCount) +{ + using compression::bloscCompressedSize; + + const size_t srcBytes = srcCount*sizeof(T); + const size_t maximumBytes = std::numeric_limits::max(); + if (srcBytes >= maximumBytes) { + OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << + maximumBytes << " bytes in voxel values.") + } + + const char* charBuffer = reinterpret_cast(srcBuf); + + // calculate voxel buffer size after compression + size_t compressedBytes = bloscCompressedSize(charBuffer, srcBytes); + + if (compressedBytes > 0) { + auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + } + else { + auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + } +} + +} // namespace io + + +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED From 6528cb82eb0691514b60f74ac4127026b53d4ee6 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 18 Mar 2026 14:18:25 -0700 Subject: [PATCH 44/70] Remove support for multi-pass I/O except PointDataLeafNode Signed-off-by: Dan Bailey --- openvdb/openvdb/Grid.h | 30 +++- openvdb/openvdb/io/io.h | 4 +- openvdb/openvdb/unittest/TestFile.cc | 240 --------------------------- 3 files changed, 27 insertions(+), 247 deletions(-) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index b364e9efc0..b2f1dd8ecd 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -1181,14 +1181,34 @@ struct TreeAdapter > //////////////////////////////////////// +namespace points { + +template class PointDataLeafNode; + +/// @brief Type trait that evaluates to true only for @c PointDataLeafNode instantiations. +template +struct IsPointDataLeafNode : std::false_type {}; + +template +struct IsPointDataLeafNode> : std::true_type {}; + +} // namespace points + + /// @brief Metafunction that specifies whether a given leaf node, tree, or grid type -/// requires multiple passes to read and write voxel data -/// @details Multi-pass I/O allows one to optimize the data layout of leaf nodes -/// for certain access patterns during delayed loading. -/// @sa io::MultiPass +/// requires multiple passes to read and write voxel data. +/// @details Multi-pass I/O allows leaf nodes to optimize their serialization layout +/// for delayed-load access patterns. Only @c PointDataLeafNode supports multi-pass I/O. +/// Defining a custom leaf node that inherits @c io::MultiPass is no longer permitted. +/// @sa points::IsPointDataLeafNode template struct HasMultiPassIO { - static const bool value = std::is_base_of::value; + static_assert( + !std::is_base_of::value + || points::IsPointDataLeafNode::value, + "Only PointDataLeafNode may inherit from io::MultiPass; " + "use points::IsPointDataLeafNode to test for multi-pass I/O support."); + static const bool value = points::IsPointDataLeafNode::value; }; // Partial specialization for Tree types diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index f2ac342ff7..96d2408eb3 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -116,8 +116,8 @@ std::ostream& operator<<(std::ostream&, const StreamMetadata::AuxDataMap&); //////////////////////////////////////// -/// @brief Leaf nodes that require multi-pass I/O must inherit from this struct. -/// @sa Grid::hasMultiPassIO() +/// @brief Tag for multi-pass I/O. Only points::PointDataLeafNode may inherit from this. +/// @sa Grid::hasMultiPassIO(), points::IsPointDataLeafNode struct MultiPass {}; diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index fad51e9567..130947fdf1 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -1522,246 +1522,6 @@ TEST_F(TestFile, testReadClippedGrid) //////////////////////////////////////// -namespace { - -template struct MultiPassLeafNode; // forward declaration - -// Dummy value type -using MultiPassValue = openvdb::PointIndex; - -// Tree configured to match the default OpenVDB configuration -using MultiPassTree = openvdb::tree::Tree< - openvdb::tree::RootNode< - openvdb::tree::InternalNode< - openvdb::tree::InternalNode< - MultiPassLeafNode, 4>, 5>>>; - -using MultiPassGrid = openvdb::Grid; - - -template -struct MultiPassLeafNode: public openvdb::tree::LeafNode, openvdb::io::MultiPass -{ - // The following had to be copied from the LeafNode class - // to make the derived class compatible with the tree structure. - - using LeafNodeType = MultiPassLeafNode; - using Ptr = openvdb::SharedPtr; - using BaseLeaf = openvdb::tree::LeafNode; - using NodeMaskType = openvdb::util::NodeMask; - using ValueType = T; - using ValueOnCIter = typename BaseLeaf::template ValueIter; - using ChildOnIter = typename BaseLeaf::template ChildIter; - using ChildOnCIter = typename BaseLeaf::template ChildIter< - typename NodeMaskType::OnIterator, const MultiPassLeafNode, typename BaseLeaf::ChildOn>; - - MultiPassLeafNode(const openvdb::Coord& coords, const T& value, bool active = false) - : BaseLeaf(coords, value, active) {} - MultiPassLeafNode(openvdb::PartialCreate, const openvdb::Coord& coords, const T& value, - bool active = false): BaseLeaf(openvdb::PartialCreate(), coords, value, active) {} - MultiPassLeafNode(const MultiPassLeafNode& rhs): BaseLeaf(rhs) {} - - ValueOnCIter cbeginValueOn() const { return ValueOnCIter(this->getValueMask().beginOn(),this); } - ChildOnCIter cbeginChildOn() const { return ChildOnCIter(this->getValueMask().endOn(), this); } - ChildOnIter beginChildOn() { return ChildOnIter(this->getValueMask().endOn(), this); } - - // Methods in use for reading and writing multiple buffers - - void readBuffers(std::istream& is, const openvdb::CoordBBox&, bool fromHalf = false) - { - this->readBuffers(is, fromHalf); - } - - void readBuffers(std::istream& is, bool /*fromHalf*/ = false) - { - const openvdb::io::StreamMetadata::Ptr meta = openvdb::io::getStreamMetadataPtr(is); - if (!meta) { - OPENVDB_THROW(openvdb::IoError, - "Cannot write out a MultiBufferLeaf without StreamMetadata."); - } - - // clamp pass to 16-bit integer - const uint32_t pass(static_cast(meta->pass())); - - // Read in the stored pass number. - uint32_t readPass; - is.read(reinterpret_cast(&readPass), sizeof(uint32_t)); - EXPECT_EQ(pass, readPass); - // Record the pass number. - mReadPasses.push_back(readPass); - - if (pass == 0) { - // Read in the node's origin. - openvdb::Coord origin; - is.read(reinterpret_cast(&origin), sizeof(openvdb::Coord)); - EXPECT_EQ(origin, this->origin()); - } - } - - void writeBuffers(std::ostream& os, bool /*toHalf*/ = false) const - { - const openvdb::io::StreamMetadata::Ptr meta = openvdb::io::getStreamMetadataPtr(os); - if (!meta) { - OPENVDB_THROW(openvdb::IoError, - "Cannot read in a MultiBufferLeaf without StreamMetadata."); - } - - // clamp pass to 16-bit integer - const uint32_t pass(static_cast(meta->pass())); - - // Leaf traversal analysis deduces the number of passes to perform for this leaf - // then updates the leaf traversal value to ensure all passes will be written. - if (meta->countingPasses()) { - if (mNumPasses > pass) meta->setPass(mNumPasses); - return; - } - - // Record the pass number. - EXPECT_TRUE(mWritePassesPtr); - const_cast&>(*mWritePassesPtr).push_back(pass); - - // Write out the pass number. - os.write(reinterpret_cast(&pass), sizeof(uint32_t)); - if (pass == 0) { - // Write out the node's origin and the pass number. - const auto origin = this->origin(); - os.write(reinterpret_cast(&origin), sizeof(openvdb::Coord)); - } - } - - - uint32_t mNumPasses = 0; - // Pointer to external vector in which to record passes as they are written - std::vector* mWritePassesPtr = nullptr; - // Vector in which to record passes as they are read - // (this needs to be internal, because leaf nodes are constructed as a grid is read) - std::vector mReadPasses; -}; // struct MultiPassLeafNode - -} // anonymous namespace - - -TEST_F(TestFile, testMultiPassIO) -{ - using namespace openvdb; - - openvdb::initialize(); - MultiPassGrid::registerGrid(); - - // Create a multi-buffer grid. - const MultiPassGrid::Ptr grid = openvdb::createGrid(); - grid->setName("test"); - grid->setTransform(math::Transform::createLinearTransform(1.0)); - MultiPassGrid::TreeType& tree = grid->tree(); - tree.setValue(Coord(0, 0, 0), 5); - tree.setValue(Coord(0, 10, 0), 5); - EXPECT_EQ(2, int(tree.leafCount())); - - const GridPtrVec grids{grid}; - - // Vector in which to record pass numbers (to ensure blocked ordering) - std::vector writePasses; - { - // Specify the required number of I/O passes for each leaf node. - MultiPassGrid::TreeType::LeafIter leafIter = tree.beginLeaf(); - leafIter->mNumPasses = 3; - leafIter->mWritePassesPtr = &writePasses; - ++leafIter; - leafIter->mNumPasses = 2; - leafIter->mWritePassesPtr = &writePasses; - } - - const char* filename = "testMultiPassIO.vdb"; - SharedPtr scopedFile(filename, ::remove); - { - // Verify that passes are written to a file in the correct order. - io::File(filename).write(grids); - EXPECT_EQ(6, int(writePasses.size())); - EXPECT_EQ(0, writePasses[0]); // leaf 0 - EXPECT_EQ(0, writePasses[1]); // leaf 1 - EXPECT_EQ(1, writePasses[2]); // leaf 0 - EXPECT_EQ(1, writePasses[3]); // leaf 1 - EXPECT_EQ(2, writePasses[4]); // leaf 0 - EXPECT_EQ(2, writePasses[5]); // leaf 1 - } - { - // Verify that passes are read in the correct order. - io::File file(filename); - file.open(); - const auto newGrid = GridBase::grid(file.readGrid("test")); - - auto leafIter = newGrid->tree().beginLeaf(); - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - ++leafIter; - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - } - { - // Verify that when using multi-pass and bbox clipping that each leaf node - // is still being read before being clipped - io::File file(filename); - file.open(); - const auto newGrid = GridBase::grid( - file.readGrid("test", BBoxd(Vec3d(0), Vec3d(1)))); - EXPECT_EQ(Index64(1), newGrid->tree().leafCount()); - - auto leafIter = newGrid->tree().beginLeaf(); - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - ++leafIter; - EXPECT_TRUE(!leafIter); // second leaf node has now been clipped - } - - // Clear the pass data. - writePasses.clear(); - - { - // Verify that passes are written to and read from a non-seekable stream - // in the correct order. - std::ostringstream ostr(std::ios_base::binary); - io::Stream(ostr).write(grids); - - EXPECT_EQ(6, int(writePasses.size())); - EXPECT_EQ(0, writePasses[0]); // leaf 0 - EXPECT_EQ(0, writePasses[1]); // leaf 1 - EXPECT_EQ(1, writePasses[2]); // leaf 0 - EXPECT_EQ(1, writePasses[3]); // leaf 1 - EXPECT_EQ(2, writePasses[4]); // leaf 0 - EXPECT_EQ(2, writePasses[5]); // leaf 1 - - std::istringstream is(ostr.str(), std::ios_base::binary); - io::Stream strm(is); - const auto streamedGrids = strm.getGrids(); - EXPECT_EQ(1, int(streamedGrids->size())); - - const auto newGrid = gridPtrCast(*streamedGrids->begin()); - EXPECT_TRUE(bool(newGrid)); - auto leafIter = newGrid->tree().beginLeaf(); - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - ++leafIter; - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - } -} - - -//////////////////////////////////////// - - TEST_F(TestFile, testHasGrid) { using namespace openvdb; From 0b95d2e9ca190858b4b7fb23b963860f138f8eeb Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 18 Mar 2026 14:16:25 -0700 Subject: [PATCH 45/70] Remove AttributeArray out-of-core atomic Signed-off-by: Dan Bailey --- openvdb/openvdb/points/AttributeArray.cc | 2 -- openvdb/openvdb/points/AttributeArray.h | 8 +++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/openvdb/openvdb/points/AttributeArray.cc b/openvdb/openvdb/points/AttributeArray.cc index 7d83051b52..19e6d14436 100644 --- a/openvdb/openvdb/points/AttributeArray.cc +++ b/openvdb/openvdb/points/AttributeArray.cc @@ -61,7 +61,6 @@ AttributeArray::AttributeArray(const AttributeArray& rhs, const tbb::spin_mutex: : mIsUniform(rhs.mIsUniform) , mFlags(rhs.mFlags) , mUsePagedRead(rhs.mUsePagedRead) - , mOutOfCore(rhs.mOutOfCore.load()) , mPageHandle() { if (mFlags & PARTIALREAD) mCompressedBytes = rhs.mCompressedBytes; @@ -78,7 +77,6 @@ AttributeArray::operator=(const AttributeArray& rhs) mIsUniform = rhs.mIsUniform; mFlags = rhs.mFlags; mUsePagedRead = rhs.mUsePagedRead; - mOutOfCore.store(rhs.mOutOfCore); if (mFlags & PARTIALREAD) mCompressedBytes = rhs.mCompressedBytes; else if (rhs.mPageHandle) mPageHandle = rhs.mPageHandle->copy(); else mPageHandle.reset(); diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index bbfba7b156..3ad2d3a48d 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -363,10 +363,12 @@ class OPENVDB_API AttributeArray mutable tbb::spin_mutex mMutex; uint8_t mFlags = 0; uint8_t mUsePagedRead = 0; +#if OPENVDB_ABI_VERSION_NUMBER < 14 std::atomic mOutOfCore{0}; // interpreted as bool +#endif /// used for out-of-core, paged reading union { - compression::PageHandle::Ptr mPageHandle; + std::unique_ptr mPageHandle; size_t mCompressedBytes; }; }; // class AttributeArray @@ -1535,8 +1537,6 @@ TypedAttributeArray::readBuffers(std::istream& is) OPENVDB_THROW(IoError, "Cannot read paged AttributeArray buffers."); } - tbb::spin_mutex::scoped_lock lock(mMutex); - this->deallocate(); uint8_t bloscCompressed(0); @@ -1586,8 +1586,6 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu OPENVDB_ASSERT(mPageHandle); - tbb::spin_mutex::scoped_lock lock(mMutex); - this->deallocate(); is.read(mPageHandle, std::streamsize(mPageHandle->size()), false); From a9a6cff2ab5a8c248c0319ee2d2545c20ac28157 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 27 Mar 2026 14:36:30 -0700 Subject: [PATCH 46/70] Rename MultiPass tag class to PointDataGridMultiPass Signed-off-by: Dan Bailey --- openvdb/openvdb/Grid.h | 6 +++--- openvdb/openvdb/io/io.h | 2 +- openvdb/openvdb/points/PointDataGrid.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index b2f1dd8ecd..41f6e58501 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -1199,14 +1199,14 @@ struct IsPointDataLeafNode> : std::true_type {}; /// requires multiple passes to read and write voxel data. /// @details Multi-pass I/O allows leaf nodes to optimize their serialization layout /// for delayed-load access patterns. Only @c PointDataLeafNode supports multi-pass I/O. -/// Defining a custom leaf node that inherits @c io::MultiPass is no longer permitted. +/// Defining a custom leaf node that inherits @c io::PointDataGridMultiPass is no longer permitted. /// @sa points::IsPointDataLeafNode template struct HasMultiPassIO { static_assert( - !std::is_base_of::value + !std::is_base_of::value || points::IsPointDataLeafNode::value, - "Only PointDataLeafNode may inherit from io::MultiPass; " + "Only PointDataLeafNode may inherit from io::PointDataGridMultiPass; " "use points::IsPointDataLeafNode to test for multi-pass I/O support."); static const bool value = points::IsPointDataLeafNode::value; }; diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index 96d2408eb3..13a9134012 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -118,7 +118,7 @@ std::ostream& operator<<(std::ostream&, const StreamMetadata::AuxDataMap&); /// @brief Tag for multi-pass I/O. Only points::PointDataLeafNode may inherit from this. /// @sa Grid::hasMultiPassIO(), points::IsPointDataLeafNode -struct MultiPass {}; +struct PointDataGridMultiPass {}; //////////////////////////////////////// diff --git a/openvdb/openvdb/points/PointDataGrid.h b/openvdb/openvdb/points/PointDataGrid.h index 946b5764c3..f186717799 100644 --- a/openvdb/openvdb/points/PointDataGrid.h +++ b/openvdb/openvdb/points/PointDataGrid.h @@ -107,7 +107,7 @@ prefetch(PointDataTreeT&, bool /*position*/ = true, bool /*otherAttributes*/ = t template -class PointDataLeafNode : public tree::LeafNode, io::MultiPass { +class PointDataLeafNode : public tree::LeafNode, io::PointDataGridMultiPass { public: using LeafNodeType = PointDataLeafNode; From 646cdaa609ae204d4095d1823a901a8c32d182b8 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 20 Apr 2026 11:25:29 -0700 Subject: [PATCH 47/70] Add PointDataCodec Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 1 + openvdb/openvdb/codecs/PointDataCodec.h | 449 ++++++++++++++++++++ openvdb/openvdb/io/Codec.cc | 2 + openvdb/openvdb/points/AttributeArray.h | 40 ++ openvdb/openvdb/points/StreamCompression.cc | 32 ++ openvdb/openvdb/points/StreamCompression.h | 8 + openvdb/openvdb/unittest/TestPointCodec.cc | 400 +++++++++++++++++ 7 files changed, 932 insertions(+) create mode 100644 openvdb/openvdb/codecs/PointDataCodec.h diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 97317b1fb7..19172ca380 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -367,6 +367,7 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES codecs/BoolCodec.h + codecs/PointDataCodec.h codecs/PointIndexCodec.h codecs/ScalarCodec.h codecs/TopologyCodec.h diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h new file mode 100644 index 0000000000..e5cc550fe4 --- /dev/null +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -0,0 +1,449 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED + +#include + +#include + +#include +#include +#include + +#include "ScalarLeafCodec.h" +#include "TopologyCodec.h" + +#include +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +/// Look up an existing paged stream or create a new one for the given attribute index +template +inline typename PagedStreamPtrT::element_type* getOrCreatePagedStream( + std::map& pagedStreams, Index attributeIndex) +{ + auto it = pagedStreams.find(attributeIndex); + if (it != pagedStreams.end()) return it->second.get(); + using PagedStreamT = typename PagedStreamPtrT::element_type; + pagedStreams[attributeIndex] = std::make_shared(); + return pagedStreams[attributeIndex].get(); +} + +//////////////////////////////////////// +// Read-side functions + +template +inline void readPointDataVoxelSizes(const std::vector& leaves, + std::istream& is, std::map& voxelBufferSizes) +{ + for (auto* leaf : leaves) { + uint16_t voxelBufferSize; + is.read(reinterpret_cast(&voxelBufferSize), sizeof(uint16_t)); + voxelBufferSizes[leaf->origin()] = voxelBufferSize; + } +} + +template +inline void readPointDataDescriptors(const std::vector& leaves, + std::istream& is) +{ + points::AttributeSet::Descriptor::Ptr sharedDescriptor; + for (auto* leaf : leaves) { + points::AttributeSet::UniquePtr attrSet = leaf->stealAttributeSet(); + if (sharedDescriptor) { + // Reuse shared descriptor from first leaf + attrSet->resetDescriptor(sharedDescriptor, /*allowMismatchingDescriptors=*/true); + } + else { + uint8_t header; + is.read(reinterpret_cast(&header), sizeof(uint8_t)); + attrSet->readDescriptor(is); + if (header & uint8_t(1)) { + // Store descriptor for subsequent leaves + sharedDescriptor = attrSet->descriptorPtr(); + } + // a forwards-compatibility mechanism for future use, + // if a 0x2 bit is set, read and skip over a specific number of bytes + if (header & uint8_t(2)) { + uint64_t bytesToSkip; + is.read(reinterpret_cast(&bytesToSkip), sizeof(uint64_t)); + if (bytesToSkip > uint64_t(0)) { + std::vector tempData(bytesToSkip); + is.read(reinterpret_cast(&tempData[0]), bytesToSkip); + } + } + // this reader is only able to read headers with 0x1 and 0x2 bits set + if (header > uint8_t(3)) { + OPENVDB_THROW(IoError, "Unrecognised header flags in PointDataLeafNode"); + } + } + attrSet->readMetadata(is); + leaf->replaceAttributeSet(attrSet.release(), /*allowMismatchingDescriptors=*/true); + } +} + +template +inline void readPointDataAttributeSizes(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(true); + array->readPagedBuffers(*pagedStream); + } + } +} + +template +inline void readPointDataVoxelData(const std::vector& leaves, + std::istream& is, bool saveFloatAsHalf, + const typename LeafT::ValueType& background, + const std::map& voxelBufferSizes) +{ + (void) voxelBufferSizes; + using BaseLeaf = typename LeafT::BaseLeaf; + for (auto* leaf : leaves) { + OPENVDB_ASSERT(voxelBufferSizes.find(leaf->origin()) != voxelBufferSizes.end()); + BaseLeaf& baseLeaf = static_cast(*leaf); + readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background); + } +} + +template +inline void readPointDataAttributeData(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(false); + array->readPagedBuffers(*pagedStream); + } + } +} + +template +inline void skipPointDataAttributeSizes(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(true); + array->skipPagedBuffers(*pagedStream); + } + } +} + +template +inline void skipPointDataAttributeData(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(false); + array->skipPagedBuffers(*pagedStream); + } + } +} + +//////////////////////////////////////// +// Write-side functions + +template +inline Index countPointDataPasses(const std::vector& leaves) +{ + Index maxRequiredPasses = 0; + for (const auto* leaf : leaves) { + const Index requiredPasses = leaf->buffers(); + if (requiredPasses > maxRequiredPasses) { + maxRequiredPasses = requiredPasses; + } + } + return maxRequiredPasses; +} + +template +inline void writePointDataVoxelSizes(const std::vector& leaves, + std::ostream& os, bool& matching, + points::AttributeSet::Descriptor::Ptr& sharedDescriptor) +{ + bool descriptorChecked = false; + matching = true; + for (const auto* leaf : leaves) { + io::writeCompressedValuesSize(os, leaf->buffer().data(), LeafT::SIZE); + + // Track descriptor matching + const auto& descriptor = leaf->attributeSet().descriptorPtr(); + if (!descriptorChecked) { + // First leaf - store descriptor + descriptorChecked = true; + sharedDescriptor = descriptor; + } + else if (matching && *sharedDescriptor != *descriptor) { + matching = false; + } + } +} + +template +inline void writePointDataDescriptors(const std::vector& leaves, + std::ostream& os, bool matching, + const points::AttributeSet::Descriptor::Ptr&) +{ + bool firstWrite = true; + for (const auto* leaf : leaves) { + const points::AttributeSet& attributeSet = leaf->attributeSet(); + if (matching) { + // Shared descriptor - only write on first leaf + if (firstWrite) { + firstWrite = false; + uint8_t header(1); + os.write(reinterpret_cast(&header), sizeof(uint8_t)); + attributeSet.writeDescriptor(os, /*transient=*/false); + } + } + else { + // Non-shared descriptor - write on every leaf + uint8_t header(0); + os.write(reinterpret_cast(&header), sizeof(uint8_t)); + attributeSet.writeDescriptor(os, /*transient=*/false); + } + attributeSet.writeMetadata(os, /*transient=*/false, /*paged=*/true); + } +} + +template +inline void writePointDataAttributeSizes(const std::vector& leaves, + std::ostream& os, Index attributeIndex) +{ + std::map pagedStreams; + for (const auto* leaf : leaves) { + const points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + leaf->attributeSet().getConst(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setOutputStream(os); + pagedStream->setSizeOnly(true); + array->writePagedBuffers(*pagedStream, /*outputTransient*/false); + } + } + // Flush paged streams to write any remaining buffered page headers + for (auto& pair : pagedStreams) { + pair.second->flush(); + } +} + +template +inline void writePointDataVoxelData(const std::vector& leaves, + std::ostream& os, bool saveFloatAsHalf) +{ + using BaseLeaf = typename LeafT::BaseLeaf; + for (const auto* leaf : leaves) { + const BaseLeaf& baseLeaf = static_cast(*leaf); + writeScalarLeafBuffers(baseLeaf, os, saveFloatAsHalf); + } +} + +template +inline void writePointDataAttributeData(const std::vector& leaves, + std::ostream& os, Index attributeIndex) +{ + std::map pagedStreams; + for (const auto* leaf : leaves) { + const points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + leaf->attributeSet().getConst(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setOutputStream(os); + pagedStream->setSizeOnly(false); + array->writePagedBuffers(*pagedStream, /*outputTransient*/false); + } + } + // Flush paged streams to write any remaining buffered page data + for (auto& pair : pagedStreams) { + pair.second->flush(); + } +} + +} // namespace internal + +/// Per-grid-type codec-specific options for PointDataCodec +/// Contains point attribute filtering options +struct OPENVDB_API PointDataCodecTypeData : public io::ReadTypedOptions +{ + // Point Attribute Options - which attributes to read + std::vector pointAttributeNames; +}; // struct PointDataCodecTypeData + +template +struct PointDataCodec : public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~PointDataCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + { + GridT& grid = static_cast(*data.grid); + + std::vector pointAttributeNames; + + // Look up point-specific options if provided + auto it = options.typeData.find(name()); + if (it != options.typeData.end()) { + auto& pointTypeData = io::ReadTypedOptions::cast(it->second); + pointAttributeNames = pointTypeData.pointAttributeNames; + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + uint16_t numPasses = 1; + is.read(reinterpret_cast(&numPasses), sizeof(uint16_t)); + const Index attributes = (numPasses - 4) / 2; + + using LeafT = typename GridT::TreeType::LeafNodeType; + std::vector leaves; + tree.getNodes(leaves); + + // Pass 0: read voxel data sizes + std::map voxelBufferSizes; + internal::readPointDataVoxelSizes(leaves, is, voxelBufferSizes); + + // Pass 1: read descriptor and attribute metadata + internal::readPointDataDescriptors(leaves, is); + + // Build set of attribute indices to skip based on pointAttributeNames. + // An empty pointAttributeNames means no filtering (read all attributes). + std::set skipIndices; + if (!pointAttributeNames.empty() && !leaves.empty()) { + const auto& nameMap = leaves[0]->attributeSet().descriptor().map(); + const std::set wantedNames( + pointAttributeNames.begin(), + pointAttributeNames.end()); + for (const auto& namePos : nameMap) { + if (wantedNames.find(namePos.first) == wantedNames.end()) { + skipIndices.insert(static_cast(namePos.second)); + } + } + } + + // Passes 2..N+1: read attribute buffer sizes + std::map pagedStreams; + for (Index i = 0; i < attributes; ++i) { + if (skipIndices.count(i)) { + internal::skipPointDataAttributeSizes(leaves, is, i, pagedStreams); + } else { + internal::readPointDataAttributeSizes(leaves, is, i, pagedStreams); + } + } + + // Pass N+2: read voxel data + internal::readPointDataVoxelData(leaves, is, saveFloatAsHalf, + tree.background(), voxelBufferSizes); + + // Passes N+3..2N+2: read attribute data buffers + for (Index i = 0; i < attributes; ++i) { + if (skipIndices.count(i)) { + internal::skipPointDataAttributeData(leaves, is, i, pagedStreams); + } else { + internal::readPointDataAttributeData(leaves, is, i, pagedStreams); + } + } + + // Drop skipped attributes from each leaf's AttributeSet + if (!skipIndices.empty() && !leaves.empty()) { + const std::vector dropPositions(skipIndices.begin(), skipIndices.end()); + auto filteredDescriptor = + leaves[0]->attributeSet().descriptorPtr()->duplicateDrop(dropPositions); + for (auto* leaf : leaves) { + leaf->dropAttributes(dropPositions, + leaf->attributeSet().descriptor(), filteredDescriptor); + } + } + + // PointDataGrid uses multiple passes, so clip after reading + // the buffers if bbox is not infinite. + if (options.clipBBox.isSorted()) { + CoordBBox indexBBox = + grid.constTransform().worldToIndexNodeCentered(options.clipBBox); + grid.tree().root().clip(indexBBox); + } + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + const GridT& grid = static_cast(gridBase); + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + using LeafT = typename GridT::TreeType::LeafNodeType; + std::vector leaves; + grid.tree().getNodes(leaves); + + // Determine how many leaf buffer passes are required for this grid + uint16_t numPasses = + static_cast(internal::countPointDataPasses(leaves)); + os.write(reinterpret_cast(&numPasses), sizeof(uint16_t)); + + const Index attributes = (numPasses - 4) / 2; + + // Pass 0: write voxel data sizes + descriptor tracking + bool matching = true; + points::AttributeSet::Descriptor::Ptr sharedDescriptor; + internal::writePointDataVoxelSizes(leaves, os, matching, sharedDescriptor); + + // Pass 1: write descriptor and attribute metadata + internal::writePointDataDescriptors(leaves, os, matching, sharedDescriptor); + + // Passes 2..N+1: write attribute buffer sizes (page headers) + for (Index i = 0; i < attributes; ++i) { + internal::writePointDataAttributeSizes(leaves, os, i); + } + + // Pass N+2: write voxel data + internal::writePointDataVoxelData(leaves, os, saveFloatAsHalf); + + // Passes N+3..2N+2: write attribute data buffers (page data) + for (Index i = 0; i < attributes; ++i) { + internal::writePointDataAttributeData(leaves, os, i); + } + } +}; // struct PointDataCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc index 3e437fab17..4873f779aa 100644 --- a/openvdb/openvdb/io/Codec.cc +++ b/openvdb/openvdb/io/Codec.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -60,6 +61,7 @@ void initialize() CodecRegistry::registerCodec>(); CodecRegistry::registerCodec>(); CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); // register the plugin that converts from scalar to mask/bool NumericGridTypes::foreach(); diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index 3ad2d3a48d..23ff8f8149 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -308,6 +308,9 @@ class OPENVDB_API AttributeArray /// Read attribute buffers from a paged stream. virtual void readPagedBuffers(compression::PagedInputStream&) = 0; + /// Skip attribute buffers in a paged stream without reading or + /// allocating any data. + void skipPagedBuffers(compression::PagedInputStream&); /// Write attribute buffers to a paged stream. /// @param outputTransient if true, write out transient attributes virtual void writePagedBuffers(compression::PagedOutputStream&, bool outputTransient) const = 0; @@ -1599,6 +1602,43 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu } +inline void +AttributeArray::skipPagedBuffers(compression::PagedInputStream& is) +{ + if (!mUsePagedRead) { + if (!is.sizeOnly()) { + // for non-paged data, seek past the raw data in the stream + std::istream& inputStream = is.getInputStream(); + uint8_t bloscCompressed(0); + if (!mIsUniform) inputStream.read(reinterpret_cast(&bloscCompressed), sizeof(uint8_t)); + inputStream.seekg(mCompressedBytes, std::ios_base::cur); + mCompressedBytes = 0; + mFlags = static_cast(mFlags & ~PARTIALREAD); + } + return; + } + + if (is.sizeOnly()) + { + size_t compressedBytes(mCompressedBytes); + mCompressedBytes = 0; + mFlags = static_cast(mFlags & ~PARTIALREAD); + OPENVDB_ASSERT(!mPageHandle); + mPageHandle = is.createHandle(compressedBytes); + return; + } + + OPENVDB_ASSERT(mPageHandle); + + is.skip(mPageHandle, std::streamsize(mPageHandle->size())); + mPageHandle.reset(); + + // clear page state + + mUsePagedRead = 0; +} + + template void TypedAttributeArray::write(std::ostream& os) const diff --git a/openvdb/openvdb/points/StreamCompression.cc b/openvdb/openvdb/points/StreamCompression.cc index 5760f3b4ae..f18d016463 100644 --- a/openvdb/openvdb/points/StreamCompression.cc +++ b/openvdb/openvdb/points/StreamCompression.cc @@ -338,6 +338,21 @@ Page::readBuffers(std::istream&is, bool delayed) } +void +Page::skipBuffers(std::istream& is) +{ + OPENVDB_ASSERT(mInfo); + + bool isCompressed = mInfo->compressedBytes > 0; + std::streamsize bytes = isCompressed ? + mInfo->compressedBytes : -mInfo->compressedBytes; + + is.seekg(bytes, std::ios_base::cur); + + mInfo.reset(); +} + + void Page::copy(const std::unique_ptr& temp, int pageSize) { @@ -438,6 +453,23 @@ PagedInputStream::read(PageHandle::Ptr& pageHandle, std::streamsize n, bool dela } +void +PagedInputStream::skip(PageHandle::Ptr& pageHandle, std::streamsize n) +{ + OPENVDB_ASSERT(mByteIndex <= mUncompressedBytes); + + Page& page = pageHandle->page(); + + if (mByteIndex == mUncompressedBytes) { + mUncompressedBytes = static_cast(page.uncompressedBytes()); + page.skipBuffers(*mIs); + mByteIndex = 0; + } + + mByteIndex += int(n); +} + + //////////////////////////////////////// diff --git a/openvdb/openvdb/points/StreamCompression.h b/openvdb/openvdb/points/StreamCompression.h index d983ea2404..2efbcda35c 100644 --- a/openvdb/openvdb/points/StreamCompression.h +++ b/openvdb/openvdb/points/StreamCompression.h @@ -142,6 +142,10 @@ class OPENVDB_API Page /// pointers will be stored to load the data lazily. void readBuffers(std::istream&, bool delayed); + /// @brief Skip the Page buffers by seeking past the compressed data + /// without reading or decompressing it. + void skipBuffers(std::istream&); + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") bool isOutOfCore() const { return false; } @@ -222,6 +226,10 @@ class OPENVDB_API PagedInputStream /// an immediate read of the data. void read(PageHandle::Ptr& pageHandle, std::streamsize n, bool delayed = true); + /// @brief Skip past the page data referenced by @a pageHandle without + /// reading or decompressing it. + void skip(PageHandle::Ptr& pageHandle, std::streamsize n); + private: int mByteIndex = 0; int mUncompressedBytes = 0; diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc index 2b8911419e..81f8ff2358 100644 --- a/openvdb/openvdb/unittest/TestPointCodec.cc +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "util.h" // for unittest_util::genPoints @@ -145,3 +146,402 @@ TEST_F(TestPointCodec, testPointIndexCodecIO) CodecRegistry::clear(); std::remove(codecPath.c_str()); } + +TEST_F(TestPointCodec, testPointDataCodecIO) +{ + using namespace openvdb; + using namespace openvdb::io; + using namespace openvdb::points; + using PointDataTree = PointDataGrid::TreeType; + + openvdb::initialize(); + CodecRegistry::clear(); + + // Helper: compare P attribute values leaf-by-leaf between two PointDataGrids + auto comparePositions = [](const PointDataGrid& a, const PointDataGrid& b) { + auto aIt = a.tree().cbeginLeaf(); + auto bIt = b.tree().cbeginLeaf(); + for (; aIt && bIt; ++aIt, ++bIt) { + EXPECT_EQ(aIt->pointCount(), bIt->pointCount()); + AttributeHandle aH(aIt->constAttributeArray("P")); + AttributeHandle bH(bIt->constAttributeArray("P")); + for (Index i = 0; i < aIt->pointCount(); ++i) { + const Vec3f av = aH.get(i); + const Vec3f bv = bH.get(i); + EXPECT_NEAR(av.x(), bv.x(), 1e-6f); + EXPECT_NEAR(av.y(), bv.y(), 1e-6f); + EXPECT_NEAR(av.z(), bv.z(), 1e-6f); + } + } + EXPECT_TRUE(!aIt && !bIt); + }; + + // ----------------------------------------------------------------------- + // Section A: Positions only + // ----------------------------------------------------------------------- + { + const std::vector positions = { + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(1.5f, 3.5f, 1.0f), + Vec3f(-1.0f, 6.0f, -2.0f), + Vec3f(1.1f, 1.25f, 0.06f) + }; + + const float voxelSize = 0.5f; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid(positions, *transform); + srcGrid->setName("pdg_positions"); + + const std::string rawPath = "testPDG_A_raw.vdb"; + + // Phase 1: write/read without codec + { + io::File f(rawPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr rawGrid; + { + io::File f(rawPath); + f.open(); + rawGrid = gridPtrCast(f.readGrid("pdg_positions")); + f.close(); + } + ASSERT_TRUE(rawGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(rawGrid->tree())); + + PointDataGrid::Ptr rawTopo; + { + io::File f(rawPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("pdg_positions")); + rawTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(rawTopo); + EXPECT_EQ(rawTopo->activeVoxelCount(), Index64(4)); + + std::remove(rawPath.c_str()); + + const std::string codecPath = "testPDG_A_codec.vdb"; + + // Phase 2: register codec, write/read with codec + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointDataGrid::gridType())); + + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + codecGrid = gridPtrCast(f.readGrid("pdg_positions")); + f.close(); + } + ASSERT_TRUE(codecGrid); + + // Phase 3: compare src vs codec + EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(codecGrid->tree())); + comparePositions(*srcGrid, *codecGrid); + + // Phase 4: TopologyOnly read + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + PointDataGrid::Ptr codecTopo; + { + io::File f(codecPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("pdg_positions", topoOpts)); + codecTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(codecTopo); + EXPECT_EQ(codecTopo->activeVoxelCount(), Index64(0)); + EXPECT_TRUE(codecTopo->tree().leafCount() == 0); + + std::remove(codecPath.c_str()); + } + + // ----------------------------------------------------------------------- + // Section B: Multiple attributes + // ----------------------------------------------------------------------- + { + const std::vector positions = { + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(1.5f, 3.5f, 1.0f), + Vec3f(-1.0f, 6.0f, -2.0f), + Vec3f(1.1f, 1.25f, 0.06f) + }; + const std::vector velocities = { + Vec3f(1.0f, 0.0f, 0.0f), + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(0.0f, 0.0f, 1.0f), + Vec3f(1.0f, 1.0f, 0.5f) + }; + const std::vector ids = {0, 1, 2, 3}; + + const float voxelSize = 0.5f; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointAttributeVector posWrapper(positions); + tools::PointIndexGrid::Ptr pointIndexGrid = + tools::createPointIndexGrid(posWrapper, *transform); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid( + *pointIndexGrid, posWrapper, *transform); + srcGrid->setName("pdg_multi"); + + PointDataTree& tree = srcGrid->tree(); + tools::PointIndexTree& indexTree = pointIndexGrid->tree(); + + appendAttribute(tree, "velocity"); + populateAttribute>( + tree, indexTree, "velocity", + PointAttributeVector(velocities)); + + appendAttribute(tree, "id"); + populateAttribute>( + tree, indexTree, "id", + PointAttributeVector(ids)); + + // Verify attribute count on src grid (P, velocity, id) + { + auto leafIt = srcGrid->tree().cbeginLeaf(); + ASSERT_TRUE(leafIt); + EXPECT_EQ(leafIt->attributeSet().size(), size_t(3)); + } + + CodecRegistry::clear(); + + const std::string rawPath = "testPDG_B_raw.vdb"; + + // Phase 1: write/read without codec + { + io::File f(rawPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr rawGrid; + { + io::File f(rawPath); + f.open(); + rawGrid = gridPtrCast(f.readGrid("pdg_multi")); + f.close(); + } + ASSERT_TRUE(rawGrid); + + std::remove(rawPath.c_str()); + + const std::string codecPath = "testPDG_B_codec.vdb"; + + // Phase 2: register codec, write/read with codec + io::internal::initialize(); + + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + codecGrid = gridPtrCast(f.readGrid("pdg_multi")); + f.close(); + } + ASSERT_TRUE(codecGrid); + + EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(codecGrid->tree())); + + // Verify attribute count on codec grid + { + auto leafIt = codecGrid->tree().cbeginLeaf(); + ASSERT_TRUE(leafIt); + EXPECT_EQ(leafIt->attributeSet().size(), size_t(3)); + } + + // Compare all three attributes leaf-by-leaf + { + auto srcIt = srcGrid->tree().cbeginLeaf(); + auto codecIt = codecGrid->tree().cbeginLeaf(); + for (; srcIt && codecIt; ++srcIt, ++codecIt) { + EXPECT_EQ(srcIt->pointCount(), codecIt->pointCount()); + AttributeHandle srcP(srcIt->constAttributeArray("P")); + AttributeHandle codecP(codecIt->constAttributeArray("P")); + AttributeHandle srcVel(srcIt->constAttributeArray("velocity")); + AttributeHandle codecVel(codecIt->constAttributeArray("velocity")); + AttributeHandle srcId(srcIt->constAttributeArray("id")); + AttributeHandle codecId(codecIt->constAttributeArray("id")); + for (Index i = 0; i < srcIt->pointCount(); ++i) { + const Vec3f rp = srcP.get(i); + const Vec3f cp = codecP.get(i); + EXPECT_NEAR(rp.x(), cp.x(), 1e-6f); + EXPECT_NEAR(rp.y(), cp.y(), 1e-6f); + EXPECT_NEAR(rp.z(), cp.z(), 1e-6f); + const Vec3f rv = srcVel.get(i); + const Vec3f cv = codecVel.get(i); + EXPECT_NEAR(rv.x(), cv.x(), 1e-6f); + EXPECT_NEAR(rv.y(), cv.y(), 1e-6f); + EXPECT_NEAR(rv.z(), cv.z(), 1e-6f); + EXPECT_EQ(srcId.get(i), codecId.get(i)); + } + } + EXPECT_TRUE(!srcIt && !codecIt); + } + + CodecRegistry::clear(); + std::remove(codecPath.c_str()); + } + + // ----------------------------------------------------------------------- + // Section C: Shared vs non-shared descriptors + // ----------------------------------------------------------------------- + { + std::vector pts; + unittest_util::genPoints(100, pts); + + std::vector positions; + positions.reserve(pts.size()); + for (const auto& p : pts) { + positions.emplace_back(float(p.x()), float(p.y()), float(p.z())); + } + + const double voxelSize = 0.1; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid(positions, *transform); + srcGrid->setName("pdg_desc"); + + // All leaves should share one Descriptor::Ptr initially + ASSERT_GT(srcGrid->tree().leafCount(), Index32(1)); + { + auto leafIt = srcGrid->tree().cbeginLeaf(); + auto firstDescPtr = leafIt->attributeSet().descriptorPtr(); + for (; leafIt; ++leafIt) { + EXPECT_EQ(leafIt->attributeSet().descriptorPtr(), firstDescPtr); + } + } + + io::internal::initialize(); + + // -- C1: Shared descriptors (default) -- + // All leaves already share one pointer; this exercises the header=1 write path. + const std::string sharedPath = "testPDG_C_shared.vdb"; + { + io::File f(sharedPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr sharedGrid; + { + io::File f(sharedPath); + f.open(); + sharedGrid = gridPtrCast(f.readGrid("pdg_desc")); + f.close(); + } + ASSERT_TRUE(sharedGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(sharedGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(sharedGrid->tree())); + comparePositions(*srcGrid, *sharedGrid); + + // -- C2: After makeDescriptorUnique -- + // makeDescriptorUnique() creates ONE new descriptor and assigns it to every + // leaf, so all leaves still share a single pointer (the new copy). + // The codec still detects matching descriptors and writes header=1; + // this is a regression check that the round-trip remains correct. + makeDescriptorUnique(srcGrid->tree()); + { + auto leafIt = srcGrid->tree().cbeginLeaf(); + auto firstDescPtr = leafIt->attributeSet().descriptorPtr(); + ++leafIt; + if (leafIt) { + // All leaves share the same new pointer + EXPECT_EQ(leafIt->attributeSet().descriptorPtr(), firstDescPtr); + } + } + + const std::string nonSharedPath = "testPDG_C_nonshared.vdb"; + { + io::File f(nonSharedPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr nonSharedGrid; + { + io::File f(nonSharedPath); + f.open(); + nonSharedGrid = gridPtrCast(f.readGrid("pdg_desc")); + f.close(); + } + ASSERT_TRUE(nonSharedGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(nonSharedGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(nonSharedGrid->tree())); + comparePositions(*sharedGrid, *nonSharedGrid); + + // -- C3: Genuinely different descriptors (exercises the header=0 write path) -- + // Add "extra" to all leaves, then drop it from only the first leaf so that + // leaf descriptors differ by value, triggering matching=false in the codec. + appendAttribute(srcGrid->tree(), "extra"); + makeDescriptorUnique(srcGrid->tree()); + srcGrid->setName("pdg_desc_diff"); + + { + auto leafIt = srcGrid->tree().beginLeaf(); + ASSERT_TRUE(leafIt); + const size_t extraIdx = + leafIt->attributeSet().descriptor().find("extra"); + ASSERT_NE(extraIdx, AttributeSet::INVALID_POS); + const std::vector dropIndices = {extraIdx}; + AttributeSet::Descriptor::Ptr newDesc = + leafIt->attributeSet().descriptor().duplicateDrop(dropIndices); + leafIt->dropAttributes( + dropIndices, leafIt->attributeSet().descriptor(), newDesc); + } + + const std::string diffPath = "testPDG_C_diff.vdb"; + { + io::File f(diffPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr diffGrid; + { + io::File f(diffPath); + f.open(); + diffGrid = gridPtrCast(f.readGrid("pdg_desc_diff")); + f.close(); + } + ASSERT_TRUE(diffGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(diffGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(diffGrid->tree())); + + // First leaf has {P} only; remaining leaves have {P, extra} + { + auto diffIt = diffGrid->tree().cbeginLeaf(); + ASSERT_TRUE(diffIt); + EXPECT_EQ(diffIt->attributeSet().size(), size_t(1)); + ++diffIt; + if (diffIt) { + EXPECT_EQ(diffIt->attributeSet().size(), size_t(2)); + } + } + + std::remove(sharedPath.c_str()); + std::remove(nonSharedPath.c_str()); + std::remove(diffPath.c_str()); + } +} From 056b3223cc203637b5417ed22654dc20dec7b697 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 14 May 2026 14:29:07 -0700 Subject: [PATCH 48/70] Address feedback Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/PointDataCodec.h | 18 ++++++++++-------- openvdb/openvdb/codecs/PointIndexCodec.h | 6 ++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index e5cc550fe4..8ad1d829db 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -12,7 +12,7 @@ #include #include -#include "ScalarLeafCodec.h" +#include "impl/ScalarLeafCodec.h" #include "TopologyCodec.h" #include @@ -32,8 +32,9 @@ inline typename PagedStreamPtrT::element_type* getOrCreatePagedStream( auto it = pagedStreams.find(attributeIndex); if (it != pagedStreams.end()) return it->second.get(); using PagedStreamT = typename PagedStreamPtrT::element_type; - pagedStreams[attributeIndex] = std::make_shared(); - return pagedStreams[attributeIndex].get(); + auto ptr = std::make_shared(); + auto& stored = (pagedStreams[attributeIndex] = std::move(ptr)); + return stored.get(); } //////////////////////////////////////// @@ -41,7 +42,7 @@ inline typename PagedStreamPtrT::element_type* getOrCreatePagedStream( template inline void readPointDataVoxelSizes(const std::vector& leaves, - std::istream& is, std::map& voxelBufferSizes) + std::istream& is, std::unordered_map& voxelBufferSizes) { for (auto* leaf : leaves) { uint16_t voxelBufferSize; @@ -110,9 +111,8 @@ template inline void readPointDataVoxelData(const std::vector& leaves, std::istream& is, bool saveFloatAsHalf, const typename LeafT::ValueType& background, - const std::map& voxelBufferSizes) + const std::unordered_map& voxelBufferSizes) { - (void) voxelBufferSizes; using BaseLeaf = typename LeafT::BaseLeaf; for (auto* leaf : leaves) { OPENVDB_ASSERT(voxelBufferSizes.find(leaf->origin()) != voxelBufferSizes.end()); @@ -302,7 +302,7 @@ struct OPENVDB_API PointDataCodecTypeData : public io::ReadTypedOptions }; // struct PointDataCodecTypeData template -struct PointDataCodec : public TopologyCodec +struct PointDataCodec final: public TopologyCodec { using Ptr = std::unique_ptr>; @@ -312,6 +312,8 @@ struct PointDataCodec : public TopologyCodec void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { + OPENVDB_ASSERT(dynamic_cast(data.grid.get())); + GridT& grid = static_cast(*data.grid); std::vector pointAttributeNames; @@ -339,7 +341,7 @@ struct PointDataCodec : public TopologyCodec tree.getNodes(leaves); // Pass 0: read voxel data sizes - std::map voxelBufferSizes; + std::unordered_map voxelBufferSizes; internal::readPointDataVoxelSizes(leaves, is, voxelBufferSizes); // Pass 1: read descriptor and attribute metadata diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h index d8d9f1ee32..851ee140fd 100644 --- a/openvdb/openvdb/codecs/PointIndexCodec.h +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -8,7 +8,7 @@ #include -#include "ScalarLeafCodec.h" +#include "impl/ScalarLeafCodec.h" #include "TopologyCodec.h" namespace openvdb { @@ -106,7 +106,7 @@ struct WritePointIndexBuffersOp } // namespace internal template -struct PointIndexCodec : public TopologyCodec +struct PointIndexCodec final: public TopologyCodec { using Ptr = std::unique_ptr>; @@ -116,6 +116,8 @@ struct PointIndexCodec : public TopologyCodec void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final { + OPENVDB_ASSERT(dynamic_cast(data.grid.get())); + GridT& grid = static_cast(*data.grid); if (grid.hasMultiPassIO()) { From 031489e38980ea92a57ef0f69df08a44d6994567 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 14 May 2026 15:01:25 -0700 Subject: [PATCH 49/70] Address feedback (fix unused error) Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/PointDataCodec.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index 8ad1d829db..f4bc713bd7 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -111,7 +111,7 @@ template inline void readPointDataVoxelData(const std::vector& leaves, std::istream& is, bool saveFloatAsHalf, const typename LeafT::ValueType& background, - const std::unordered_map& voxelBufferSizes) + [[maybe_unused]] const std::unordered_map& voxelBufferSizes) { using BaseLeaf = typename LeafT::BaseLeaf; for (auto* leaf : leaves) { From c1118e298aef7dc45bd307c722aa3f4ab086f99c Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 9 Jun 2026 21:26:27 -0700 Subject: [PATCH 50/70] Fix underflow bug when reading or writing empty grids Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/PointDataCodec.h | 10 ++++-- openvdb/openvdb/unittest/TestPointCodec.cc | 42 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index f4bc713bd7..ac3d353f77 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -334,7 +334,11 @@ struct PointDataCodec final: public TopologyCodec uint16_t numPasses = 1; is.read(reinterpret_cast(&numPasses), sizeof(uint16_t)); - const Index attributes = (numPasses - 4) / 2; + // The pass layout is: voxel sizes (1) + descriptors (1) + attribute + // sizes (N) + voxel data (1) + attribute data (N) = 2N + 4 passes. + // A leafless grid stores numPasses == 0, and malformed files may store + // numPasses < 4; guard against unsigned underflow in either case. + const Index attributes = numPasses >= 4 ? Index(numPasses - 4) / 2 : 0; using LeafT = typename GridT::TreeType::LeafNodeType; std::vector leaves; @@ -419,7 +423,9 @@ struct PointDataCodec final: public TopologyCodec static_cast(internal::countPointDataPasses(leaves)); os.write(reinterpret_cast(&numPasses), sizeof(uint16_t)); - const Index attributes = (numPasses - 4) / 2; + // See readBuffers(): a leafless grid yields numPasses == 0, so guard + // against unsigned underflow rather than computing (numPasses - 4) / 2. + const Index attributes = numPasses >= 4 ? Index(numPasses - 4) / 2 : 0; // Pass 0: write voxel data sizes + descriptor tracking bool matching = true; diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc index 81f8ff2358..52cb6a34bc 100644 --- a/openvdb/openvdb/unittest/TestPointCodec.cc +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -545,3 +545,45 @@ TEST_F(TestPointCodec, testPointDataCodecIO) std::remove(diffPath.c_str()); } } + +// A leafless PointDataGrid stores numPasses == 0. The attribute count is +// derived as (numPasses - 4) / 2; without an underflow guard this wraps to a +// huge unsigned value and the attribute loops spin ~4.3e9 times, hanging +// both write and read. This test exercises an empty grid round-trip. +TEST_F(TestPointCodec, testPointDataCodecEmptyGrid) +{ + using namespace openvdb; + using namespace openvdb::io; + using namespace openvdb::points; + + openvdb::initialize(); + CodecRegistry::clear(); + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointDataGrid::gridType())); + + PointDataGrid::Ptr srcGrid = PointDataGrid::create(); + srcGrid->setName("pdg_empty"); + EXPECT_EQ(srcGrid->tree().leafCount(), Index32(0)); + + const std::string codecPath = "testPDG_empty_codec.vdb"; + + { + io::File f(codecPath); + EXPECT_NO_THROW(f.write(GridPtrVec{srcGrid})); + } + + PointDataGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("pdg_empty")); + codecGrid = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(codecGrid); + EXPECT_EQ(codecGrid->tree().leafCount(), Index32(0)); + EXPECT_EQ(codecGrid->activeVoxelCount(), Index64(0)); + + std::remove(codecPath.c_str()); +} From 74db979826464fe734f29ba76ab0ec815b4f4619 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 9 Jun 2026 21:43:57 -0700 Subject: [PATCH 51/70] Fix an issue in the old version codepath for reading topology where the value mask was used instead of the child mask Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/TopologyCodec.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index e602f176ce..8b64d60420 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -194,8 +194,12 @@ struct ReadTopologyOp // Copy values from the array into this node's table. if (oldVersion) { + // The node's member child mask is still empty at this point + // (PartialCreate; setChildUnsafe runs below), so iterate the + // local childMask's off-bits to match the legacy ordering and + // avoid over-reading the countOff-sized values array. Index n = 0; - for (auto iter = node.beginValueAll(); iter; ++iter) { + for (auto iter = childMask.beginOff(); iter; ++iter) { node.setValueOnlyUnsafe(iter.pos(), static_cast(values[n++])); } OPENVDB_ASSERT(n == numValues); From b8f7f6c6176d9a3c4a05b21ad7973fe5709851ce Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 09:10:35 -0700 Subject: [PATCH 52/70] Fix issue where instanced grids were not being clipped using the correct transform Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 19 +++++++- openvdb/openvdb/unittest/TestFile.cc | 71 ++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 3aaa31c5b1..d380cfb50a 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -490,11 +490,26 @@ File::readGrid(const Name& name, const io::ReadOptions& readOptions) << " in file " << mFilename); } + // Read the parent without clipping. Archive::readGrid() converts the + // world-space clip region into index space using the grid's own + // transform, but an instance has its own transform that may differ + // from the parent's. Instead, read the full parent tree and clip the + // assembled instance below using the instance's transform, so that the + // retained region matches the requested world-space bbox. + io::ReadOptions parentOptions = readOptions; + parentOptions.clipBBox = BBoxd(); + GridBase::Ptr parent; OPENVDB_ASSERT(inputHasGridOffsets()); parentIt->second.seekToGrid(inputStream()); - parent = Archive::readGrid(parentIt->second, inputStream(), readOptions, mReadDiagnostics); - if (parent) grid->setTree(parent->baseTreePtr()); + parent = Archive::readGrid(parentIt->second, inputStream(), parentOptions, mReadDiagnostics); + if (parent) { + grid->setTree(parent->baseTreePtr()); + const auto& clipBBox = readOptions.clipBBox; + if (clipBBox.isSorted()) { + grid->clipGrid(clipBBox); + } + } } return grid; } diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index 130947fdf1..9aa7693d6a 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -557,6 +557,77 @@ TEST_F(TestFile, testWriteInstancedGrids) } +// Verify that clipping an instanced grid uses the instance's own transform, +// not the parent's. The bug was that Archive::readGrid() converted the +// world-space bbox to index space with the parent's transform before the +// instance's transform was applied, so the clipped region was wrong when the +// two transforms differed. +TEST_F(TestFile, testReadClippedInstancedGrid) +{ + using namespace openvdb; + + const char* filename = "testReadClippedInstancedGrid.vdb"; + SharedPtr scopedFile(filename, ::remove); + + // Parent grid: voxel size 1.0. Fill index [-5, 5] with value 1. + FloatTree::Ptr tree(new FloatTree(0.0f)); + tree->fill(CoordBBox(Coord(-5), Coord(5)), 1.0f, /*active=*/true); + + GridBase::Ptr parent = FloatGrid::create(tree); + parent->setName("parent"); + parent->setTransform(math::Transform::createLinearTransform(1.0)); + + // Instance grid: same tree, but voxel size 2.0. + // Index coord n → world coord 2n (double the parent's world-space extent). + GridBase::Ptr instance = FloatGrid::create(tree); + instance->setName("instance"); + instance->setTransform(math::Transform::createLinearTransform(2.0)); + + GridPtrVec grids; + grids.push_back(parent); + grids.push_back(instance); + + { + io::File vdbfile(filename); + vdbfile.write(grids); + } + + // World-space clip: [0, 6]. + // Via parent transform (voxel 1.0): index [0, 6] → voxels 0..5 survive. + // Via instance transform (voxel 2.0): index [0, 3] → voxels 0..3 survive. + // The correct answer uses the instance's transform. + const BBoxd clipBox(Vec3d(0.0), Vec3d(6.0)); + + io::File vdbfile(filename); + vdbfile.open(); + + GridBase::Ptr readGrid = vdbfile.readGrid("instance", clipBox); + EXPECT_TRUE(readGrid.get() != nullptr); + FloatGrid::Ptr clipped = gridPtrCast(readGrid); + EXPECT_TRUE(clipped.get() != nullptr); + + const CoordBBox bbox = clipped->evalActiveVoxelBoundingBox(); + // The instance's transform maps world [0,6] to index [0,3]. + EXPECT_EQ(Coord(0, 0, 0), bbox.min()); + EXPECT_EQ(Coord(3, 3, 3), bbox.max()); + + // No active voxels should survive outside [0,3] in any axis. + FloatGrid::ConstAccessor acc = clipped->getConstAccessor(); + for (int i = -5; i <= 5; ++i) { + for (int j = -5; j <= 5; ++j) { + for (int k = -5; k <= 5; ++k) { + const Coord xyz(i, j, k); + if (i >= 0 && j >= 0 && k >= 0 && i <= 3 && j <= 3 && k <= 3) { + EXPECT_EQ(1.0f, acc.getValue(xyz)); + } else { + EXPECT_EQ(0.0f, acc.getValue(xyz)); + } + } + } + } +} + + void TestFile::testReadGridDescriptors() { From 98d9d0e4f279dadff347ac900dc42457c431260a Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 20:58:13 -0700 Subject: [PATCH 53/70] Fix a bug where the Stream assignment operator was not calling the Archive assignment operator Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Stream.cc | 1 + openvdb/openvdb/unittest/TestStream.cc | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 938cc321b0..91b3ca4b7d 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -83,6 +83,7 @@ Stream& Stream::operator=(const Stream& other) { if (&other != this) { + Archive::operator=(other); mMeta = other.mMeta; mGrids = other.mGrids; mOutputStream = other.mOutputStream; diff --git a/openvdb/openvdb/unittest/TestStream.cc b/openvdb/openvdb/unittest/TestStream.cc index 35850fab3d..6091e60c2b 100644 --- a/openvdb/openvdb/unittest/TestStream.cc +++ b/openvdb/openvdb/unittest/TestStream.cc @@ -216,3 +216,20 @@ TestStream::testFileReadFromStream() verifyTestGrids(grids, meta); } TEST_F(TestStream, testFileReadFromStream) { testFileReadFromStream(); } + + +TEST_F(TestStream, testAssignmentPreservesArchiveFlags) +{ + using namespace openvdb; + + std::ostringstream os1(std::ios_base::binary), os2(std::ios_base::binary); + io::Stream src(os1); + src.setCompression(io::COMPRESS_ZIP); + src.setInstancingEnabled(false); + + io::Stream dst(os2); + dst = src; + + EXPECT_EQ(src.compression(), dst.compression()); + EXPECT_EQ(src.isInstancingEnabled(), dst.isInstancingEnabled()); +} From 8cb8cebbce641b03a6706ff95762fad9abd1ad4c Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 16:59:50 -0700 Subject: [PATCH 54/70] Fix topology-only and data race condition Signed-off-by: Dan Bailey --- openvdb/openvdb/Grid.h | 22 +++++++++++++++++++++ openvdb/openvdb/codecs/TopologyCodec.h | 14 +++++++++++++ openvdb/openvdb/io/Archive.cc | 15 +++++++++++++- openvdb/openvdb/io/Codec.h | 14 +++++++++---- openvdb/openvdb/io/File.cc | 4 ++-- openvdb/openvdb/io/io.h | 5 +++++ openvdb/openvdb/tree/InternalNode.h | 4 ++++ openvdb/openvdb/tree/LeafBuffer.h | 23 +++++++++++++++------- openvdb/openvdb/tree/LeafNode.h | 3 +++ openvdb/openvdb/unittest/TestCodec.cc | 13 ++++++++++-- openvdb/openvdb/unittest/TestPointCodec.cc | 21 ++++++++++++++++---- 11 files changed, 118 insertions(+), 20 deletions(-) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index 41f6e58501..3d7a34e63b 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -9,6 +9,7 @@ #include "Types.h" #include "io/io.h" #include "math/Transform.h" +#include "tree/LeafManager.h" #include "tree/Tree.h" #include "util/Assert.h" #include "util/logging.h" @@ -1630,6 +1631,27 @@ inline void Grid::readTopology(std::istream& is) { tree().readTopology(is, saveFloatAsHalf()); + // When called from the legacy (non-codec) TopologyOnly path, the stream + // metadata carries a flag requesting allocation+zero-fill of leaf buffers + // (PartialCreate leaves them unallocated after readTopology). + if (io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is)) { + if (meta->allocateLeafBuffers()) { + meta->setAllocateLeafBuffers(false); + if constexpr (!std::is_void_v) { + const auto background = tree().root().background(); + tree::LeafManager leafManager(tree()); + leafManager.foreach([&background](auto& leaf, size_t) { + using LeafType = std::decay_t; + if constexpr (!std::is_same_v) { + if (leaf.buffer().empty()) { + leaf.buffer().allocate(); + leaf.buffer().fill(background); + } + } + }); + } + } + } } diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 8b64d60420..cbccf5eef2 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -291,6 +292,19 @@ void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::R if (options.readMode == io::ReadMode::TopologyOnly) { internal::setTilesToBackground(grid.tree()); + // allocate leaf buffers in parallel and fill with the background value; + // ReadTopologyOp uses PartialCreate which leaves buffers unallocated. + const auto background = grid.tree().root().background(); + tree::LeafManager leafManager(grid.tree()); + leafManager.foreach([&background](auto& leaf, size_t) { + using LeafType = std::decay_t; + if constexpr (!std::is_same_v) { + if (leaf.buffer().empty()) { + leaf.buffer().allocate(); + leaf.buffer().fill(background); + } + } + }); return; } } diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 86dd2755cc..e0fc5dc599 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -182,6 +182,7 @@ struct StreamMetadata::Impl bool mDelayedLoadMeta = false; uint64_t mLeaf = 0; uint32_t mTest = 0; // for testing only + bool mAllocateLeafBuffers = false; }; // struct StreamMetadata @@ -243,6 +244,7 @@ bool StreamMetadata::writeGridStats() const { return mImpl->mWriteGr bool StreamMetadata::seekable() const { return mImpl->mSeekable; } bool StreamMetadata::delayedLoadMeta() const { return mImpl->mDelayedLoadMeta; } bool StreamMetadata::countingPasses() const { return mImpl->mCountingPasses; } +bool StreamMetadata::allocateLeafBuffers() const { return mImpl->mAllocateLeafBuffers; } uint32_t StreamMetadata::pass() const { return mImpl->mPass; } MetaMap& StreamMetadata::gridMetadata() { return mImpl->mGridMetadata; } const MetaMap& StreamMetadata::gridMetadata() const { return mImpl->mGridMetadata; } @@ -260,6 +262,7 @@ void StreamMetadata::setHalfFloat(bool b) { mImpl->mHalfFloat = b; void StreamMetadata::setWriteGridStats(bool b) { mImpl->mWriteGridStats = b; } void StreamMetadata::setSeekable(bool b) { mImpl->mSeekable = b; } void StreamMetadata::setCountingPasses(bool b) { mImpl->mCountingPasses = b; } +void StreamMetadata::setAllocateLeafBuffers(bool b) { mImpl->mAllocateLeafBuffers = b; } void StreamMetadata::setPass(uint32_t i) { mImpl->mPass = i; } void StreamMetadata::__setTest(uint32_t t) { mImpl->mTest = t; } @@ -1000,13 +1003,23 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti io::setGridClass(is, gridClass); grid->readTransform(is); - if (readOptions.readMode != io::ReadMode::TopologyOnly && !gd.isInstance()) { + const bool readTopology = readOptions.readMode != io::ReadMode::MetadataOnly && !gd.isInstance(); + const bool readBuffers = readTopology && readOptions.readMode != io::ReadMode::TopologyOnly; + if (readTopology) { // read topology if (codec) { codec->readTopology(is, *codecData, readOptions, diagnostics); } else { + if (readOptions.readMode == io::ReadMode::TopologyOnly) { + // signal Grid::readTopology to allocate+zero-fill leaf buffers + if (auto meta = io::getStreamMetadataPtr(is)) { + meta->setAllocateLeafBuffers(true); + } + } grid->readTopology(is); } + } + if (readBuffers) { // read buffers if (codec) { codec->readBuffers(is, *codecData, readOptions, diagnostics); diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 4fbf3412d1..f236732cc9 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -79,10 +79,14 @@ enum class ReadMode { Mask, /// Deserialize topology only; value buffers are skipped. The resulting /// grid has a valid tree structure (active/inactive state, node - /// hierarchy) but leaf buffer data is left at its default (background) - /// value. Useful when only the active-voxel mask is needed and - /// avoiding the cost of reading large value buffers is desirable. - TopologyOnly + /// hierarchy) and all leaf buffers are allocated and zero-filled. + /// Useful when only the active-voxel mask is needed and avoiding the + /// cost of reading large value buffers is desirable. + TopologyOnly, + /// Deserialize grid metadata and transform only; no topology, no value + /// buffers. The codec is still used to construct the correct grid type, + /// but its @c readTopology()/@c readBuffers() are not called. + MetadataOnly }; /// @brief Base class for per-grid-type, codec-specific read options. @@ -144,6 +148,8 @@ struct OPENVDB_API ReadTypedOptions /// in-place type conversion as data is read. /// @c ReadMode::TopologyOnly skips value buffers entirely, which can be /// significantly faster when only the active-voxel mask is needed. +/// @c ReadMode::MetadataOnly skips both topology and value buffers, +/// reading only grid metadata and transform. /// /// @par Per-type options /// @c typeData allows callers to attach codec-specific configuration for diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index d380cfb50a..b783008986 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -386,7 +386,7 @@ File::readAllGridMetadata() // Seek to the grid in the file. gd.seekToGrid(inputStream()); io::ReadOptions readOptions; - readOptions.readMode = io::ReadMode::TopologyOnly; + readOptions.readMode = io::ReadMode::MetadataOnly; GridBase::ConstPtr grid = Archive::readGrid(gd, inputStream(), readOptions); // Return copies of the grids, but with empty trees. // (As of 0.98.0, at least, it would suffice to just const cast @@ -426,7 +426,7 @@ File::readGridMetadata(const Name& name) const GridDescriptor& gd = it->second; gd.seekToGrid(inputStream()); io::ReadOptions readOptions; - readOptions.readMode = io::ReadMode::TopologyOnly; + readOptions.readMode = io::ReadMode::MetadataOnly; ret = Archive::readGrid(gd, inputStream(), readOptions); } return ret->copyGridWithNewTree(); diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index 13a9134012..26e513b7f6 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -73,6 +73,11 @@ class OPENVDB_API StreamMetadata bool countingPasses() const; void setCountingPasses(bool); + /// @brief Return @c true if readTopology() should allocate and zero-fill + /// leaf buffers after loading the tree structure (topology-only read mode). + bool allocateLeafBuffers() const; + void setAllocateLeafBuffers(bool); + uint32_t pass() const; void setPass(uint32_t); diff --git a/openvdb/openvdb/tree/InternalNode.h b/openvdb/openvdb/tree/InternalNode.h index fb8c5a8ef9..7dbe5349dc 100644 --- a/openvdb/openvdb/tree/InternalNode.h +++ b/openvdb/openvdb/tree/InternalNode.h @@ -82,6 +82,10 @@ class InternalNode /// @param active State assigned to all the tiles InternalNode(const Coord& origin, const ValueType& fillValue, bool active = false); + /// @brief Construct a node without allocating child memory. Children are + /// left unallocated and must be populated single-threaded, or with external + /// synchronization. The valid advanced pattern is: create all nodes + /// single-threaded, then allocate leaf buffers in parallel across distinct leaves. InternalNode(PartialCreate, const Coord&, const ValueType& fillValue, bool active = false); /// @brief Deep copy constructor diff --git a/openvdb/openvdb/tree/LeafBuffer.h b/openvdb/openvdb/tree/LeafBuffer.h index bb6222cf84..7d1a965144 100644 --- a/openvdb/openvdb/tree/LeafBuffer.h +++ b/openvdb/openvdb/tree/LeafBuffer.h @@ -44,7 +44,12 @@ class LeafBuffer explicit inline LeafBuffer(const ValueType&); /// Copy constructor inline LeafBuffer(const LeafBuffer&); - /// Construct a buffer but don't allocate memory for the full array of values. + /// @brief Construct a buffer without allocating the value array. + /// The buffer is left unallocated; call @c allocate() before use. + /// Populating the buffer (via @c allocate() or @c data()) must be done + /// single-threaded per leaf, or externally synchronized. The valid + /// advanced pattern is: create all nodes single-threaded, then call + /// @c allocate() in parallel across distinct leaves. LeafBuffer(PartialCreate, const ValueType&): mData(nullptr) {} /// Destructor inline ~LeafBuffer(); @@ -87,11 +92,15 @@ class LeafBuffer static Index size() { return SIZE; } /// @brief Return a const pointer to the array of voxel values. - /// @details This method guarantees that the buffer is allocated and loaded. + /// @warning The buffer must already be allocated (call @c allocate() first). + /// First-touch allocation via @c data() is not thread-safe; concurrent access + /// to an unallocated buffer is a programming error flagged in debug builds. /// @warning This method should only be used by experts seeking low-level optimizations. const ValueType* data() const; /// @brief Return a pointer to the array of voxel values. - /// @details This method guarantees that the buffer is allocated and loaded. + /// @warning The buffer must already be allocated (call @c allocate() first). + /// First-touch allocation via @c data() is not thread-safe; concurrent access + /// to an unallocated buffer is a programming error flagged in debug builds. /// @warning This method should only be used by experts seeking low-level optimizations. ValueType* data(); @@ -229,9 +238,10 @@ template inline const typename LeafBuffer::ValueType* LeafBuffer::data() const { + OPENVDB_ASSERT(mData != nullptr); if (mData == nullptr) { LeafBuffer* self = const_cast(this); - if (mData == nullptr) self->mData = new ValueType[SIZE]; + self->mData = new ValueType[SIZE]; } return mData; } @@ -240,9 +250,8 @@ template inline typename LeafBuffer::ValueType* LeafBuffer::data() { - if (mData == nullptr) { - if (mData == nullptr) mData = new ValueType[SIZE]; - } + OPENVDB_ASSERT(mData != nullptr); + if (mData == nullptr) mData = new ValueType[SIZE]; return mData; } diff --git a/openvdb/openvdb/tree/LeafNode.h b/openvdb/openvdb/tree/LeafNode.h index 12692e76f1..4a53a3927f 100644 --- a/openvdb/openvdb/tree/LeafNode.h +++ b/openvdb/openvdb/tree/LeafNode.h @@ -83,6 +83,9 @@ class LeafNode /// @param value a value with which to fill the buffer /// @param active the active state to which to initialize all voxels /// @details This constructor does not allocate memory for voxel values. + /// Call @c buffer().allocate() before accessing voxel data. The valid + /// advanced pattern is: create all leaves single-threaded, then call + /// @c allocate() in parallel across distinct leaves. LeafNode(PartialCreate, const Coord& coords, const ValueType& value = zeroVal(), diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index d8981ca3f7..c9e96add01 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -219,8 +219,17 @@ void testIOImpl( f.close(); } ASSERT_TRUE(readTopo); - EXPECT_EQ(readTopo->activeVoxelCount(), Index64(0)); - EXPECT_TRUE(readTopo->tree().leafCount() == 0); + // TopologyOnly: full tree structure is read (topology + active-voxel masks), + // leaf buffers are allocated and zero-filled, values are not read. + EXPECT_EQ(readTopo->tree().leafCount(), srcGrid->tree().leafCount()); + EXPECT_TRUE(readTopo->tree().leafCount() > 0); + EXPECT_EQ(readTopo->activeVoxelCount(), srcGrid->activeVoxelCount()); + // verify leaf buffers are allocated (bool/mask buffers are always present, skip empty() check) + if constexpr (!std::is_same_v) { + for (auto leafIter = readTopo->tree().cbeginLeaf(); leafIter; ++leafIter) { + EXPECT_FALSE(leafIter->buffer().empty()); + } + } EXPECT_EQ(readTopo->getName(), gridName); // Cleanup diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc index 52cb6a34bc..9c8660a4a6 100644 --- a/openvdb/openvdb/unittest/TestPointCodec.cc +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -138,8 +138,15 @@ TEST_F(TestPointCodec, testPointIndexCodecIO) f.close(); } ASSERT_TRUE(codecTopo); - EXPECT_EQ(codecTopo->activeVoxelCount(), Index64(0)); - EXPECT_TRUE(codecTopo->tree().leafCount() == 0); + // TopologyOnly: topology and active state are preserved; voxel values are zero-filled + EXPECT_EQ(codecTopo->tree().leafCount(), srcGrid->tree().leafCount()); + EXPECT_EQ(codecTopo->activeVoxelCount(), srcGrid->activeVoxelCount()); + for (auto leafIt = codecTopo->tree().cbeginLeaf(); leafIt; ++leafIt) { + EXPECT_FALSE(leafIt->buffer().empty()); + for (auto voxIt = leafIt->cbeginValueOn(); voxIt; ++voxIt) { + EXPECT_EQ(*voxIt, PointIndexGrid::ValueType(0)); + } + } EXPECT_EQ(codecTopo->getName(), std::string("point_index_grid")); // Cleanup @@ -265,8 +272,14 @@ TEST_F(TestPointCodec, testPointDataCodecIO) f.close(); } ASSERT_TRUE(codecTopo); - EXPECT_EQ(codecTopo->activeVoxelCount(), Index64(0)); - EXPECT_TRUE(codecTopo->tree().leafCount() == 0); + // TopologyOnly: topology and active state are preserved; voxel values are zero-filled + EXPECT_EQ(codecTopo->tree().leafCount(), srcGrid->tree().leafCount()); + EXPECT_EQ(codecTopo->activeVoxelCount(), srcGrid->activeVoxelCount()); + for (auto leafIt = codecTopo->tree().cbeginLeaf(); leafIt; ++leafIt) { + for (auto voxIt = leafIt->cbeginValueOn(); voxIt; ++voxIt) { + EXPECT_EQ(*voxIt, PointDataGrid::ValueType(0)); + } + } std::remove(codecPath.c_str()); } From 863900fe2c6f9dfce32d4d9efe489ced5da5ee30 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 20:44:44 -0700 Subject: [PATCH 55/70] Add conversion codec fallback and read diagnostics Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/TopologyCodec.h | 17 +++++++++++++++++ openvdb/openvdb/io/Archive.cc | 9 +++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index cbccf5eef2..6f704cbe89 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -339,6 +339,23 @@ struct TopologyCodec : public io::Codec void readTopology(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final { + // Warn when a conversion readMode was requested but this codec is a + // non-conversion instance (GridT == StorageGridT), meaning no conversion + // codec was registered for this grid type and we are falling back to the + // original type. + if constexpr (std::is_same_v) { + if (options.readMode == io::ReadMode::Half || + options.readMode == io::ReadMode::Bool || + options.readMode == io::ReadMode::Mask) + { + const std::string modeStr = + options.readMode == io::ReadMode::Half ? "Half" : + options.readMode == io::ReadMode::Bool ? "Bool" : "Mask"; + diagnostics.addWarning(data.grid->getName(), + "ReadMode::" + modeStr + " conversion is not supported for grid type '" + + GridT::gridType() + "'; reading as original type"); + } + } internal::topologyCodecReadTopology(*data.grid, is, options, diagnostics); } diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index e0fc5dc599..987013e9c1 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -893,14 +893,15 @@ Archive::findCodec(const std::string& gridType, const io::ReadOptions& options) // if readMode is Half, then search for a codec that converts // from the storage grid type to the grid type if (options.readMode == ReadMode::Half) { - return io::CodecRegistry::get(gridType + "_to_half"); + if (auto* codec = io::CodecRegistry::get(gridType + "_to_half")) return codec; } else if (options.readMode == ReadMode::Bool) { - return io::CodecRegistry::get(gridType + "_to_bool"); + if (auto* codec = io::CodecRegistry::get(gridType + "_to_bool")) return codec; } else if (options.readMode == ReadMode::Mask) { - return io::CodecRegistry::get(gridType + "_to_mask"); + if (auto* codec = io::CodecRegistry::get(gridType + "_to_mask")) return codec; } - // Determine the I/O codec to use to read this grid + // Determine the I/O codec to use to read this grid (also the fallback when + // no conversion codec is registered for a Half/Bool/Mask readMode). return io::CodecRegistry::get(gridType); } From 8dceecf27e4eb4103bae45703195c6442f2474f1 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 20:46:48 -0700 Subject: [PATCH 56/70] Fix a bug in PointDataGrid where read-and-discard not being used when stream is not seekable Signed-off-by: Dan Bailey --- openvdb/openvdb/points/AttributeArray.h | 8 +- openvdb/openvdb/points/StreamCompression.cc | 9 +- openvdb/openvdb/unittest/TestPointCodec.cc | 127 ++++++++++++++++++++ 3 files changed, 142 insertions(+), 2 deletions(-) diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index 23ff8f8149..26f03b71cc 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -1611,7 +1611,13 @@ AttributeArray::skipPagedBuffers(compression::PagedInputStream& is) std::istream& inputStream = is.getInputStream(); uint8_t bloscCompressed(0); if (!mIsUniform) inputStream.read(reinterpret_cast(&bloscCompressed), sizeof(uint8_t)); - inputStream.seekg(mCompressedBytes, std::ios_base::cur); + auto meta = io::getStreamMetadataPtr(inputStream); + if (meta && meta->seekable()) { + inputStream.seekg(mCompressedBytes, std::ios_base::cur); + } else { + std::vector tempData(mCompressedBytes); + inputStream.read(tempData.data(), mCompressedBytes); + } mCompressedBytes = 0; mFlags = static_cast(mFlags & ~PARTIALREAD); } diff --git a/openvdb/openvdb/points/StreamCompression.cc b/openvdb/openvdb/points/StreamCompression.cc index f18d016463..fb99554b70 100644 --- a/openvdb/openvdb/points/StreamCompression.cc +++ b/openvdb/openvdb/points/StreamCompression.cc @@ -7,6 +7,7 @@ #include #include #include +#include #ifdef OPENVDB_USE_BLOSC #include #endif @@ -347,7 +348,13 @@ Page::skipBuffers(std::istream& is) std::streamsize bytes = isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes; - is.seekg(bytes, std::ios_base::cur); + auto meta = io::getStreamMetadataPtr(is); + if (meta && meta->seekable()) { + is.seekg(bytes, std::ios_base::cur); + } else { + std::vector tempData(bytes); + is.read(tempData.data(), bytes); + } mInfo.reset(); } diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc index 9c8660a4a6..25ffab976a 100644 --- a/openvdb/openvdb/unittest/TestPointCodec.cc +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -1,11 +1,16 @@ // Copyright Contributors to the OpenVDB Project // SPDX-License-Identifier: Apache-2.0 +#include #include #include +#include +#include +#include #include #include #include +#include #include #include "util.h" // for unittest_util::genPoints @@ -600,3 +605,125 @@ TEST_F(TestPointCodec, testPointDataCodecEmptyGrid) std::remove(codecPath.c_str()); } + +// Regression test for the fix in AttributeArray::skipPagedBuffers and +// Page::skipBuffers: when the stream is not seekable (written via io::Stream), +// skip must read-and-discard rather than seekg. Without the fix, both code +// paths called seekg unconditionally, corrupting the stream position on +// non-seekable streams. +TEST_F(TestPointCodec, testPointDataCodecSkipNonSeekable) +{ + using namespace openvdb; + using namespace openvdb::io; + using namespace openvdb::points; + + openvdb::initialize(); + CodecRegistry::clear(); + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointDataGrid::gridType())); + + const std::vector positions = { + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(1.5f, 3.5f, 1.0f), + Vec3f(-1.0f, 6.0f, -2.0f), + Vec3f(1.1f, 1.25f, 0.06f) + }; + const std::vector velocities = { + Vec3f(1.0f, 0.0f, 0.0f), + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(0.0f, 0.0f, 1.0f), + Vec3f(1.0f, 1.0f, 0.5f) + }; + const std::vector ids = {0, 1, 2, 3}; + + const float voxelSize = 0.5f; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointAttributeVector posWrapper(positions); + tools::PointIndexGrid::Ptr pointIndexGrid = + tools::createPointIndexGrid(posWrapper, *transform); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid( + *pointIndexGrid, posWrapper, *transform); + srcGrid->setName("pdg_skip"); + + PointDataTree& tree = srcGrid->tree(); + tools::PointIndexTree& indexTree = pointIndexGrid->tree(); + + appendAttribute(tree, "velocity"); + populateAttribute>( + tree, indexTree, "velocity", + PointAttributeVector(velocities)); + + appendAttribute(tree, "id"); + populateAttribute>( + tree, indexTree, "id", + PointAttributeVector(ids)); + + // Write via io::Stream (seekable=false by construction) + std::ostringstream ostr(std::ios_base::binary); + io::Stream(ostr).write(GridPtrVec{srcGrid}); + + // Build ReadOptions requesting only "P" — "velocity" and "id" will be skipped + io::ReadOptions readOptions; + auto typeData = std::make_shared(); + typeData->pointAttributeNames = {"P"}; + readOptions.typeData[PointDataGrid::gridType()] = typeData; + + // Parse the stream manually, replicating Stream::Stream(istream&) but using + // our custom ReadOptions so that the skip path is exercised with seekable=false. + std::istringstream is(ostr.str(), std::ios_base::binary); + + io::Stream headerReader(is); + // StreamMetadata created without setSeekable(true) => seekable() == false + StreamMetadata::Ptr streamMeta(new StreamMetadata); + EXPECT_FALSE(streamMeta->seekable()); + + std::istringstream isFinal(ostr.str(), std::ios_base::binary); + headerReader.readHeader(isFinal); + io::setStreamMetadataPtr(isFinal, streamMeta, /*transfer=*/false); + io::setVersion(isFinal, headerReader.libraryVersion(), headerReader.fileVersion()); + io::setDataCompression(isFinal, headerReader.compression()); + + MetaMap fileMeta; + fileMeta.readMeta(isFinal); + + const int32_t gridCount = Archive::readGridCount(isFinal); + ASSERT_EQ(gridCount, 1); + + GridDescriptor gd; + gd.readHeader(isFinal); + gd.readStreamPos(isFinal); + + GridBase::Ptr baseGrid = Archive::readGrid(gd, isFinal, readOptions); + ASSERT_TRUE(baseGrid); + + PointDataGrid::Ptr resultGrid = gridPtrCast(baseGrid); + ASSERT_TRUE(resultGrid); + + // Only "P" should be present; "velocity" and "id" were skipped + { + auto leafIt = resultGrid->tree().cbeginLeaf(); + ASSERT_TRUE(leafIt); + EXPECT_EQ(leafIt->attributeSet().size(), size_t(1)); + EXPECT_NE(leafIt->attributeSet().find("P"), AttributeSet::INVALID_POS); + EXPECT_EQ(leafIt->attributeSet().find("velocity"), AttributeSet::INVALID_POS); + EXPECT_EQ(leafIt->attributeSet().find("id"), AttributeSet::INVALID_POS); + } + + // All 4 points should be present and readable + EXPECT_EQ(pointCount(resultGrid->tree()), Index64(4)); + { + std::vector readPositions; + for (auto it = resultGrid->tree().cbeginLeaf(); it; ++it) { + AttributeHandle posHandle(it->constAttributeArray("P")); + for (Index i = 0; i < it->pointCount(); ++i) { + readPositions.push_back(posHandle.get(i)); + } + } + EXPECT_EQ(readPositions.size(), size_t(4)); + } +} From 132ee154040d72f48a3e20f576ae6c9110c119d0 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 20:52:32 -0700 Subject: [PATCH 57/70] Calling initialize will now clear the registry making it idempotent and align with all the other initialize methods Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Codec.cc | 1 + openvdb/openvdb/unittest/TestCodec.cc | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc index 4873f779aa..e2a3213814 100644 --- a/openvdb/openvdb/io/Codec.cc +++ b/openvdb/openvdb/io/Codec.cc @@ -55,6 +55,7 @@ struct RegisterConvertCodec { void initialize() { + CodecRegistry::clear(); NumericGridTypes::foreach(); Vec3GridTypes::foreach(); diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index c9e96add01..db7de5c70d 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -78,6 +78,24 @@ TEST_F(TestCodec, testCodecRegistry) } +TEST_F(TestCodec, testInitializeIdempotent) +{ + using namespace openvdb::io; + + // Calling initialize() twice without uninitialize() in between must not throw. + // Previously registerCodecByName() threw KeyError on the duplicate registration. + CodecRegistry::clear(); + EXPECT_NO_THROW(internal::initialize()); + EXPECT_NO_THROW(internal::initialize()); + + // Codecs must still be registered after the second call. + EXPECT_TRUE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType())); + EXPECT_TRUE(CodecRegistry::isRegistered(openvdb::FloatGrid::gridType())); + + internal::uninitialize(); +} + + TEST_F(TestCodec, testReadDiagnostics) { using namespace openvdb; From 6624c4dce40eed2eb67e569a8510a83041c6e475 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 20:54:27 -0700 Subject: [PATCH 58/70] Fix a potential memory leak when reading topology Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/TopologyCodec.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 6f704cbe89..6970b99bcd 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -212,11 +212,12 @@ struct ReadTopologyOp } // Read in all child nodes and insert them into the table at their proper locations. + // Register the child before recursing so that node's destructor frees it on a throw. for (auto iter = childMask.beginOn(); iter; ++iter) { Coord origin = node.offsetToGlobalCoord(iter.pos()); auto* child = new ChildT(PartialCreate(), origin, background); - (*this)(*child); node.setChildUnsafe(iter.pos(), child); + (*this)(*child); } } From f173582e16b5eb9a25d55bad69491c37307e6f3c Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 20:55:03 -0700 Subject: [PATCH 59/70] Throw an error if the attribute descriptor is not homogeneous Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/PointDataCodec.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index ac3d353f77..512eb0b67a 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -355,7 +355,17 @@ struct PointDataCodec final: public TopologyCodec // An empty pointAttributeNames means no filtering (read all attributes). std::set skipIndices; if (!pointAttributeNames.empty() && !leaves.empty()) { - const auto& nameMap = leaves[0]->attributeSet().descriptor().map(); + // Attribute filtering requires homogeneous descriptors across all + // leaves because skip decisions are made per-index across all leaves. + const auto* firstDesc = &leaves[0]->attributeSet().descriptor(); + for (size_t i = 1; i < leaves.size(); ++i) { + if (&leaves[i]->attributeSet().descriptor() != firstDesc) { + OPENVDB_THROW(IoError, + "Attribute filtering is not supported for PointDataGrids " + "with heterogeneous descriptors"); + } + } + const auto& nameMap = firstDesc->map(); const std::set wantedNames( pointAttributeNames.begin(), pointAttributeNames.end()); From 4208d6b1f3c5c22dad63c53c5855f4bd8400cddd Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 9 Jun 2026 21:39:58 -0700 Subject: [PATCH 60/70] Fix an issue where the background value was previously using ValueT instead of StorageValueT causing inactive values to be read incorrectly Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/PointDataCodec.h | 9 +- openvdb/openvdb/codecs/PointIndexCodec.h | 12 ++- openvdb/openvdb/codecs/ScalarCodec.h | 31 ++++--- openvdb/openvdb/codecs/TopologyCodec.h | 53 ++++++++---- openvdb/openvdb/codecs/impl/ScalarLeafCodec.h | 24 +++--- openvdb/openvdb/io/Compression.h | 27 +++--- openvdb/openvdb/unittest/TestCodec.cc | 82 +++++++++++++++++++ 7 files changed, 182 insertions(+), 56 deletions(-) diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index 512eb0b67a..ab203fe4de 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -111,13 +111,14 @@ template inline void readPointDataVoxelData(const std::vector& leaves, std::istream& is, bool saveFloatAsHalf, const typename LeafT::ValueType& background, - [[maybe_unused]] const std::unordered_map& voxelBufferSizes) + [[maybe_unused]] const std::unordered_map& voxelBufferSizes, + const typename LeafT::ValueType* storageBackground = nullptr) { using BaseLeaf = typename LeafT::BaseLeaf; for (auto* leaf : leaves) { OPENVDB_ASSERT(voxelBufferSizes.find(leaf->origin()) != voxelBufferSizes.end()); BaseLeaf& baseLeaf = static_cast(*leaf); - readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background); + readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background, /*skip=*/false, /*clipBBox=*/nullptr, storageBackground); } } @@ -387,8 +388,10 @@ struct PointDataCodec final: public TopologyCodec } // Pass N+2: read voxel data + using ValueT = typename GridT::TreeType::ValueType; + auto& topoData = static_cast&>(data); internal::readPointDataVoxelData(leaves, is, saveFloatAsHalf, - tree.background(), voxelBufferSizes); + tree.background(), voxelBufferSizes, &topoData.storageBackground); // Passes N+3..2N+2: read attribute data buffers for (Index i = 0; i < attributes; ++i) { diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h index 851ee140fd..0dec4fd77b 100644 --- a/openvdb/openvdb/codecs/PointIndexCodec.h +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -26,10 +26,11 @@ struct ReadPointIndexBuffersOp using ValueT = typename TreeT::ValueType; ReadPointIndexBuffersOp(std::istream& _is, bool _saveFloatAsHalf, - const ValueT& _background) + const ValueT& _background, const ValueT* _storageBackground = nullptr) : is(_is) , saveFloatAsHalf(_saveFloatAsHalf) - , background(_background) { } + , background(_background) + , storageBackground(_storageBackground) { } template void operator()(NodeT&, size_t) { } @@ -40,7 +41,7 @@ struct ReadPointIndexBuffersOp // Read the value mask and voxel data via base class BaseLeaf& baseLeaf = static_cast(leaf); - readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background, /*skip=*/false, /*clipBBox=*/nullptr); + readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background, /*skip=*/false, /*clipBBox=*/nullptr, storageBackground); // Read the number of indices. Index64 numIndices = Index64(0); @@ -63,6 +64,7 @@ struct ReadPointIndexBuffersOp std::istream& is; const bool saveFloatAsHalf; const ValueT& background; + const ValueT* storageBackground = nullptr; }; // struct ReadPointIndexBuffersOp template @@ -135,7 +137,9 @@ struct PointIndexCodec final: public TopologyCodec diagnostics.addWarning(grid.getName(), "bounding box clipping is not supported for PointIndexGrids"); } - internal::ReadPointIndexBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background()); + using ValueT = typename GridT::TreeType::ValueType; + auto& topoData = static_cast&>(data); + internal::ReadPointIndexBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), &topoData.storageBackground); tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); } diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h index 114c575bcf..440325d2b3 100644 --- a/openvdb/openvdb/codecs/ScalarCodec.h +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -23,21 +23,26 @@ template struct WriteBuffersOp { using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; - WriteBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + WriteBuffersOp(std::ostream& _os, bool _saveFloatAsHalf, const ValueT& _background) : os(_os) - , saveFloatAsHalf(_saveFloatAsHalf) { } + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) { } template void operator()(const NodeT&, size_t) { } void operator()(const LeafT& leaf, size_t) { - writeScalarLeafBuffers(leaf, os, saveFloatAsHalf); + // Pass the background explicitly so leaf compression does not depend on + // the stream's background pointer (which the codec path no longer sets). + writeScalarLeafBuffers(leaf, os, saveFloatAsHalf, &background); } std::ostream& os; const bool saveFloatAsHalf; + const ValueT& background; }; // struct WriteBuffersOp @@ -48,13 +53,15 @@ struct ReadBuffersOp using LeafT = typename TreeT::LeafNodeType; using ValueT = typename TreeT::ValueType; using StorageLeafT = typename StorageTreeT::LeafNodeType; + using StorageValueT = typename StorageTreeT::ValueType; ReadBuffersOp(std::istream& _is, bool _saveFloatAsHalf, const ValueT& _background, - const CoordBBox* _clipBBox) + const CoordBBox* _clipBBox, const StorageValueT* _storageBackground = nullptr) : is(_is) , saveFloatAsHalf(_saveFloatAsHalf) , background(_background) - , clipBBox(_clipBBox) { } + , clipBBox(_clipBBox) + , storageBackground(_storageBackground) { } void operator()(RootT& root, size_t) { @@ -73,20 +80,22 @@ struct ReadBuffersOp void operator()(LeafT& leaf, size_t) { - readScalarLeafBuffers(leaf, is, saveFloatAsHalf, background, /*skip=*/false, clipBBox); + readScalarLeafBuffers(leaf, is, saveFloatAsHalf, background, /*skip=*/false, clipBBox, storageBackground); } std::istream& is; const bool saveFloatAsHalf; const ValueT& background; const CoordBBox* clipBBox = nullptr; + const StorageValueT* storageBackground = nullptr; }; // struct ReadBuffersOp // Free-standing function for both standard and conversion codec cases // Uses StorageGridT = GridT by default, but allows different storage type for conversions template -void scalarCodecReadBuffers(GridT& grid, std::istream& is, const io::ReadOptions& options) +void scalarCodecReadBuffers(GridT& grid, std::istream& is, const io::ReadOptions& options, + const typename StorageGridT::TreeType::ValueType* storageBackground) { if (grid.hasMultiPassIO()) { OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ScalarCodec"); @@ -109,7 +118,7 @@ void scalarCodecReadBuffers(GridT& grid, std::istream& is, const io::ReadOptions // Works for both standard (TreeT == StorageTreeT) and conversion cases ReadBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), - clipIndexBBox.get()); + clipIndexBBox.get(), storageBackground); tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); } @@ -123,7 +132,7 @@ void scalarCodecWriteBuffers(const GridT& grid, std::ostream& os) OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ScalarCodec"); } - WriteBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + WriteBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf(), grid.tree().background()); tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); } @@ -152,8 +161,10 @@ struct ScalarCodec final: public TopologyCodec void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { + using StorageValueT = typename StorageGridT::TreeType::ValueType; GridT& grid = static_cast(*data.grid); - internal::scalarCodecReadBuffers(grid, is, options); + auto& topoData = static_cast&>(data); + internal::scalarCodecReadBuffers(grid, is, options, &topoData.storageBackground); } void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 6970b99bcd..305d9c0129 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -16,6 +16,16 @@ namespace openvdb { OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { namespace codecs { + +/// Per-read-operation state shared between readTopology() and readBuffers(). +/// Stores the storage-typed background so readBuffers() can pass it directly +/// to readCompressedValues(), bypassing the stream background ptr entirely. +template +struct TopologyCodecData : public io::CodecData +{ + StorageValueT storageBackground{}; +}; // struct TopologyCodecData + namespace internal { template @@ -41,7 +51,6 @@ struct WriteTopologyOp ValueT truncatedVal = io::truncateRealToHalf(*background); os.write(reinterpret_cast(&truncatedVal), sizeof(ValueT)); } - io::setGridBackgroundValuePtr(os, background); const Index numTiles = root.tileCount(), numChildren = root.childCount(); os.write(reinterpret_cast(&numTiles), sizeof(Index)); @@ -84,7 +93,7 @@ struct WriteTopologyOp values[i] = (node.isChildMaskOff(i) ? node.getValueUnsafe(i) : zero); } // Compress (optionally) and write out the contents of the array. - io::writeCompressedValues(os, values, NodeT::NUM_VALUES, valueMask, childMask, saveFloatAsHalf); + io::writeCompressedValues(os, values, NodeT::NUM_VALUES, valueMask, childMask, saveFloatAsHalf, background); } } @@ -130,16 +139,8 @@ struct ReadTopologyOp // Read a RootNode that was stored in the current format. - if constexpr (std::is_same_v) { - is.read(reinterpret_cast(&background), sizeof(ValueT)); - } else { - StorageValueT _background; - is.read(reinterpret_cast(&_background), sizeof(StorageValueT)); - background = static_cast(_background); - } - - root.setBackground(background, false); - io::setGridBackgroundValuePtr(is, &root.background()); + is.read(reinterpret_cast(&storageBackground), sizeof(StorageValueT)); + background = static_cast(storageBackground); Index numTiles = 0, numChildren = 0; is.read(reinterpret_cast(&numTiles), sizeof(Index)); @@ -191,7 +192,7 @@ struct ReadTopologyOp // into a contiguous array. std::unique_ptr valuePtr(new StorageValueT[numValues]); StorageValueT* values = valuePtr.get(); - io::readCompressedValues(is, values, numValues, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, values, numValues, valueMask, saveFloatAsHalf, &storageBackground); // Copy values from the array into this node's table. if (oldVersion) { @@ -231,6 +232,7 @@ struct ReadTopologyOp std::istream& is; bool saveFloatAsHalf; ValueT background; + StorageValueT storageBackground; io::ReadDiagnostics& diagnostics; std::string gridName; }; // struct ReadTopologyOp @@ -279,9 +281,12 @@ void setTilesToBackground(TreeT& tree) nodeManager.foreachTopDown(op); } -// Free-standing function for read case (supports type conversion via StorageGridT) +// Free-standing function for read case (supports type conversion via StorageGridT). +// codecData receives the storage-typed background value so readBuffers() callers +// can pass it explicitly to readCompressedValues(), avoiding the stream background ptr. template -void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) +void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::ReadOptions& options, + io::ReadDiagnostics& diagnostics, TopologyCodecData& codecData) { io::checkFormatVersion(is); @@ -291,6 +296,18 @@ void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::R internal::ReadTopologyOp readTopologyOp(is, grid.saveFloatAsHalf(), diagnostics, grid.getName()); readTopologyOp(grid.tree().root()); + // Restore the (value-typed) background on the root. ReadTopologyOp only reads + // the on-disk background into a local; without this the grid would retain the + // default background from GridT::create(). Pass updateChildNodes=false so the + // already-populated child nodes are left untouched. + grid.tree().root().setBackground(readTopologyOp.background, /*updateChildNodes=*/false); + + // Copy storageBackground out of the stack-local ReadTopologyOp into codecData + // so it stays alive until readBuffers() completes. readBuffers() passes it + // explicitly to readCompressedValues(), so the stream background ptr is never + // used and setGridBackgroundValuePtr() is not needed in the codec path. + codecData.storageBackground = readTopologyOp.storageBackground; + if (options.readMode == io::ReadMode::TopologyOnly) { internal::setTilesToBackground(grid.tree()); // allocate leaf buffers in parallel and fill with the background value; @@ -326,13 +343,14 @@ void topologyCodecWriteTopology(const GridBase& gridBase, std::ostream& os) template struct TopologyCodec : public io::Codec { + using StorageValueT = typename StorageGridT::TreeType::ValueType; using Ptr = std::unique_ptr>; ~TopologyCodec() noexcept = default; io::CodecData::Ptr createData() override { - auto data = std::make_unique(); + auto data = std::make_unique>(); data->grid = GridT::create(); return data; } @@ -357,7 +375,8 @@ struct TopologyCodec : public io::Codec + GridT::gridType() + "'; reading as original type"); } } - internal::topologyCodecReadTopology(*data.grid, is, options, diagnostics); + auto& topoData = static_cast&>(data); + internal::topologyCodecReadTopology(*data.grid, is, options, diagnostics, topoData); } void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final diff --git a/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h b/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h index ad9e088604..2288f8a5a2 100644 --- a/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h +++ b/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h @@ -13,7 +13,8 @@ namespace codecs { namespace internal { template -void writeScalarLeafBuffers(const LeafT& leaf, std::ostream& os, bool saveFloatAsHalf) +void writeScalarLeafBuffers(const LeafT& leaf, std::ostream& os, bool saveFloatAsHalf, + const typename LeafT::ValueType* background = nullptr) { using NodeMaskT = typename LeafT::NodeMaskType; @@ -23,14 +24,15 @@ void writeScalarLeafBuffers(const LeafT& leaf, std::ostream& os, bool saveFloatA leaf.buffer().data(); // load values io::writeCompressedValues(os, leaf.buffer().data(), LeafT::SIZE, - leaf.getValueMask(), /*childMask=*/NodeMaskT(), saveFloatAsHalf); + leaf.getValueMask(), /*childMask=*/NodeMaskT(), saveFloatAsHalf, background); } template void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, const typename LeafT::ValueType& background, bool skip = false, - const math::CoordBBox* clipBBox = nullptr) + const math::CoordBBox* clipBBox = nullptr, + const typename StorageLeafT::ValueType* storageBackground = nullptr) { using ValueT = typename LeafT::ValueType; using NodeMaskT = typename LeafT::NodeMaskType; @@ -65,10 +67,10 @@ void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, if (skip) { if (seekable) { - io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf, storageBackground); } else { StorageBufferT storageTemp; - io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); } // Clear the value mask so that the skipped leaf has no active // voxels. Without this, the leaf retains its on-disk active @@ -81,18 +83,18 @@ void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, // ValueMask leaf: value == active state, already captured in the value mask above. // Seek/consume past the storage buffer without populating any separate leaf buffer. if (seekable) { - io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf, storageBackground); } else { StorageBufferT storageTemp; - io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); } } else if constexpr (std::is_same_v) { // Bool leaf: must read storage values regardless of seekability, then convert. if constexpr (std::is_same_v) { - io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); } else { StorageBufferT storageTemp; - io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); for (Index i = 0; i < SIZE; ++i) { leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); } @@ -100,10 +102,10 @@ void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, } else { leaf.buffer().allocate(); if constexpr (std::is_same_v) { - io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); } else { StorageBufferT storageTemp; - io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); for (Index i = 0; i < SIZE; ++i) { leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); } diff --git a/openvdb/openvdb/io/Compression.h b/openvdb/openvdb/io/Compression.h index ae572c4105..ade464275d 100644 --- a/openvdb/openvdb/io/Compression.h +++ b/openvdb/openvdb/io/Compression.h @@ -451,7 +451,7 @@ struct HalfWriter { template inline void readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, - const MaskT& valueMask, bool fromHalf) + const MaskT& valueMask, bool fromHalf, const ValueT* background = nullptr) { checkFormatVersion(is); @@ -475,13 +475,15 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, } } - ValueT background = zeroVal(); - if (const void* bgPtr = getGridBackgroundValuePtr(is)) { - background = *static_cast(bgPtr); + ValueT bgValue = zeroVal(); + if (background) { + bgValue = *background; + } else if (const void* bgPtr = getGridBackgroundValuePtr(is)) { + bgValue = *static_cast(bgPtr); } - ValueT inactiveVal1 = background; + ValueT inactiveVal1 = bgValue; ValueT inactiveVal0 = - ((metadata == NO_MASK_OR_INACTIVE_VALS) ? background : math::negative(background)); + ((metadata == NO_MASK_OR_INACTIVE_VALS) ? bgValue : math::negative(bgValue)); if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL || metadata == MASK_AND_ONE_INACTIVE_VAL || @@ -620,7 +622,8 @@ writeCompressedValuesSize(ValueT* srcBuf, Index srcCount, template inline void writeCompressedValues(std::ostream& os, const ValueT* srcBuf, Index srcCount, - const MaskT& valueMask, const MaskT& childMask, bool toHalf) + const MaskT& valueMask, const MaskT& childMask, bool toHalf, + const ValueT* background = nullptr) { // Get the stream's compression settings. const uint32_t compress = getDataCompression(os); @@ -642,12 +645,14 @@ writeCompressedValues(std::ostream& os, const ValueT* srcBuf, Index srcCount, // an inside/outside bitmask. const ValueT zero = zeroVal(); - ValueT background = zero; - if (const void* bgPtr = getGridBackgroundValuePtr(os)) { - background = *static_cast(bgPtr); + ValueT bgValue = zero; + if (background) { + bgValue = *background; + } else if (const void* bgPtr = getGridBackgroundValuePtr(os)) { + bgValue = *static_cast(bgPtr); } - MaskCompress maskCompressData(valueMask, childMask, srcBuf, background); + MaskCompress maskCompressData(valueMask, childMask, srcBuf, bgValue); metadata = maskCompressData.metadata; os.write(reinterpret_cast(&metadata), /*bytes=*/1); diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index db7de5c70d..cfa14de91e 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -430,3 +430,85 @@ TEST_F(TestCodec, testNumericToMaskCodecConversion) testConvertCodecImpl(); testConvertCodecImpl(); } + +// Regression test for the dangling storageBackground pointer bug. +// +// ReadTopologyOp stores storageBackground on its stack frame and registers +// &storageBackground with the stream. Before the fix, topologyCodecReadTopology +// returned and destroyed ReadTopologyOp before readBuffers() ran; readCompressedValues +// then dereferenced the dead pointer to reconstruct inactive voxels under +// COMPRESS_ACTIVE_MASK, producing garbage inactive values. +// +// The test is deliberately structured to maximize the chance that the freed +// stack frame has been overwritten: a non-zero background (3.0f / 5) forces the +// reconstructed inactive value to be wrong if the pointer is stale, and +// COMPRESS_ACTIVE_MASK (flag 0x2, always on by default) is the code path that +// uses the background pointer. +TEST_F(TestCodec, testInactiveValuesAfterReadBuffers) +{ + using namespace openvdb; + using namespace openvdb::io; + + openvdb::io::CodecRegistry::clear(); + openvdb::io::internal::initialize(); + + // Float: non-zero background, active region surrounded by inactive background voxels. + { + const float bg = 3.0f; + FloatGrid::Ptr src = FloatGrid::create(bg); + src->setName("float_bg"); + src->fill(CoordBBox(Coord(0), Coord(15)), 1.0f, /*active=*/true); + src->fill(CoordBBox(Coord(4), Coord(11)), bg, /*active=*/false); + + const std::string path = "testInactiveVals_float.vdb"; + { + io::File f(path); + f.setCompression(COMPRESS_ACTIVE_MASK); + f.write(GridPtrVec{src}); + } + FloatGrid::Ptr result; + { + io::File f(path); + f.open(); + result = gridPtrCast(f.readGrid("float_bg")); + f.close(); + } + ASSERT_TRUE(result); + EXPECT_EQ(result->background(), bg); + FloatGrid::ConstAccessor refAcc = src->getConstAccessor(); + for (FloatGrid::ValueAllCIter it = result->cbeginValueAll(); it; ++it) { + EXPECT_EQ(*it, refAcc.getValue(it.getCoord())); + } + std::remove(path.c_str()); + } + + // Int32: non-zero background (5), verify inactive values round-trip. + { + const int bg = 5; + Int32Grid::Ptr src = Int32Grid::create(bg); + src->setName("int_bg"); + src->fill(CoordBBox(Coord(0), Coord(15)), 99, /*active=*/true); + src->fill(CoordBBox(Coord(4), Coord(11)), bg, /*active=*/false); + + const std::string path = "testInactiveVals_int.vdb"; + { + io::File f(path); + f.setCompression(COMPRESS_ACTIVE_MASK); + f.write(GridPtrVec{src}); + } + Int32Grid::Ptr result; + { + io::File f(path); + f.open(); + result = gridPtrCast(f.readGrid("int_bg")); + f.close(); + } + ASSERT_TRUE(result); + EXPECT_EQ(result->background(), bg); + Int32Grid::ConstAccessor refAcc = src->getConstAccessor(); + for (Int32Grid::ValueAllCIter it = result->cbeginValueAll(); it; ++it) { + EXPECT_EQ(*it, refAcc.getValue(it.getCoord())); + } + std::remove(path.c_str()); + } +} From ca04e210b54c1cbf0346512ecdc46b9297ca6265 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 11 Jun 2026 15:25:56 -0700 Subject: [PATCH 61/70] Add ReadOptions argument to Stream Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/ScalarCodec.h | 13 ++++++--- openvdb/openvdb/codecs/TopologyCodec.h | 11 ++++--- openvdb/openvdb/io/Stream.cc | 8 ++++- openvdb/openvdb/io/Stream.h | 8 ++++- openvdb/openvdb/points/PointDataIO.h | 6 ++-- openvdb/openvdb/unittest/TestPointCodec.cc | 34 +++++----------------- 6 files changed, 41 insertions(+), 39 deletions(-) diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h index 440325d2b3..0999707068 100644 --- a/openvdb/openvdb/codecs/ScalarCodec.h +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -169,10 +169,15 @@ struct ScalarCodec final: public TopologyCodec void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final { - if constexpr (Mode == io::CodecMode::ReadOnly) return; - - const GridT& grid = static_cast(gridBase); - internal::scalarCodecWriteBuffers(grid, os); + // Note: the write body must live inside the negated if constexpr branch + // so it is not instantiated for read-only codecs. A bare + // `if constexpr (Mode == ReadOnly) return;` would still instantiate the + // code that follows, which fails to compile for the scalar-to-mask/bool + // convert codecs (their leaf buffers expose WordType*, not ValueType*). + if constexpr (Mode != io::CodecMode::ReadOnly) { + const GridT& grid = static_cast(gridBase); + internal::scalarCodecWriteBuffers(grid, os); + } } }; // struct ScalarCodec diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 305d9c0129..eee4ead6e5 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -381,10 +381,13 @@ struct TopologyCodec : public io::Codec void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final { - // disable implementation when read only - if constexpr (Mode == io::CodecMode::ReadOnly) return; - - internal::topologyCodecWriteTopology(gridBase, os); + // Disable implementation when read only. The body must live inside the + // negated if constexpr branch so it is not instantiated for read-only + // codecs; a bare `if constexpr (...) return;` still instantiates what + // follows. + if constexpr (Mode != io::CodecMode::ReadOnly) { + internal::topologyCodecWriteTopology(gridBase, os); + } } }; // struct TopologyCodec diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 91b3ca4b7d..5b7ae46601 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -21,6 +21,12 @@ namespace io { Stream::Stream(std::istream& is) + : Stream(is, io::ReadOptions{}) +{ +} + + +Stream::Stream(std::istream& is, const io::ReadOptions& readOptions) { if (!is) return; @@ -51,7 +57,7 @@ Stream::Stream(std::istream& is) gd.readHeader(is); gd.readStreamPos(is); descriptors.push_back(gd); - GridBase::Ptr grid = Archive::readGrid(gd, is, io::ReadOptions{}); + GridBase::Ptr grid = Archive::readGrid(gd, is, readOptions); mGrids->push_back(grid); namedGrids[gd.uniqueName()] = grid; } diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index 418a06b142..fd302c8547 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -23,7 +23,13 @@ class OPENVDB_API Stream: public Archive public: /// @brief Read grids from an input stream. /// @param is The input stream to read from - explicit Stream(std::istream&); + explicit Stream(std::istream& is); + + /// @brief Read grids from an input stream using the given read options. + /// @param is The input stream to read from + /// @param readOptions Options controlling how grids are read (e.g. attribute + /// skipping for point data grids) + Stream(std::istream& is, const io::ReadOptions& readOptions); OPENVDB_DEPRECATED_MESSAGE("Use Stream(std::istream&) instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") Stream(std::istream& is, bool /*delayLoad*/) : Stream(is) { } diff --git a/openvdb/openvdb/points/PointDataIO.h b/openvdb/openvdb/points/PointDataIO.h index fddbaef6c4..110b93972b 100644 --- a/openvdb/openvdb/points/PointDataIO.h +++ b/openvdb/openvdb/points/PointDataIO.h @@ -21,7 +21,8 @@ namespace io template<> inline void readCompressedValues( std::istream& is, PointDataIndex32* destBuf, Index destCount, - const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/) + const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/, + const PointDataIndex32* /*background*/) { using compression::bloscDecompress; @@ -82,7 +83,8 @@ template<> inline void writeCompressedValues( std::ostream& os, const PointDataIndex32* srcBuf, Index srcCount, const util::NodeMask<3>& /*valueMask*/, - const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/) + const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/, + const PointDataIndex32* /*background*/) { using compression::bloscCompress; diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc index 25ffab976a..0729f6cc98 100644 --- a/openvdb/openvdb/unittest/TestPointCodec.cc +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -673,35 +673,15 @@ TEST_F(TestPointCodec, testPointDataCodecSkipNonSeekable) typeData->pointAttributeNames = {"P"}; readOptions.typeData[PointDataGrid::gridType()] = typeData; - // Parse the stream manually, replicating Stream::Stream(istream&) but using - // our custom ReadOptions so that the skip path is exercised with seekable=false. + // Read via io::Stream (non-seekable by construction) with our ReadOptions, so the + // skip path is exercised with seekable == false. std::istringstream is(ostr.str(), std::ios_base::binary); + io::Stream strm(is, readOptions); + GridPtrVecPtr grids = strm.getGrids(); + ASSERT_TRUE(grids); + ASSERT_EQ(grids->size(), size_t(1)); - io::Stream headerReader(is); - // StreamMetadata created without setSeekable(true) => seekable() == false - StreamMetadata::Ptr streamMeta(new StreamMetadata); - EXPECT_FALSE(streamMeta->seekable()); - - std::istringstream isFinal(ostr.str(), std::ios_base::binary); - headerReader.readHeader(isFinal); - io::setStreamMetadataPtr(isFinal, streamMeta, /*transfer=*/false); - io::setVersion(isFinal, headerReader.libraryVersion(), headerReader.fileVersion()); - io::setDataCompression(isFinal, headerReader.compression()); - - MetaMap fileMeta; - fileMeta.readMeta(isFinal); - - const int32_t gridCount = Archive::readGridCount(isFinal); - ASSERT_EQ(gridCount, 1); - - GridDescriptor gd; - gd.readHeader(isFinal); - gd.readStreamPos(isFinal); - - GridBase::Ptr baseGrid = Archive::readGrid(gd, isFinal, readOptions); - ASSERT_TRUE(baseGrid); - - PointDataGrid::Ptr resultGrid = gridPtrCast(baseGrid); + PointDataGrid::Ptr resultGrid = gridPtrCast((*grids)[0]); ASSERT_TRUE(resultGrid); // Only "P" should be present; "velocity" and "id" were skipped From 392162bbcec5cccd32af9491e9229201cd5c8531 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 24 Jun 2026 09:41:35 -0700 Subject: [PATCH 62/70] Add an Index64 size to readBuffers() to aid seeking for faster performance Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/BoolCodec.h | 2 +- openvdb/openvdb/codecs/PointDataCodec.h | 2 +- openvdb/openvdb/codecs/PointIndexCodec.h | 2 +- openvdb/openvdb/codecs/ScalarCodec.h | 2 +- openvdb/openvdb/codecs/ValueMaskCodec.h | 2 +- openvdb/openvdb/io/Archive.cc | 4 +++- openvdb/openvdb/io/Codec.h | 5 +++-- 7 files changed, 11 insertions(+), 8 deletions(-) diff --git a/openvdb/openvdb/codecs/BoolCodec.h b/openvdb/openvdb/codecs/BoolCodec.h index e8537fa662..82c7ae5a38 100644 --- a/openvdb/openvdb/codecs/BoolCodec.h +++ b/openvdb/openvdb/codecs/BoolCodec.h @@ -110,7 +110,7 @@ struct BoolCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { GridT& grid = static_cast(*data.grid); diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index ab203fe4de..7cb6219cb7 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -311,7 +311,7 @@ struct PointDataCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { OPENVDB_ASSERT(dynamic_cast(data.grid.get())); diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h index 0dec4fd77b..d1694d5b96 100644 --- a/openvdb/openvdb/codecs/PointIndexCodec.h +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -116,7 +116,7 @@ struct PointIndexCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final + void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final { OPENVDB_ASSERT(dynamic_cast(data.grid.get())); diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h index 0999707068..60b1eb6425 100644 --- a/openvdb/openvdb/codecs/ScalarCodec.h +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -159,7 +159,7 @@ struct ScalarCodec final: public TopologyCodec } } - void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { using StorageValueT = typename StorageGridT::TreeType::ValueType; GridT& grid = static_cast(*data.grid); diff --git a/openvdb/openvdb/codecs/ValueMaskCodec.h b/openvdb/openvdb/codecs/ValueMaskCodec.h index 27f6289cb9..3d2b8d7bd0 100644 --- a/openvdb/openvdb/codecs/ValueMaskCodec.h +++ b/openvdb/openvdb/codecs/ValueMaskCodec.h @@ -102,7 +102,7 @@ struct ValueMaskCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { GridT& grid = static_cast(*data.grid); diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 61e498c48c..0fd0db1dab 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -1033,7 +1033,9 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti if (readBuffers) { // read buffers if (codec) { - codec->readBuffers(is, *codecData, readOptions, diagnostics); + OPENVDB_ASSERT(gd.getEndPos() >= gd.getGridPos()); + const Index64 size = static_cast(gd.getEndPos() - gd.getGridPos()); + codec->readBuffers(is, size, *codecData, readOptions, diagnostics); } else { const auto& worldBBox = readOptions.clipBBox; const bool clip = worldBBox.isSorted(); diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index f236732cc9..86b93b6b3c 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -408,8 +408,9 @@ struct OPENVDB_API Codec /// that mode internally for safety. If @c options.clipBBox is non-empty, /// restrict the loaded data to the region that intersects it; if the codec /// cannot honour clipping natively, fall back to a post-process and record - /// a warning via @a diagnostics. - virtual void readBuffers(std::istream& /*is*/, CodecData& /*data*/, + /// a warning via @a diagnostics. The @c size argument is the number of bytes + /// occupied by the entire readBuffers data section. + virtual void readBuffers(std::istream& /*is*/, Index64 /*size*/, CodecData& /*data*/, const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) { } /// @brief Serialize the grid topology (tree structure and active-voxel From b846e9351bb2663d09f921c839b5720c94ae9b2f Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 5 Aug 2026 10:38:42 -0700 Subject: [PATCH 63/70] Address feedback Signed-off-by: Dan Bailey --- openvdb/openvdb/Grid.h | 5 +-- openvdb/openvdb/io/Archive.cc | 20 +++++++++--- openvdb/openvdb/io/Codec.h | 7 ++-- openvdb/openvdb/io/Stream.cc | 15 +++++++++ openvdb/openvdb/io/Stream.h | 3 ++ openvdb/openvdb/unittest/TestStream.cc | 45 ++++++++++++++++++++++++++ 6 files changed, 86 insertions(+), 9 deletions(-) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index 3d7a34e63b..133a5866db 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -1632,8 +1632,9 @@ Grid::readTopology(std::istream& is) { tree().readTopology(is, saveFloatAsHalf()); // When called from the legacy (non-codec) TopologyOnly path, the stream - // metadata carries a flag requesting allocation+zero-fill of leaf buffers - // (PartialCreate leaves them unallocated after readTopology). + // metadata carries a flag requesting that leaf buffers be allocated and + // filled with the background value (PartialCreate leaves them + // unallocated after readTopology). if (io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is)) { if (meta->allocateLeafBuffers()) { meta->setAllocateLeafBuffers(false); diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 0fd0db1dab..d1d2451943 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -1021,13 +1021,25 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti if (codec) { codec->readTopology(is, *codecData, readOptions, diagnostics); } else { + io::StreamMetadata::Ptr allocateLeafBuffersMeta; if (readOptions.readMode == io::ReadMode::TopologyOnly) { - // signal Grid::readTopology to allocate+zero-fill leaf buffers - if (auto meta = io::getStreamMetadataPtr(is)) { - meta->setAllocateLeafBuffers(true); + // Signal Grid::readTopology to allocate leaf buffers and + // fill them with the background value. + allocateLeafBuffersMeta = io::getStreamMetadataPtr(is); + if (allocateLeafBuffersMeta) { + allocateLeafBuffersMeta->setAllocateLeafBuffers(true); } } - grid->readTopology(is); + try { + grid->readTopology(is); + } catch (...) { + // Grid::readTopology() clears the flag on success, but if + // it throws the flag must not leak into subsequent grid reads. + if (allocateLeafBuffersMeta) { + allocateLeafBuffersMeta->setAllocateLeafBuffers(false); + } + throw; + } } } if (readBuffers) { diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 86b93b6b3c..70eea1c468 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -79,9 +79,10 @@ enum class ReadMode { Mask, /// Deserialize topology only; value buffers are skipped. The resulting /// grid has a valid tree structure (active/inactive state, node - /// hierarchy) and all leaf buffers are allocated and zero-filled. - /// Useful when only the active-voxel mask is needed and avoiding the - /// cost of reading large value buffers is desirable. + /// hierarchy) and all leaf buffers are allocated and filled with the + /// grid's background value. Useful when only the active-voxel mask is + /// needed and avoiding the cost of reading large value buffers is + /// desirable. TopologyOnly, /// Deserialize grid metadata and transform only; no topology, no value /// buffers. The codec is still used to construct the correct grid type, diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 5b7ae46601..87f9f99058 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -28,6 +28,21 @@ Stream::Stream(std::istream& is) Stream::Stream(std::istream& is, const io::ReadOptions& readOptions) { + // Read modes that stop before consuming all of a grid's data are not + // supported, because a stream is read sequentially - the position after + // a partial read is still inside the previous grid's data, so the next + // grid header would be read from the wrong offset. + // TODO: Skip over the unread bytes of each grid instead of disallowing + // these read modes. This is best implemented alongside the extension that + // adds support for byte skipping in non-seekable streams. + if (readOptions.readMode == io::ReadMode::MetadataOnly || + readOptions.readMode == io::ReadMode::TopologyOnly) { + OPENVDB_THROW(ValueError, "io::ReadMode::" + << (readOptions.readMode == io::ReadMode::MetadataOnly + ? "MetadataOnly" : "TopologyOnly") + << " is not supported when reading from a stream"); + } + if (!is) return; // Delayed loading has been removed - always read directly from the stream diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index fd302c8547..432e5eb9a9 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -29,6 +29,9 @@ class OPENVDB_API Stream: public Archive /// @param is The input stream to read from /// @param readOptions Options controlling how grids are read (e.g. attribute /// skipping for point data grids) + /// @throw ValueError if @a readOptions requests @c io::ReadMode::MetadataOnly + /// or @c io::ReadMode::TopologyOnly. These modes leave a grid's data + /// partially unread, which a sequentially read stream cannot skip over. Stream(std::istream& is, const io::ReadOptions& readOptions); OPENVDB_DEPRECATED_MESSAGE("Use Stream(std::istream&) instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") diff --git a/openvdb/openvdb/unittest/TestStream.cc b/openvdb/openvdb/unittest/TestStream.cc index 6091e60c2b..6146b8f96e 100644 --- a/openvdb/openvdb/unittest/TestStream.cc +++ b/openvdb/openvdb/unittest/TestStream.cc @@ -218,6 +218,51 @@ TestStream::testFileReadFromStream() TEST_F(TestStream, testFileReadFromStream) { testFileReadFromStream(); } +TEST_F(TestStream, testUnsupportedReadModes) +{ + using namespace openvdb; + + Int32Grid::Ptr grid1 = Int32Grid::create(0); + grid1->setName("first"); + grid1->tree().setValue(Coord(0, 0, 0), 1); + + FloatGrid::Ptr grid2 = FloatGrid::create(0.0f); + grid2->setName("second"); + grid2->tree().setValue(Coord(1, 2, 3), 2.0f); + + std::ostringstream ostr(std::ios_base::binary); + io::Stream(ostr).write(GridPtrVec{grid1, grid2}); + + // A stream is read sequentially, so read modes that leave part of a grid + // unread cannot be supported - the next grid header would be read from + // the wrong offset. + for (auto readMode: {io::ReadMode::MetadataOnly, io::ReadMode::TopologyOnly}) { + io::ReadOptions readOptions; + readOptions.readMode = readMode; + + std::istringstream is(ostr.str(), std::ios_base::binary); + EXPECT_THROW({ io::Stream strm(is, readOptions); }, ValueError); + } + + // Modes that read every byte of each grid are supported. + for (auto readMode: {io::ReadMode::Original, io::ReadMode::Half, + io::ReadMode::Bool, io::ReadMode::Mask}) + { + io::ReadOptions readOptions; + readOptions.readMode = readMode; + + std::istringstream is(ostr.str(), std::ios_base::binary); + io::Stream strm(is, readOptions); + + GridPtrVecPtr grids = strm.getGrids(); + ASSERT_TRUE(grids); + ASSERT_EQ(grids->size(), size_t(2)); + EXPECT_EQ((*grids)[0]->getName(), std::string("first")); + EXPECT_EQ((*grids)[1]->getName(), std::string("second")); + } +} + + TEST_F(TestStream, testAssignmentPreservesArchiveFlags) { using namespace openvdb; From 6bff7fa0d1337ad0ac737f50bce9d36cf6f8f12e Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 11 Aug 2026 17:12:57 -0700 Subject: [PATCH 64/70] Remove delayed loading option in vdb tool Signed-off-by: Dan Bailey --- openvdb_cmd/vdb_tool/include/Tool.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openvdb_cmd/vdb_tool/include/Tool.h b/openvdb_cmd/vdb_tool/include/Tool.h index 2833bf1505..bc9824c4e6 100644 --- a/openvdb_cmd/vdb_tool/include/Tool.h +++ b/openvdb_cmd/vdb_tool/include/Tool.h @@ -622,8 +622,7 @@ void Tool::init() mParser.addAction( {"read", "import", "load", "i"}, "Read one or more geometry or VDB files from disk or STDIN.", {{"files", "", "{file|stdin}.{obj|ply|abc|stl|off|pts|xyz|e57|vdb|nvdb|gltf|glb|geo|usd|usda|usdc|usdz}", "list of files or the input stream, e.g. file.vdb,stdin.vdb. Note that \"files=\" is optional since any argument without \"=\" is intrepreted as a file and appended to \"files\""}, - {"grids", "*", "*|grid_name,...", "list of VDB grids name to be imported (defaults to \"*\", i.e. import all available grids)"}, - {"delayed", "true", "1|0|true|false", "toggle delayed loading of VDB grids (enabled by default). This option is ignored by other file types"}}, + {"grids", "*", "*|grid_name,...", "list of VDB grids name to be imported (defaults to \"*\", i.e. import all available grids)"}}, [](){}, [&](){this->read();}, 0);// anonymous options are treated as to the first option,i.e. "files" mParser.addAction( @@ -1466,7 +1465,7 @@ void Tool::readVDB(const std::string &fileName) } else { if (mParser.verbose) mTimer.start("Reading VDB grid(s) from file named \""+fileName+"\""); io::File file(fileName); - file.open(mParser.get("delayed")); + file.open(); grids = file.getGrids(); } const size_t count = mGrid.size(); From 5354d1408010559281228b7a352e7954ffe85257 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 11 Aug 2026 18:20:31 -0700 Subject: [PATCH 65/70] Fix doxygen errors Signed-off-by: Dan Bailey --- doc/changes.txt | 4 ++-- openvdb/openvdb/io/Archive.h | 2 ++ openvdb/openvdb/io/Compression.h | 2 ++ openvdb/openvdb/points/PointDataGrid.h | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/changes.txt b/doc/changes.txt index 3c05074c81..033231bbaa 100644 --- a/doc/changes.txt +++ b/doc/changes.txt @@ -2034,7 +2034,7 @@ Bug fixes: New features: - Added @vdblink{tools::FindActiveValues,FindActiveValues}, which counts the active values in a tree that intersect a given bounding box. -- Added @vdblink{io::DelayedLoadMetadata,DelayedLoadMetadata}, which stores +- Added @c io::DelayedLoadMetadata, which stores mask offsets and compression sizes on write to accelerate delayed load reading. @@ -2933,7 +2933,7 @@ New features: - Added a toggle to the @vdblink::tools::clip() clip@endlink tool to invert the clipping mask. - Custom leaf node implementations may now optimize their file layout - by inheriting from @vdblink::io::MultiPass io::MultiPass@endlink. + by inheriting from @c io::MultiPass. Voxel data for grids with such leaf nodes will be written and read in multiple passes, allowing blocks of related data to be stored contiguously. [Contributed by Double Negative] diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index e0414f166a..5d15b9cdc2 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -162,12 +162,14 @@ class OPENVDB_API Archive /// Write the given grid descriptor and grid to an output stream /// and update the GridDescriptor offsets. /// @param seekable if true, the output stream supports seek operations + /// @param writeOptions options controlling how grid data is written void writeGrid(GridDescriptor&, GridBase::ConstPtr, std::ostream&, bool seekable, const io::WriteOptions& writeOptions = io::WriteOptions{}) const; /// Write the given grid descriptor and grid metadata to an output stream /// and update the GridDescriptor offsets, but don't write the grid's tree, /// since it is shared with another grid. /// @param seekable if true, the output stream supports seek operations + /// @param writeOptions options controlling how grid data is written void writeGridInstance(GridDescriptor&, GridBase::ConstPtr, std::ostream&, bool seekable, const io::WriteOptions& writeOptions = io::WriteOptions{}) const; diff --git a/openvdb/openvdb/io/Compression.h b/openvdb/openvdb/io/Compression.h index ade464275d..a81b1ef1e5 100644 --- a/openvdb/openvdb/io/Compression.h +++ b/openvdb/openvdb/io/Compression.h @@ -448,6 +448,7 @@ struct HalfWriter { /// which positions in the buffer correspond to active values /// @param fromHalf if true, read 16-bit half floats from the input stream /// and convert them to full floats +/// @param background optional background value used when mask compressed template inline void readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, @@ -619,6 +620,7 @@ writeCompressedValuesSize(ValueT* srcBuf, Index srcCount, /// @param childMask a bitmask (typically, a node's child mask) indicating /// which positions in the buffer correspond to child node pointers /// @param toHalf if true, convert floating-point values to 16-bit half floats +/// @param background optional background value used when mask compressed template inline void writeCompressedValues(std::ostream& os, const ValueT* srcBuf, Index srcCount, diff --git a/openvdb/openvdb/points/PointDataGrid.h b/openvdb/openvdb/points/PointDataGrid.h index f186717799..35988239af 100644 --- a/openvdb/openvdb/points/PointDataGrid.h +++ b/openvdb/openvdb/points/PointDataGrid.h @@ -94,7 +94,7 @@ makeDescriptorUnique(PointDataTreeT& tree); template OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") inline void -setStreamingMode(PointDataTreeT&, bool /*on*/ = true) { } +setStreamingMode(PointDataTreeT& tree, bool on = true) { (void)tree; (void)on; } template From d85dead9094e28c0fcf8aa760865bc4f6261e1e2 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 11 Aug 2026 19:35:28 -0700 Subject: [PATCH 66/70] Retry vcpkg install in Windows CI when a download fails Signed-off-by: Dan Bailey --- ci/install_windows.ps1 | 52 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/ci/install_windows.ps1 b/ci/install_windows.ps1 index 6054cb96bf..a4d44da353 100644 --- a/ci/install_windows.ps1 +++ b/ci/install_windows.ps1 @@ -21,20 +21,52 @@ $vcpkgPackages = @( "nanobind" ) +$maxAttempts = 3 + +# curl's schannel backend reports an unreachable CRL/OCSP responder as a +# certificate verification failure (error 60), which vcpkg then treats as +# permanent. Downgrade a missing revocation answer to a warning; the rest of +# certificate validation still applies. +$env:VCPKG_SSL_REVOKE_BEST_EFFORT = "1" + # Update vcpkg vcpkg update -# Allow the vcpkg command to fail once so we can retry with the latest -try { - vcpkg install $vcpkgPackages -} catch { - Write-Host "vcpkg install failed, retrying with latest ports..." - # Retry the installation with updated ports - Push-Location $env:VCPKG_INSTALLATION_ROOT - git pull - Pop-Location - vcpkg update +$installed = $false + +for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { vcpkg install $vcpkgPackages + + # A failing native command does not raise a terminating error, so the exit + # code has to be inspected explicitly rather than relying on try/catch. + if ($LASTEXITCODE -eq 0) { + $installed = $true + break + } + + if ($attempt -eq $maxAttempts) { + break + } + + # vcpkg fetches port sources directly from upstream hosts and won't retry + # downloads it classifies as permanent failures, so a single flaky TLS + # handshake aborts the whole install. + Write-Host "vcpkg install failed (attempt $attempt of $maxAttempts), retrying..." + Start-Sleep -Seconds 15 + + # Refresh the ports before the last attempt in case the failure is caused + # by a stale port rather than the network. + if ($attempt -eq ($maxAttempts - 1)) { + Write-Host "Retrying with latest ports..." + Push-Location $env:VCPKG_INSTALLATION_ROOT + git pull + Pop-Location + vcpkg update + } +} + +if (-not $installed) { + throw "vcpkg install failed after $maxAttempts attempts" } Write-Host "vcpkg install completed successfully" From 610e8cce31b6d6ab6696838e74ea1b9d00ba7d21 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 12 Aug 2026 11:51:25 -0700 Subject: [PATCH 67/70] Add pending changes Signed-off-by: Dan Bailey --- pendingchanges/codecs.txt | 7 +++++++ pendingchanges/delayedloading.txt | 6 ++++++ 2 files changed, 13 insertions(+) create mode 100644 pendingchanges/codecs.txt create mode 100644 pendingchanges/delayedloading.txt diff --git a/pendingchanges/codecs.txt b/pendingchanges/codecs.txt new file mode 100644 index 0000000000..1e049f78e4 --- /dev/null +++ b/pendingchanges/codecs.txt @@ -0,0 +1,7 @@ +OpenVDB: + Features: + - Introduced new openvdb::codecs subsystem and CodecRegistry for stream encoding and decoding. + - Added support for implicit codec conversion fallback (e.g. float to half, scalar to mask) + and ReadDiagnostics. + API changes: + - Extracted Point Data I/O function overloads from PointDataGrid.h into new openvdb/points/PointDataIO.h header. diff --git a/pendingchanges/delayedloading.txt b/pendingchanges/delayedloading.txt new file mode 100644 index 0000000000..6a445fe74f --- /dev/null +++ b/pendingchanges/delayedloading.txt @@ -0,0 +1,6 @@ +OpenVDB: + Highlights: + - Completely removed support for out-of-core / delayed loading across OpenVDB core, + including removal of the OPENVDB_USE_DELAYED_LOADING build option, DelayedLoadMetadata, + and TempFile. This was done to improve performance, reduce codebase complexity and to + entirely eliminate the optional Boost dependency. From fc294a0ab09aee30975564e9bf9aa70eada0b8e3 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 12 Aug 2026 12:27:15 -0700 Subject: [PATCH 68/70] More pending changes Signed-off-by: Dan Bailey --- pendingchanges/ioreadconversion.txt | 5 +++++ pendingchanges/removedelayedloading.txt | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 pendingchanges/ioreadconversion.txt create mode 100644 pendingchanges/removedelayedloading.txt diff --git a/pendingchanges/ioreadconversion.txt b/pendingchanges/ioreadconversion.txt new file mode 100644 index 0000000000..46cddca5db --- /dev/null +++ b/pendingchanges/ioreadconversion.txt @@ -0,0 +1,5 @@ +OpenVDB: + New features: + - io::ReadOptions now supports read-time grid type conversion, allowing a grid + to be read as a different registered type than the one stored in the file + (for example, reading a FloatGrid as a HalfGrid). diff --git a/pendingchanges/removedelayedloading.txt b/pendingchanges/removedelayedloading.txt new file mode 100644 index 0000000000..7d31734800 --- /dev/null +++ b/pendingchanges/removedelayedloading.txt @@ -0,0 +1,20 @@ +OpenVDB: + API changes: + - Delayed loading has been removed. The OPENVDB_USE_DELAYED_LOADING CMake option, + MappedFile support and DelayedLoadMetadata have all been removed. + - io::File::open() and io::Stream constructors that accepted delayed-loading + parameters are deprecated and now behave as their non-delayed-loading + equivalents. + - Out-of-core tree APIs are deprecated, including LeafBuffer::isOutOfCore(), + memUsageIfLoaded() and readNonResidentBuffers(). The latter are now no-ops + and memUsageIfLoaded() forwards to memUsage(). + - Free functions that previously accepted an inCoreOnly parameter are deprecated + in favor of overloads without that parameter. + + Improvements: + - OpenVDB no longer depends on Boost. The boost::iostreams and Boost + interprocess optional dependencies have been removed along with io::TempFile. + + Build: + - Removed the OPENVDB_USE_DELAYED_LOADING CMake option. + - OpenVDB no longer requires or links against Boost. From 53af683310d0ac962b89b47e5e71814c003751a6 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 17 Aug 2026 23:13:05 -0700 Subject: [PATCH 69/70] Make Codec read/write methods const Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/BoolCodec.h | 4 ++-- openvdb/openvdb/codecs/PointDataCodec.h | 4 ++-- openvdb/openvdb/codecs/PointIndexCodec.h | 4 ++-- openvdb/openvdb/codecs/ScalarCodec.h | 4 ++-- openvdb/openvdb/codecs/TopologyCodec.h | 4 ++-- openvdb/openvdb/codecs/ValueMaskCodec.h | 4 ++-- openvdb/openvdb/io/Codec.h | 22 ++++++++++++---------- 7 files changed, 24 insertions(+), 22 deletions(-) diff --git a/openvdb/openvdb/codecs/BoolCodec.h b/openvdb/openvdb/codecs/BoolCodec.h index 82c7ae5a38..3da1839101 100644 --- a/openvdb/openvdb/codecs/BoolCodec.h +++ b/openvdb/openvdb/codecs/BoolCodec.h @@ -110,7 +110,7 @@ struct BoolCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) const final { GridT& grid = static_cast(*data.grid); @@ -134,7 +134,7 @@ struct BoolCodec final: public TopologyCodec tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); } - void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final { const GridT& grid = static_cast(gridBase); diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index 7cb6219cb7..18484facd1 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -311,7 +311,7 @@ struct PointDataCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) const final { OPENVDB_ASSERT(dynamic_cast(data.grid.get())); @@ -422,7 +422,7 @@ struct PointDataCodec final: public TopologyCodec } } - void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final { const GridT& grid = static_cast(gridBase); bool saveFloatAsHalf = grid.saveFloatAsHalf(); diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h index d1694d5b96..5b4d6b62cf 100644 --- a/openvdb/openvdb/codecs/PointIndexCodec.h +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -116,7 +116,7 @@ struct PointIndexCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) const final { OPENVDB_ASSERT(dynamic_cast(data.grid.get())); @@ -143,7 +143,7 @@ struct PointIndexCodec final: public TopologyCodec tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); } - void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final { const GridT& grid = static_cast(gridBase); diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h index 60b1eb6425..199a376232 100644 --- a/openvdb/openvdb/codecs/ScalarCodec.h +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -159,7 +159,7 @@ struct ScalarCodec final: public TopologyCodec } } - void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) const final { using StorageValueT = typename StorageGridT::TreeType::ValueType; GridT& grid = static_cast(*data.grid); @@ -167,7 +167,7 @@ struct ScalarCodec final: public TopologyCodec internal::scalarCodecReadBuffers(grid, is, options, &topoData.storageBackground); } - void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final { // Note: the write body must live inside the negated if constexpr branch // so it is not instantiated for read-only codecs. A bare diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index eee4ead6e5..47f07b7ba0 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -356,7 +356,7 @@ struct TopologyCodec : public io::Codec } void readTopology(std::istream& is, io::CodecData& data, const io::ReadOptions& options, - io::ReadDiagnostics& diagnostics) final + io::ReadDiagnostics& diagnostics) const final { // Warn when a conversion readMode was requested but this codec is a // non-conversion instance (GridT == StorageGridT), meaning no conversion @@ -379,7 +379,7 @@ struct TopologyCodec : public io::Codec internal::topologyCodecReadTopology(*data.grid, is, options, diagnostics, topoData); } - void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final { // Disable implementation when read only. The body must live inside the // negated if constexpr branch so it is not instantiated for read-only diff --git a/openvdb/openvdb/codecs/ValueMaskCodec.h b/openvdb/openvdb/codecs/ValueMaskCodec.h index 3d2b8d7bd0..d3e3e2a938 100644 --- a/openvdb/openvdb/codecs/ValueMaskCodec.h +++ b/openvdb/openvdb/codecs/ValueMaskCodec.h @@ -102,7 +102,7 @@ struct ValueMaskCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) const final { GridT& grid = static_cast(*data.grid); @@ -126,7 +126,7 @@ struct ValueMaskCodec final: public TopologyCodec tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); } - void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final { const GridT& grid = static_cast(gridBase); diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 70eea1c468..473102b57a 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -284,7 +284,7 @@ struct OPENVDB_API ReadDiagnostics { /// In the read and write methods, downcast the reference back to the concrete /// type to access the extra fields: /// @code -/// void readBuffers(std::istream& is, io::CodecData& data, ...) override { +/// void readBuffers(std::istream& is, int64_t size, io::CodecData& data, ...) const override { /// auto& myData = static_cast(data); /// MyGridType& grid = static_cast(*myData.grid); /// // Use myData.intermediateBuffer, grid, etc. @@ -350,15 +350,15 @@ struct OPENVDB_API CodecData /// @code /// void readTopology(std::istream& is, io::CodecData& data, /// const io::ReadOptions& options, -/// io::ReadDiagnostics& diagnostics) override +/// io::ReadDiagnostics& diagnostics) const override /// { /// MyGridType& grid = static_cast(*data.grid); /// // Deserialize the tree structure into grid... /// } /// -/// void readBuffers(std::istream& is, io::CodecData& data, +/// void readBuffers(std::istream& is, int64_t size, CodecData& data, /// const io::ReadOptions& options, -/// io::ReadDiagnostics& diagnostics) override +/// io::ReadDiagnostics& diagnostics) const override /// { /// if (options.readMode == io::ReadMode::TopologyOnly) return; /// MyGridType& grid = static_cast(*data.grid); @@ -397,7 +397,7 @@ struct OPENVDB_API Codec /// @brief Deserialize the grid topology (tree structure and active-voxel /// mask) from @a is into the grid held by @a data. virtual void readTopology(std::istream& /*is*/, CodecData& /*data*/, - const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) { } + const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) const { } /// @brief Deserialize all voxel-value buffers from @a is into the grid /// held by @a data. @@ -410,9 +410,11 @@ struct OPENVDB_API Codec /// restrict the loaded data to the region that intersects it; if the codec /// cannot honour clipping natively, fall back to a post-process and record /// a warning via @a diagnostics. The @c size argument is the number of bytes - /// occupied by the entire readBuffers data section. - virtual void readBuffers(std::istream& /*is*/, Index64 /*size*/, CodecData& /*data*/, - const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) { } + /// occupied by the readBuffers data section, measured from the stream + /// position at which this method is called, or -1 when the size is not known + /// (a non-seekable stream, or a stream without grid offsets). + virtual void readBuffers(std::istream& /*is*/, int64_t /*size*/, CodecData& /*data*/, + const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) const { } /// @brief Serialize the grid topology (tree structure and active-voxel /// mask) from @a grid to @a os. @@ -421,14 +423,14 @@ struct OPENVDB_API Codec /// topology as a distinct section that precedes the value buffers in the /// stream. virtual void writeTopology(std::ostream& /*os*/, const GridBase& /*grid*/, - const WriteOptions& /*options*/) { } + const WriteOptions& /*options*/) const { } /// @brief Serialize all voxel-value buffers from @a grid to @a os. /// /// The default implementation is a no-op. Override to write the leaf-node /// value buffers that follow the topology section. virtual void writeBuffers(std::ostream& /*os*/, const GridBase& /*grid*/, - const WriteOptions& /*options*/) { } + const WriteOptions& /*options*/) const { } }; // struct Codec /// @brief A thread-safe, process-global registry that maps codec names to From d940d205f82c5b712690966db861782466f5ad9a Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 18 Aug 2026 15:47:03 -0700 Subject: [PATCH 70/70] Skip readBuffers() logic in ScalarCodec Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/ScalarCodec.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h index 199a376232..e181f54df7 100644 --- a/openvdb/openvdb/codecs/ScalarCodec.h +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -163,6 +163,11 @@ struct ScalarCodec final: public TopologyCodec { using StorageValueT = typename StorageGridT::TreeType::ValueType; GridT& grid = static_cast(*data.grid); + + // readTopology() has already set the tiles and leaf buffers to the + // background, so there is nothing left to read. + if (options.readMode == io::ReadMode::TopologyOnly) return; + auto& topoData = static_cast&>(data); internal::scalarCodecReadBuffers(grid, is, options, &topoData.storageBackground); }