diff --git a/packages/on_demand_video_decoder/accvlab/on_demand_video_decoder/_internal/decoder.py b/packages/on_demand_video_decoder/accvlab/on_demand_video_decoder/_internal/decoder.py index ccbfe461..8c7c8328 100644 --- a/packages/on_demand_video_decoder/accvlab/on_demand_video_decoder/_internal/decoder.py +++ b/packages/on_demand_video_decoder/accvlab/on_demand_video_decoder/_internal/decoder.py @@ -271,8 +271,9 @@ def CreateGopDecoder( Args: maxfiles: Maximum number of unique files that can be processed concurrently iGpu: GPU device ID to use for decoding (0 for primary GPU) - suppressNoColorRangeWarning: Suppress warning when no color range can be - extracted from video files (limited/MPEG range is assumed) + suppressNoColorRangeWarning: Suppress the warning emitted during RGB/BGR conversion + when the input color range is unspecified. Limited/MPEG + range is assumed regardless of this option. gopCacheCapacity: Maximum number of filepath entries kept in the Python GOP cache. ``None`` defaults to ``maxfiles``. This capacity only affects calls with ``useGOPCache=True``; each filepath stores the most diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/CMakeLists.txt b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/CMakeLists.txt index 2fc59591..d8c85835 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/CMakeLists.txt +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/CMakeLists.txt @@ -49,6 +49,7 @@ set(PY_SOURCES src/PyNvOnDemandDecoder.cpp src/PyCAIMemoryView.cpp src/PyDecodedFrameExt.cpp + src/FrameOutput.cpp src/PyNvGopDecoder_common.cpp src/PyNvGopDecoder_constructors.cpp src/PyNvGopDecoder_random_decoder.cpp diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/FrameOutput.hpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/FrameOutput.hpp new file mode 100644 index 00000000..1d0fce31 --- /dev/null +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/FrameOutput.hpp @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +class NvDecoder; +struct DecodedFrameExt; +class RGBFrame; + +enum Pixel_Format { + Pixel_Format_UNDEFINED = 0, + Pixel_Format_NV12 = 3, + Pixel_Format_YUV444 = 4, + Pixel_Format_P016 = 5, + Pixel_Format_YUV444_16Bit = 6 + +}; + +namespace accvlab::on_demand_video_decoder::internal::frame_output { + +// Tightly packed layouts exposed by the public frame-output APIs. RGB8 also +// covers BGR8 because channel order does not change the allocation size. +enum class FrameOutputFormat : uint8_t { + RGB8, + NV12, + P016, + YUV444, + YUV444_16Bit, +}; + +FrameOutputFormat output_format_from_pixel_format(Pixel_Format format); +Pixel_Format pixel_format_from_surface(cudaVideoSurfaceFormat surface_format); +FrameOutputFormat output_format_from_surface(cudaVideoSurfaceFormat surface_format); +FrameOutputFormat output_format_from_av_pixel_format(AVPixelFormat pixel_format); + +// The single size-calculation entry point for all tightly packed RGB/YUV +// frames exposed by this package. +size_t frame_bytes(FrameOutputFormat format, size_t height, size_t width); + +// Convert one NVDEC surface into an RGB/BGR frame backed by output_buffer. +// When is_async is true, work is only enqueued on decoder.GetStream(); the caller +// owns the terminal synchronization and must keep both buffers alive until then. +RGBFrame convert_decoded_frame_to_rgb(NvDecoder& decoder, const uint8_t* decoded_surface, + CUdeviceptr output_buffer, AVColorRange color_range, bool as_bgr, + bool is_async); + +// Copy one NVDEC surface into output_buffer and expose its native YUV planes. +DecodedFrameExt copy_decoded_frame_to_yuv(NvDecoder& decoder, const uint8_t* decoded_surface, + CUdeviceptr output_buffer, AVColorRange color_range, + int64_t timestamp, bool is_async); + +// Copy an existing RGB/BGR frame into aggregator-owned storage. +RGBFrame copy_rgb_frame(const RGBFrame& source, CUdeviceptr destination, CUstream destination_stream, + bool as_bgr, bool is_async); + +// Copy an existing tightly packed YUV frame into aggregator-owned storage. +DecodedFrameExt copy_yuv_frame(const DecodedFrameExt& source, CUdeviceptr destination, + CUstream destination_stream, bool is_async); + +} // namespace accvlab::on_demand_video_decoder::internal::frame_output diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyCAIMemoryView.hpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyCAIMemoryView.hpp index 5d4f5df2..e86be2c0 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyCAIMemoryView.hpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyCAIMemoryView.hpp @@ -15,12 +15,12 @@ */ #pragma once -#include "ExternalBuffer.hpp" #include "NvCodecUtils.h" #include "nvEncodeAPI.h" #include #include #include +#include #include #include #include @@ -78,15 +78,6 @@ class CuCtxGuard { #define DEF_CONSTANT(s) attr(ENUM_VALUE_STRINGIFY(s)) = py::cast(s) #define DEF_READWRITE(type, s) def_readwrite(ENUM_VALUE_STRINGIFY(s), &type::s) -enum Pixel_Format { - Pixel_Format_UNDEFINED = 0, - Pixel_Format_NV12 = 3, - Pixel_Format_YUV444 = 4, - Pixel_Format_P016 = 5, - Pixel_Format_YUV444_16Bit = 6 - -}; - struct CAIMemoryView { std::vector shape; std::vector stride; @@ -147,139 +138,5 @@ struct CAIMemoryView { } }; -struct DecodedFrame { - int64_t timestamp; - std::vector views; - Pixel_Format format; - std::shared_ptr extBuf; - DecodedFrame() { extBuf = std::make_shared(); } - static void Export(py::module& m) { - py::class_>(m, "DecodedFrame", py::module_local()) - .def_readonly("timestamp", &DecodedFrame::timestamp) - .def_readonly("format", &DecodedFrame::format) - .def("__repr__", - [](std::shared_ptr& self) { - std::stringstream ss; - ss << "views)); - ss << "]>"; - return ss.str(); - }) - .def( - "framesize", - [](std::shared_ptr& self) { - int height = self->views.at(0).shape.at(0); - int width = self->views.at(0).shape.at(1); - int framesize = width * height * 1.5; - switch (self->format) { - case Pixel_Format_NV12: - break; - case Pixel_Format_P016: - framesize = width * height * 3; - break; - case Pixel_Format_YUV444: - framesize = width * height * 3; - break; - case Pixel_Format_YUV444_16Bit: - framesize = width * height * 6; - break; - default: - break; - } - return framesize; - }, - R"pbdoc( - return underlying views which implement CAI - :param None: None - )pbdoc") - .def( - "cuda", [](std::shared_ptr& self) { return self->views; }, - R"pbdoc( - return underlying views which implement CAI - :param None: None - )pbdoc") - .def( - "nvcv_image", - [](std::shared_ptr& self) { - switch (self->format) { - case Pixel_Format_NV12: { - size_t width = self->views.at(0).shape[1]; - size_t height = self->views.at(0).shape[0] * 1.5; - CUdeviceptr data = self->views.at(0).data; - CUstream stream = self->views.at(0).stream; - self->views.clear(); - self->views.push_back( - CAIMemoryView{{height, width, 1}, - {width, 2, 1}, - "|u1", - reinterpret_cast(stream), - (data), - false}); // hack for cvcuda tensor represenation - } break; - case Pixel_Format_YUV444: { - size_t width = self->views.at(0).shape[1]; - size_t height = self->views.at(0).shape[0] * 3; - CUdeviceptr data = self->views.at(0).data; - CUstream stream = self->views.at(0).stream; - self->views.clear(); - self->views.push_back( - CAIMemoryView{{height, width, 1}, - {width, 3, 1}, - "|u1", - reinterpret_cast(stream), - (data), - false}); // hack for cvcuda tensor represenation - } break; - default: - throw std::invalid_argument("only nv12 and yuv444 supported as of now"); - break; - } - return self->views; - }, - R"pbdoc( - return underlying views which implement CAI - :param None: None - )pbdoc") - - // DL Pack Tensor - .def_property_readonly( - "shape", [](std::shared_ptr& self) { return self->extBuf->shape(); }, - "Get the shape of the buffer as an array") - .def_property_readonly( - "strides", [](std::shared_ptr& self) { return self->extBuf->strides(); }, - "Get the strides of the buffer") - .def_property_readonly( - "dtype", [](std::shared_ptr& self) { return self->extBuf->dtype(); }, - "Get the data type of the buffer") - .def( - "__dlpack__", - [](std::shared_ptr& self, py::object stream) { - return self->extBuf->dlpack(stream); - }, - py::arg("stream") = NULL, "Export the buffer as a DLPack tensor") - .def( - "__dlpack_device__", - [](std::shared_ptr& self) { - // DLDevice ctx; - // ctx.device_type = DLDeviceType::kDLCUDA; - // ctx.device_id = 0; - return py::make_tuple(py::int_(static_cast(DLDeviceType::kDLCUDA)), - py::int_(static_cast(0))); - }, - "Get the device associated with the buffer") - .def( - "GetPtrToPlane", - - [](std::shared_ptr& self, int planeIdx) { return self->views[planeIdx].data; }, - R"pbdoc( - return pointer to base address for plane index - :param planeIdx : index to the plane - )pbdoc"); - // TODO add __iter__ interface on DecodedFrame - } -}; - CAIMemoryView coerceToCudaArrayView(py::object cuda_array, NV_ENC_BUFFER_FORMAT bufferFormat, size_t width, size_t height, int planeIdx = 0); diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyDecodedFrameExt.hpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyDecodedFrameExt.hpp index 98f74bf6..44872fbb 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyDecodedFrameExt.hpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyDecodedFrameExt.hpp @@ -18,12 +18,14 @@ #include +#include "ExternalBuffer.hpp" +#include "FrameOutput.hpp" #include "PyCAIMemoryView.hpp" #include "cuviddec.h" #include -struct DecodedFrameExt : public DecodedFrame { +struct DecodedFrameExt { enum class VideoSurfaceFormat { UNSPECIFIED = 0, NV12 = 1, @@ -46,8 +48,12 @@ struct DecodedFrameExt : public DecodedFrame { ColorRange_FULL = 2, }; + int64_t timestamp; + std::vector views; + Pixel_Format format; + std::shared_ptr extBuf; ColorRange color_range = ColorRange::ColorRange_UNSPECIFIED; - DecodedFrameExt() = default; + DecodedFrameExt() { extBuf = std::make_shared(); } VideoSurfaceFormat GetVideoSurfaceFormat() const; void SetVideoSurfaceFormat(cudaVideoSurfaceFormat video_format_in); @@ -93,27 +99,17 @@ struct DecodedFrameExt : public DecodedFrame { [](std::shared_ptr& self) { int height = self->views.at(0).shape.at(0); int width = self->views.at(0).shape.at(1); - int framesize = width * height * 1.5; - switch (self->format) { - case Pixel_Format_NV12: - break; - case Pixel_Format_P016: - framesize = width * height * 3; - break; - case Pixel_Format_YUV444: - framesize = width * height * 3; - break; - case Pixel_Format_YUV444_16Bit: - framesize = width * height * 6; - break; - default: - break; - } + int framesize = static_cast( + accvlab::on_demand_video_decoder::internal::frame_output::frame_bytes( + accvlab::on_demand_video_decoder::internal::frame_output:: + output_format_from_pixel_format(self->format), + height, width)); return framesize; }, R"pbdoc( - return underlying views which implement CAI - :param None: None + Return the total size in bytes of the tightly packed decoded frame buffer. + + The size includes all YUV planes. )pbdoc") .def( "cuda", [](std::shared_ptr& self) { return self->views; }, @@ -127,7 +123,8 @@ struct DecodedFrameExt : public DecodedFrame { switch (self->format) { case Pixel_Format_NV12: { size_t width = self->views.at(0).shape[1]; - size_t height = self->views.at(0).shape[0] * 1.5; + size_t luma_height = self->views.at(0).shape[0]; + size_t height = luma_height + (luma_height + 1) / 2; CUdeviceptr data = self->views.at(0).data; CUstream stream = self->views.at(0).stream; self->views.clear(); diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvBatchAsyncGopDecoder.hpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvBatchAsyncGopDecoder.hpp index d5f0cbd4..177c7ddc 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvBatchAsyncGopDecoder.hpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvBatchAsyncGopDecoder.hpp @@ -140,15 +140,6 @@ class PyNvBatchAsyncGopDecoder { std::vector filepaths, std::vector> frame_ids_2d, bool as_bgr, bool is_rgb); - // Returns the total contiguous byte size of one YUV frame for the given - // pixel format and dimensions. Matches the layout produced by GetYUVFromFrame. - static size_t compute_yuv_frame_bytes(Pixel_Format fmt, size_t H, size_t W); - - // Reconstruct a DecodedFrameExt whose views point into aggregator pool memory. - static void build_yuv_frame(Pixel_Format fmt, size_t H, size_t W, int64_t timestamp, - DecodedFrameExt::ColorRange color_range, CUdeviceptr dst_ptr, CUstream stream, - DecodedFrameExt& out); - private: bool suppress_no_color_range_warning_ = false; bool destroy_context_ = false; diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvGopDecoder.hpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvGopDecoder.hpp index f8b9c22d..4b734120 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvGopDecoder.hpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvGopDecoder.hpp @@ -40,6 +40,7 @@ #include #include #include +#include #include #define MAX_SIZE 2000 @@ -518,8 +519,8 @@ class PyNvGopDecoder { * 2. **Packet Processing**: Consumes packets from queue until completion signal * 3. **Frame Decoding**: Calls NvDecoder with packet data and retrieval flags * 4. **Format Conversion**: - * - RGB: Calls `GetRGBFromFrame()` with color space conversion - * - YUV: Calls `GetYUVFromFrame()` with direct GPU memory copy + * - RGB: Uses the shared FrameOutput color-conversion path + * - YUV: Uses the shared FrameOutput GPU-copy and view-construction path * 5. **State Updates**: Maintains decoder state for cross-session optimization * 6. **Validation**: Ensures output frame count matches expected frame count * @@ -693,12 +694,14 @@ class PyNvGopDecoder { std::vector& sorted_frame_ids, std::vector& first_frame_ids, std::vector& gop_length); - static Pixel_Format GetNativeFormat(const cudaVideoSurfaceFormat inputFormat); - private: + void WarnIfColorRangeUnspecified(AVColorRange color_range, const std::string& filename); + int max_num_files = 0; bool suppress_no_color_range_given_warning = false; + std::mutex no_color_range_warning_mutex; + std::unordered_set warned_no_color_range_files; bool destroy_context = false; CUcontext cu_context = NULL; @@ -772,32 +775,6 @@ class PyNvGopDecoder { const std::vector>& all_first_frame_ids, const std::vector>>>& vpacket_queue, const std::vector>>& vpacket_array); - - /** - * Convert decoded frame to RGB format - * @param decoder The decoder instance - * @param pFrame Pointer to the decoded frame data - * @param pFrame_buffer Pointer to the output RGB buffer - * @param color_range Color range of the input frame - * @param use_bgr_format Whether to use BGR format instead of RGB - * @param rgb_frame Output reference to construct the RGB frame directly - * @return 0 on success, -1 on error - */ - static int GetRGBFromFrame(NvDecoder* decoder, const uint8_t* pFrame, uint8_t* pFrame_buffer, - AVColorRange color_range, bool use_bgr_format, RGBFrame& rgb_frame); - - /** - * Create a DecodedFrameExt object from decoded frame data - * @param decoder The decoder instance - * @param pFrame Pointer to the decoded frame data - * @param pFrame_buffer Pointer to the output frame buffer - * @param color_range Color range of the input frame - * @param timestamp Timestamp of the frame - * @param decoded_frame Output reference to construct the DecodedFrameExt object - * @return 0 on success, -1 on error - */ - static int GetYUVFromFrame(NvDecoder* decoder, const uint8_t* pFrame, uint8_t* pFrame_buffer, - AVColorRange color_range, int64_t timestamp, DecodedFrameExt& decoded_frame); }; /** diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvGopDemuxer.hpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvGopDemuxer.hpp index 57e19aca..35e2007a 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvGopDemuxer.hpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvGopDemuxer.hpp @@ -43,7 +43,7 @@ class PyNvGopDemuxer { uint32_t GetWidth() { return demuxer->GetWidth(); } - uint32_t GetFrameSize() { return demuxer->GetFrameSize(); } + AVPixelFormat GetPixelFormat() const { return demuxer->GetPixelFormat(); } FFmpegDemuxer* GetDemuxer() { return demuxer.get(); } diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvVideoReader.hpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvVideoReader.hpp index 878d02a5..a3a009f7 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvVideoReader.hpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/inc/PyNvVideoReader.hpp @@ -26,6 +26,7 @@ #include "PyRGBFrame.hpp" #include #include +#include #include #include #include @@ -67,8 +68,6 @@ class PyNvVideoReader { */ void ReleaseMemPools(); - static Pixel_Format GetNativeFormat(const cudaVideoSurfaceFormat inputFormat); - std::vector run(const std::vector frame_ids); DecodedFrameExt run_single(const int frame_id); @@ -89,9 +88,6 @@ class PyNvVideoReader { void releasePacketArray(); void startNewGop(); void startNextGop(); - DecodedFrameExt returnYUVFrame(void* pFrame_buffer, void* pFrame); - RGBFrame returnRGBFrame(void* pFrame_buffer, void* pFrame, bool use_bgr_format, - bool& file_added_for_warning); void run_single_frame_internal(const int frame_ids, bool convert_to_rgb, bool as_bgr, DecodedFrameExt* out_if_no_color_conversion, RGBFrame* out_if_color_converted); @@ -99,17 +95,20 @@ class PyNvVideoReader { void fetchNewGop(int cur_keyframe); void decodeNextPacket(); bool processDecodedFrames(const int frame_id, uint8_t* pFrame, uint8_t* pReturnFrame, bool convert_to_rgb, - bool use_bgr_format, bool& file_added_for_warning, - RGBFrame* out_if_color_converted, DecodedFrameExt* out_if_no_color_conversion); + bool use_bgr_format, RGBFrame* out_if_color_converted, + DecodedFrameExt* out_if_no_color_conversion); static void demuxGopProcZeroLen(PyNvGopDemuxer* demuxer, ConcurrentQueue>* packet_queue, const int key_frame_ids, std::vector& packet_array, bool seeking); private: + void WarnIfColorRangeUnspecified(AVColorRange color_range); + std::string filename = {}; bool suppress_no_color_range_given_warning = false; + std::atomic has_warned_no_color_range_ = false; bool destroy_context = false; CUcontext cu_context = NULL; bool owner_stream = false; diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/FrameOutput.cpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/FrameOutput.cpp new file mode 100644 index 00000000..dfb14c96 --- /dev/null +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/FrameOutput.cpp @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "FrameOutput.hpp" + +#include "ColorConvertKernels.cuh" +#include "NvCodecUtils.h" +#include "NvDecoder/NvDecoder.h" +#include "PyDecodedFrameExt.hpp" +#include "PyRGBFrame.hpp" +#include "nvtx3/nvtx3.hpp" + +#include +#include +#include +#include +#include + +namespace accvlab::on_demand_video_decoder::internal::frame_output { + +namespace { + +constexpr size_t chroma_height_420(size_t height) { return (height + 1) / 2; } + +} // namespace + +FrameOutputFormat output_format_from_pixel_format(Pixel_Format format) { + switch (format) { + case Pixel_Format_NV12: + return FrameOutputFormat::NV12; + case Pixel_Format_P016: + return FrameOutputFormat::P016; + case Pixel_Format_YUV444: + return FrameOutputFormat::YUV444; + case Pixel_Format_YUV444_16Bit: + return FrameOutputFormat::YUV444_16Bit; + default: + throw std::invalid_argument("Unsupported pixel format for frame output: " + + std::to_string(static_cast(format))); + } +} + +Pixel_Format pixel_format_from_surface(cudaVideoSurfaceFormat surface_format) { + switch (surface_format) { + case cudaVideoSurfaceFormat_NV12: + return Pixel_Format_NV12; + case cudaVideoSurfaceFormat_P016: + return Pixel_Format_P016; + case cudaVideoSurfaceFormat_YUV444: + return Pixel_Format_YUV444; + case cudaVideoSurfaceFormat_YUV444_16Bit: + return Pixel_Format_YUV444_16Bit; + default: + return Pixel_Format_UNDEFINED; + } +} + +FrameOutputFormat output_format_from_surface(cudaVideoSurfaceFormat surface_format) { + return output_format_from_pixel_format(pixel_format_from_surface(surface_format)); +} + +FrameOutputFormat output_format_from_av_pixel_format(AVPixelFormat pixel_format) { + switch (pixel_format) { + case AV_PIX_FMT_YUV420P: + case AV_PIX_FMT_YUVJ420P: + case AV_PIX_FMT_YUVJ422P: + case AV_PIX_FMT_YUVJ444P: + case AV_PIX_FMT_GRAY8: + return FrameOutputFormat::NV12; + case AV_PIX_FMT_YUV420P10LE: + case AV_PIX_FMT_YUV420P12LE: + case AV_PIX_FMT_GRAY10LE: + return FrameOutputFormat::P016; + case AV_PIX_FMT_YUV444P: + return FrameOutputFormat::YUV444; + case AV_PIX_FMT_YUV444P10LE: + case AV_PIX_FMT_YUV444P12LE: + return FrameOutputFormat::YUV444_16Bit; + default: + throw std::invalid_argument("Unsupported FFmpeg pixel format for frame output: " + + std::to_string(static_cast(pixel_format))); + } +} + +size_t frame_bytes(FrameOutputFormat format, size_t height, size_t width) { + switch (format) { + case FrameOutputFormat::RGB8: + return height * width * 3; + case FrameOutputFormat::NV12: + return width * (height + chroma_height_420(height)); + case FrameOutputFormat::P016: + return 2 * width * (height + chroma_height_420(height)); + case FrameOutputFormat::YUV444: + return height * width * 3; + case FrameOutputFormat::YUV444_16Bit: + return height * width * 6; + default: + throw std::invalid_argument("Unsupported frame output format: " + + std::to_string(static_cast(format))); + } +} + +namespace { + +RGBFrame make_rgb_frame_view(size_t height, size_t width, CUdeviceptr data, CUstream stream) { + const std::vector shape{height, width, 3}; + const std::vector stride{width * 3, 3, 1}; + return RGBFrame(shape, stride, "|u1", reinterpret_cast(stream), data, false, false); +} + +DecodedFrameExt make_yuv_frame_view(Pixel_Format format, size_t height, size_t width, int64_t timestamp, + DecodedFrameExt::ColorRange color_range, CUdeviceptr data, + CUstream stream) { + DecodedFrameExt frame; + frame.format = format; + frame.timestamp = timestamp; + frame.color_range = color_range; + const size_t stream_id = reinterpret_cast(stream); + + switch (format) { + case Pixel_Format_NV12: { + const size_t chroma_height = chroma_height_420(height); + frame.views.push_back( + CAIMemoryView{{height, width, 1}, {width, 1, 1}, "|u1", stream_id, data, false}); + frame.views.push_back(CAIMemoryView{{chroma_height, width / 2, 2}, + {width / 2 * 2, 2, 1}, + "|u1", + stream_id, + data + height * width, + false}); + frame.extBuf->LoadDLPack({height + chroma_height, width}, {width, 1}, "|u1", stream_id, data, + false); + break; + } + case Pixel_Format_P016: { + const size_t chroma_height = chroma_height_420(height); + frame.views.push_back( + CAIMemoryView{{height, width, 1}, {width * 2, 2, 2}, "|u2", stream_id, data, false}); + frame.views.push_back(CAIMemoryView{{chroma_height, width / 2, 2}, + {width * 2, 4, 2}, + "|u2", + stream_id, + data + 2 * height * width, + false}); + break; + } + case Pixel_Format_YUV444: + frame.views.push_back( + CAIMemoryView{{height, width, 1}, {width, 1, 1}, "|u1", stream_id, data, false}); + frame.views.push_back(CAIMemoryView{ + {height, width, 1}, {width, 1, 1}, "|u1", stream_id, data + height * width, false}); + frame.views.push_back(CAIMemoryView{ + {height, width, 1}, {width, 1, 1}, "|u1", stream_id, data + 2 * height * width, false}); + break; + case Pixel_Format_YUV444_16Bit: + frame.views.push_back( + CAIMemoryView{{height, width, 1}, {width * 2, 2, 2}, "|u2", stream_id, data, false}); + frame.views.push_back(CAIMemoryView{ + {height, width, 1}, {width * 2, 2, 2}, "|u2", stream_id, data + 2 * height * width, false}); + frame.views.push_back(CAIMemoryView{ + {height, width, 1}, {width * 2, 2, 2}, "|u2", stream_id, data + 4 * height * width, false}); + break; + default: + throw std::invalid_argument("Unsupported pixel format for YUV output: " + + std::to_string(static_cast(format))); + } + + return frame; +} + +} // namespace + +RGBFrame convert_decoded_frame_to_rgb(NvDecoder& decoder, const uint8_t* decoded_surface, + CUdeviceptr output_buffer, AVColorRange color_range, bool as_bgr, + bool is_async) { + const Pixel_Format format = pixel_format_from_surface(decoder.GetOutputFormat()); + const size_t width = static_cast(decoder.GetWidth()); + const size_t height = static_cast(decoder.GetHeight()); + RGBFrame output = make_rgb_frame_view(height, width, output_buffer, decoder.GetStream()); + + if (format != Pixel_Format_NV12) { + throw std::invalid_argument("[ERROR] Conversion to RGB/BGR only supported for videos in NV12-format"); + } + + const CAIMemoryView y_view{{height, width, 1}, + {width, 1, 1}, + "|u1", + reinterpret_cast(decoder.GetStream()), + reinterpret_cast(decoded_surface), + false}; + const CAIMemoryView uv_view{{chroma_height_420(height), width / 2, 2}, + {width / 2 * 2, 2, 1}, + "|u1", + reinterpret_cast(decoder.GetStream()), + reinterpret_cast(decoded_surface + width * height), + false}; + + { + nvtx3::scoped_range range{"Color convert"}; + const bool is_full_range = color_range == AVColorRange::AVCOL_RANGE_JPEG; + convert_nv12_to_rgb(y_view, uv_view, output, is_full_range, as_bgr); + } + + if (!is_async) { + CUDA_DRVAPI_CALL(cuStreamSynchronize(decoder.GetStream())); + } + return output; +} + +DecodedFrameExt copy_decoded_frame_to_yuv(NvDecoder& decoder, const uint8_t* decoded_surface, + CUdeviceptr output_buffer, AVColorRange color_range, + int64_t timestamp, bool is_async) { + const Pixel_Format format = pixel_format_from_surface(decoder.GetOutputFormat()); + if (format == Pixel_Format_UNDEFINED) { + throw std::runtime_error("[ERROR] Unsupported pixel format for YUV output"); + } + + const size_t height = static_cast(decoder.GetHeight()); + const size_t width = static_cast(decoder.GetWidth()); + const size_t output_bytes = frame_bytes(output_format_from_pixel_format(format), height, width); + CUDA_DRVAPI_CALL(cuMemcpyDtoDAsync(output_buffer, reinterpret_cast(decoded_surface), + output_bytes, decoder.GetStream())); + + DecodedFrameExt output = + make_yuv_frame_view(format, height, width, timestamp, DecodedFrameExt::ConvertColorRange(color_range), + output_buffer, decoder.GetStream()); + + if (!is_async) { + CUDA_DRVAPI_CALL(cuStreamSynchronize(decoder.GetStream())); + } + return output; +} + +RGBFrame copy_rgb_frame(const RGBFrame& source, CUdeviceptr destination, CUstream destination_stream, + bool as_bgr, bool is_async) { + const size_t height = std::get<0>(source.shape); + const size_t width = std::get<1>(source.shape); + + const size_t output_bytes = frame_bytes(FrameOutputFormat::RGB8, height, width); + CUDA_DRVAPI_CALL(cuMemcpyDtoDAsync(destination, source.data, output_bytes, destination_stream)); + + const std::vector shape{height, width, 3}; + const std::vector stride{std::get<0>(source.stride), std::get<1>(source.stride), + std::get<2>(source.stride)}; + RGBFrame output(shape, stride, source.typestr, reinterpret_cast(destination_stream), destination, + false, as_bgr); + + if (!is_async) { + CUDA_DRVAPI_CALL(cuStreamSynchronize(destination_stream)); + } + return output; +} + +DecodedFrameExt copy_yuv_frame(const DecodedFrameExt& source, CUdeviceptr destination, + CUstream destination_stream, bool is_async) { + const size_t height = source.views[0].shape[0]; + const size_t width = source.views[0].shape[1]; + const size_t output_bytes = frame_bytes(output_format_from_pixel_format(source.format), height, width); + CUDA_DRVAPI_CALL(cuMemcpyDtoDAsync(destination, source.views[0].data, output_bytes, destination_stream)); + + DecodedFrameExt output = make_yuv_frame_view(source.format, height, width, source.timestamp, + source.color_range, destination, destination_stream); + + if (!is_async) { + CUDA_DRVAPI_CALL(cuStreamSynchronize(destination_stream)); + } + return output; +} + +} // namespace accvlab::on_demand_video_decoder::internal::frame_output diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyDecodedFrameExt.cpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyDecodedFrameExt.cpp index 49d0b50c..1ce4358a 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyDecodedFrameExt.cpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyDecodedFrameExt.cpp @@ -15,8 +15,11 @@ */ #include "PyDecodedFrameExt.hpp" +#include "FrameOutput.hpp" #include +namespace frame_output = accvlab::on_demand_video_decoder::internal::frame_output; + DecodedFrameExt::VideoSurfaceFormat DecodedFrameExt::GetVideoSurfaceFormat() const { const VideoSurfaceFormat res = ConvertPixelFormatToVideoSurfaceFormatOut(this->format); return res; @@ -55,25 +58,12 @@ DecodedFrameExt::VideoSurfaceFormat DecodedFrameExt::ConvertPixelFormatToVideoSu Pixel_Format DecodedFrameExt::ConvertVideoSurfaceFormatInToPixelFormat( cudaVideoSurfaceFormat video_format_in) { - Pixel_Format res; - switch (video_format_in) { - case cudaVideoSurfaceFormat::cudaVideoSurfaceFormat_NV12: - res = Pixel_Format::Pixel_Format_NV12; - break; - case cudaVideoSurfaceFormat::cudaVideoSurfaceFormat_P016: - res = Pixel_Format::Pixel_Format_P016; - break; - case cudaVideoSurfaceFormat::cudaVideoSurfaceFormat_YUV444: - res = Pixel_Format::Pixel_Format_YUV444; - break; - case cudaVideoSurfaceFormat::cudaVideoSurfaceFormat_YUV444_16Bit: - res = Pixel_Format::Pixel_Format_YUV444_16Bit; - break; - default: - throw std::invalid_argument("Got unexpected value " + std::to_string(video_format_in) + - " for input argument `video_format_in`."); + const Pixel_Format result = frame_output::pixel_format_from_surface(video_format_in); + if (result == Pixel_Format_UNDEFINED) { + throw std::invalid_argument("Got unexpected value " + std::to_string(video_format_in) + + " for input argument `video_format_in`."); } - return res; + return result; } DecodedFrameExt::ColorRange DecodedFrameExt::ConvertColorRange(AVColorRange color_range_in) { diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvBatchAsyncGopDecoder.cpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvBatchAsyncGopDecoder.cpp index 35ff6e73..207d4224 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvBatchAsyncGopDecoder.cpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvBatchAsyncGopDecoder.cpp @@ -15,6 +15,7 @@ */ #include "PyNvBatchAsyncGopDecoder.hpp" +#include "FrameOutput.hpp" #include #include @@ -33,6 +34,7 @@ #include "nvtx3/nvtx3.hpp" namespace py = pybind11; +namespace frame_output = accvlab::on_demand_video_decoder::internal::frame_output; // --------------------------------------------------------------------------- // Constructor / Destructor @@ -225,78 +227,6 @@ void PyNvBatchAsyncGopDecoder::validate_decode_input( } } -// static -size_t PyNvBatchAsyncGopDecoder::compute_yuv_frame_bytes(Pixel_Format fmt, size_t H, size_t W) { - switch (fmt) { - case Pixel_Format_NV12: - // Y: H*W bytes + UV interleaved: (H/2)*W bytes = H*W*3/2 - return H * W + (H / 2) * W; - case Pixel_Format_P016: - // Y: H*W*2 bytes + UV interleaved: (H/2)*W*2 bytes = H*W*3 - return H * W * 3; - case Pixel_Format_YUV444: - // Y + U planes (matching GetYUVFromFrame which adds 2 views), but the full - // 3-plane buffer (Y+U+V = 3*H*W) must be copied for correctness. - return H * W * 3; - case Pixel_Format_YUV444_16Bit: - // Y + U + V, 2 bytes each: 3*H*W*2 - return H * W * 6; - default: - throw std::runtime_error("compute_yuv_frame_bytes: unsupported pixel format " + - std::to_string(static_cast(fmt))); - } -} - -// static -void PyNvBatchAsyncGopDecoder::build_yuv_frame(Pixel_Format fmt, size_t H, size_t W, int64_t timestamp, - DecodedFrameExt::ColorRange color_range, CUdeviceptr dst_ptr, - CUstream stream, DecodedFrameExt& out) { - out.format = fmt; - out.timestamp = timestamp; - out.color_range = color_range; - const size_t stream_id = reinterpret_cast(stream); - - switch (fmt) { - case Pixel_Format_NV12: - out.views.push_back(CAIMemoryView{{H, W, 1}, {W, 1, 1}, "|u1", stream_id, dst_ptr, false}); - out.views.push_back(CAIMemoryView{ - {H / 2, W / 2, 2}, {W / 2 * 2, 2, 1}, "|u1", stream_id, dst_ptr + H * W, false}); - out.extBuf->LoadDLPack({static_cast(H * 1.5), W}, {W, 1}, "|u1", stream_id, dst_ptr, - false); - break; - // TODO(P016): LoadDLPack rejects "|u2" typestr, so no DLPack tensor can be built. - case Pixel_Format_P016: - out.views.push_back(CAIMemoryView{{H, W, 1}, {W * 2, 2, 2}, "|u2", stream_id, dst_ptr, false}); - out.views.push_back(CAIMemoryView{ - {H / 2, W / 2, 2}, {W * 2, 4, 2}, "|u2", stream_id, dst_ptr + 2 * H * W, false}); - break; - // TODO(YUV444): needs a flat (H*3, W) DLPack view and extBuf support for 3-plane layouts. - case Pixel_Format_YUV444: - out.views.push_back(CAIMemoryView{{H, W, 1}, {W, 1, 1}, "|u1", stream_id, dst_ptr, false}); - out.views.push_back( - CAIMemoryView{{H, W, 1}, {W, 1, 1}, "|u1", stream_id, dst_ptr + H * W, false}); - out.views.push_back( - CAIMemoryView{{H, W, 1}, {W, 1, 1}, "|u1", stream_id, dst_ptr + 2 * H * W, false}); - break; - // TODO(YUV444_16Bit): same as P016 — LoadDLPack rejects "|u2". - case Pixel_Format_YUV444_16Bit: - out.views.push_back(CAIMemoryView{{H, W, 1}, {W * 2, 2, 2}, "|u2", stream_id, dst_ptr, false}); - out.views.push_back( - CAIMemoryView{{H, W, 1}, {W * 2, 2, 2}, "|u2", stream_id, dst_ptr + 2 * H * W, false}); - out.views.push_back( - CAIMemoryView{{H, W, 1}, {W * 2, 2, 2}, "|u2", stream_id, dst_ptr + 4 * H * W, false}); - break; - default: - // Only NV12 is currently supported. Returning a DecodedFrameExt with an empty extBuf - // for other formats would let torch.as_tensor() silently produce a 0-dim null-pointer - // CUDA tensor, so we fail fast here instead. - throw std::runtime_error( - "PyNvBatchAsyncGopDecoder: DecodeFromGOPList (YUV path) only supports " - "Pixel_Format_NV12. Got pixel format " + - std::to_string(static_cast(fmt)) + ". Use DecodeFromGOPListRGB for other formats."); - } -} - // --------------------------------------------------------------------------- // Common async submission path // --------------------------------------------------------------------------- @@ -472,7 +402,8 @@ void PyNvBatchAsyncGopDecoder::submit_work(std::vector(frames_f[v].shape); const size_t W = std::get<1>(frames_f[v].shape); - const size_t frame_bytes = H * W * 3; + const size_t frame_bytes = + frame_output::frame_bytes(frame_output::FrameOutputFormat::RGB8, H, W); // Size each video's pool once (first frame-slot), then append. if (f == 0) @@ -480,18 +411,10 @@ void PyNvBatchAsyncGopDecoder::submit_work(std::vector(dst), - frames_f[v].data, frame_bytes, cu_stream_)); - const std::vector shape_vec = {H, W, 3}; - const std::vector stride_vec = {std::get<0>(frames_f[v].stride), - std::get<1>(frames_f[v].stride), - std::get<2>(frames_f[v].stride)}; // Place at the original (unsorted) output index. result.decoded_rgb_frames[v][perm_2d[v][f]] = - RGBFrame(shape_vec, stride_vec, frames_f[v].typestr, - reinterpret_cast(cu_stream_), reinterpret_cast(dst), - /*readOnly=*/false, - /*isBGR=*/as_bgr); + frame_output::copy_rgb_frame(frames_f[v], reinterpret_cast(dst), + cu_stream_, as_bgr, /*is_async=*/true); } } else { @@ -512,22 +435,17 @@ void PyNvBatchAsyncGopDecoder::submit_work(std::vector(F) * frame_bytes, false); void* dst = yuv_agg_pools_[v].AddElement(frame_bytes); - // Source is contiguous in gop_dec_'s pool (views[0].data = base). - CUDA_DRVAPI_CALL(cuMemcpyDtoDAsync(reinterpret_cast(dst), - frames_f[v].views[0].data, frame_bytes, - cu_stream_)); - DecodedFrameExt frame; - build_yuv_frame(fmt, H, W, frames_f[v].timestamp, frames_f[v].color_range, - reinterpret_cast(dst), cu_stream_, frame); // Place at the original (unsorted) output index. - result.decoded_yuv_frames[v][perm_2d[v][f]] = std::move(frame); + result.decoded_yuv_frames[v][perm_2d[v][f]] = frame_output::copy_yuv_frame( + frames_f[v], reinterpret_cast(dst), cu_stream_, /*is_async=*/true); } } } diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvBatchAsyncStreamReader.cpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvBatchAsyncStreamReader.cpp index b915f837..c88bea31 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvBatchAsyncStreamReader.cpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvBatchAsyncStreamReader.cpp @@ -15,6 +15,7 @@ */ #include "PyNvBatchAsyncStreamReader.hpp" +#include "FrameOutput.hpp" #include #include @@ -73,6 +74,8 @@ std::vector process_frames_in_parallel(const std::vector& filepa } } // namespace +namespace frame_output = accvlab::on_demand_video_decoder::internal::frame_output; + PyNvBatchAsyncStreamReader::PyNvBatchAsyncStreamReader(int num_of_set, int num_of_file, int max_frames_per_decode_call, int iGpu, bool bSuppressNoColorRangeWarning) @@ -308,7 +311,8 @@ std::vector PyNvBatchAsyncStreamReader::run_rgb_out_1d(const std::vect // Only allocate a new reader when there's room AND the file isn't already // cached, matching PyNvSampleReader::run_rgb_out's memory-leak guard. if (reader_map.notFull() && !reader_map.contains(filepaths[i])) { - video_reader = new PyNvVideoReader(filepaths[i], this->gpu_id, this->cu_context, this->cu_stream); + video_reader = new PyNvVideoReader(filepaths[i], this->gpu_id, this->cu_context, this->cu_stream, + this->suppress_no_color_range_given_warning); } auto cur_video_reader = reader_map.find(filepaths[i], video_reader); video_readers[i] = cur_video_reader; @@ -388,8 +392,6 @@ void PyNvBatchAsyncStreamReader::Decode(const std::vector& filepath // Videos in a single Decode() call may differ in resolution; each // video's own F frames must be uniform (always true for one mp4). std::vector> ref_shape(V); - std::vector> ref_stride(V); - std::vector ref_typestr(V); std::vector v_bytes(V, 0); for (int f = 0; f < F; ++f) { @@ -407,11 +409,9 @@ void PyNvBatchAsyncStreamReader::Decode(const std::vector& filepath // triggers a re-alloc automatically; same-or-smaller // resolutions reuse the existing allocation. ref_shape[v] = frames[v].shape; - ref_stride[v] = frames[v].stride; - ref_typestr[v] = frames[v].typestr; const size_t H = std::get<0>(ref_shape[v]); const size_t W = std::get<1>(ref_shape[v]); - v_bytes[v] = H * W * 3; + v_bytes[v] = frame_output::frame_bytes(frame_output::FrameOutputFormat::RGB8, H, W); // TODO: eliminate this D2D copy by having the underlying VideoReader write // directly into agg_pools[v]. Requires GPUMemoryPool move semantics and a @@ -435,17 +435,9 @@ void PyNvBatchAsyncStreamReader::Decode(const std::vector& filepath } void* dst = agg_pools[v].AddElement(v_bytes[v]); - CUDA_DRVAPI_CALL(cuMemcpyDtoDAsync(reinterpret_cast(dst), frames[v].data, - v_bytes[v], cu_stream)); - - const std::vector shape_vec = {std::get<0>(ref_shape[v]), - std::get<1>(ref_shape[v]), 3}; - const std::vector stride_vec = { - std::get<0>(ref_stride[v]), std::get<1>(ref_stride[v]), std::get<2>(ref_stride[v])}; - result.decoded_frames[v].emplace_back(shape_vec, stride_vec, ref_typestr[v], - reinterpret_cast(cu_stream), - reinterpret_cast(dst), - /*readOnly=*/false, /*isBGR=*/as_bgr_cap); + result.decoded_frames[v].emplace_back( + frame_output::copy_rgb_frame(frames[v], reinterpret_cast(dst), cu_stream, + as_bgr_cap, /*is_async=*/true)); } } diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_common.cpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_common.cpp index 109bc037..687686fc 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_common.cpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_common.cpp @@ -15,6 +15,7 @@ */ #include "PyNvGopDecoder.hpp" +#include "FrameOutput.hpp" #include "GopDecoderUtils.hpp" #include @@ -34,181 +35,8 @@ #include "nvtx3/nvtx3.hpp" -#include "ColorConvertKernels.cuh" - namespace fs = std::filesystem; - -Pixel_Format PyNvGopDecoder::GetNativeFormat(const cudaVideoSurfaceFormat inputFormat) { - switch (inputFormat) { - case cudaVideoSurfaceFormat_NV12: - return Pixel_Format_NV12; - case cudaVideoSurfaceFormat_P016: - return Pixel_Format_P016; - case cudaVideoSurfaceFormat_YUV444: - return Pixel_Format_YUV444; - case cudaVideoSurfaceFormat_YUV444_16Bit: - return Pixel_Format_YUV444_16Bit; - default: - break; - } - return Pixel_Format_UNDEFINED; -} - -int PyNvGopDecoder::GetRGBFromFrame(NvDecoder* decoder, const uint8_t* pFrame, uint8_t* pFrame_buffer, - AVColorRange color_range, bool use_bgr_format, RGBFrame& rgb_frame) { - Pixel_Format format = GetNativeFormat(decoder->GetOutputFormat()); - auto width = size_t(decoder->GetWidth()); - auto height = size_t(decoder->GetHeight()); - - const std::vector frame_shape{height, width, 3}; - const std::vector frame_stride{width * 3, 3, 1}; - - rgb_frame = RGBFrame(frame_shape, frame_stride, "|u1", reinterpret_cast(decoder->GetStream()), - reinterpret_cast(pFrame_buffer), false, false); - - switch (format) { - case Pixel_Format_NV12: { - const CAIMemoryView y_view{{height, width, 1}, - {width, 1, 1}, - "|u1", - reinterpret_cast(decoder->GetStream()), - reinterpret_cast(pFrame), - false}; - const CAIMemoryView uv_view{{height / 2, width / 2, 2}, - {width / 2 * 2, 2, 1}, - "|u1", - reinterpret_cast(decoder->GetStream()), - reinterpret_cast(pFrame + width * height), - false}; // todo: data+width*height assumes both planes are - // contiguous. Actual NVENC allocation can have padding? - - nvtxRangePushA("Color convert"); - bool is_full_range = color_range == AVColorRange::AVCOL_RANGE_JPEG; - if ((color_range != AVColorRange::AVCOL_RANGE_JPEG && - color_range != AVColorRange::AVCOL_RANGE_MPEG)) { - // LOG(WARNING) << "Color range is not supported with color range: " << color_range; - is_full_range = false; - } - - convert_nv12_to_rgb(y_view, uv_view, rgb_frame, is_full_range, use_bgr_format); - nvtxRangePop(); - } break; - default: { - LOG(ERROR) << "Conversion to RGB/BGR only supported for videos in NV12-format"; - return -1; - } - } - - return 0; -} - -int PyNvGopDecoder::GetYUVFromFrame(NvDecoder* decoder, const uint8_t* pFrame, uint8_t* pFrame_buffer, - AVColorRange color_range, int64_t timestamp, - DecodedFrameExt& decoded_frame) { - decoded_frame.format = GetNativeFormat(decoder->GetOutputFormat()); - auto width = size_t(decoder->GetWidth()); - auto height = size_t(decoder->GetHeight()); - decoded_frame.timestamp = timestamp; - decoded_frame.SetColorRange(color_range); - - // Queue the decode-buffer copy on the decoder stream. DecProc synchronizes - // this stream before returning the Python-visible frame. - CUDA_DRVAPI_CALL(cuMemcpyDtoDAsync((CUdeviceptr)pFrame_buffer, (CUdeviceptr)pFrame, - decoder->GetFrameSize(), decoder->GetStream())); - - switch (decoded_frame.format) { - case Pixel_Format_NV12: { - decoded_frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width, 1, 1}, - "|u1", - reinterpret_cast(decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer), - false}); - decoded_frame.views.push_back( - CAIMemoryView{{height / 2, width / 2, 2}, - {width / 2 * 2, 2, 1}, - "|u1", - reinterpret_cast(decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer + width * height), - false}); // todo: data+width*height assumes both planes are - // contiguous. Actual NVENC allocation can have padding? - // Load DLPack Tensor - std::vector shape{(size_t)(height * 1.5), width}; - std::vector stride{size_t(width), 1}; - int returntype = decoded_frame.extBuf->LoadDLPack(shape, stride, "|u1", - reinterpret_cast(decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer), false); - } break; - case Pixel_Format_P016: { - decoded_frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width * 2, 2, 2}, - "|u2", - reinterpret_cast(decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer), - false}); - decoded_frame.views.push_back( - CAIMemoryView{{height / 2, width / 2, 2}, - {width * 2, 4, 2}, - "|u2", - reinterpret_cast(decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer + 2 * (width * height)), - false}); // todo: data+width*height assumes both planes are - // contiguous. Actual NVENC allocation can have padding? - } break; - case Pixel_Format_YUV444: { - decoded_frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width, 1, 1}, - "|u1", - reinterpret_cast(decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer), - false}); - decoded_frame.views.push_back( - CAIMemoryView{{height, width, 1}, - {width, 1, 1}, - "|u1", - reinterpret_cast(decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer + width * height), - false}); // todo: data+width*height assumes both planes are - // contiguous. Actual NVENC allocation can have padding? - decoded_frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width, 1, 1}, - "|u1", - reinterpret_cast(decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer + 2 * width * height), - false}); - } break; - case Pixel_Format_YUV444_16Bit: { - decoded_frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width * 2, 2, 2}, - "|u2", - reinterpret_cast(decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer), - false}); - decoded_frame.views.push_back( - CAIMemoryView{{height, width, 1}, - {width * 2, 2, 2}, - "|u2", - reinterpret_cast(decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer + 2 * (width * height)), - false}); // todo: data+width*height assumes both planes are - // contiguous. Actual NVENC allocation can have padding? - decoded_frame.views.push_back( - CAIMemoryView{{height, width, 1}, - {width * 2, 2, 2}, - "|u2", - reinterpret_cast(decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer + 4 * (width * height)), - false}); // todo: data+width*height assumes both planes are - // contiguous. Actual NVENC allocation can have padding? - } break; - default: { - LOG(ERROR) << "Unsupported pixel format for DecodedFrameExt creation"; - return -1; - } - } - - return 0; -} +namespace frame_output = accvlab::on_demand_video_decoder::internal::frame_output; /*The video packet is ordered in decoding order, For example @@ -459,19 +287,25 @@ void PyNvGopDecoder::DecProc(AVColorRange color_range, NvDecoder* decoder, if (timestamp % 2 || timestamp / 2 == *frame_id_iter) { OutputFrame output_frame; if constexpr (std::is_same_v) { - int st = PyNvGopDecoder::GetRGBFromFrame(decoder, pFrame, *pFrame_iter, color_range, - use_bgr_format, output_frame); - if (st) { + if (frame_output::pixel_format_from_surface(decoder->GetOutputFormat()) != + Pixel_Format_NV12) { + LOG(ERROR) << "Conversion to RGB/BGR only supported for videos in NV12-format"; throw std::runtime_error("[ERROR] Failed to convert frame to RGB for file: " + filename); } + output_frame = frame_output::convert_decoded_frame_to_rgb( + *decoder, pFrame, reinterpret_cast(*pFrame_iter), color_range, + use_bgr_format, /*is_async=*/true); } else { - int st = PyNvGopDecoder::GetYUVFromFrame(decoder, pFrame, *pFrame_iter, color_range, - timestamp, output_frame); - if (st) { + if (frame_output::pixel_format_from_surface(decoder->GetOutputFormat()) == + Pixel_Format_UNDEFINED) { + LOG(ERROR) << "Unsupported pixel format for DecodedFrameExt creation"; throw std::runtime_error("[ERROR] Failed to convert frame to YUV for file: " + filename); } + output_frame = frame_output::copy_decoded_frame_to_yuv( + *decoder, pFrame, reinterpret_cast(*pFrame_iter), color_range, timestamp, + /*is_async=*/true); } output_frames.push_back(std::move(output_frame)); ++frame_id_iter; @@ -504,6 +338,22 @@ void PyNvGopDecoder::DecProc(AVColorRange color_range, NvDecoder* decoder, nvtxRangePop(); } +void PyNvGopDecoder::WarnIfColorRangeUnspecified(AVColorRange color_range, const std::string& filename) { + if (suppress_no_color_range_given_warning || color_range == AVColorRange::AVCOL_RANGE_JPEG || + color_range == AVColorRange::AVCOL_RANGE_MPEG) { + return; + } + + std::lock_guard lock(no_color_range_warning_mutex); + if (!warned_no_color_range_files.insert(filename).second) { + return; + } + + std::cout << "WARNING: PyNvGopDecoder could not obtain color range for:\n" + << " " << (filename.empty() ? "" : filename) << "\n" + << " --> Limited range (MPEG range) assumed for this file." << std::endl; +} + void PyNvGopDecoder::CreateDemuxer(std::unique_ptr& demuxer, const std::string& filename, const FastStreamInfo* fastStreamInfo) { if (fastStreamInfo) { @@ -626,7 +476,9 @@ int PyNvGopDecoder::InitGpuMemPool(const std::vector& heights, const std::v for (int i = 0; i < len; ++i) { if (convert_to_rgb) { - needed_size += widths[i] * heights[i] * 3; + needed_size += + frame_output::frame_bytes(frame_output::FrameOutputFormat::RGB8, + static_cast(heights[i]), static_cast(widths[i])); } else { needed_size += frame_sizes[i]; } @@ -803,8 +655,9 @@ int PyNvGopDecoder::GetFileFrameBuffers(const std::vector* widths, const st nvtxRangePushA("Frame memory allocation"); uint8_t* pFrame; if (convert_to_rgb) { - pFrame = - reinterpret_cast(this->gpu_mem_pool.AddElement(widths->at(i) * heights->at(i) * 3)); + pFrame = reinterpret_cast(this->gpu_mem_pool.AddElement(frame_output::frame_bytes( + frame_output::FrameOutputFormat::RGB8, static_cast(heights->at(i)), + static_cast(widths->at(i))))); } else { pFrame = reinterpret_cast(this->gpu_mem_pool.AddElement(frame_sizes->at(i))); } diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_constructors.cpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_constructors.cpp index d9575039..2ed78982 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_constructors.cpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_constructors.cpp @@ -256,7 +256,6 @@ PyNvGopDecoder::~PyNvGopDecoder() { void Init_PyNvGopDecoder(py::module& m) { ExternalBuffer::Export(m); - DecodedFrame::Export(m); CAIMemoryView::Export(m); DecodedFrameExt::Export(m); RGBFrame::Export(m); @@ -309,38 +308,10 @@ void Init_PyNvGopDecoder(py::module& m) { }, py::arg("maxfiles"), py::arg("iGpu") = 0, py::arg("suppressNoColorRangeWarning") = false, R"pbdoc( - Create a GPU-accelerated video decoder with GOP-level random access. + Create the native GPU decoder. - Factory function for the on-demand video decoding module, which provides: - - - **Random frame access**: Decode any frame by index without sequential scanning, - using :meth:`PyNvGopDecoder.Decode` or :meth:`PyNvGopDecoder.DecodeN12ToRGB`. - - **Demux/decode separation**: Extract serialized GOP bundles first via - :meth:`PyNvGopDecoder.GetGOPList`, then decode on - GPU later. This enables caching, prefetching, and DataLoader-friendly pipelines. - - **GOP persistence**: Save serialized GOP bundles to disk with :func:`SaveGopToFile` - and reload it with :meth:`PyNvGopDecoder.LoadGopsToList`, avoiding redundant - demuxing across training runs. - - Args: - maxfiles: Maximum number of video files that can be processed concurrently. - iGpu: GPU device ID to use for decoding (0 for primary GPU) - suppressNoColorRangeWarning: Suppress warning when no color range information - can be extracted from video files (limited/MPEG - range is assumed in that case). Currently has no - effect in this decoder. - - Returns: - :class:`PyNvGopDecoder` instance configured with the specified parameters - - Raises: - RuntimeError: If parameters are invalid - - Example: - >>> decoder = CreateGopDecoder(maxfiles=3, iGpu=0) - >>> frames = decoder.Decode(['v0.mp4', 'v1.mp4', 'v2.mp4'], [0, 10, 20]) - >>> # Convert to PyTorch tensors on GPU (NV12 layout: (height * 3 // 2, width), uint8) - >>> nv12_tensors = [torch.as_tensor(frame).clone() for frame in frames] + See :func:`~accvlab.on_demand_video_decoder.CreateGopDecoder` for the public API + and full documentation. )pbdoc"); m.def( diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_random_decoder.cpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_random_decoder.cpp index 92489895..80e44b27 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_random_decoder.cpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_random_decoder.cpp @@ -15,6 +15,7 @@ */ #include "PyNvGopDecoder.hpp" +#include "FrameOutput.hpp" #include #include @@ -35,6 +36,7 @@ #define MAX_SIZE 2000 namespace fs = std::filesystem; +namespace frame_output = accvlab::on_demand_video_decoder::internal::frame_output; void PyNvGopDecoder::decode_from_video(const std::vector& filepaths, const std::vector frame_ids, bool convert_to_rgb, bool as_bgr, @@ -94,7 +96,9 @@ void PyNvGopDecoder::decode_from_video(const std::vector& filepaths codec_ids[i] = demuxers[i]->GetNvCodecId(); widths[i] = demuxers[i]->GetWidth(); heights[i] = demuxers[i]->GetHeight(); - frame_sizes[i] = demuxers[i]->GetFrameSize(); + frame_sizes[i] = static_cast(frame_output::frame_bytes( + frame_output::output_format_from_av_pixel_format(demuxers[i]->GetPixelFormat()), + static_cast(heights[i]), static_cast(widths[i]))); } st = InitGpuMemPool(heights, widths, frame_sizes, convert_to_rgb); @@ -145,6 +149,7 @@ void PyNvGopDecoder::decode_from_video(const std::vector& filepaths } } if (convert_to_rgb) { + WarnIfColorRangeUnspecified(demuxers[i]->GetColorRange(), filepaths[i]); rgb_frames[i].reserve(1); } else { decodedFrames[i].reserve(1); diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_separate_decoder.cpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_separate_decoder.cpp index b95a5c5a..361abf69 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_separate_decoder.cpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvGopDecoder_separate_decoder.cpp @@ -15,6 +15,7 @@ */ #include "PyNvGopDecoder.hpp" +#include "FrameOutput.hpp" #include #include @@ -33,6 +34,8 @@ #include "ColorConvertKernels.cuh" +namespace frame_output = accvlab::on_demand_video_decoder::internal::frame_output; + void PyNvGopDecoder::get_gop_internal( const std::vector& filepaths, const std::vector frame_ids, const FastStreamInfo* fastStreamInfos, std::vector>& demuxers, @@ -809,6 +812,7 @@ int PyNvGopDecoder::main_decode_groups( try { const size_t slot_idx = decoder_slots[group_idx]; const AVColorRange color_range = static_cast(color_ranges[group_idx]); + WarnIfColorRangeUnspecified(color_range, source_names[group_idx]); rgb_frames[group_idx].reserve(frame_id_groups[group_idx].size()); #ifdef PROCESS_SYNC DecProc(color_range, vdec[slot_idx].get(), rgb_frames[group_idx], @@ -914,6 +918,7 @@ int PyNvGopDecoder::main_decode( try { std::vector sorted_frame_ids = {frame_ids[i]}; if (convert_to_rgb) { + WarnIfColorRangeUnspecified(static_cast(color_ranges[i]), filepaths[i]); rgb_frames[i].reserve(sorted_frame_ids.size()); } else { decodedFrames[i].reserve(sorted_frame_ids.size()); @@ -1139,7 +1144,9 @@ SerializedPacketBundle PyNvGopDecoder::createSerializedPacketBundle( ptr += sizeof(int32_t); // width *reinterpret_cast(ptr) = demuxers[i]->GetHeight(); ptr += sizeof(int32_t); // height - *reinterpret_cast(ptr) = demuxers[i]->GetFrameSize(); + *reinterpret_cast(ptr) = static_cast(frame_output::frame_bytes( + frame_output::output_format_from_av_pixel_format(demuxers[i]->GetPixelFormat()), + static_cast(demuxers[i]->GetHeight()), static_cast(demuxers[i]->GetWidth()))); ptr += sizeof(int32_t); // frame_size *reinterpret_cast(ptr) = all_gop_lens[i][0]; ptr += sizeof(int32_t); // gop_len @@ -1187,7 +1194,11 @@ SerializedPacketBundle PyNvGopDecoder::createSerializedPacketBundle( printf(" codec_id: %d\n", demuxers[i]->GetNvCodecId()); printf(" width: %d\n", demuxers[i]->GetWidth()); printf(" height: %d\n", demuxers[i]->GetHeight()); - printf(" frame_size: %d\n", demuxers[i]->GetFrameSize()); + printf( + " frame_size: %zu\n", + frame_output::frame_bytes( + frame_output::output_format_from_av_pixel_format(demuxers[i]->GetPixelFormat()), + static_cast(demuxers[i]->GetHeight()), static_cast(demuxers[i]->GetWidth()))); printf(" gop_len: %d\n", all_gop_lens[i][0]); printf(" first_frame_id: %d\n", all_first_frame_ids[i][0]); diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvSampleReader.cpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvSampleReader.cpp index d17be147..2ea2ec56 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvSampleReader.cpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvSampleReader.cpp @@ -194,7 +194,8 @@ std::vector PyNvSampleReader::run_rgb_out(const std::vectorgpu_id, this->cu_context, this->cu_stream); + video_reader = new PyNvVideoReader(filepaths[i], this->gpu_id, this->cu_context, this->cu_stream, + this->suppress_no_color_range_given_warning); } auto cur_video_reader = reader_map.find(filepaths[i], video_reader); @@ -237,7 +238,8 @@ std::vector PyNvSampleReader::run(const std::vectorgpu_id, this->cu_context, this->cu_stream); + video_reader = new PyNvVideoReader(filepaths[i], this->gpu_id, this->cu_context, this->cu_stream, + this->suppress_no_color_range_given_warning); } auto cur_video_reader = reader_map.find(filepaths[i], video_reader); diff --git a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvVideoReader.cpp b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvVideoReader.cpp index f216e9fc..35806e04 100644 --- a/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvVideoReader.cpp +++ b/packages/on_demand_video_decoder/ext_impl/src/PyNvOnDemandDecoder/src/PyNvVideoReader.cpp @@ -15,6 +15,7 @@ */ #include "PyNvVideoReader.hpp" +#include "FrameOutput.hpp" #include "GopDecoderUtils.hpp" #include @@ -30,11 +31,10 @@ #include "nvtx3/nvtx3.hpp" -#include "ColorConvertKernels.cuh" - #define MAX_SIZE 2000 namespace fs = std::filesystem; +namespace frame_output = accvlab::on_demand_video_decoder::internal::frame_output; /* If a frame is a I-frame and key_frame in the same time, then the frame is the start of a new GOP The keyframe is the frame which has flag AV_FRAME_FLAG_KEY @@ -99,22 +99,6 @@ void PyNvVideoReader::parse_keyframe_idx(std::vector& key_frame_ids, std::m } } -Pixel_Format PyNvVideoReader::GetNativeFormat(const cudaVideoSurfaceFormat inputFormat) { - switch (inputFormat) { - case cudaVideoSurfaceFormat_NV12: - return Pixel_Format_NV12; - case cudaVideoSurfaceFormat_P016: - return Pixel_Format_P016; - case cudaVideoSurfaceFormat_YUV444: - return Pixel_Format_YUV444; - case cudaVideoSurfaceFormat_YUV444_16Bit: - return Pixel_Format_YUV444_16Bit; - default: - break; - } - return Pixel_Format_UNDEFINED; -} - PyNvVideoReader::PyNvVideoReader(const std::string filename, int iGpu, CUcontext cu_context, CUstream cu_stream, bool bSuppressNoColorRangeWarning) : filename(filename), gpu_id(iGpu), suppress_no_color_range_given_warning(bSuppressNoColorRangeWarning) { @@ -277,6 +261,7 @@ void PyNvVideoReader::startNextGop() { this->releasePacketArray(); } void PyNvVideoReader::ReplaceWithFile(const std::string filename) { this->releasePacketArray(); + this->has_warned_no_color_range_.store(false); nvtxRangePushA("Reset Demuxer"); this->demuxer.reset(new PyNvGopDemuxer(filename.c_str())); nvtxRangePop(); @@ -516,30 +501,21 @@ void PyNvVideoReader::run_single_frame_internal(const int frame_id, bool convert ck(cuCtxPushCurrent(this->cu_context)); nvtxRangePushA("Decode Single Frame"); - size_t needed_size = 0; - - if (convert_to_rgb) { - needed_size = this->demuxer->GetWidth() * this->demuxer->GetHeight() * 3; - } else { - needed_size = this->demuxer->GetFrameSize(); - } + const frame_output::FrameOutputFormat output_format = + convert_to_rgb ? frame_output::FrameOutputFormat::RGB8 + : frame_output::output_format_from_av_pixel_format(demuxer->GetPixelFormat()); + const size_t needed_size = frame_output::frame_bytes( + output_format, static_cast(demuxer->GetHeight()), static_cast(demuxer->GetWidth())); gpu_mem_pool.EnsureSizeAndSoftReset(needed_size, false); // Note that the following difference needs to be computed with signed // integers as the difference may be negative - // TODO, the size of the allocated buffer for decoded Frames should be - // dec->GetFrameSize() + // Output allocations use the package-wide tightly packed frame size. uint8_t *pReturnFrame = nullptr, *pFrame = nullptr; int64_t timestamp = 0; - bool file_added_for_warning = false; - if (convert_to_rgb) { - pFrame = reinterpret_cast( - this->gpu_mem_pool.AddElement(this->demuxer->GetWidth() * this->demuxer->GetHeight() * 3)); - } else { - pFrame = reinterpret_cast(this->gpu_mem_pool.AddElement(this->demuxer->GetFrameSize())); - } + pFrame = reinterpret_cast(this->gpu_mem_pool.AddElement(needed_size)); if (frame_id >= this->next_keyframe_) { auto cur_keyframe_ = demuxer->getKeyFrameId(frame_id); @@ -565,8 +541,7 @@ void PyNvVideoReader::run_single_frame_internal(const int frame_id, bool convert while (this->cur_frame_ < frame_id) { // First try to process any pending decoded frames if (this->processDecodedFrames(frame_id, pFrame, pReturnFrame, convert_to_rgb, use_bgr_format, - file_added_for_warning, out_if_color_converted, - out_if_no_color_conversion)) { + out_if_color_converted, out_if_no_color_conversion)) { nvtxRangePop(); ck(cuCtxPopCurrent(NULL)); return; @@ -581,27 +556,28 @@ void PyNvVideoReader::run_single_frame_internal(const int frame_id, bool convert // Process any newly decoded frames if (processDecodedFrames(frame_id, pFrame, pReturnFrame, convert_to_rgb, use_bgr_format, - file_added_for_warning, out_if_color_converted, - out_if_no_color_conversion)) { + out_if_color_converted, out_if_no_color_conversion)) { nvtxRangePop(); ck(cuCtxPopCurrent(NULL)); return; } } - if (!suppress_no_color_range_given_warning) { - if (file_added_for_warning) { - std::cout << "WARNING: PyNVGopDecoder could not obtain color range for " - "the following files:\n"; - std::cout << " " << this->filename << "\n"; - std::cout << " --> Limited range (MPEG range) assumed for these files." << std::endl; - } - } - ck(cuCtxPopCurrent(NULL)); nvtxRangePop(); } +void PyNvVideoReader::WarnIfColorRangeUnspecified(AVColorRange color_range) { + if (suppress_no_color_range_given_warning || color_range == AVColorRange::AVCOL_RANGE_JPEG || + color_range == AVColorRange::AVCOL_RANGE_MPEG || has_warned_no_color_range_.exchange(true)) { + return; + } + + std::cout << "WARNING: PyNvVideoReader could not obtain color range for:\n" + << " " << filename << "\n" + << " --> Limited range (MPEG range) assumed for this file." << std::endl; +} + void PyNvVideoReader::fetchNextGop() { auto next_next_key_frame = demuxer->getNextKeyFrameId(this->next_keyframe_); auto gop_len = next_next_key_frame - this->next_keyframe_; @@ -655,7 +631,7 @@ void PyNvVideoReader::decodeNextPacket() { bool PyNvVideoReader::processDecodedFrames(const int frame_id, uint8_t* pFrame, uint8_t* pReturnFrame, bool convert_to_rgb, bool use_bgr_format, - bool& file_added_for_warning, RGBFrame* out_if_color_converted, + RGBFrame* out_if_color_converted, DecodedFrameExt* out_if_no_color_conversion) { while (this->return_frames_) { int64_t timestamp; @@ -674,156 +650,18 @@ bool PyNvVideoReader::processDecodedFrames(const int frame_id, uint8_t* pFrame, // TODO This line is important, but sometimes the timestamp returned by // nvdec is wrong if (convert_to_rgb) { - *out_if_color_converted = - this->returnRGBFrame(pFrame, pReturnFrame, use_bgr_format, file_added_for_warning); + const AVColorRange color_range = this->demuxer->GetColorRange(); + WarnIfColorRangeUnspecified(color_range); + *out_if_color_converted = frame_output::convert_decoded_frame_to_rgb( + *decoder, pReturnFrame, reinterpret_cast(pFrame), color_range, + use_bgr_format, /*is_async=*/false); } else { - *out_if_no_color_conversion = this->returnYUVFrame(pFrame, pReturnFrame); + *out_if_no_color_conversion = frame_output::copy_decoded_frame_to_yuv( + *decoder, pReturnFrame, reinterpret_cast(pFrame), + this->demuxer->GetColorRange(), /*timestamp=*/0, /*is_async=*/false); } return true; } } return false; } - -DecodedFrameExt PyNvVideoReader::returnYUVFrame(void* pFrame_buffer, void* pFrame) { - DecodedFrameExt frame; - frame.format = GetNativeFormat(this->decoder->GetOutputFormat()); - auto width = size_t(this->decoder->GetWidth()); - auto height = size_t(this->decoder->GetHeight()); - // Currently no timestamp, any bad ? - // frame.timestamp = timestamp; - frame.SetColorRange(demuxer->GetColorRange()); - - // Copy the decode frames from device - CUDA_DRVAPI_CALL( - cuMemcpyDtoD((CUdeviceptr)pFrame_buffer, (CUdeviceptr)pFrame, this->decoder->GetFrameSize())); - - switch (frame.format) { - case Pixel_Format_NV12: { - frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width, 1, 1}, - "|u1", - reinterpret_cast(this->decoder->GetStream()), - (CUdeviceptr)pFrame_buffer, - false}); - frame.views.push_back(CAIMemoryView{{height / 2, width / 2, 2}, - {width / 2 * 2, 2, 1}, - "|u1", - reinterpret_cast(this->decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer + width * height), - false}); // todo: data+width*height assumes both planes - // are Load DLPack Tensor - std::vector shape{(size_t)(height * 1.5), width}; - std::vector stride{size_t(width), 1}; - int returntype = frame.extBuf->LoadDLPack(shape, stride, "|u1", - reinterpret_cast(this->decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer), false); - } break; - case Pixel_Format_P016: { - frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width * 2, 2, 2}, - "|u2", - reinterpret_cast(this->decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer), - false}); - frame.views.push_back(CAIMemoryView{{height / 2, width / 2, 2}, - {width * 2, 4, 2}, - "|u2", - reinterpret_cast(this->decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer + 2 * (width * height)), - false}); // todo: data+width*height assumes both planes are - // contiguous. Actual NVENC allocation can have padding? - } break; - case Pixel_Format_YUV444: { - frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width, 1, 1}, - "|u1", - reinterpret_cast(this->decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer), - false}); - frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width, 1, 1}, - "|u1", - reinterpret_cast(this->decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer + width * height), - false}); // todo: data+width*height assumes both planes are - // contiguous. Actual NVENC allocation can have padding? - frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width, 1, 1}, - "|u1", - reinterpret_cast(this->decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer + 2 * width * height), - false}); - } break; - case Pixel_Format_YUV444_16Bit: { - frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width * 2, 2, 2}, - "|u2", - reinterpret_cast(this->decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer), - false}); - frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width * 2, 2, 2}, - "|u2", - reinterpret_cast(this->decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer + 2 * (width * height)), - false}); // todo: data+width*height assumes both planes are - // contiguous. Actual NVENC allocation can have padding? - frame.views.push_back(CAIMemoryView{{height, width, 1}, - {width * 2, 2, 2}, - "|u2", - reinterpret_cast(this->decoder->GetStream()), - (CUdeviceptr)(pFrame_buffer + 4 * (width * height)), - false}); // todo: data+width*height assumes both planes are - // contiguous. Actual NVENC allocation can have padding? - } break; - default: - throw std::runtime_error("[ERROR] Unsupported pixel format for YUV output"); - } - CUDA_DRVAPI_CALL(cuStreamSynchronize(this->decoder->GetStream())); - return frame; -} - -RGBFrame PyNvVideoReader::returnRGBFrame(void* pFrame_buffer, void* pFrame, bool use_bgr_format, - bool& file_added_for_warning) { - Pixel_Format format = GetNativeFormat(this->decoder->GetOutputFormat()); - auto width = size_t(this->decoder->GetWidth()); - auto height = size_t(this->decoder->GetHeight()); - - const std::vector frame_shape{height, width, 3}; - const std::vector frame_stride{width * 3, 3, 1}; - RGBFrame rgb_frame(frame_shape, frame_stride, "|u1", reinterpret_cast(decoder->GetStream()), - reinterpret_cast(pFrame_buffer), false, false); - - switch (format) { - case Pixel_Format_NV12: { - const CAIMemoryView y_view{{height, width, 1}, - {width, 1, 1}, - "|u1", - reinterpret_cast(this->decoder->GetStream()), - reinterpret_cast(pFrame), - false}; - const CAIMemoryView uv_view{{height / 2, width / 2, 2}, - {width / 2 * 2, 2, 1}, - "|u1", - reinterpret_cast(this->decoder->GetStream()), - reinterpret_cast(pFrame + width * height), - false}; // todo: data+width*height assumes both planes are - // contiguous. Actual NVENC allocation can have padding? - const AVColorRange color_range = this->demuxer->GetColorRange(); - if (!file_added_for_warning && (color_range != AVColorRange::AVCOL_RANGE_JPEG && - color_range != AVColorRange::AVCOL_RANGE_MPEG)) { - file_added_for_warning = true; - } - const bool is_full_range = color_range == AVColorRange::AVCOL_RANGE_JPEG; - convert_nv12_to_rgb(y_view, uv_view, rgb_frame, is_full_range, use_bgr_format); - } break; - default: { - throw std::invalid_argument( - "[ERROR] Conversion to RGB/BGR only supported " - "for videos in NV12-format"); - } - } - CUDA_DRVAPI_CALL(cuStreamSynchronize(this->decoder->GetStream())); - return rgb_frame; -} diff --git a/packages/on_demand_video_decoder/tests/common/test_color_range_warning.py b/packages/on_demand_video_decoder/tests/common/test_color_range_warning.py new file mode 100644 index 00000000..5e30a68f --- /dev/null +++ b/packages/on_demand_video_decoder/tests/common/test_color_range_warning.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import utils +import accvlab.on_demand_video_decoder as nvc + + +def _video_without_color_range(filename="moving_shape_circle_h265.mp4"): + path = os.path.join(utils.get_data_dir(), "sample_clip", filename) + assert os.path.exists(path), f"test data missing: {path}" + return path + + +def _get_gop_data(filepath, frame_id=0): + decoder = nvc.CreateGopDecoder(maxfiles=1, iGpu=0) + gop_data, _first_frame_ids, _gop_lens = decoder.GetGOPList([filepath], [frame_id])[0] + return gop_data + + +def test_sample_reader_warning_emitted_once_per_file(capfd): + path = _video_without_color_range() + reader = nvc.CreateSampleReader(num_of_set=1, num_of_file=1, iGpu=0) + + reader.DecodeN12ToRGB([path], [0], False) + reader.DecodeN12ToRGB([path], [1], False) + + output = capfd.readouterr().out + warning = "PyNvVideoReader could not obtain color range" + assert output.count(warning) == 1 + assert path in output + + +def test_sample_reader_warning_can_be_suppressed(capfd): + path = _video_without_color_range() + reader = nvc.CreateSampleReader( + num_of_set=1, + num_of_file=1, + iGpu=0, + suppressNoColorRangeWarning=True, + ) + + reader.DecodeN12ToRGB([path], [0], False) + + assert "PyNvVideoReader could not obtain color range" not in capfd.readouterr().out + + +def test_sample_reader_warning_resets_when_cache_slot_changes_file(capfd): + first_path = _video_without_color_range("moving_shape_circle_h265.mp4") + second_path = _video_without_color_range("moving_shape_ellipse_h265.mp4") + reader = nvc.CreateSampleReader(num_of_set=1, num_of_file=1, iGpu=0) + + reader.DecodeN12ToRGB([first_path], [0], False) + reader.DecodeN12ToRGB([second_path], [0], False) + + output = capfd.readouterr().out + assert output.count("PyNvVideoReader could not obtain color range") == 2 + assert first_path in output + assert second_path in output + + +def test_batch_async_stream_reader_warning_can_be_suppressed(capfd): + path = _video_without_color_range() + reader = nvc.CreateBatchAsyncStreamReader( + num_of_set=1, + num_of_file=1, + max_frames_per_decode_call=1, + iGpu=0, + suppressNoColorRangeWarning=True, + ) + + reader.Decode([path], [[0]], False) + reader.GetBuffer([path], [[0]], False) + + assert "PyNvVideoReader could not obtain color range" not in capfd.readouterr().out + + +def test_gop_decoder_warning_emitted_once_per_file(capfd): + path = _video_without_color_range() + decoder = nvc.CreateGopDecoder(maxfiles=1, iGpu=0) + + decoder.DecodeN12ToRGB([path], [0]) + decoder.DecodeN12ToRGB([path], [1]) + + output = capfd.readouterr().out + warning = "PyNvGopDecoder could not obtain color range" + assert output.count(warning) == 1 + assert path in output + + +def test_gop_decoder_warning_can_be_suppressed(capfd): + path = _video_without_color_range() + decoder = nvc.CreateGopDecoder(maxfiles=1, iGpu=0, suppressNoColorRangeWarning=True) + + decoder.DecodeN12ToRGB([path], [0]) + + assert "PyNvGopDecoder could not obtain color range" not in capfd.readouterr().out + + +def test_batch_async_gop_decoder_warning_emitted_once_per_file(capfd): + path = _video_without_color_range() + gop_data = _get_gop_data(path) + frame_ids = [[0, 1]] + decoder = nvc.CreateBatchAsyncGopDecoder(maxfiles=1, max_frames_per_decode_call=2, iGpu=0) + + decoder.DecodeFromGOPListRGB([[gop_data]], [path], frame_ids, False) + decoder.DecodeFromGOPListRGBGetBuffer([path], frame_ids, False) + + output = capfd.readouterr().out + warning = "PyNvGopDecoder could not obtain color range" + assert output.count(warning) == 1 + assert path in output + + +def test_batch_async_gop_decoder_warning_can_be_suppressed(capfd): + path = _video_without_color_range() + gop_data = _get_gop_data(path) + frame_ids = [[0]] + decoder = nvc.CreateBatchAsyncGopDecoder( + maxfiles=1, + max_frames_per_decode_call=1, + iGpu=0, + suppressNoColorRangeWarning=True, + ) + + decoder.DecodeFromGOPListRGB([[gop_data]], [path], frame_ids, False) + decoder.DecodeFromGOPListRGBGetBuffer([path], frame_ids, False) + + assert "PyNvGopDecoder could not obtain color range" not in capfd.readouterr().out diff --git a/packages/on_demand_video_decoder/tests/test_pix_fmt_detection.py b/packages/on_demand_video_decoder/tests/test_pix_fmt_detection.py index 0cdf073b..9d4bc8f2 100644 --- a/packages/on_demand_video_decoder/tests/test_pix_fmt_detection.py +++ b/packages/on_demand_video_decoder/tests/test_pix_fmt_detection.py @@ -128,6 +128,12 @@ def test_decode_from_gop_round_trip( assert frames[0].format == expected_format planes = frames[0].cuda() assert tuple(tuple(plane.shape) for plane in planes) == expected_shapes + expected_frame_bytes = { + 3: 256 * 256 * 3 // 2, # NV12 + 4: 256 * 256 * 3, # YUV444 + 5: 256 * 256 * 3, # P016 + }[expected_format] + assert frames[0].framesize() == expected_frame_bytes expected_bytes_per_sample = 2 if bit_depth >= 10 else 1 luma_width = expected_shapes[0][1] @@ -153,3 +159,12 @@ def test_decode_from_gop_round_trip( f"Y plane element size mismatch for {filename}: got " f"{actual_bytes_per_sample}B, expected {expected_bytes_per_sample}B" ) + + if expected_format in (3, 4): + nvcv_views = frames[0].nvcv_image() + expected_nvcv_shape = (256 + 128, 256, 1) if expected_format == 3 else (256 * 3, 256, 1) + expected_nvcv_stride = (256, 2, 1) if expected_format == 3 else (256, 3, 1) + assert len(nvcv_views) == 1 + assert tuple(nvcv_views[0].shape) == expected_nvcv_shape + assert tuple(nvcv_views[0].stride) == expected_nvcv_stride + assert nvcv_views[0].__cuda_array_interface__["typestr"] == "|u1"