From f7d71e006f49bb886e09068f32cff76f58b6e163 Mon Sep 17 00:00:00 2001 From: furionw Date: Sun, 23 Aug 2026 18:44:43 -0700 Subject: [PATCH] feat(vllm-omni): add MiniMax-H3 request controls --- .../dynamo/common/protocols/video_protocol.py | 121 +++++++++- .../common/tests/test_video_protocol.py | 96 ++++++++ components/src/dynamo/vllm/omni/args.py | 61 +++++ .../src/dynamo/vllm/omni/omni_handler.py | 138 ++++++++++- .../src/dynamo/vllm/omni/video_references.py | 11 +- .../dynamo/vllm/tests/omni/test_omni_args.py | 17 ++ .../vllm/tests/omni/test_omni_base_handler.py | 23 ++ .../vllm/tests/omni/test_omni_handler.py | 208 +++++++++++++++++ .../vllm/tests/omni/test_video_references.py | 74 +++++- lib/llm/src/protocols/openai/videos.rs | 217 +++++++++++++++++- lib/llm/src/protocols/openai/videos/nvext.rs | 96 +++++++- 11 files changed, 1045 insertions(+), 17 deletions(-) diff --git a/components/src/dynamo/common/protocols/video_protocol.py b/components/src/dynamo/common/protocols/video_protocol.py index e47a321e78f0..1a6d3acdde41 100644 --- a/components/src/dynamo/common/protocols/video_protocol.py +++ b/components/src/dynamo/common/protocols/video_protocol.py @@ -8,10 +8,17 @@ """ # TODO: Replace these Pydantic models with Python bindings to the Rust protocol types once PyO3 bindings are available. -from typing import Literal, Optional +from typing import Literal, Optional, Union from pydantic import BaseModel, model_validator +H3Task = Literal["t2va", "fl2va", "ref2va"] + + +def is_minimax_h3_model_name(model: str) -> bool: + """Return whether a public model identifier names MiniMax-H3.""" + return "minimax-h3" in model.lower().replace("_", "-") + class VideoInputReference(BaseModel): """Typed conditioning input for video generation.""" @@ -56,6 +63,51 @@ class VideoNvExt(BaseModel): guidance_scale_2: Optional[float] = None """CFG scale for the low-noise expert (vLLM-Omni I2V dual-guidance).""" + task: Optional[H3Task] = None + """MiniMax-H3 task routed to its FL2VA or Ref2VA transformer.""" + + duration: Optional[float] = None + """Requested MiniMax-H3 duration in seconds (4 through 15).""" + + flow_shift: Optional[float] = None + """MiniMax-H3 video sigma shift.""" + + audio_flow_shift: Optional[float] = None + """MiniMax-H3 audio sigma shift.""" + + aspect_ratio: Optional[str] = None + """MiniMax-H3 output aspect ratio.""" + + short_edge: Optional[int] = None + """MiniMax-H3 output canvas short edge.""" + + frame_indices: Optional[list[int]] = None + """FL2VA keyframe positions: [0], [-1], or [0, -1].""" + + start_time_seconds: Optional[Union[float, list[float]]] = None + """Start offset for one reference video, or one offset per video.""" + + num_outputs_per_prompt: Optional[int] = None + """Number of generated videos (MiniMax-H3 supports 1 through 10).""" + + quality: Optional[Literal["lossless", "high"]] = None + """MiniMax-H3 request-scoped quality policy.""" + + @model_validator(mode="after") + def validate_h3_fields(self) -> "VideoNvExt": + if self.duration is not None and not 4 <= self.duration <= 15: + raise ValueError("duration must be between 4 and 15 seconds") + if self.num_outputs_per_prompt is not None and not ( + 1 <= self.num_outputs_per_prompt <= 10 + ): + raise ValueError("num_outputs_per_prompt must be between 1 and 10") + if self.task is not None and self.fps is not None and self.fps != 24: + raise ValueError("MiniMax-H3 fps is fixed at 24") + if self.task == "fl2va" and self.frame_indices is not None: + if self.frame_indices not in ([0], [-1], [0, -1]): + raise ValueError("FL2VA frame_indices must be [0], [-1], or [0, -1]") + return self + class NvCreateVideoRequest(BaseModel): """Request for video generation (/v1/videos endpoint). @@ -108,8 +160,75 @@ def validate_input_references(self) -> "NvCreateVideoRequest": ) if self.input_references is not None and not self.input_references: raise ValueError("input_references must not be empty") + if self.input_references is not None and len(self.input_references) > 12: + raise ValueError("input_references accepts at most 12 references") + + task = self.nvext.task if self.nvext is not None else None + if task is not None or is_minimax_h3_model_name(self.model): + self.validate_h3_reference_contract(task or self.infer_h3_task()) return self + def infer_h3_task(self) -> H3Task: + """Match MiniMax-H3's task inference from the supplied references.""" + if self.nvext is not None and self.nvext.task is not None: + return self.nvext.task + reference_types = {reference.type for reference in self.input_references or []} + if reference_types.intersection({"video", "audio"}): + return "ref2va" + if self.input_reference is not None or "image" in reference_types: + return "fl2va" + return "t2va" + + def validate_h3_reference_contract(self, task: H3Task) -> None: + """Validate controls and references for a resolved MiniMax-H3 task.""" + nvext = self.nvext or VideoNvExt() + if nvext.fps is not None and nvext.fps != 24: + raise ValueError("MiniMax-H3 fps is fixed at 24") + if task == "fl2va" and nvext.frame_indices is not None: + if nvext.frame_indices not in ([0], [-1], [0, -1]): + raise ValueError("FL2VA frame_indices must be [0], [-1], or [0, -1]") + + counts = {kind: 0 for kind in ("image", "video", "audio")} + if self.input_reference is not None: + counts["image"] = 1 + for reference in self.input_references or []: + counts[reference.type] += 1 + total = sum(counts.values()) + + if task == "t2va" and total: + raise ValueError("t2va does not accept input references") + if task == "fl2va": + if counts["video"] or counts["audio"] or not counts["image"]: + raise ValueError("fl2va accepts only one or two image references") + if counts["image"] > 2: + raise ValueError("fl2va accepts at most two image references") + if ( + nvext.frame_indices is not None + and len(nvext.frame_indices) != counts["image"] + ): + raise ValueError("fl2va requires one frame index per image reference") + if task == "ref2va": + if not counts["image"] and not counts["video"]: + raise ValueError( + "ref2va requires at least one image or video reference" + ) + if counts["image"] > 9: + raise ValueError("ref2va accepts at most 9 image references") + if counts["video"] > 3: + raise ValueError("ref2va accepts at most 3 video references") + if counts["audio"] > 3: + raise ValueError("ref2va accepts at most 3 audio references") + + start_times = nvext.start_time_seconds + if isinstance(start_times, list) and len(start_times) != counts["video"]: + raise ValueError( + "start_time_seconds requires one value per video reference" + ) + if isinstance(start_times, float) and counts["video"] != 1: + raise ValueError( + "scalar start_time_seconds requires exactly one video reference" + ) + class VideoData(BaseModel): """Video data in response. diff --git a/components/src/dynamo/common/tests/test_video_protocol.py b/components/src/dynamo/common/tests/test_video_protocol.py index b22cafaebf14..fad440ee9a0f 100644 --- a/components/src/dynamo/common/tests/test_video_protocol.py +++ b/components/src/dynamo/common/tests/test_video_protocol.py @@ -68,6 +68,102 @@ def test_video_request_rejects_invalid_reference_combinations(input_references): NvCreateVideoRequest(prompt="cat", model="video-model", **kwargs) +def test_video_request_rejects_more_than_twelve_references(): + with pytest.raises(ValueError, match="at most 12"): + NvCreateVideoRequest( + prompt="cat", + model="video-model", + input_references=[ + {"type": "image", "source": f"https://example.com/{index}.png"} + for index in range(13) + ], + ) + + +@pytest.mark.parametrize( + ("task", "references", "nvext", "message"), + [ + ( + "t2va", + [{"type": "image", "source": "https://example.com/cat.png"}], + {}, + "does not accept", + ), + ( + "fl2va", + [{"type": "audio", "source": "https://example.com/cat.wav"}], + {}, + "only one or two image", + ), + ( + "fl2va", + [{"type": "image", "source": "https://example.com/cat.png"}], + {"frame_indices": [0, -1]}, + "one frame index per image", + ), + ( + "ref2va", + [{"type": "audio", "source": "https://example.com/cat.wav"}], + {}, + "at least one image or video", + ), + ( + "ref2va", + [{"type": "video", "source": "https://example.com/cat.mp4"}], + {"start_time_seconds": [0.0, 1.0]}, + "one value per video", + ), + ], +) +def test_video_request_rejects_invalid_h3_reference_contract( + task, references, nvext, message +): + with pytest.raises(ValueError, match=message): + NvCreateVideoRequest( + prompt="cat", + model="MiniMaxAI/MiniMax-H3", + input_references=references, + nvext={"task": task, **nvext}, + ) + + +@pytest.mark.parametrize( + "references", + [ + [{"type": "audio", "source": "https://example.com/cat.wav"}], + [ + {"type": "image", "source": f"https://example.com/{index}.png"} + for index in range(3) + ], + [ + {"type": "video", "source": f"https://example.com/{index}.mp4"} + for index in range(4) + ], + ], +) +def test_video_request_rejects_invalid_taskless_h3_reference_contract(references): + with pytest.raises(ValueError): + NvCreateVideoRequest( + prompt="cat", + model="MiniMaxAI/MiniMax-H3", + input_references=references, + ) + + +@pytest.mark.parametrize( + "nvext", + [ + {"task": "t2va", "fps": 16}, + {"task": "fl2va", "frame_indices": [1]}, + {"task": "ref2va", "duration": 3}, + {"task": "ref2va", "num_outputs_per_prompt": 11}, + ], +) +def test_video_request_rejects_invalid_h3_controls(nvext): + with pytest.raises(ValueError): + NvCreateVideoRequest(prompt="cat", model="MiniMaxAI/MiniMax-H3", nvext=nvext) + + def test_video_response_wire_shape(): response = NvVideosResponse( id="r1", diff --git a/components/src/dynamo/vllm/omni/args.py b/components/src/dynamo/vllm/omni/args.py index 6b3321e0b966..9dedf3fb97cb 100644 --- a/components/src/dynamo/vllm/omni/args.py +++ b/components/src/dynamo/vllm/omni/args.py @@ -49,6 +49,12 @@ class OmniDiffusionKwargs: cache_config: Optional[str] = None enable_cache_dit_summary: bool = False enable_cpu_offload: bool = False + task_type: Optional[str] = None + diffusion_attention_backend: Optional[str] = None + diffusion_attention_config: Optional[str] = None + enable_distributed_layerwise_offload: bool = False + dlo_use_allgather: bool = True + dlo_resident_layers: int = 0 enforce_eager: bool = False @@ -184,6 +190,55 @@ def add_arguments(self, parser) -> None: default=False, help="Enable CPU offloading for diffusion models to reduce GPU memory usage.", ) + add_argument( + g, + flag_name="--task-type", + env_var="DYN_OMNI_TASK_TYPE", + default=None, + help=( + "Model-defined startup task partition. MiniMax-H3 accepts " + "'fl2va' or 'ref2va'; omit it to load both partitions." + ), + ) + add_argument( + g, + flag_name="--diffusion-attention-backend", + env_var="DYN_OMNI_DIFFUSION_ATTENTION_BACKEND", + default=None, + help="Default vLLM-Omni diffusion attention backend.", + ) + add_argument( + g, + flag_name="--diffusion-attention-config", + env_var="DYN_OMNI_DIFFUSION_ATTENTION_CONFIG", + default=None, + help="vLLM-Omni diffusion attention configuration as JSON.", + ) + add_negatable_bool_argument( + g, + flag_name="--enable-distributed-layerwise-offload", + env_var="DYN_OMNI_ENABLE_DISTRIBUTED_LAYERWISE_OFFLOAD", + default=False, + help="Enable distributed layerwise DiT offload.", + ) + add_negatable_bool_argument( + g, + flag_name="--dlo-use-allgather", + env_var="DYN_OMNI_DLO_USE_ALLGATHER", + default=True, + help="Reconstruct distributed offload shards with AllGather.", + ) + add_argument( + g, + flag_name="--dlo-resident-layers", + env_var="DYN_OMNI_DLO_RESIDENT_LAYERS", + default=0, + arg_type=int, + help=( + "Number of leading main-DiT blocks kept resident with DLO; " + "positive values require --no-dlo-use-allgather." + ), + ) add_negatable_bool_argument( g, flag_name="--enforce-eager", @@ -436,6 +491,12 @@ def validate(self) -> None: raise ValueError("--text-encoder-tp-size must be > 0") if not (0 < self.diffusion.boundary_ratio <= 1): raise ValueError("--boundary-ratio must be in (0, 1]") + if self.diffusion.dlo_resident_layers < 0: + raise ValueError("--dlo-resident-layers must be >= 0") + if self.diffusion.dlo_resident_layers > 0 and self.diffusion.dlo_use_allgather: + raise ValueError( + "--dlo-resident-layers > 0 requires --no-dlo-use-allgather" + ) if self.stage_configs_path is None: if self.stage_id is not None: raise ValueError("--stage-id requires --stage-configs-path") diff --git a/components/src/dynamo/vllm/omni/omni_handler.py b/components/src/dynamo/vllm/omni/omni_handler.py index 14a4d49efb4a..079ec2c73ebf 100644 --- a/components/src/dynamo/vllm/omni/omni_handler.py +++ b/components/src/dynamo/vllm/omni/omni_handler.py @@ -28,7 +28,11 @@ from dynamo.common.multimodal import ImageLoader from dynamo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest from dynamo.common.protocols.image_protocol import ImageNvExt, NvCreateImageRequest -from dynamo.common.protocols.video_protocol import NvCreateVideoRequest, VideoNvExt +from dynamo.common.protocols.video_protocol import ( + H3Task, + NvCreateVideoRequest, + VideoNvExt, +) from dynamo.common.rl import RLAdminValidationError from dynamo.common.utils.output_modalities import ( RequestType, @@ -58,8 +62,6 @@ logger = logging.getLogger(__name__) -DEFAULT_VIDEO_FPS = 16 - @dataclass class EngineInputs: @@ -229,6 +231,39 @@ def __init__( media_output_http_url=media_output_http_url, ) + def _is_minimax_h3_model(self) -> bool: + """Return whether this worker serves a MiniMax-H3 checkpoint.""" + get_diffusion_config = getattr( + self.engine_client, "get_diffusion_od_config", None + ) + if callable(get_diffusion_config): + diffusion_config = get_diffusion_config() + model_class_name = getattr(diffusion_config, "model_class_name", None) + if isinstance(model_class_name, str): + return model_class_name in { + "MiniMaxH3Pipeline", + "MiniMaxH3ModularPipeline", + } + + candidates = ( + getattr(self.config, "model", None), + getattr(self.config, "served_model_name", None), + *(getattr(self.config, "served_model_aliases", ()) or ()), + ) + return any( + "minimax-h3" in str(candidate).lower().replace("_", "-") + for candidate in candidates + if candidate + ) + + def _resolve_h3_task(self, request: NvCreateVideoRequest) -> H3Task | None: + """Resolve H3 from an explicit task or the loaded pipeline metadata.""" + if request.nvext is not None and request.nvext.task is not None: + return request.nvext.task + if self._is_minimax_h3_model(): + return request.infer_h3_task() + return None + @functools.cached_property def _lora_enabled(self) -> bool: # Match non-Omni LoRA gating: engine must be started with LoRA support @@ -333,6 +368,18 @@ async def _generate_openai_mode( parsed_request_raw, ) + resolved_h3_task = None + if request_type == RequestType.VIDEO_GENERATION and isinstance( + parsed_request, NvCreateVideoRequest + ): + resolved_h3_task = self._resolve_h3_task(parsed_request) + if resolved_h3_task is not None: + try: + parsed_request.validate_h3_reference_contract(resolved_h3_task) + except ValueError as e: + yield self._error_chunk(request_id, str(e), request_type) + return + # Pre-load input image for I2V requests (async I/O before sync build) image = None materialized_references = None @@ -363,7 +410,10 @@ async def _generate_openai_mode( ): try: materialized_references = ( - await self._video_reference_materializer.materialize(parsed_request) + await self._video_reference_materializer.materialize( + parsed_request, + h3_task=resolved_h3_task, + ) ) except Exception as e: logger.warning("Failed to materialize input_references: %s", e) @@ -380,7 +430,9 @@ async def _generate_openai_mode( request_type, image=image, multi_modal_data=( - materialized_references.as_omni_data() + materialized_references.as_omni_data( + allow_multiple=resolved_h3_task is not None + ) if materialized_references is not None else None ), @@ -738,15 +790,50 @@ def _engine_inputs_from_video( I2V pipeline pre-process can use it. multi_modal_data: Typed reference paths adapted for the pipeline. """ - width, height = parse_size(req.size) nvext = req.nvext or VideoNvExt() - num_frames = compute_num_frames( - num_frames=nvext.num_frames, - seconds=req.seconds, - fps=nvext.fps, - default_fps=DEFAULT_VIDEO_FPS, + is_h3_request = nvext.task is not None or self._is_minimax_h3_model() + if is_h3_request and nvext.fps is not None and nvext.fps != 24: + raise ValueError("MiniMax-H3 fps is fixed at 24") + + if req.size is None and is_h3_request: + width, height = None, None + else: + width, height = parse_size(req.size) + + h3_task = nvext.task + if is_h3_request and h3_task is None: + if multi_modal_data is not None and ( + multi_modal_data.get("video") is not None + or multi_modal_data.get("audio") is not None + ): + h3_task = "ref2va" + elif image is not None or ( + multi_modal_data is not None + and multi_modal_data.get("image") is not None + ): + h3_task = "fl2va" + else: + h3_task = "t2va" + + default_fps = ( + 24 if is_h3_request else getattr(self.config, "default_video_fps", 16) ) - fps = nvext.fps if nvext.fps is not None else DEFAULT_VIDEO_FPS + fps = nvext.fps if nvext.fps is not None else default_fps + + if nvext.duration is not None and nvext.num_frames is None: + num_frames = round(nvext.duration * fps) + elif is_h3_request and nvext.num_frames is None and req.seconds is None: + # Let H3 select its task-specific native length (209 frames for + # T2VA/FL2VA, 124 for Ref2VA) instead of injecting Dynamo's + # generic 97-frame default. + num_frames = None + else: + num_frames = compute_num_frames( + num_frames=nvext.num_frames, + seconds=req.seconds, + fps=nvext.fps, + default_fps=default_fps, + ) prompt = OmniTextPrompt(prompt=req.prompt) if nvext.negative_prompt is not None: @@ -782,6 +869,33 @@ def _engine_inputs_from_video( self._update_if_not_none(sp, "boundary_ratio", nvext.boundary_ratio) self._update_if_not_none(sp, "guidance_scale_2", nvext.guidance_scale_2) self._update_if_not_none(sp, "fps", fps) + self._update_if_not_none( + sp, "num_outputs_per_prompt", nvext.num_outputs_per_prompt + ) + self._update_if_not_none(sp, "quality", nvext.quality) + + extra_args = { + key: value + for key, value in { + "task": nvext.task, + "duration": nvext.duration, + "flow_shift": nvext.flow_shift, + "audio_flow_shift": nvext.audio_flow_shift, + "aspect_ratio": ( + nvext.aspect_ratio + if nvext.aspect_ratio is not None + else "16:9" + if is_h3_request and h3_task == "t2va" + else None + ), + "short_edge": nvext.short_edge, + "frame_indices": nvext.frame_indices, + "start_time_seconds": nvext.start_time_seconds, + }.items() + if value is not None + } + if extra_args: + sp.extra_args.update(extra_args) sampling_params_list = self._build_sampling_params_list(sp) lora_request = self._resolve_and_apply_lora(req.model, sampling_params_list) diff --git a/components/src/dynamo/vllm/omni/video_references.py b/components/src/dynamo/vllm/omni/video_references.py index 8b1f1b5f68d6..411af39e0252 100644 --- a/components/src/dynamo/vllm/omni/video_references.py +++ b/components/src/dynamo/vllm/omni/video_references.py @@ -14,8 +14,10 @@ from dynamo.common.http.url_validator import UrlValidationPolicy, validate_media_url from dynamo.common.multimodal.media_source import read_local_media_bytes from dynamo.common.protocols.video_protocol import ( + H3Task, NvCreateVideoRequest, VideoInputReference, + is_minimax_h3_model_name, ) _REFERENCE_LIMITS = { @@ -85,7 +87,7 @@ def __init__( self._url_policy = url_policy or UrlValidationPolicy.from_env() async def materialize( - self, request: NvCreateVideoRequest + self, request: NvCreateVideoRequest, *, h3_task: H3Task | None = None ) -> MaterializedVideoReferences | None: references = request.input_references if references is None: @@ -95,6 +97,13 @@ async def materialize( f"input_references accepts at most {_MAX_REFERENCE_COUNT} references" ) + if h3_task is None and ( + (request.nvext is not None and request.nvext.task is not None) + or is_minimax_h3_model_name(request.model) + ): + h3_task = request.infer_h3_task() + if h3_task is not None: + request.validate_h3_reference_contract(h3_task) temporary_directory = tempfile.TemporaryDirectory( prefix="dynamo_video_references_" ) diff --git a/components/src/dynamo/vllm/tests/omni/test_omni_args.py b/components/src/dynamo/vllm/tests/omni/test_omni_args.py index d1fce69ef32a..64ac6caafe18 100644 --- a/components/src/dynamo/vllm/tests/omni/test_omni_args.py +++ b/components/src/dynamo/vllm/tests/omni/test_omni_args.py @@ -129,6 +129,23 @@ def test_omni_config_valid_boundary_ratio(ratio): config.validate() +def test_negative_dlo_resident_layers_rejected(): + config = _make_omni_config(dlo_resident_layers=-1) + with pytest.raises(ValueError, match="--dlo-resident-layers must be >= 0"): + config.validate() + + +def test_resident_dlo_layers_reject_allgather(): + config = _make_omni_config(dlo_resident_layers=1, dlo_use_allgather=True) + with pytest.raises(ValueError, match="--no-dlo-use-allgather"): + config.validate() + + +def test_resident_dlo_layers_allow_sharded_reconstruction(): + config = _make_omni_config(dlo_resident_layers=1, dlo_use_allgather=False) + config.validate() + + def test_negative_stage_id_rejected(): config = _make_omni_config(stage_id=-1, stage_configs_path="/fake/path.yaml") with pytest.raises(ValueError, match="--stage-id must be >= 0"): diff --git a/components/src/dynamo/vllm/tests/omni/test_omni_base_handler.py b/components/src/dynamo/vllm/tests/omni/test_omni_base_handler.py index 7d2f82983de9..25c0a48c5b6a 100644 --- a/components/src/dynamo/vllm/tests/omni/test_omni_base_handler.py +++ b/components/src/dynamo/vllm/tests/omni/test_omni_base_handler.py @@ -108,6 +108,29 @@ def test_output_modalities_forwarded_to_async_omni(self): assert kwargs["output_modalities"] == ["image"] + def test_h3_startup_fields_forwarded_to_async_omni(self): + config = _make_config() + config.diffusion = dataclasses.replace( + OmniDiffusionKwargs(), + task_type="ref2va", + diffusion_attention_backend="TRTLLM_ATTN", + diffusion_attention_config='{"default":{"backend":"TRTLLM_ATTN"}}', + enable_distributed_layerwise_offload=True, + dlo_use_allgather=False, + dlo_resident_layers=2, + ) + + kwargs = _build_kwargs(config) + + assert kwargs["task_type"] == "ref2va" + assert kwargs["diffusion_attention_backend"] == "TRTLLM_ATTN" + assert kwargs["diffusion_attention_config"] == ( + '{"default":{"backend":"TRTLLM_ATTN"}}' + ) + assert kwargs["enable_distributed_layerwise_offload"] is True + assert kwargs["dlo_use_allgather"] is False + assert kwargs["dlo_resident_layers"] == 2 + def test_lora_disabled_resolves_no_capacity(self): config = _make_config() handler = BaseOmniHandler.__new__(BaseOmniHandler) diff --git a/components/src/dynamo/vllm/tests/omni/test_omni_handler.py b/components/src/dynamo/vllm/tests/omni/test_omni_handler.py index 6879b9953888..f24767a02722 100644 --- a/components/src/dynamo/vllm/tests/omni/test_omni_handler.py +++ b/components/src/dynamo/vllm/tests/omni/test_omni_handler.py @@ -44,6 +44,7 @@ def _make_handler(stage_types=("diffusion",)): config.model = "test-model" config.served_model_name = None config.output_modalities = ["text"] + config.default_video_fps = 16 config.enable_lora = False # Disable LoRA for tests unless explicitly set config.engine_args = SimpleNamespace(enable_lora=False) handler.config = config @@ -59,6 +60,7 @@ def _make_handler(stage_types=("diffusion",)): engine_client = MagicMock() engine_client.default_sampling_params_list = defaults + engine_client.get_diffusion_od_config.return_value = None engine_client.engine.get_stage_metadata.side_effect = lambda i: SimpleNamespace( stage_type=stage_types[i] ) @@ -86,6 +88,14 @@ def _make_handler(stage_types=("diffusion",)): return handler +def _make_h3_handler(): + handler = _make_handler() + handler.config.model = "MiniMaxAI/MiniMax-H3" + handler._served_model_name = handler.config.model + handler.engine_args.model = handler.config.model + return handler + + class TestEngineInputs: def test_defaults(self): """EngineInputs uses CHAT_COMPLETION, fps=0, and None optionals by default.""" @@ -268,6 +278,204 @@ async def test_request_maps_typed_references(self): assert result.fps == 16 +class TestMiniMaxH3EngineInputs: + @pytest.mark.asyncio + async def test_explicit_h3_task_works_with_local_path_and_custom_alias(self): + handler = _make_handler() + handler.config.model = "/models/h3" + handler.config.served_model_name = "h3" + req = NvCreateVideoRequest( + prompt="a speaking astronaut", + model="h3", + nvext=VideoNvExt(task="t2va"), + ) + + result = await handler.build_engine_inputs(req, RequestType.VIDEO_GENERATION) + sp = result.sampling_params_list[0] + + assert result.fps == 24 + assert sp.num_frames is None + assert sp.extra_args["aspect_ratio"] == "16:9" + + @pytest.mark.asyncio + async def test_taskless_local_h3_uses_pipeline_metadata(self): + handler = _make_handler() + handler.config.model = "/models/h3" + handler.config.served_model_name = "h3" + handler.engine_client.get_diffusion_od_config.return_value = SimpleNamespace( + model_class_name="MiniMaxH3Pipeline" + ) + req = NvCreateVideoRequest(prompt="a speaking astronaut", model="h3") + + result = await handler.build_engine_inputs(req, RequestType.VIDEO_GENERATION) + sp = result.sampling_params_list[0] + + assert result.fps == 24 + assert sp.num_frames is None + assert sp.extra_args["aspect_ratio"] == "16:9" + + @pytest.mark.asyncio + async def test_h3_request_maps_references_and_extra_args(self): + handler = _make_h3_handler() + req = NvCreateVideoRequest( + prompt="a talking cat", + model="MiniMaxAI/MiniMax-H3", + response_format="b64_json", + output_format="mp4", + input_references=[ + {"type": "image", "source": "/tmp/cat.png"}, + {"type": "video", "source": "/tmp/motion.mp4"}, + {"type": "audio", "source": "/tmp/voice.wav"}, + ], + nvext=VideoNvExt( + task="ref2va", + duration=4.0, + flow_shift=12.0, + audio_flow_shift=3.0, + aspect_ratio="16:9", + short_edge=768, + start_time_seconds=[0.5], + num_outputs_per_prompt=2, + quality="lossless", + num_inference_steps=50, + seed=7, + ), + ) + references = { + "image": ["/tmp/cat.png"], + "video": ["/tmp/motion.mp4"], + "audio": ["/tmp/voice.wav"], + } + + result = await handler.build_engine_inputs( + req, + RequestType.VIDEO_GENERATION, + multi_modal_data=references, + ) + + assert result.prompt["multi_modal_data"] == references + assert result.fps == 24 + assert result.response_format == "b64_json" + assert result.output_format == "mp4" + sp = result.sampling_params_list[0] + assert sp.width is None + assert sp.height is None + assert sp.num_frames == 96 + assert sp.num_inference_steps == 50 + assert sp.num_outputs_per_prompt == 2 + assert sp.quality == "lossless" + assert sp.extra_args == { + "task": "ref2va", + "duration": 4.0, + "flow_shift": 12.0, + "audio_flow_shift": 3.0, + "aspect_ratio": "16:9", + "short_edge": 768, + "start_time_seconds": [0.5], + } + + @pytest.mark.asyncio + async def test_h3_fl2va_forwards_frame_indices(self): + handler = _make_h3_handler() + req = NvCreateVideoRequest( + prompt="transition", + model="MiniMaxAI/MiniMax-H3", + input_references=[ + {"type": "image", "source": "/tmp/first.png"}, + {"type": "image", "source": "/tmp/last.png"}, + ], + nvext=VideoNvExt(task="fl2va", duration=4, frame_indices=[0, -1]), + ) + + result = await handler.build_engine_inputs( + req, + RequestType.VIDEO_GENERATION, + multi_modal_data={"image": ["/tmp/first.png", "/tmp/last.png"]}, + ) + + assert result.sampling_params_list[0].extra_args["frame_indices"] == [0, -1] + + @pytest.mark.asyncio + async def test_h3_uses_native_defaults_without_generic_overrides(self): + handler = _make_h3_handler() + req = NvCreateVideoRequest( + prompt="a speaking astronaut", + model="MiniMaxAI/MiniMax-H3", + nvext=VideoNvExt(task="t2va"), + ) + + result = await handler.build_engine_inputs(req, RequestType.VIDEO_GENERATION) + sp = result.sampling_params_list[0] + + assert result.fps == 24 + assert sp.fps == 24 + assert sp.num_frames is None + assert sp.extra_args["aspect_ratio"] == "16:9" + + @pytest.mark.asyncio + async def test_taskless_h3_t2va_uses_h3_defaults_without_forcing_task(self): + handler = _make_h3_handler() + req = NvCreateVideoRequest( + prompt="a speaking astronaut", + model="MiniMaxAI/MiniMax-H3", + ) + + result = await handler.build_engine_inputs(req, RequestType.VIDEO_GENERATION) + sp = result.sampling_params_list[0] + + assert result.fps == 24 + assert sp.num_frames is None + assert sp.extra_args["aspect_ratio"] == "16:9" + assert "task" not in sp.extra_args + + @pytest.mark.asyncio + async def test_taskless_h3_image_request_uses_fl2va_defaults(self): + handler = _make_h3_handler() + req = NvCreateVideoRequest( + prompt="a cat at a piano", + model="MiniMaxAI/MiniMax-H3", + ) + references = {"image": ["/tmp/cat.png"]} + + result = await handler.build_engine_inputs( + req, + RequestType.VIDEO_GENERATION, + multi_modal_data=references, + ) + sp = result.sampling_params_list[0] + + assert result.fps == 24 + assert sp.num_frames is None + assert result.prompt["multi_modal_data"] == references + assert "task" not in sp.extra_args + assert "aspect_ratio" not in sp.extra_args + + @pytest.mark.asyncio + async def test_taskless_h3_video_audio_request_uses_ref2va_defaults(self): + handler = _make_h3_handler() + req = NvCreateVideoRequest( + prompt="a cat at a piano", + model="MiniMaxAI/MiniMax-H3", + ) + references = { + "video": ["/tmp/cat.mp4"], + "audio": ["/tmp/piano.wav"], + } + + result = await handler.build_engine_inputs( + req, + RequestType.VIDEO_GENERATION, + multi_modal_data=references, + ) + sp = result.sampling_params_list[0] + + assert result.fps == 24 + assert sp.num_frames is None + assert result.prompt["multi_modal_data"] == references + assert "task" not in sp.extra_args + assert "aspect_ratio" not in sp.extra_args + + class TestBuildSamplingParamsList: def test_single_diffusion_stage(self): handler = _make_handler(stage_types=("diffusion",)) diff --git a/components/src/dynamo/vllm/tests/omni/test_video_references.py b/components/src/dynamo/vllm/tests/omni/test_video_references.py index eb0c7d837f76..fbcf3041296c 100644 --- a/components/src/dynamo/vllm/tests/omni/test_video_references.py +++ b/components/src/dynamo/vllm/tests/omni/test_video_references.py @@ -8,7 +8,7 @@ try: from dynamo.common.http.url_validator import UrlValidationPolicy - from dynamo.common.protocols.video_protocol import NvCreateVideoRequest + from dynamo.common.protocols.video_protocol import NvCreateVideoRequest, VideoNvExt from dynamo.vllm.omni.video_references import VideoReferenceMaterializer except ImportError: pytest.skip("vLLM omni dependencies not available", allow_module_level=True) @@ -155,3 +155,75 @@ def test_unknown_remote_suffix_uses_media_type_default(): ).input_references[0] assert VideoReferenceMaterializer._suffix(reference) == ".mp4" + + +@pytest.mark.parametrize( + ("task", "references", "message"), + [ + ( + "t2va", + [{"type": "image", "source": "data:image/png;base64,AA=="}], + "does not accept", + ), + ( + "fl2va", + [{"type": "audio", "source": "data:audio/wav;base64,AA=="}], + "only one or two image", + ), + ( + "ref2va", + [{"type": "audio", "source": "data:audio/wav;base64,AA=="}], + "at least one image or video", + ), + ], +) +@pytest.mark.asyncio +async def test_rejects_invalid_h3_reference_contract(task, references, message): + with pytest.raises(ValueError, match=message): + request = NvCreateVideoRequest( + prompt="cat", + model="MiniMaxAI/MiniMax-H3", + input_references=references, + nvext=VideoNvExt(task=task), + ) + await VideoReferenceMaterializer().materialize(request) + + +@pytest.mark.parametrize( + ("references", "message"), + [ + ( + [{"type": "audio", "source": "data:audio/wav;base64,AA=="}], + "at least one image or video", + ), + ( + [ + {"type": "image", "source": f"data:image/png;base64,{index}A=="} + for index in range(3) + ], + "at most two image", + ), + ( + [ + {"type": "video", "source": f"data:video/mp4;base64,{index}A=="} + for index in range(4) + ], + "at most 3 video", + ), + ], +) +@pytest.mark.asyncio +async def test_rejects_invalid_taskless_h3_contract_for_custom_alias( + references, message +): + request = NvCreateVideoRequest( + prompt="cat", + model="h3", + input_references=references, + ) + + with pytest.raises(ValueError, match=message): + await VideoReferenceMaterializer().materialize( + request, + h3_task=request.infer_h3_task(), + ) diff --git a/lib/llm/src/protocols/openai/videos.rs b/lib/llm/src/protocols/openai/videos.rs index 2639ff22db21..10ad685feb75 100644 --- a/lib/llm/src/protocols/openai/videos.rs +++ b/lib/llm/src/protocols/openai/videos.rs @@ -8,7 +8,7 @@ use validator::{Validate, ValidationError}; mod aggregator; mod nvext; -pub use nvext::{NvExt, NvExtProvider}; +pub use nvext::{NvExt, NvExtProvider, StartTimeSeconds}; /// Media type for a video-generation conditioning reference. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -74,6 +74,7 @@ pub struct NvCreateVideoRequest { /// NVIDIA extensions #[serde(skip_serializing_if = "Option::is_none")] + #[validate(nested)] pub nvext: Option, } @@ -111,6 +112,107 @@ fn validate_video_request(request: &NvCreateVideoRequest) -> Result<(), Validati { return Err(ValidationError::new("input_references_empty")); } + if request + .input_references + .as_ref() + .is_some_and(|references| references.len() > 12) + { + return Err(ValidationError::new("too_many_input_references")); + } + + let mut image_count = usize::from(request.input_reference.is_some()); + let mut video_count = 0; + let mut audio_count = 0; + for reference in request.input_references.iter().flatten() { + match reference.reference_type { + VideoInputReferenceType::Image => image_count += 1, + VideoInputReferenceType::Video => video_count += 1, + VideoInputReferenceType::Audio => audio_count += 1, + } + } + let total = image_count + video_count + audio_count; + + let explicit_task = request + .nvext + .as_ref() + .and_then(|nvext| nvext.task.as_deref()); + let is_named_h3 = request + .model + .to_ascii_lowercase() + .replace('_', "-") + .contains("minimax-h3"); + if explicit_task.is_none() && !is_named_h3 { + return Ok(()); + } + let task = explicit_task.unwrap_or({ + if video_count != 0 || audio_count != 0 { + "ref2va" + } else if image_count != 0 { + "fl2va" + } else { + "t2va" + } + }); + + if request + .nvext + .as_ref() + .and_then(|nvext| nvext.fps) + .is_some_and(|fps| fps != 24) + { + return Err(ValidationError::new("invalid_h3_fps")); + } + if task == "fl2va" + && request + .nvext + .as_ref() + .and_then(|nvext| nvext.frame_indices.as_deref()) + .is_some_and(|indices| !matches!(indices, [0] | [-1] | [0, -1])) + { + return Err(ValidationError::new("invalid_fl2va_frame_indices")); + } + + match task { + "t2va" if total != 0 => { + return Err(ValidationError::new("t2va_references_not_allowed")); + } + "fl2va" if !(1..=2).contains(&image_count) || video_count != 0 || audio_count != 0 => { + return Err(ValidationError::new("invalid_fl2va_references")); + } + "fl2va" + if request + .nvext + .as_ref() + .and_then(|nvext| nvext.frame_indices.as_ref()) + .is_some_and(|indices| indices.len() != image_count) => + { + return Err(ValidationError::new("invalid_fl2va_frame_index_count")); + } + "ref2va" + if image_count + video_count == 0 + || image_count > 9 + || video_count > 3 + || audio_count > 3 + || total > 12 => + { + return Err(ValidationError::new("invalid_ref2va_references")); + } + _ => {} + } + + if let Some(start_times) = request + .nvext + .as_ref() + .and_then(|nvext| nvext.start_time_seconds.as_ref()) + { + let valid = match start_times { + StartTimeSeconds::Scalar(_) => video_count == 1, + StartTimeSeconds::List(values) => values.len() == video_count, + }; + if !valid { + return Err(ValidationError::new("invalid_start_time_count")); + } + } Ok(()) } @@ -312,6 +414,119 @@ mod tests { assert!(req.validate().is_err()); } + #[test] + fn video_request_h3_controls_round_trip() { + let json = r#"{ + "prompt":"cat", + "model":"MiniMaxAI/MiniMax-H3", + "input_references":[ + {"type":"image","source":"https://example.com/cat.png"} + ], + "nvext":{ + "task":"ref2va", + "duration":4.0, + "audio_flow_shift":3.0, + "quality":"high" + } + }"#; + let req: NvCreateVideoRequest = serde_json::from_str(json).unwrap(); + assert!(req.validate().is_ok()); + + let out = serde_json::to_string(&req).unwrap(); + assert!(out.contains("\"task\":\"ref2va\"")); + } + + #[test] + fn video_request_rejects_invalid_h3_fps() { + let json = r#"{ + "prompt":"cat", + "model":"MiniMaxAI/MiniMax-H3", + "nvext":{"task":"t2va","fps":16} + }"#; + let req: NvCreateVideoRequest = serde_json::from_str(json).unwrap(); + assert!(req.validate().is_err()); + } + + #[test] + fn video_request_rejects_too_many_typed_references() { + let references = (0..13) + .map(|index| { + serde_json::json!({ + "type": "image", + "source": format!("https://example.com/{index}.png") + }) + }) + .collect::>(); + let request = serde_json::json!({ + "prompt": "cat", + "model": "video-model", + "input_references": references, + }); + let req: NvCreateVideoRequest = serde_json::from_value(request).unwrap(); + + assert!(req.validate().is_err()); + } + + #[test] + fn video_request_rejects_invalid_h3_reference_contracts() { + for request in [ + serde_json::json!({ + "prompt": "cat", + "model": "MiniMaxAI/MiniMax-H3", + "input_references": [{"type": "image", "source": "cat.png"}], + "nvext": {"task": "t2va"}, + }), + serde_json::json!({ + "prompt": "cat", + "model": "MiniMaxAI/MiniMax-H3", + "input_references": [{"type": "image", "source": "cat.png"}], + "nvext": {"task": "fl2va", "frame_indices": [0, -1]}, + }), + serde_json::json!({ + "prompt": "cat", + "model": "MiniMaxAI/MiniMax-H3", + "input_references": [{"type": "video", "source": "cat.mp4"}], + "nvext": {"task": "ref2va", "start_time_seconds": [0.0, 1.0]}, + }), + ] { + let req: NvCreateVideoRequest = serde_json::from_value(request).unwrap(); + assert!(req.validate().is_err()); + } + } + + #[test] + fn video_request_rejects_invalid_taskless_h3_reference_contracts() { + for request in [ + serde_json::json!({ + "prompt": "cat", + "model": "MiniMaxAI/MiniMax-H3", + "input_references": [{"type": "audio", "source": "cat.wav"}], + }), + serde_json::json!({ + "prompt": "cat", + "model": "MiniMaxAI/MiniMax-H3", + "input_references": [ + {"type": "image", "source": "1.png"}, + {"type": "image", "source": "2.png"}, + {"type": "image", "source": "3.png"}, + ], + }), + serde_json::json!({ + "prompt": "cat", + "model": "MiniMaxAI/MiniMax-H3", + "input_references": [ + {"type": "video", "source": "1.mp4"}, + {"type": "video", "source": "2.mp4"}, + {"type": "video", "source": "3.mp4"}, + {"type": "video", "source": "4.mp4"}, + ], + }), + ] { + let req: NvCreateVideoRequest = serde_json::from_value(request).unwrap(); + assert!(req.validate().is_err()); + } + } + // --- VideoData --- #[test] diff --git a/lib/llm/src/protocols/openai/videos/nvext.rs b/lib/llm/src/protocols/openai/videos/nvext.rs index e2de581ddf0e..45a6fb522017 100644 --- a/lib/llm/src/protocols/openai/videos/nvext.rs +++ b/lib/llm/src/protocols/openai/videos/nvext.rs @@ -10,6 +10,13 @@ pub trait NvExtProvider { fn nvext(&self) -> Option<&NvExt>; } +#[derive(ToSchema, Serialize, Deserialize, Debug, Clone)] +#[serde(untagged)] +pub enum StartTimeSeconds { + Scalar(f32), + List(Vec), +} + /// NVIDIA extensions to the OpenAI Videos API #[derive(ToSchema, Serialize, Deserialize, Builder, Validate, Debug, Clone)] #[validate(schema(function = "validate_nv_ext"))] @@ -60,6 +67,56 @@ pub struct NvExt { #[serde(skip_serializing_if = "Option::is_none")] #[builder(default, setter(strip_option))] pub guidance_scale_2: Option, + + /// MiniMax-H3 task routed to its FL2VA or Ref2VA transformer. + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default, setter(strip_option))] + pub task: Option, + + /// Requested MiniMax-H3 duration in seconds (4 through 15). + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default, setter(strip_option))] + pub duration: Option, + + /// MiniMax-H3 video sigma shift. + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default, setter(strip_option))] + pub flow_shift: Option, + + /// MiniMax-H3 audio sigma shift. + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default, setter(strip_option))] + pub audio_flow_shift: Option, + + /// MiniMax-H3 output aspect ratio. + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default, setter(strip_option))] + pub aspect_ratio: Option, + + /// MiniMax-H3 output canvas short edge. + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default, setter(strip_option))] + pub short_edge: Option, + + /// FL2VA keyframe positions: [0], [-1], or [0, -1]. + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default, setter(strip_option))] + pub frame_indices: Option>, + + /// Start offset for one reference video, or one offset per video. + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default, setter(strip_option))] + pub start_time_seconds: Option, + + /// Number of generated videos (MiniMax-H3 supports 1 through 10). + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default, setter(strip_option))] + pub num_outputs_per_prompt: Option, + + /// MiniMax-H3 request-scoped quality policy. + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default, setter(strip_option))] + pub quality: Option, } impl Default for NvExt { @@ -74,7 +131,44 @@ impl NvExt { } } -fn validate_nv_ext(_nv_ext: &NvExt) -> Result<(), ValidationError> { +fn validate_nv_ext(nv_ext: &NvExt) -> Result<(), ValidationError> { + if nv_ext + .task + .as_deref() + .is_some_and(|task| !matches!(task, "t2va" | "fl2va" | "ref2va")) + { + return Err(ValidationError::new("invalid_h3_task")); + } + if nv_ext + .duration + .is_some_and(|duration| !duration.is_finite() || !(4.0..=15.0).contains(&duration)) + { + return Err(ValidationError::new("invalid_h3_duration")); + } + if nv_ext + .num_outputs_per_prompt + .is_some_and(|count| !(1..=10).contains(&count)) + { + return Err(ValidationError::new("invalid_h3_output_count")); + } + if nv_ext.task.is_some() && nv_ext.fps.is_some_and(|fps| fps != 24) { + return Err(ValidationError::new("invalid_h3_fps")); + } + if nv_ext.task.as_deref() == Some("fl2va") + && nv_ext + .frame_indices + .as_deref() + .is_some_and(|indices| !matches!(indices, [0] | [-1] | [0, -1])) + { + return Err(ValidationError::new("invalid_fl2va_frame_indices")); + } + if nv_ext + .quality + .as_deref() + .is_some_and(|quality| !matches!(quality, "lossless" | "high")) + { + return Err(ValidationError::new("invalid_h3_quality")); + } Ok(()) }