[Experimental] Add NVFP4 E5M3 support and related components for model-free quantization

auto_round/data_type/nvfp.py

@@ -206,10 +206,15 @@ def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):
     scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))
     scale = torch.clip(scale, 0, FLOAT8_UE5M3_MAX)
     scale = cast_to_ue5m3_ste(scale).to(torch.float32)
-    output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
-    scaled_x = x.to(torch.float32) * output_scale + v
+    dequant_scale = scale * get_reciprocal(global_scale)
+    scaled_x = torch.where(
+        dequant_scale == 0,
+        torch.zeros_like(x, dtype=torch.float32),
+        x.to(torch.float32) / dequant_scale,
+    )
+    scaled_x = scaled_x + v
     clipped_x = torch.clamp(scaled_x, -6.0, 6.0)
-    return (cast_to_fp4(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale
+    return (cast_to_fp4(clipped_x) * dequant_scale).reshape(m, n), scale
 
 
 @register_dtype("fp4_v2_with_global_scale")

auto_round/envs.py

@@ -28,6 +28,7 @@
     AR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = None
     AR_AUTO_SCHEME_CACHE: Optional[str] = None
     AR_ENABLE_AUTO_SCHEME_PARALLEL: bool = True
+    AR_NVFP4_E5M3_CACHE_HP_WEIGHT: bool = False
     AR_DISK_STREAM_MODEL: bool = False
     AR_RESUME_DIR: Optional[str] = None
 
@@ -101,6 +102,13 @@ def _get_optional_positive_int_env(name: str) -> Optional[int]:
     # set it to 0 when workers could exhaust host RAM or device memory.
     "AR_ENABLE_AUTO_SCHEME_PARALLEL": lambda: os.getenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "1").lower()
     in ("1", "true", "yes"),
+    # Controls whether NVFP4 E5M3 quant linear caches a dequantized high-
+    # precision weight after the first forward instead of dequantizing on
+    # every call. When enabled, the packed weight buffers are released after
+    # the cache is materialized, trading lower runtime overhead for higher
+    # steady-state memory usage.
+    "AR_NVFP4_E5M3_CACHE_HP_WEIGHT": lambda: os.getenv("AR_NVFP4_E5M3_CACHE_HP_WEIGHT", "0").lower()
+    in ("1", "true", "yes", "on"),
     # When set, the model is built as a meta-device skeleton and streamed
     # block-by-block from disk during quantization instead of being fully
     # materialized on CPU RAM up front.

auto_round/experimental/qmodules/__init__.py

@@ -19,5 +19,7 @@
     MXINT4QuantLinear,
 )
 
+from auto_round.experimental.qmodules.fake import FakeActQuantLinear
 from auto_round.experimental.qmodules.nvfp4 import NVFP4QuantLinear
+from auto_round.experimental.qmodules.nvfp4_e5m3 import CuteNVFP4E5M3QuantLinear, NVFP4E5M3QuantLinear
 from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLinear

auto_round/experimental/qmodules/fake.py

@@ -0,0 +1,87 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+import torch
+
+from auto_round.data_type.utils import get_quant_func
+from auto_round.experimental.qmodules.base import QModuleBase
+from auto_round.schemes import QuantizationScheme
+
+__all__ = ["FakeActQuantLinear"]
+
+
+class FakeActQuantLinear(QModuleBase):
+    """Linear with high-precision QDQ weights and runtime activation QDQ."""
+
+    def __init__(
+        self,
+        in_features: int,
+        out_features: int,
+        config: QuantizationScheme,
+        weight: Optional[torch.Tensor] = None,
+        bias: Optional[torch.Tensor] = None,
+        dtype: torch.dtype = torch.bfloat16,
+    ):
+        super().__init__()
+        self.in_features = in_features
+        self.out_features = out_features
+        self.config = config
+        if weight is None:
+            weight = torch.empty((out_features, in_features), dtype=dtype)
+        self.weight = torch.nn.Parameter(weight, requires_grad=False)
+        if bias is None:
+            self.register_parameter("bias", None)
+        else:
+            self.bias = torch.nn.Parameter(bias, requires_grad=False)
+
+    @classmethod
+    def from_original(cls, config: QuantizationScheme, original_layer: torch.nn.Linear):
+        return cls(
+            in_features=original_layer.in_features,
+            out_features=original_layer.out_features,
+            config=config,
+            weight=original_layer.weight,
+            bias=original_layer.bias,
+            dtype=original_layer.weight.dtype,
+        )
+
+    @classmethod
+    def get_min_capability(cls) -> int:
+        return 0
+
+    def process_weights_after_loading(self, layer: torch.nn.Module):
+        return
+
+    def post_init(self):
+        return
+
+    def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:
+        quant_func, _ = get_quant_func(
+            dtype=self.config.act_data_type,
+            bits=self.config.act_bits,
+            sym=self.config.act_sym,
+        )
+        qdq_activation, _, _ = quant_func(
+            tensor=activation,
+            bits=self.config.act_bits,
+            group_size=self.config.act_group_size,
+        )
+        return qdq_activation.to(activation.dtype)
+
+    @torch.inference_mode()
+    def forward(self, activation: torch.Tensor) -> torch.Tensor:
+        qdq_activation = self.qdq_input(activation)
+        return torch.nn.functional.linear(qdq_activation, self.weight.to(qdq_activation.dtype), self.bias)

auto_round/experimental/qmodules/nvfp4_e5m3.py

@@ -0,0 +1,161 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import Optional, Union
+
+import torch
+
+from auto_round.data_type.nvfp import e5m3_to_float_tensor, fp4_v2
+from auto_round.experimental.qmodules.base import QModuleBase
+from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme
+from auto_round_extension.cuda.cute_nvfp4_e5m3 import (
+    try_cute_fp4_v2_qdq,
+    try_cute_nvfp4_e5m3_linear,
+    try_cute_nvfp4_e5m3_weight_dq,
+)
+
+__all__ = ["CuteNVFP4E5M3QuantLinear", "NVFP4E5M3QuantLinear"]
+
+_CACHE_WEIGHT_ENV = "AR_NVFP4_E5M3_CACHE_HP_WEIGHT"
+
+
+def _resolve_cache_weight(cache_weight: Optional[bool], default: bool) -> bool:
+    if cache_weight is not None:
+        return cache_weight
+    value = os.getenv(_CACHE_WEIGHT_ENV)
+    if value is None:
+        return default
+    return value.strip().lower() not in {"0", "false", "no", "off"}
+
+
+class NVFP4E5M3QuantLinear(QModuleBase):
+    """FP4 E2M1 weights and activations with unsigned E5M3 block scales."""
+
+    SUPPORTED_COMPUTE_DTYPE = [torch.bfloat16, torch.float16, torch.float32]
+    DEFAULT_CACHE_WEIGHT = False
+
+    def __init__(
+        self,
+        in_features: int,
+        out_features: int,
+        config: QuantizationScheme,
+        weight: Optional[torch.Tensor] = None,
+        weight_scale: Optional[torch.Tensor] = None,
+        bias: Union[torch.Tensor, bool, None] = None,
+        dtype=torch.bfloat16,
+        cache_weight: Optional[bool] = None,
+    ):
+        super().__init__()
+        assert dtype in self.SUPPORTED_COMPUTE_DTYPE
+        assert config.group_size == 16 and config.act_group_size == 16
+        self.in_features = in_features
+        self.out_features = out_features
+        self.group_size = config.group_size
+        self.config = config
+        self.dtype = dtype
+        self.cache_weight = _resolve_cache_weight(cache_weight, self.DEFAULT_CACHE_WEIGHT)
+        self._cached_weight = None
+
+        packed_weight = torch.zeros((out_features, in_features // 2), dtype=torch.uint8) if weight is None else weight
+        self.register_buffer("weight_packed", packed_weight)
+        scale = (
+            torch.empty((out_features, in_features // self.group_size), dtype=torch.uint8)
+            if weight_scale is None
+            else weight_scale
+        )
+        self.register_buffer("weight_scale", scale)
+
+        if bias is not None:
+            if isinstance(bias, bool):
+                bias = torch.zeros((out_features,), dtype=dtype)
+            self.bias = torch.nn.Parameter(bias, requires_grad=False)
+        else:
+            self.register_parameter("bias", None)
+
+    @classmethod
+    def get_min_capability(cls) -> int:
+        logger.warning_once("NVFP4 E5M3 quantization uses reference PyTorch inference and may be slow.")
+        return 0
+
+    def dequant_weight_online(self) -> torch.Tensor:
+        unpacked = unpack_fp4_from_uint8(self.weight_packed, self.out_features, self.in_features, dtype=self.dtype).to(
+            torch.float32
+        )
+        scale = e5m3_to_float_tensor(self.weight_scale).reshape(-1, 1)
+        return (unpacked.reshape(-1, self.group_size) * scale).reshape(self.out_features, self.in_features)
+
+    @property
+    def weight(self) -> torch.Tensor:
+        if self._cached_weight is None:
+            self._cached_weight = self.dequant_weight_online()
+            if self.cache_weight:
+                self.weight_packed = None
+                self.weight_scale = None
+        return self._cached_weight
+
+    def clear_weight_cache(self) -> None:
+        if self.weight_packed is None:
+            raise RuntimeError("Cannot clear the cached weight after quantized weight buffers have been released.")
+        self._cached_weight = None
+
+    def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:
+        original_dtype = activation.dtype
+        qdq_activation, _, _ = fp4_v2(
+            activation.to(torch.float32), bits=self.config.act_bits, group_size=self.config.act_group_size
+        )
+        return qdq_activation.to(original_dtype)
+
+    @torch.inference_mode()
+    def forward(self, input: torch.Tensor) -> torch.Tensor:
+        qdq_input = self.qdq_input(input)
+        weight = self.weight if self.cache_weight else self.dequant_weight_online()
+        return torch.nn.functional.linear(qdq_input, weight.to(qdq_input.dtype), self.bias)
+
+    @classmethod
+    def from_original(cls, config: QuantizationScheme, original_layer: torch.nn.Linear):
+        return cls(
+            in_features=original_layer.in_features,
+            out_features=original_layer.out_features,
+            config=config,
+            bias=original_layer.bias,
+            dtype=original_layer.weight.dtype,
+        )
+
+
+class CuteNVFP4E5M3QuantLinear(NVFP4E5M3QuantLinear):
+    """NVFP4 E5M3 linear that dispatches activation QDQ and GEMM to CuTe."""
+
+    def dequant_weight_online(self) -> torch.Tensor:
+        cute_weight = try_cute_nvfp4_e5m3_weight_dq(self.weight_packed, self.weight_scale, self.dtype)
+        if cute_weight is not None:
+            return cute_weight
+        return super().dequant_weight_online()
+
+    def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:
+        cute_qdq_activation = try_cute_fp4_v2_qdq(activation, self.config.act_group_size)
+        if cute_qdq_activation is not None:
+            return cute_qdq_activation
+        return super().qdq_input(activation)
+
+    @torch.inference_mode()
+    def forward(self, input: torch.Tensor) -> torch.Tensor:
+        if self.cache_weight:
+            return super().forward(input)
+        fused_output = try_cute_nvfp4_e5m3_linear(input, self.weight_packed, self.weight_scale, self.bias)
+        if fused_output is not None:
+            return fused_output
+        return super().forward(input)

auto_round/export/export_to_autoround/qlinear_fp.py

@@ -36,7 +36,7 @@
 import auto_round.envs as envs
 from auto_round.compressors.utils import BackendDataType, is_mx_fp, is_nv_fp
 from auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMAL
-from auto_round.data_type.nvfp import cast_to_fp4, get_reciprocal
+from auto_round.data_type.nvfp import cast_to_fp4, float_to_e5m3_frexp, get_reciprocal
 from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
 from auto_round.utils import get_packing_device, logger
 
@@ -73,14 +73,15 @@ def __init__(
             raise NotImplementedError("Only 4,8 bits are supported.")
         self.is_mx = is_mx_fp(data_type)
         self.is_nv = is_nv_fp(data_type)
+        self.is_nvfp4_e5m3 = data_type == "fp4_v2"
         if self.is_mx:
             if group_size != 32:
                 raise NotImplementedError(f"Only group_size 32 are supported for {BackendDataType.MX_FP} data type.")
             if infeatures % group_size != 0:
                 raise NotImplementedError(
                     f"in_feature must be divisible by {group_size} for {BackendDataType.MX_FP} data type."
                 )
-        if self.is_nv:
+        if self.is_nv or self.is_nvfp4_e5m3:
             if group_size % 16 != 0:
                 raise NotImplementedError(f"Only group_size 16 are supported for {BackendDataType.NV_FP} data type.")
             if infeatures % group_size != 0:
@@ -159,11 +160,17 @@ def pack(self, linear, scales, zeros=None, g_idx=None, global_scale=None, input_
             )
             scaled_tensor.clamp_(-6.0, 6.0)
             scaled_tensor = cast_to_fp4(scaled_tensor)
+        elif self.is_nvfp4_e5m3:
+            scaled_tensor = tensor / scales.reshape(tensor.shape[0], -1)
+            scaled_tensor.clamp_(-6.0, 6.0)
+            scaled_tensor = cast_to_fp4(scaled_tensor)
         else:
             scaled_tensor = tensor / (2 ** scales.reshape(tensor.shape[0], -1))
         scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)
         if self.is_mx:
             final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)
+        elif self.is_nvfp4_e5m3:
+            final_scale = float_to_e5m3_frexp(scales.to(torch.float32))
         else:
             final_scale = scales.to(torch.float8_e4m3fn)
 

auto_round/export/export_to_llmcompressor/config.py

@@ -102,3 +102,38 @@ def initialize_quantization(scheme, targets=["Linear"], config_groups=None, kv_c
         quantization_status=QuantizationStatus.COMPRESSED,
         ignore=ignore,
     )
+
+
+def initialize_nvfp4_e5m3_quantization(ignore=None):
+    """Build stable compressed-tensors metadata for global-scale-free NVFP4 E5M3."""
+
+    def quant_args(dynamic):
+        return {
+            "actorder": None,
+            "block_structure": None,
+            "dynamic": dynamic,
+            "group_size": 16,
+            "num_bits": 4,
+            "observer": "minmax",
+            "observer_kwargs": {},
+            "strategy": "tensor_group",
+            "symmetric": True,
+            "type": "float",
+        }
+
+    return {
+        "config_groups": {
+            "group_0": {
+                "input_activations": quant_args("local"),
+                "output_activations": None,
+                "targets": ["Linear"],
+                "weights": quant_args(False),
+            }
+        },
+        "format": "nvfp4-e5m3-pack-quantized",
+        "global_compression_ratio": None,
+        "ignore": list(dict.fromkeys(ignore or ["lm_head"])),
+        "kv_cache_scheme": None,
+        "quant_method": "compressed-tensors",
+        "quantization_status": "compressed",
+    }