diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5f0e653 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,166 @@ +name: CI + +"on": + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + +jobs: + quality: + name: quality + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install stable Rust + uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # stable + with: + toolchain: stable + components: clippy,rustfmt + + - name: Check formatting + run: cargo fmt --check + + - name: Lint the stable library and unit tests + run: cargo clippy --lib --tests -- -D warnings + + - name: Check public documentation + env: + RUSTDOCFLAGS: -D warnings + run: cargo doc --no-deps + + stable-test: + name: stable-test (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install stable Rust + uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # stable + with: + toolchain: stable + + - name: Test the stable library + run: cargo test --lib + + - name: Test public documentation + run: cargo test --doc + + - name: Check release library compilation + run: cargo check --release --lib + + msrv: + name: msrv (1.85.0) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install MSRV + uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # 1.85.0 + with: + toolchain: 1.85.0 + + - name: Check package and tests on MSRV + run: | + cargo check --lib + cargo test --lib + cargo test --doc + + no-std-targets: + name: no-std (${{ matrix.target }}) + strategy: + fail-fast: false + matrix: + target: + - aarch64-unknown-none + - riscv64gc-unknown-none-elf + - wasm32-unknown-unknown + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install stable Rust and portable target + uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # stable + with: + toolchain: stable + targets: ${{ matrix.target }} + + - name: Compile the default no_std API + run: cargo check --no-default-features --target ${{ matrix.target }} + + nightly-features: + name: nightly-features + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install nightly Rust + uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # nightly + with: + toolchain: nightly + + - name: Test every opt-in nightly path + run: cargo test --all-targets --all-features + + - name: Check nightly public documentation + env: + RUSTDOCFLAGS: -D warnings + run: cargo doc --all-features --no-deps + + publish-dry-run: + name: publish-dry-run + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install stable Rust + uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # stable + with: + toolchain: stable + + - name: Verify the extracted publication archive + run: bash ci/publish-dry-run.sh diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..2def44c --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,59 @@ +name: Publish crate + +"on": + release: + types: + - published + +permissions: + contents: read + +concurrency: + group: crates-io-release + cancel-in-progress: false + +env: + CARGO_TERM_COLOR: always + +jobs: + publish: + name: publish-crates-io + runs-on: ubuntu-latest + timeout-minutes: 30 + # Trusted Publishing grants this job only the ability to read the released + # source and to mint the short-lived OIDC identity that crates.io checks. + permissions: + contents: read + id-token: write + steps: + # Pin checkout to the SHA captured by the release event. The tag name is + # mutable, so it is checked separately rather than used as the source. + - name: Check out the released commit + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + ref: ${{ github.sha }} + + - name: Install stable Rust + uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # stable + with: + toolchain: stable + + - name: Require a matching tag on the default branch + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + RELEASE_SHA: ${{ github.sha }} + run: bash ci/check-release.sh "${{ github.event.release.tag_name }}" + + - name: Verify the extracted publication archive + run: bash ci/publish-dry-run.sh + + - name: Authenticate with crates.io through Trusted Publishing + id: crates_io_auth + uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5 + + - name: Publish the verified archive + env: + CARGO_REGISTRY_TOKEN: ${{ steps.crates_io_auth.outputs.token }} + run: cargo publish --locked diff --git a/.gitignore b/.gitignore index ea8c4bf..0592392 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +.DS_Store diff --git a/Cargo.toml b/Cargo.toml index 5448b59..7200270 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ name = "unsigned-float" version = "0.2.2" edition = "2024" +rust-version = "1.85" description = "Unsigned floating-point formats for non-negative numeric domains." license = "MIT" repository = "https://github.com/MicroPerceptron/ufloat" @@ -14,7 +15,8 @@ default = [] soft-float = [] f16 = [] f128 = [] -nightly = ["f16", "f128"] +simd = [] +nightly = ["f16", "f128", "simd"] [dependencies] libm = "0.2" diff --git a/README.md b/README.md index 69e6733..0781783 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Crates.io](https://img.shields.io/crates/v/unsigned-float.svg)](https://crates.io/crates/unsigned-float) [![docs.rs](https://docs.rs/unsigned-float/badge.svg)](https://docs.rs/unsigned-float) +[![CI](https://github.com/MicroPerceptron/ufloat/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/MicroPerceptron/ufloat/actions/workflows/ci.yml) [![GitHub last commit](https://img.shields.io/github/last-commit/MicroPerceptron/ufloat.svg)](https://github.com/MicroPerceptron/ufloat/commits/main) [![GitHub license](https://img.shields.io/github/license/MicroPerceptron/ufloat.svg)](https://github.com/MicroPerceptron/ufloat/blob/main/LICENSE) [![GitHub issues](https://img.shields.io/github/issues/MicroPerceptron/ufloat.svg)](https://github.com/MicroPerceptron/ufloat/issues) @@ -186,7 +187,8 @@ output. | ------------ | ------- | ----------------------------------------------------------------------------------------------------------- | | `f16` | No | Uses nightly primitive `f16` for `Uf8` arithmetic dispatch. Requires nightly Rust. | | `f128` | No | Enables `Uf64`/`Uf64E11M52` and promotes its arithmetic through nightly primitive `f128`. | -| `nightly` | No | Convenience feature enabling both `f16` and `f128`. | +| `simd` | No | Enables nightly portable-SIMD bulk conversion and elementwise APIs for UF16/UF32. | +| `nightly` | No | Convenience feature enabling `f16`, `f128`, and `simd`. | | `soft-float` | No | Forces the software/LUT dispatch path where available. If combined with `f16`, `soft-float` wins for `Uf8`. | The default `Uf8` arithmetic path uses generated 256x256 lookup tables for @@ -199,6 +201,42 @@ generated by `build.rs` into Cargo's `OUT_DIR` and embedded with The library is `#![no_std]`. The build script and benchmarks use `std`, but the crate API itself does not require allocation or the standard library. +## Development and release checks + +The supported stable baseline is Rust 1.85. CI tests that baseline and current +stable on Linux, macOS, and Windows; compiles the default `no_std` API for +AArch64, RISC-V, and WebAssembly; and runs the nightly-only feature set +separately. It also packages the crate, extracts the archive, and reruns the +library, documentation, and documentation tests from the extracted source. + +Before submitting a change, run the stable baseline checks: + +```sh +cargo fmt --check +cargo clippy --lib --tests -- -D warnings +cargo test --lib +cargo test --doc +RUSTDOCFLAGS='-D warnings' cargo doc --no-deps +``` + +Changes to an opt-in nightly implementation should additionally run: + +```sh +cargo +nightly test --all-features +RUSTDOCFLAGS='-D warnings' cargo +nightly doc --all-features --no-deps +``` + +Releases are tag-driven: GitHub release tag `vX.Y.Z` must name the exact +`Cargo.toml` version, and its current target must still match the immutable +commit SHA captured when that release was published; that commit must be +reachable from `main`. The release workflow repeats the extracted-package +verification before publishing through crates.io Trusted Publishing: it +exchanges the workflow's GitHub OIDC identity for a short-lived token rather +than storing a long-lived crates.io credential in the repository. Before the +first automated release, configure the crate's trusted publisher as GitHub +owner `MicroPerceptron`, repository `ufloat`, and workflow filename +`publish.yml`. + ## Benchmarks The benchmark suite uses nightly's built-in `test` harness and covers @@ -214,6 +252,24 @@ Each command benchmarks the dispatch path selected by that feature set. For example, `--features f16` measures the `Uf8` primitive-`f16` path, while the default command measures the generated LUT path. +With nightly, `simd` adds allocation-free slice APIs that widen normal finite +UF16 lanes to native F32 and UF32 lanes to native F64. The bulk operators keep +the exact scalar result for every lane: blocks containing subnormals, NaNs, +infinities, negative results, or an encoding round-carry fall back to the +scalar conversion rather than changing the numeric contract. + +```rust +use unsigned_float::{Uf16, simd}; + +let packed = [Uf16::from_f32(0.75); 256]; +let mut linear = [0.0_f32; 256]; +simd::decode_uf16_to_f32(&packed, &mut linear).unwrap(); + +let mut doubled = [Uf16::ZERO; 256]; +simd::add_uf16(&packed, &packed, &mut doubled).unwrap(); +assert_eq!(doubled[0].to_f32(), 1.5); +``` + ## Status Implemented: @@ -230,13 +286,14 @@ Implemented: - `Add`, `Sub`, `Mul`, and `Div` - raw-bit `Ord`/`PartialOrd` - generated UF8 arithmetic and exponentiation lookup tables +- opt-in SIMD bulk conversion and arithmetic for UF16/UF32, with scalar-exact + edge-lane fallback - benchmarks across conversions, arithmetic, and ordering Still worth exploring: - native baseline benchmarks against `f32` and `f64` - configurable UF8 dispatch for direct LUT versus promote-to-native comparisons -- SIMD bulk operations - more explicit NaN payload policy - broader property tests for all finite `Uf16` layout edge cases diff --git a/benches/arithmetic.rs b/benches/arithmetic.rs index ff8af9b..f1e89f2 100644 --- a/benches/arithmetic.rs +++ b/benches/arithmetic.rs @@ -5,8 +5,13 @@ extern crate test; use test::{Bencher, black_box}; #[cfg(feature = "f128")] use unsigned_float::Uf64; +#[cfg(feature = "simd")] +use unsigned_float::simd; use unsigned_float::{Pow1mUf, PowUf, Uf8, Uf8E5M3, Uf16, Uf16E6M10, Uf32}; +#[cfg(feature = "simd")] +const SIMD_BULK_LEN: usize = 4_096; + const F32_INPUTS: [f32; 16] = [ 0.0, 0.000_976_562_5, @@ -513,3 +518,85 @@ bench_pow1muf_f64!(f64_pow1muf_uf16_e6m10, UF16_E6M10_INPUTS); bench_pow1muf_f64!(f64_pow1muf_uf32, UF32_INPUTS); #[cfg(feature = "f128")] bench_pow1muf_f64!(f64_pow1muf_uf64, UF64_INPUTS); + +#[cfg(feature = "simd")] +#[bench] +fn scalar_bulk_uf16_decode(b: &mut Bencher) { + let source: [Uf16; SIMD_BULK_LEN] = + core::array::from_fn(|index| Uf16::from_bits(0x6800 | index as u16 & 0x07ff)); + let mut output = [0.0_f32; SIMD_BULK_LEN]; + b.iter(|| { + for (source, output) in source.iter().zip(&mut output) { + *output = black_box(*source).to_f32(); + } + black_box(&output); + }); +} + +#[cfg(feature = "simd")] +#[bench] +fn simd_bulk_uf16_decode(b: &mut Bencher) { + let source: [Uf16; SIMD_BULK_LEN] = + core::array::from_fn(|index| Uf16::from_bits(0x6800 | index as u16 & 0x07ff)); + let mut output = [0.0_f32; SIMD_BULK_LEN]; + b.iter(|| { + simd::decode_uf16_to_f32(black_box(&source), &mut output).unwrap(); + black_box(&output); + }); +} + +#[cfg(feature = "simd")] +#[bench] +fn scalar_bulk_uf16_add(b: &mut Bencher) { + let left: [Uf16; SIMD_BULK_LEN] = + core::array::from_fn(|index| Uf16::from_bits(0x6800 | index as u16 & 0x07ff)); + let right: [Uf16; SIMD_BULK_LEN] = + core::array::from_fn(|index| Uf16::from_bits(0x7000 | index as u16 & 0x07ff)); + let mut output = [Uf16::ZERO; SIMD_BULK_LEN]; + b.iter(|| { + for ((left, right), output) in left.iter().zip(&right).zip(&mut output) { + *output = black_box(*left) + black_box(*right); + } + black_box(&output); + }); +} + +#[cfg(feature = "simd")] +#[bench] +fn simd_bulk_uf16_add(b: &mut Bencher) { + let left: [Uf16; SIMD_BULK_LEN] = + core::array::from_fn(|index| Uf16::from_bits(0x6800 | index as u16 & 0x07ff)); + let right: [Uf16; SIMD_BULK_LEN] = + core::array::from_fn(|index| Uf16::from_bits(0x7000 | index as u16 & 0x07ff)); + let mut output = [Uf16::ZERO; SIMD_BULK_LEN]; + b.iter(|| { + simd::add_uf16(black_box(&left), black_box(&right), &mut output).unwrap(); + black_box(&output); + }); +} + +#[cfg(feature = "simd")] +#[bench] +fn scalar_bulk_uf32_decode(b: &mut Bencher) { + let source: [Uf32; SIMD_BULK_LEN] = + core::array::from_fn(|index| Uf32::from_bits(0x7800_0000 | index as u32)); + let mut output = [0.0_f64; SIMD_BULK_LEN]; + b.iter(|| { + for (source, output) in source.iter().zip(&mut output) { + *output = black_box(*source).to_f64(); + } + black_box(&output); + }); +} + +#[cfg(feature = "simd")] +#[bench] +fn simd_bulk_uf32_decode(b: &mut Bencher) { + let source: [Uf32; SIMD_BULK_LEN] = + core::array::from_fn(|index| Uf32::from_bits(0x7800_0000 | index as u32)); + let mut output = [0.0_f64; SIMD_BULK_LEN]; + b.iter(|| { + simd::decode_uf32_to_f64(black_box(&source), &mut output).unwrap(); + black_box(&output); + }); +} diff --git a/ci/check-release.sh b/ci/check-release.sh new file mode 100644 index 0000000..12554a5 --- /dev/null +++ b/ci/check-release.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Validate that a release is an immutable, reviewed snapshot of main and that +# its tag agrees exactly with the published Cargo package version. +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 vX.Y.Z" >&2 + exit 2 +fi + +tag="$1" +repo_root="$(git rev-parse --show-toplevel)" +manifest="$repo_root/Cargo.toml" +version="$(sed -n 's/^version = "\([^"]*\)"/\1/p' "$manifest" | head -n 1)" +expected_tag="v$version" +release_sha="${RELEASE_SHA:?RELEASE_SHA must be the immutable release-event commit SHA}" + +if [[ "$tag" != "$expected_tag" ]]; then + echo "release tag $tag does not match package version $version (expected $expected_tag)" >&2 + exit 1 +fi + +release_commit="$(git rev-parse --verify "$release_sha^{commit}")" +head_commit="$(git rev-parse HEAD)" +if [[ "$head_commit" != "$release_commit" ]]; then + echo "checked-out commit $head_commit does not match release-event commit $release_commit" >&2 + exit 1 +fi + +tag_commit="$(git rev-parse --verify "$tag^{commit}")" +if [[ "$tag_commit" != "$release_commit" ]]; then + echo "release tag $tag resolves to $tag_commit, not release-event commit $release_commit" >&2 + exit 1 +fi + +default_branch="${DEFAULT_BRANCH:-main}" +git merge-base --is-ancestor "$release_commit" "origin/$default_branch" diff --git a/ci/publish-dry-run.sh b/ci/publish-dry-run.sh new file mode 100644 index 0000000..41929f4 --- /dev/null +++ b/ci/publish-dry-run.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Package the public crate, then prove the published source still builds, tests, +# and documents independently of this checkout. Cargo's own verification only +# builds the library target, so it cannot substitute for these extracted-source +# checks. +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +manifest="$repo_root/Cargo.toml" +version="$(sed -n 's/^version = "\([^"]*\)"/\1/p' "$manifest" | head -n 1)" + +if [[ -z "$version" ]]; then + echo "could not determine package version from $manifest" >&2 + exit 1 +fi + +cd "$repo_root" +cargo package --locked + +archive="$repo_root/target/package/unsigned-float-$version.crate" +if [[ ! -f "$archive" ]]; then + echo "cargo package did not produce $archive" >&2 + exit 1 +fi + +package_root="$(mktemp -d)" +trap 'rm -rf "$package_root"' EXIT +tar -xzf "$archive" -C "$package_root" +cd "$package_root/unsigned-float-$version" + +cargo test --lib +cargo test --doc +RUSTDOCFLAGS="-D warnings" cargo doc --no-deps diff --git a/src/lib.rs b/src/lib.rs index 5be530f..a82846d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,7 @@ //! concrete layouts [`Uf8E4M4`], [`Uf16E5M11`], and [`Uf32E8M24`]. Alternate //! layouts such as [`Uf8E5M3`] and [`Uf16E6M10`] are exported as distinct types //! so their range and precision tradeoffs stay explicit. -//! With the `f128` feature enabled, [`Uf64`] is also available and promotes +//! With the `f128` feature enabled, `Uf64` is also available and promotes //! through nightly primitive `f128`. //! //! # Conversions @@ -50,6 +50,7 @@ #![no_std] #![cfg_attr(feature = "f16", feature(f16))] #![cfg_attr(feature = "f128", feature(f128))] +#![cfg_attr(feature = "simd", feature(portable_simd))] #[cfg(test)] extern crate std; @@ -57,6 +58,8 @@ extern crate std; mod convert; mod dispatch; mod pow; +#[cfg(feature = "simd")] +pub mod simd; mod uf16; mod uf32; #[cfg(feature = "f128")] diff --git a/src/simd.rs b/src/simd.rs new file mode 100644 index 0000000..fb428b5 --- /dev/null +++ b/src/simd.rs @@ -0,0 +1,721 @@ +//! Bulk SIMD operations for the unsigned floating-point storage formats. +//! +//! The scalar newtypes intentionally retain their precise, portable +//! promote/compute/demote contract. SIMD is therefore exposed as an opt-in +//! slice API: storage lanes are widened into native `f32` (`Uf16`) or `f64` +//! (`Uf32`) vectors, computed there, and rounded back to the exact scalar +//! encoding. Normal finite lanes take the register-only bit-expansion path; +//! subnormals, infinities, NaNs, overflows, and negative results fall back to +//! the scalar conversion for bit-for-bit compatibility. +//! +//! This feature uses nightly `portable_simd`. The chosen lane widths follow +//! the portable `axnn-cpu` tier: 128-bit F32/F64 vectors by default and +//! 256-bit vectors when the crate is compiled with AVX2 or AVX-512 enabled. + +use core::fmt; +use core::simd::prelude::*; + +use crate::{Uf16, Uf16E5M11, Uf16E6M10, Uf32}; + +/// F32 lanes processed by the UF16 path in one vector operation. +#[cfg(any(target_feature = "avx2", target_feature = "avx512f"))] +pub const UF16_LANES: usize = 8; +/// F32 lanes processed by the UF16 path in one vector operation. +#[cfg(not(any(target_feature = "avx2", target_feature = "avx512f")))] +pub const UF16_LANES: usize = 4; + +/// F64 lanes processed by the UF32 path in one vector operation. +#[cfg(any(target_feature = "avx2", target_feature = "avx512f"))] +pub const UF32_LANES: usize = 4; +/// F64 lanes processed by the UF32 path in one vector operation. +#[cfg(not(any(target_feature = "avx2", target_feature = "avx512f")))] +pub const UF32_LANES: usize = 2; + +/// Invalid source/destination slice relationship passed to a bulk operation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SimdError { + /// Two input planes that must be elementwise aligned have different sizes. + InputLengthMismatch { left: usize, right: usize }, + /// The output plane does not have one element for each input element. + OutputLengthMismatch { input: usize, output: usize }, +} + +impl fmt::Display for SimdError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InputLengthMismatch { left, right } => { + write!(formatter, "SIMD input lengths differ ({left} and {right})") + } + Self::OutputLengthMismatch { input, output } => { + write!( + formatter, + "SIMD output length is {output}, expected {input}" + ) + } + } + } +} + +trait Uf16Layout: Copy { + const EXPONENT_BITS: u32; + const MANTISSA_BITS: u32; + /// Native F32 exponent corresponding to an encoded exponent of zero. + const F32_EXPONENT_BIAS: u32; + + fn from_bits(bits: u16) -> Self; + fn to_bits(self) -> u16; + fn from_f32(value: f32) -> Self; + fn to_f32(self) -> f32; +} + +impl Uf16Layout for Uf16E5M11 { + const EXPONENT_BITS: u32 = 5; + const MANTISSA_BITS: u32 = 11; + const F32_EXPONENT_BIAS: u32 = 112; + + fn from_bits(bits: u16) -> Self { + Self::from_bits(bits) + } + + fn to_bits(self) -> u16 { + self.to_bits() + } + + fn from_f32(value: f32) -> Self { + Self::from_f32(value) + } + + fn to_f32(self) -> f32 { + self.to_f32() + } +} + +impl Uf16Layout for Uf16E6M10 { + const EXPONENT_BITS: u32 = 6; + const MANTISSA_BITS: u32 = 10; + const F32_EXPONENT_BIAS: u32 = 96; + + fn from_bits(bits: u16) -> Self { + Self::from_bits(bits) + } + + fn to_bits(self) -> u16 { + self.to_bits() + } + + fn from_f32(value: f32) -> Self { + Self::from_f32(value) + } + + fn to_f32(self) -> f32 { + self.to_f32() + } +} + +fn output_len(input: usize, output: usize) -> Result<(), SimdError> { + if input == output { + Ok(()) + } else { + Err(SimdError::OutputLengthMismatch { input, output }) + } +} + +fn binary_len(left: usize, right: usize, output: usize) -> Result<(), SimdError> { + if left != right { + return Err(SimdError::InputLengthMismatch { left, right }); + } + output_len(left, output) +} + +fn uf16_max_exponent() -> u32 { + (1 << T::EXPONENT_BITS) - 1 +} + +fn can_decode_uf16(value: T) -> bool { + let exponent = (value.to_bits() as u32 >> T::MANTISSA_BITS) & uf16_max_exponent::(); + exponent != 0 && exponent != uf16_max_exponent::() +} + +fn can_encode_uf16(value: f32) -> bool { + let bits = value.to_bits(); + let exponent = (bits >> 23) & 0xff; + let max_normal = uf16_max_exponent::() - 1; + bits >> 31 == 0 + && exponent > T::F32_EXPONENT_BIAS + // Keep the largest normal bin scalar: a round carry there can become + // infinity, while every lane accepted here stays normal after RNE. + && exponent < T::F32_EXPONENT_BIAS + max_normal +} + +fn decode_uf16_fast(src: &[T]) -> Simd { + debug_assert_eq!(src.len(), UF16_LANES); + debug_assert!(src.iter().copied().all(can_decode_uf16::)); + let raw = Simd::::from_array(core::array::from_fn(|lane| { + src[lane].to_bits() as u32 + })); + let bits = + (raw << Simd::splat(23 - T::MANTISSA_BITS)) + Simd::splat(T::F32_EXPONENT_BIAS << 23); + Simd::::from_bits(bits) +} + +fn encode_uf16_fast(src: Simd, dst: &mut [T]) { + debug_assert_eq!(dst.len(), UF16_LANES); + debug_assert!(src.to_array().into_iter().all(can_encode_uf16::)); + let bits = src.to_bits(); + let fraction = bits & Simd::splat(0x007f_ffff_u32); + let drop = 23 - T::MANTISSA_BITS; + let mantissa = fraction >> Simd::splat(drop); + let discarded = fraction & Simd::splat((1_u32 << drop) - 1); + // Round-to-nearest-even without a per-lane comparison. + let rounding = + (discarded + Simd::splat((1_u32 << (drop - 1)) - 1) + (mantissa & Simd::splat(1))) + >> Simd::splat(drop); + let rounded = mantissa + rounding; + let carry = rounded >> Simd::splat(T::MANTISSA_BITS); + let exponent = (bits >> Simd::splat(23)) - Simd::splat(T::F32_EXPONENT_BIAS) + carry; + let raw = (exponent << Simd::splat(T::MANTISSA_BITS)) + | (rounded & Simd::splat((1_u32 << T::MANTISSA_BITS) - 1)); + for (lane, bits) in raw.to_array().into_iter().enumerate() { + dst[lane] = T::from_bits(bits as u16); + } +} + +fn decode_uf16(src: &[T], dst: &mut [f32]) -> Result<(), SimdError> { + output_len(src.len(), dst.len())?; + let vector_end = src.len() / UF16_LANES * UF16_LANES; + for offset in (0..vector_end).step_by(UF16_LANES) { + let input = &src[offset..offset + UF16_LANES]; + if input.iter().copied().all(can_decode_uf16::) { + decode_uf16_fast(input).copy_to_slice(&mut dst[offset..]); + } else { + for lane in 0..UF16_LANES { + dst[offset + lane] = input[lane].to_f32(); + } + } + } + for index in vector_end..src.len() { + dst[index] = src[index].to_f32(); + } + Ok(()) +} + +fn encode_uf16(src: &[f32], dst: &mut [T]) -> Result<(), SimdError> { + output_len(src.len(), dst.len())?; + let vector_end = src.len() / UF16_LANES * UF16_LANES; + for offset in (0..vector_end).step_by(UF16_LANES) { + let input = Simd::::from_slice(&src[offset..]); + if input.to_array().into_iter().all(can_encode_uf16::) { + encode_uf16_fast(input, &mut dst[offset..offset + UF16_LANES]); + } else { + for lane in 0..UF16_LANES { + dst[offset + lane] = T::from_f32(src[offset + lane]); + } + } + } + for index in vector_end..src.len() { + dst[index] = T::from_f32(src[index]); + } + Ok(()) +} + +fn binary_uf16( + left: &[T], + right: &[T], + output: &mut [T], + vector: impl Fn(Simd, Simd) -> Simd, + scalar: impl Fn(f32, f32) -> f32, +) -> Result<(), SimdError> { + binary_len(left.len(), right.len(), output.len())?; + let vector_end = left.len() / UF16_LANES * UF16_LANES; + for offset in (0..vector_end).step_by(UF16_LANES) { + let lhs = &left[offset..offset + UF16_LANES]; + let rhs = &right[offset..offset + UF16_LANES]; + if lhs.iter().copied().all(can_decode_uf16::) + && rhs.iter().copied().all(can_decode_uf16::) + { + let result = vector(decode_uf16_fast(lhs), decode_uf16_fast(rhs)); + if result.to_array().into_iter().all(can_encode_uf16::) { + encode_uf16_fast(result, &mut output[offset..offset + UF16_LANES]); + continue; + } + } + for lane in 0..UF16_LANES { + output[offset + lane] = T::from_f32(scalar(lhs[lane].to_f32(), rhs[lane].to_f32())); + } + } + for index in vector_end..left.len() { + output[index] = T::from_f32(scalar(left[index].to_f32(), right[index].to_f32())); + } + Ok(()) +} + +/// Decode packed [`Uf16`] lanes to F32. Normal finite blocks use SIMD bit expansion. +pub fn decode_uf16_to_f32(src: &[Uf16], dst: &mut [f32]) -> Result<(), SimdError> { + decode_uf16(src, dst) +} + +/// Encode F32 lanes as [`Uf16`] with the scalar constructor's exact RNE behavior. +pub fn encode_f32_to_uf16(src: &[f32], dst: &mut [Uf16]) -> Result<(), SimdError> { + encode_uf16(src, dst) +} + +/// Decode packed [`Uf16E6M10`] lanes to F32. +pub fn decode_uf16e6m10_to_f32(src: &[Uf16E6M10], dst: &mut [f32]) -> Result<(), SimdError> { + decode_uf16(src, dst) +} + +/// Encode F32 lanes as [`Uf16E6M10`] with scalar-equivalent rounding. +pub fn encode_f32_to_uf16e6m10(src: &[f32], dst: &mut [Uf16E6M10]) -> Result<(), SimdError> { + encode_uf16(src, dst) +} + +/// Elementwise [`Uf16`] addition through native F32 SIMD lanes. +pub fn add_uf16(left: &[Uf16], right: &[Uf16], output: &mut [Uf16]) -> Result<(), SimdError> { + binary_uf16( + left, + right, + output, + |left, right| left + right, + |left, right| left + right, + ) +} + +/// Elementwise [`Uf16`] subtraction through native F32 SIMD lanes. +pub fn sub_uf16(left: &[Uf16], right: &[Uf16], output: &mut [Uf16]) -> Result<(), SimdError> { + binary_uf16( + left, + right, + output, + |left, right| left - right, + |left, right| left - right, + ) +} + +/// Elementwise [`Uf16`] multiplication through native F32 SIMD lanes. +pub fn mul_uf16(left: &[Uf16], right: &[Uf16], output: &mut [Uf16]) -> Result<(), SimdError> { + binary_uf16( + left, + right, + output, + |left, right| left * right, + |left, right| left * right, + ) +} + +/// Elementwise [`Uf16`] division through native F32 SIMD lanes. +pub fn div_uf16(left: &[Uf16], right: &[Uf16], output: &mut [Uf16]) -> Result<(), SimdError> { + binary_uf16( + left, + right, + output, + |left, right| left / right, + |left, right| left / right, + ) +} + +/// Elementwise [`Uf16E6M10`] addition through native F32 SIMD lanes. +pub fn add_uf16e6m10( + left: &[Uf16E6M10], + right: &[Uf16E6M10], + output: &mut [Uf16E6M10], +) -> Result<(), SimdError> { + binary_uf16( + left, + right, + output, + |left, right| left + right, + |left, right| left + right, + ) +} + +/// Elementwise [`Uf16E6M10`] subtraction through native F32 SIMD lanes. +pub fn sub_uf16e6m10( + left: &[Uf16E6M10], + right: &[Uf16E6M10], + output: &mut [Uf16E6M10], +) -> Result<(), SimdError> { + binary_uf16( + left, + right, + output, + |left, right| left - right, + |left, right| left - right, + ) +} + +/// Elementwise [`Uf16E6M10`] multiplication through native F32 SIMD lanes. +pub fn mul_uf16e6m10( + left: &[Uf16E6M10], + right: &[Uf16E6M10], + output: &mut [Uf16E6M10], +) -> Result<(), SimdError> { + binary_uf16( + left, + right, + output, + |left, right| left * right, + |left, right| left * right, + ) +} + +/// Elementwise [`Uf16E6M10`] division through native F32 SIMD lanes. +pub fn div_uf16e6m10( + left: &[Uf16E6M10], + right: &[Uf16E6M10], + output: &mut [Uf16E6M10], +) -> Result<(), SimdError> { + binary_uf16( + left, + right, + output, + |left, right| left / right, + |left, right| left / right, + ) +} + +fn can_decode_uf32(value: Uf32) -> bool { + let exponent = value.to_bits() >> 24; + exponent != 0 && exponent != 0xff +} + +fn can_encode_uf32(value: f64) -> bool { + let bits = value.to_bits(); + let exponent = (bits >> 52) & 0x7ff; + // An encoded exponent of 254 is kept scalar because rounding may carry to infinity. + bits >> 63 == 0 && exponent > 896 && exponent < 1150 +} + +fn decode_uf32_fast(src: &[Uf32]) -> Simd { + debug_assert_eq!(src.len(), UF32_LANES); + debug_assert!(src.iter().copied().all(can_decode_uf32)); + let raw = Simd::::from_array(core::array::from_fn(|lane| { + src[lane].to_bits() as u64 + })); + let bits = (raw << Simd::splat(28)) + Simd::splat(896_u64 << 52); + Simd::::from_bits(bits) +} + +fn encode_uf32_fast(src: Simd, dst: &mut [Uf32]) { + debug_assert_eq!(dst.len(), UF32_LANES); + debug_assert!(src.to_array().into_iter().all(can_encode_uf32)); + let bits = src.to_bits(); + let fraction = bits & Simd::splat(0x000f_ffff_ffff_ffff_u64); + let mantissa = fraction >> Simd::splat(28); + let discarded = fraction & Simd::splat((1_u64 << 28) - 1); + let rounding = (discarded + Simd::splat((1_u64 << 27) - 1) + (mantissa & Simd::splat(1))) + >> Simd::splat(28); + let rounded = mantissa + rounding; + let carry = rounded >> Simd::splat(24); + let exponent = (bits >> Simd::splat(52)) - Simd::splat(896_u64) + carry; + let raw = (exponent << Simd::splat(24)) | (rounded & Simd::splat(0x00ff_ffff_u64)); + for (lane, bits) in raw.to_array().into_iter().enumerate() { + dst[lane] = Uf32::from_bits(bits as u32); + } +} + +/// Decode packed [`Uf32`] lanes to F64. Normal finite blocks use SIMD bit expansion. +pub fn decode_uf32_to_f64(src: &[Uf32], dst: &mut [f64]) -> Result<(), SimdError> { + output_len(src.len(), dst.len())?; + let vector_end = src.len() / UF32_LANES * UF32_LANES; + for offset in (0..vector_end).step_by(UF32_LANES) { + let input = &src[offset..offset + UF32_LANES]; + if input.iter().copied().all(can_decode_uf32) { + decode_uf32_fast(input).copy_to_slice(&mut dst[offset..]); + } else { + for lane in 0..UF32_LANES { + dst[offset + lane] = input[lane].to_f64(); + } + } + } + for index in vector_end..src.len() { + dst[index] = src[index].to_f64(); + } + Ok(()) +} + +/// Encode F64 lanes as [`Uf32`] with the scalar constructor's exact RNE behavior. +pub fn encode_f64_to_uf32(src: &[f64], dst: &mut [Uf32]) -> Result<(), SimdError> { + output_len(src.len(), dst.len())?; + let vector_end = src.len() / UF32_LANES * UF32_LANES; + for offset in (0..vector_end).step_by(UF32_LANES) { + let input = Simd::::from_slice(&src[offset..]); + if input.to_array().into_iter().all(can_encode_uf32) { + encode_uf32_fast(input, &mut dst[offset..offset + UF32_LANES]); + } else { + for lane in 0..UF32_LANES { + dst[offset + lane] = Uf32::from_f64(src[offset + lane]); + } + } + } + for index in vector_end..src.len() { + dst[index] = Uf32::from_f64(src[index]); + } + Ok(()) +} + +fn binary_uf32( + left: &[Uf32], + right: &[Uf32], + output: &mut [Uf32], + vector: impl Fn(Simd, Simd) -> Simd, + scalar: impl Fn(f64, f64) -> f64, +) -> Result<(), SimdError> { + binary_len(left.len(), right.len(), output.len())?; + let vector_end = left.len() / UF32_LANES * UF32_LANES; + for offset in (0..vector_end).step_by(UF32_LANES) { + let lhs = &left[offset..offset + UF32_LANES]; + let rhs = &right[offset..offset + UF32_LANES]; + if lhs.iter().copied().all(can_decode_uf32) && rhs.iter().copied().all(can_decode_uf32) { + let result = vector(decode_uf32_fast(lhs), decode_uf32_fast(rhs)); + if result.to_array().into_iter().all(can_encode_uf32) { + encode_uf32_fast(result, &mut output[offset..offset + UF32_LANES]); + continue; + } + } + for lane in 0..UF32_LANES { + output[offset + lane] = Uf32::from_f64(scalar(lhs[lane].to_f64(), rhs[lane].to_f64())); + } + } + for index in vector_end..left.len() { + output[index] = Uf32::from_f64(scalar(left[index].to_f64(), right[index].to_f64())); + } + Ok(()) +} + +/// Elementwise [`Uf32`] addition through native F64 SIMD lanes. +pub fn add_uf32(left: &[Uf32], right: &[Uf32], output: &mut [Uf32]) -> Result<(), SimdError> { + binary_uf32( + left, + right, + output, + |left, right| left + right, + |left, right| left + right, + ) +} + +/// Elementwise [`Uf32`] subtraction through native F64 SIMD lanes. +pub fn sub_uf32(left: &[Uf32], right: &[Uf32], output: &mut [Uf32]) -> Result<(), SimdError> { + binary_uf32( + left, + right, + output, + |left, right| left - right, + |left, right| left - right, + ) +} + +/// Elementwise [`Uf32`] multiplication through native F64 SIMD lanes. +pub fn mul_uf32(left: &[Uf32], right: &[Uf32], output: &mut [Uf32]) -> Result<(), SimdError> { + binary_uf32( + left, + right, + output, + |left, right| left * right, + |left, right| left * right, + ) +} + +/// Elementwise [`Uf32`] division through native F64 SIMD lanes. +pub fn div_uf32(left: &[Uf32], right: &[Uf32], output: &mut [Uf32]) -> Result<(), SimdError> { + binary_uf32( + left, + right, + output, + |left, right| left / right, + |left, right| left / right, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::vec; + use std::vec::Vec; + + fn lcg(state: &mut u64) -> u64 { + *state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + *state + } + + fn verify_uf16_conversions() { + let source: Vec = (u16::MIN..=u16::MAX).map(T::from_bits).collect(); + let mut decoded = vec![0.0; source.len()]; + decode_uf16(&source, &mut decoded).unwrap(); + for (value, actual) in source.iter().copied().zip(decoded) { + assert_eq!(actual.to_bits(), value.to_f32().to_bits()); + } + + let mut state = 0x6f_1d_5eed_u64; + let mut input = vec![0.0; 32_771]; + input[0] = 0.0; + input[1] = -0.0; + input[2] = f32::INFINITY; + input[3] = f32::NEG_INFINITY; + input[4] = f32::NAN; + for value in &mut input[5..] { + *value = f32::from_bits(lcg(&mut state) as u32); + } + let mut encoded = vec![T::from_bits(0); input.len()]; + encode_uf16(&input, &mut encoded).unwrap(); + for (value, actual) in input.into_iter().zip(encoded) { + assert_eq!(actual.to_bits(), T::from_f32(value).to_bits()); + } + } + + fn verify_uf16_binary() { + let mut state = 0x9a_6d_ef_41_u64; + let left: Vec = (0..(UF16_LANES * 19 + 3)) + .map(|_| T::from_bits(lcg(&mut state) as u16)) + .collect(); + let right: Vec = (0..left.len()) + .map(|_| T::from_bits(lcg(&mut state) as u16)) + .collect(); + let mut output = vec![T::from_bits(0); left.len()]; + + binary_uf16( + &left, + &right, + &mut output, + |left, right| left + right, + |left, right| left + right, + ) + .unwrap(); + for ((left, right), actual) in left.iter().zip(&right).zip(&output) { + assert_eq!( + actual.to_bits(), + T::from_f32(left.to_f32() + right.to_f32()).to_bits() + ); + } + + binary_uf16( + &left, + &right, + &mut output, + |left, right| left - right, + |left, right| left - right, + ) + .unwrap(); + for ((left, right), actual) in left.iter().zip(&right).zip(&output) { + assert_eq!( + actual.to_bits(), + T::from_f32(left.to_f32() - right.to_f32()).to_bits() + ); + } + + binary_uf16( + &left, + &right, + &mut output, + |left, right| left * right, + |left, right| left * right, + ) + .unwrap(); + for ((left, right), actual) in left.iter().zip(&right).zip(&output) { + assert_eq!( + actual.to_bits(), + T::from_f32(left.to_f32() * right.to_f32()).to_bits() + ); + } + + binary_uf16( + &left, + &right, + &mut output, + |left, right| left / right, + |left, right| left / right, + ) + .unwrap(); + for ((left, right), actual) in left.iter().zip(&right).zip(&output) { + assert_eq!( + actual.to_bits(), + T::from_f32(left.to_f32() / right.to_f32()).to_bits() + ); + } + } + + #[test] + fn uf16e5m11_bulk_paths_are_bit_exact() { + verify_uf16_conversions::(); + verify_uf16_binary::(); + } + + #[test] + fn uf16e6m10_bulk_paths_are_bit_exact() { + verify_uf16_conversions::(); + verify_uf16_binary::(); + } + + #[test] + fn uf32_bulk_paths_are_bit_exact() { + let mut state = 0x03_2d_99_ef_u64; + let mut source = vec![Uf32::ZERO, Uf32::MIN_POSITIVE, Uf32::INFINITY, Uf32::NAN]; + source.extend((0..32_767).map(|_| Uf32::from_bits(lcg(&mut state) as u32))); + let mut decoded = vec![0.0; source.len()]; + decode_uf32_to_f64(&source, &mut decoded).unwrap(); + for (value, actual) in source.iter().copied().zip(decoded) { + assert_eq!(actual.to_bits(), value.to_f64().to_bits()); + } + + let mut encoded_input = vec![0.0; 32_771]; + encoded_input[0] = 0.0; + encoded_input[1] = -0.0; + encoded_input[2] = f64::INFINITY; + encoded_input[3] = f64::NAN; + for value in &mut encoded_input[4..] { + *value = f64::from_bits(lcg(&mut state)); + } + let mut encoded = vec![Uf32::ZERO; encoded_input.len()]; + encode_f64_to_uf32(&encoded_input, &mut encoded).unwrap(); + for (value, actual) in encoded_input.into_iter().zip(encoded) { + assert_eq!(actual.to_bits(), Uf32::from_f64(value).to_bits()); + } + + let left: Vec = (0..(UF32_LANES * 19 + 1)) + .map(|_| Uf32::from_bits(lcg(&mut state) as u32)) + .collect(); + let right: Vec = (0..left.len()) + .map(|_| Uf32::from_bits(lcg(&mut state) as u32)) + .collect(); + let mut output = vec![Uf32::ZERO; left.len()]; + macro_rules! assert_uf32_binary { + ($vector:expr, $scalar:expr) => {{ + binary_uf32(&left, &right, &mut output, $vector, $scalar).unwrap(); + for ((left, right), actual) in left.iter().zip(&right).zip(&output) { + assert_eq!( + actual.to_bits(), + Uf32::from_f64($scalar(left.to_f64(), right.to_f64())).to_bits() + ); + } + }}; + } + assert_uf32_binary!(|left, right| left + right, |left: f64, right: f64| left + + right); + assert_uf32_binary!(|left, right| left - right, |left: f64, right: f64| left + - right); + assert_uf32_binary!(|left, right| left * right, |left: f64, right: f64| left + * right); + assert_uf32_binary!(|left, right| left / right, |left: f64, right: f64| left + / right); + } + + #[test] + fn bulk_operations_reject_mismatched_planes() { + let input = [Uf16::ONE; 2]; + let other = [Uf16::ONE; 1]; + let mut output = [Uf16::ZERO; 2]; + assert_eq!( + add_uf16(&input, &other, &mut output), + Err(SimdError::InputLengthMismatch { left: 2, right: 1 }) + ); + let mut short = [0.0; 1]; + assert_eq!( + decode_uf16_to_f32(&input, &mut short), + Err(SimdError::OutputLengthMismatch { + input: 2, + output: 1 + }) + ); + } +}