diff --git a/Justfile b/Justfile index d146313..d2b45e1 100644 --- a/Justfile +++ b/Justfile @@ -726,6 +726,7 @@ package-test: sys_entry_count=$(wc -l < "$sys_list") safe_package_size=$(stat -c '%s' "$safe_package") safe_unpacked_size=$(du -sb "$safe_root" | cut -f1) + safe_package_sha256=$(sha256sum "$safe_package" | awk '{ print $1 }') safe_entry_count=$(wc -l < "$safe_list") ((sys_package_size <= 6 * 1024 * 1024)) ((sys_unpacked_size <= 40 * 1024 * 1024)) @@ -871,8 +872,68 @@ package-test: printf 'sys package: %d entries, %d regular-file bytes, %d du bytes, %d compressed bytes, sha256 %s\n' \ "$sys_entry_count" "$sys_regular_file_bytes" "$sys_unpacked_size" \ "$sys_package_size" "$sys_package_sha256" - printf 'safe package: %d entries, %d bytes unpacked, %d bytes compressed\n' \ - "$safe_entry_count" "$safe_unpacked_size" "$safe_package_size" + printf 'safe package: %d entries, %d bytes unpacked, %d bytes compressed, sha256 %s\n' \ + "$safe_entry_count" "$safe_unpacked_size" "$safe_package_size" \ + "$safe_package_sha256" + +[private] +_native-capi-known-flake output: + #!/usr/bin/env bash + set -euo pipefail + python3 - {{ quote(output) }} <<'PY' + import re + import sys + + text = open(sys.argv[1], encoding="utf-8", errors="replace").read() + + cases = re.findall(r"^\s*TEST CASE:\s*(.*?)\s*$", text, re.MULTILINE) + if cases != ["capi: vllm_complete_stream early-stop tears the request down cleanly"]: + raise SystemExit(f"native C API failure was not the sole known test case: {cases}") + + errors = re.findall( + r"^.*test_capi\.cpp:(\d+): ERROR:\s*(.*?)\s*$", text, re.MULTILINE + ) + expected_error = [("680", "CHECK( acc.deltas == 2 ) is NOT correct!")] + if errors != expected_error: + raise SystemExit(f"native C API failure did not have the exact known assertion: {errors}") + + values = re.findall(r"^\s*values:\s*(.*?)\s*$", text, re.MULTILINE) + if values != ["CHECK( 1 == 2 )"]: + raise SystemExit(f"native C API failure did not observe exactly 1 == 2: {values}") + + case_summaries = re.findall( + r"^\[doctest\]\s+test cases:\s*(\d+)\s*\|\s*(\d+) passed\s*\|\s*" + r"(\d+) failed\s*\|\s*(\d+) skipped\s*$", + text, + re.MULTILINE, + ) + if case_summaries != [("49", "48", "1", "0")]: + raise SystemExit(f"native C API test-case summary was not the known sole failure: {case_summaries}") + + assertion_summaries = re.findall( + r"^\[doctest\]\s+assertions:\s*(\d+)\s*\|\s*(\d+) passed\s*\|\s*" + r"(\d+) failed\s*\|\s*$", + text, + re.MULTILINE, + ) + if len(assertion_summaries) != 1: + raise SystemExit(f"native C API assertion summary was ambiguous: {assertion_summaries}") + total, passed, failed = map(int, assertion_summaries[0]) + if failed != 1 or passed + failed != total: + raise SystemExit(f"native C API assertion summary was not one failure: {assertion_summaries}") + + ctest_summaries = re.findall( + r"^(\d+)% tests passed, (\d+) tests failed out of (\d+)$", text, re.MULTILINE + ) + if ctest_summaries != [("0", "1", "1")]: + raise SystemExit(f"CTest summary was not the exact test_capi failure: {ctest_summaries}") + + failed_tests = re.findall( + r"^\s*[0-9]+\s+-\s+(\S+)\s+\(([^)]+)\)\s*$", text, re.MULTILINE + ) + if failed_tests != [("test_capi", "Failed")]: + raise SystemExit(f"CTest failure list was not exactly test_capi: {failed_tests}") + PY # Build and run the focused native CPU C API fixture gate. native-capi: @@ -904,14 +965,34 @@ native-capi: listing=$(mktemp) output=$(mktemp) - trap 'rm -f "$listing" "$output"' EXIT + retry_output=$(mktemp) + trap 'rm -f "$listing" "$output" "$retry_output"' EXIT ctest --test-dir "$build" -N --tests-regex '^test_capi$' | tee "$listing" test_count=$(grep -Ec '^[[:space:]]*Test #[0-9]+: test_capi$' "$listing") [[ $test_count -eq 1 ]] grep -Fxq 'Total Tests: 1' "$listing" - ctest --test-dir "$build" --output-on-failure --tests-regex '^test_capi$' \ - | tee "$output" - grep -Fxq '100% tests passed, 0 tests failed out of 1' "$output" + + set +e + env -u VT_ASYNC_SCHED -u VT_ASYNC_RUNNER \ + ctest --test-dir "$build" --output-on-failure --tests-regex '^test_capi$' \ + 2>&1 | tee "$output" + default_status=${PIPESTATUS[0]} + set -e + if [[ $default_status -eq 0 ]]; then + grep -Fxq '100% tests passed, 0 tests failed out of 1' "$output" + exit 0 + fi + + just --justfile {{ quote(root + "/Justfile") }} \ + _native-capi-known-flake "$output" + # The synchronous scheduler makes pending-delta delivery cardinality + # deterministic while the complete suite still exercises early-stop abort, + # request teardown, and engine reuse through the unchanged C ABI test. + echo 'retrying complete test_capi once with VT_ASYNC_SCHED=0' >&2 + env -u VT_ASYNC_RUNNER VT_ASYNC_SCHED=0 \ + ctest --test-dir "$build" --output-on-failure --tests-regex '^test_capi$' \ + 2>&1 | tee "$retry_output" + grep -Fxq '100% tests passed, 0 tests failed out of 1' "$retry_output" # Run sys-first crates.io publication checks without uploading. publish-dry-run: diff --git a/RELEASING.md b/RELEASING.md index d4f0ab9..bd7c87a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -4,7 +4,27 @@ Releases are prepared and published manually. A candidate pass or dry-run does n ## Prepare a candidate -1. Start from the exact independently reviewed release commit. Require a clean root worktree and index and a clean, detached native submodule: +1. Obtain `VLLM_CPP_RELEASE_COMMIT` from the independent approval of the exact release commit. Never derive the expected value from the current checkout. Require a lowercase, full 40-character SHA, verify that it names a commit object, and compare it with the root checkout before any candidate gate: + + ```bash + set -euo pipefail + : "${VLLM_CPP_RELEASE_COMMIT:?set this to the independently approved release commit}" + if [[ ! $VLLM_CPP_RELEASE_COMMIT =~ ^[0-9a-f]{40}$ ]]; then + echo 'VLLM_CPP_RELEASE_COMMIT must be exactly 40 lowercase hexadecimal characters' >&2 + exit 1 + fi + git cat-file -e "$VLLM_CPP_RELEASE_COMMIT^{commit}" + actual_release_commit=$(git rev-parse HEAD) + if [[ $actual_release_commit != "$VLLM_CPP_RELEASE_COMMIT" ]]; then + printf 'reviewed release commit mismatch: expected %s, found %s\n' \ + "$VLLM_CPP_RELEASE_COMMIT" "$actual_release_commit" >&2 + exit 1 + fi + ``` + + Preserve the externally supplied value and rerun this guard immediately before each separately authorized upload. + +2. Require a clean root worktree and index and a clean, detached native submodule: ```console test -z "$(git status --short --untracked-files=all)" @@ -15,7 +35,7 @@ Releases are prepared and published manually. A candidate pass or dry-run does n test -z "$(git -C vllm-cpp-sys/vllm.cpp status --short --untracked-files=all)" ``` -2. Verify native identity directly, never with `git describe`: +3. Verify native identity directly, never with `git describe`: ```console native=vllm-cpp-sys/vllm.cpp @@ -25,18 +45,19 @@ Releases are prepared and published manually. A candidate pass or dry-run does n test "$(git -C "$native" rev-parse 'HEAD^{tree}')" = 28df226f0ef9924e67d563c3bef4712d0e628c5a ``` -3. Use `cargo metadata --locked --no-deps --format-version 1` and normalized package manifests to require both Rust crates at `0.0.2` and the safe dependency requirement exactly `=0.0.2`. Confirm both local package records in `Cargo.lock`. Separately parse native `project(vllm_cpp VERSION 0.0.2 LANGUAGES CXX)` from `CMakeLists.txt`. Rust and native versions are independent release identities that happen to both be `0.0.2` here; equality is not a universal policy. -4. Require `VLLM_ABI_VERSION == 17` in the pinned header and generated bindings and exactly 35 stable C functions. Run binding-drift, C11/C++20 header, every C/Rust layout and signature, runtime ABI, all-function link, and exact dynamic-export checks. ABI-10 system libraries are incompatible. -5. Keep `Unreleased` empty above the dated release entry. Describe only validated support and preserve known limitations. -6. Audit root and crate dual-license metadata, `LICENSE-MIT`, `LICENSE-APACHE`, native `LICENSE`/`NOTICE`, `THIRD_PARTY.md`, and every imported license text against the exact package inventories. Reject models, media fixtures, build output, caches, SDKs, external CUTLASS trees, internal records, and repository-local paths. +4. Use `cargo metadata --locked --no-deps --format-version 1` and normalized package manifests to require both Rust crates at `0.0.2` and the safe dependency requirement exactly `=0.0.2`. Confirm both local package records in `Cargo.lock`. Separately parse native `project(vllm_cpp VERSION 0.0.2 LANGUAGES CXX)` from `CMakeLists.txt`. Rust and native versions are independent release identities that happen to both be `0.0.2` here; equality is not a universal policy. +5. Require `VLLM_ABI_VERSION == 17` in the pinned header and generated bindings and exactly 35 stable C functions. Run binding-drift, C11/C++20 header, every C/Rust layout and signature, runtime ABI, all-function link, and exact dynamic-export checks. ABI-10 system libraries are incompatible. +6. Keep `Unreleased` empty above the dated release entry. Describe only validated support and preserve known limitations. +7. Audit root and crate dual-license metadata, `LICENSE-MIT`, `LICENSE-APACHE`, native `LICENSE`/`NOTICE`, `THIRD_PARTY.md`, and every imported license text against the exact package inventories. Reject models, media fixtures, build output, caches, SDKs, external CUTLASS trees, internal records, and repository-local paths. ## Validate -Run the mandatory Linux x86_64 CPU gates from the pinned shell: +Run the mandatory Linux x86_64 CPU gates directly with the native Cargo and Just workflows: ```console -env -u VLLM_CPP_TEST_MODEL nix develop -c just ci -nix develop .#msrv -c just msrv +env -u VLLM_CPP_TEST_MODEL just ci +RUSTUP_TOOLCHAIN=1.85.0 just msrv +# Equivalent exact-toolchain invocation: rustup run 1.85.0 just msrv cargo check --locked --workspace --all-targets --features vllm-cpp/serde RUSTDOCFLAGS='-D warnings' cargo doc --locked --workspace --no-deps --features vllm-cpp/serde git diff --check @@ -44,6 +65,8 @@ git diff --check `just ci` includes formatting, warnings-denied lint/docs, model-free workspace tests, generated bindings and ABI conformance, four CPU link modes, the native C API fixture, model-free ASan/UBSan/leak checks, package/extracted/downstream validation, and no-upload publish dry-run. Run the exact MSRV gate separately so stable-toolchain success cannot mask it. +Nix support remains an optional convenience, not a prerequisite. Maintainers who choose it may run `nix develop -c env -u VLLM_CPP_TEST_MODEL just ci` and `nix develop .#msrv -c just msrv`; `nix flake check --no-build` is an additional Nix-specific evaluation check, not an ordinary release gate. + Prepared-Qwen inference/sanitizers, native-only TSan, successful Rust MiniMax-H3 generation, Miri, Linux ARM64, Apple ARM64, Vulkan, CUDA/CUTLASS/Triton, Metal/MLX, and accelerator runtime are optional or deferred. Record one only when it ran against the exact candidate; configured workflows and older results are not candidate evidence. ## Inspect packages @@ -68,19 +91,108 @@ Require identical sorted inventories and semantically identical normalized manif `just publish-dry-run` uses Cargo's sys-first workspace order with `--no-verify` and never uploads. It cannot provide the safe crate's full registry-resolution verification before exact sys `0.0.2` is available from crates.io. Check both crate-version slots are available before any future upload; do not reserve or publish them during candidate preparation. +## Inspect registry state + +Use the exact-version crates.io API read-only. This helper classifies only HTTP 200 with matching crate/version JSON as `accepted` and only HTTP 404 as `absent`; network errors, redirects that do not finish in either status, other HTTP statuses, and malformed or mismatched JSON are `ambiguous`. Stop on `ambiguous`. + +```bash +set -euo pipefail +command -v curl >/dev/null +command -v jq >/dev/null +registry_tmp=$(mktemp -d) +trap 'rm -rf "$registry_tmp"' EXIT HUP INT TERM + +registry_state() { + local crate=$1 + local version=$2 + local body status + if [[ ! $crate =~ ^[a-z0-9][a-z0-9_-]*$ || ! $version =~ ^[0-9A-Za-z.+-]+$ ]]; then + printf '%s\n' ambiguous + return + fi + if ! body=$(mktemp "$registry_tmp/response.XXXXXX"); then + printf '%s\n' ambiguous + return + fi + if ! status=$(curl --silent --show-error --location \ + --connect-timeout 10 --max-time 30 --retry 0 \ + --output "$body" --write-out '%{http_code}' -- \ + "https://crates.io/api/v1/crates/$crate/$version"); then + printf '%s\n' ambiguous + return + fi + case $status in + 200) + if jq -e --arg crate "$crate" --arg version "$version" \ + '.version.crate == $crate and .version.num == $version' \ + "$body" >/dev/null 2>&1; then + printf '%s\n' accepted + else + printf '%s\n' ambiguous + fi + ;; + 404) printf '%s\n' absent ;; + *) printf '%s\n' ambiguous ;; + esac +} + +sys_state=$(registry_state vllm-cpp-sys 0.0.2) +safe_state=$(registry_state vllm-cpp 0.0.2) +rm -rf "$registry_tmp" +trap - EXIT HUP INT TERM +printf 'vllm-cpp-sys 0.0.2: %s\nvllm-cpp 0.0.2: %s\n' \ + "$sys_state" "$safe_state" +``` + +The helper performs no registry mutation. Preserve its output with the release evidence. Before an initial sys upload, both states must be `absent`. Before any safe upload or retry, sys must be `accepted` and safe must be `absent`. If safe is already `accepted`, never republish it. Any other combination requires stopping for diagnosis. + ## Publish -Only a separately authorized maintainer may publish from the exact reviewed commit with a clean root and detached submodule. Publish sys first: +Only a separately authorized maintainer may publish from the exact reviewed commit with a clean root and detached submodule. Rerun the reviewed-root guard immediately before each upload. Publish sys first: ```console cargo publish -p vllm-cpp-sys --locked -# Wait until crates.io serves exact vllm-cpp-sys 0.0.2. +# Wait until the exact-version helper reports sys accepted and safe absent. cargo publish -p vllm-cpp --locked --dry-run cargo publish -p vllm-cpp --locked ``` The full safe dry-run must resolve registry sys `0.0.2` before the safe upload. After both uploads, verify registry metadata, archives, docs.rs, licenses, and a clean downstream build. Create a tag and GitHub release only after separate authorization and only for the exact published commit. +### Retry or verify the safe crate + +A safe upload retry is allowed only for the same independently approved archive bytes. Obtain the approved hash externally; never copy a dirty-candidate hash into this document or derive the expected value from a rebuilt archive. + +```bash +set -euo pipefail +: "${VLLM_CPP_SAFE_ARCHIVE_SHA256:?set this to the approved safe archive SHA-256}" +if [[ ! $VLLM_CPP_SAFE_ARCHIVE_SHA256 =~ ^[0-9a-f]{64}$ ]]; then + echo 'VLLM_CPP_SAFE_ARCHIVE_SHA256 must be exactly 64 lowercase hexadecimal characters' >&2 + exit 1 +fi +safe_archive=target/package/vllm-cpp-0.0.2.crate +actual_safe_sha256=$(sha256sum "$safe_archive" | awk '{print $1}') +if [[ $actual_safe_sha256 != "$VLLM_CPP_SAFE_ARCHIVE_SHA256" ]]; then + echo 'safe archive differs from the approved bytes; prepare a new coordinated version' >&2 + exit 1 +fi +``` + +Rerun the registry helper after this check. Retry only when `sys_state=accepted` and `safe_state=absent`, and only after separate upload authorization. `ambiguous` means stop. If `safe_state=accepted`, do not upload; instead verify the accepted archive against the same approved hash before post-publication checks: + +```bash +set -euo pipefail +accepted_safe=$(mktemp) +trap 'rm -f "$accepted_safe"' EXIT HUP INT TERM +curl --fail --silent --show-error --location --connect-timeout 10 --max-time 60 \ + --retry 0 --output "$accepted_safe" -- \ + 'https://crates.io/api/v1/crates/vllm-cpp/0.0.2/download' +printf '%s %s\n' "$VLLM_CPP_SAFE_ARCHIVE_SHA256" "$accepted_safe" \ + | sha256sum --check --strict +``` + +A registry archive mismatch or any need to change the safe bytes requires a new coordinated version; never reuse `0.0.2` for different bytes. + ## Abort and recovery - Before upload, abort on any mismatch, failed gate, unexpected file, dirty state, changed lockfile, native identity/ABI/export drift, inaccurate support statement, or unavailable version. Fix it in a separately reviewed commit and restart. diff --git a/vllm-cpp/src/abi.rs b/vllm-cpp/src/abi.rs index 1361eb1..ec41451 100644 --- a/vllm-cpp/src/abi.rs +++ b/vllm-cpp/src/abi.rs @@ -62,54 +62,6 @@ impl Compatibility { pub(crate) fn check_with(abi_version: impl FnOnce() -> i32) -> Result { Self::from_actual(abi_version()) } - - #[cfg(test)] - fn model_params_default_with( - &self, - default: impl FnOnce() -> ffi::vllm_model_params, - ) -> ffi::vllm_model_params { - default() - } - - #[cfg(test)] - fn sampling_params_default_with( - &self, - default: impl FnOnce() -> ffi::vllm_sampling_params, - ) -> ffi::vllm_sampling_params { - default() - } - - #[cfg(test)] - fn transcription_params_default_with( - &self, - default: impl FnOnce() -> ffi::vllm_transcription_params, - ) -> ffi::vllm_transcription_params { - default() - } - - #[cfg(test)] - fn video_model_params_default_with( - &self, - default: impl FnOnce() -> ffi::vllm_video_model_params, - ) -> ffi::vllm_video_model_params { - default() - } - - #[cfg(test)] - fn video_params_default_with( - &self, - default: impl FnOnce() -> ffi::vllm_video_params, - ) -> ffi::vllm_video_params { - default() - } - - #[cfg(test)] - fn video_mux_params_default_with( - &self, - default: impl FnOnce() -> ffi::vllm_video_mux_params, - ) -> ffi::vllm_video_mux_params { - default() - } } #[cfg(test)] @@ -136,58 +88,4 @@ mod tests { )); assert_eq!(*calls.borrow(), ["abi"]); } - - #[test] - fn compatibility_precedes_all_by_value_defaults() { - let calls = RefCell::new(Vec::new()); - let compatibility = Compatibility::check_with(|| { - calls.borrow_mut().push("abi"); - 17 - }) - .expect("matching compatibility token"); - - compatibility.model_params_default_with(|| { - calls.borrow_mut().push("model_default"); - // SAFETY: every field in this generated C struct permits zero. - unsafe { std::mem::zeroed() } - }); - compatibility.sampling_params_default_with(|| { - calls.borrow_mut().push("sampling_default"); - // SAFETY: every field in this generated C struct permits zero. - unsafe { std::mem::zeroed() } - }); - compatibility.transcription_params_default_with(|| { - calls.borrow_mut().push("transcription_default"); - // SAFETY: every field in this generated C struct permits zero. - unsafe { std::mem::zeroed() } - }); - compatibility.video_model_params_default_with(|| { - calls.borrow_mut().push("video_model_default"); - // SAFETY: every field in this generated C struct permits zero. - unsafe { std::mem::zeroed() } - }); - compatibility.video_params_default_with(|| { - calls.borrow_mut().push("video_default"); - // SAFETY: every field in this generated C struct permits zero. - unsafe { std::mem::zeroed() } - }); - compatibility.video_mux_params_default_with(|| { - calls.borrow_mut().push("video_mux_default"); - // SAFETY: every field in this generated C struct permits zero. - unsafe { std::mem::zeroed() } - }); - - assert_eq!( - *calls.borrow(), - [ - "abi", - "model_default", - "sampling_default", - "transcription_default", - "video_model_default", - "video_default", - "video_mux_default" - ] - ); - } } diff --git a/vllm-cpp/src/engine.rs b/vllm-cpp/src/engine.rs index f7630f8..6d82599 100644 --- a/vllm-cpp/src/engine.rs +++ b/vllm-cpp/src/engine.rs @@ -249,6 +249,30 @@ pub struct EngineBuilder { config: ModelConfig, } +/// Restricted builder for a blocking transcription engine. +/// +/// Transcription uses native model defaults except for an optional device +/// selection. Text-generation, memory, and scheduler controls are intentionally +/// unavailable because the native transcription pipeline ignores them. This is +/// an ordinary cloneable, `Send + Sync` configuration value; the loaded owner is +/// thread-local and performs blocking operations with exclusive access. +#[derive(Clone, Debug)] +pub struct TranscriptionEngineBuilder { + config: ModelConfig, +} + +/// Restricted builder for a blocking embedding engine. +/// +/// This builder exposes only capacity, prefix-cache, device, and memory controls +/// used by native embedding loads. Text parsers, speculative decoding, scheduler +/// policy, KV transfer, and jump-forward settings are intentionally unavailable. +/// This is an ordinary cloneable, `Send + Sync` configuration value; the loaded +/// owner is thread-local and performs blocking operations with exclusive access. +#[derive(Clone, Debug)] +pub struct EmbeddingEngineBuilder { + config: ModelConfig, +} + #[derive(Clone, Debug)] struct ModelConfig { model_path: PathBuf, @@ -1508,16 +1532,44 @@ impl EngineBuilder { } } +impl TranscriptionEngineBuilder { + #[must_use] + pub fn new(model_path: impl Into) -> Self { + Self { + config: ModelConfig::new(model_path), + } + } + + /// Selects the required native device. + /// + /// Native transcription-only checkpoints currently reject [`Device::Cuda`] + /// rather than silently falling back to CPU. + #[must_use] + pub fn device(mut self, value: Device) -> Self { + self.config.device = Some(value); + self + } + + pub fn load(self) -> Result { + Ok(TranscriptionEngine { + inner: load_engine::(self.config)?, + }) + } +} + impl TranscriptionEngine { + /// Starts a restricted transcription engine configuration. + pub fn builder(model_path: impl Into) -> TranscriptionEngineBuilder { + TranscriptionEngineBuilder::new(model_path) + } + /// Loads a native engine owner with a transcription-only Rust method surface. /// /// ABI 17 cannot inspect the resolved task at load time. This constructor does /// not probe or infer checkpoint architecture; native task selection and /// wrong-task diagnostics remain authoritative. pub fn load(model_path: impl Into) -> Result { - Ok(Self { - inner: load_engine::(ModelConfig::new(model_path))?, - }) + Self::builder(model_path).load() } /// Runs one blocking transcription and returns Rust-owned text and token IDs. @@ -1539,16 +1591,98 @@ impl TranscriptionEngine { } } +impl EmbeddingEngineBuilder { + #[must_use] + pub fn new(model_path: impl Into) -> Self { + Self { + config: ModelConfig::new(model_path), + } + } + + #[must_use] + pub fn block_size(mut self, value: u32) -> Self { + self.config.block_size = Some(value); + self + } + + #[must_use] + pub fn num_blocks(mut self, value: u32) -> Self { + self.config.num_blocks = Some(value); + self + } + + #[must_use] + pub fn max_model_len(mut self, value: u32) -> Self { + self.config.max_model_len = Some(value); + self + } + + #[must_use] + pub fn max_num_seqs(mut self, value: u32) -> Self { + self.config.max_num_seqs = Some(value); + self + } + + #[must_use] + pub fn max_num_batched_tokens(mut self, value: u32) -> Self { + self.config.max_num_batched_tokens = Some(value); + self + } + + #[must_use] + pub fn prefix_caching(mut self, value: Toggle) -> Self { + self.config.prefix_caching = value; + self + } + + /// Selects the required native device without fallback. + #[must_use] + pub fn device(mut self, value: Device) -> Self { + self.config.device = Some(value); + self + } + + /// Sets the native fraction used by GPU memory profiling. + /// + /// The value must be finite and strictly positive. An explicit block count + /// takes precedence over absolute KV-cache bytes, which take precedence over + /// this utilization/profile setting. + #[must_use] + pub fn gpu_memory_utilization(mut self, value: f64) -> Self { + self.config.gpu_memory_utilization = Some(value); + self + } + + /// Sets an absolute KV-cache memory budget in bytes. + /// + /// The value must be nonzero and fit the native signed 64-bit field. An + /// explicit block count takes precedence over this budget. + #[must_use] + pub fn kv_cache_memory_bytes(mut self, value: u64) -> Self { + self.config.kv_cache_memory_bytes = Some(value); + self + } + + pub fn load(self) -> Result { + Ok(EmbeddingEngine { + inner: load_engine::(self.config)?, + }) + } +} + impl EmbeddingEngine { + /// Starts a restricted embedding engine configuration. + pub fn builder(model_path: impl Into) -> EmbeddingEngineBuilder { + EmbeddingEngineBuilder::new(model_path) + } + /// Loads a native engine owner with an embedding-only Rust method surface. /// /// ABI 17 cannot inspect the resolved task at load time. This constructor does /// not probe or infer checkpoint architecture; native task selection and /// wrong-task diagnostics remain authoritative. pub fn load(model_path: impl Into) -> Result { - Ok(Self { - inner: load_engine::(ModelConfig::new(model_path))?, - }) + Self::builder(model_path).load() } /// Runs one blocking native embedding batch. @@ -2358,11 +2492,12 @@ mod tests { embedding_from_raw, generate_video_with, load_engine_with, load_video_engine_with, token_completion_from_raw, transcribe_with, transcription_from_raw, validate_embedding_count, validate_pcm_input, validate_pointer_count, validate_token_input, - video_mux_argv_from_raw, Device, EmbeddingTask, MarshaledEmbeddingInput, - MarshaledModelParams, MarshaledTranscriptionInput, MarshaledVideoGenerationParams, - MarshaledVideoModelParams, ModelConfig, SchedulerPolicy, TextTask, Toggle, - TranscriptionInput, TranscriptionTask, VideoDevice, VideoEngine, VideoGenerationParams, - VideoModelConfig, VideoMuxParams, VideoPartition, + video_mux_argv_from_raw, Device, EmbeddingEngineBuilder, EmbeddingTask, + MarshaledEmbeddingInput, MarshaledModelParams, MarshaledTranscriptionInput, + MarshaledVideoGenerationParams, MarshaledVideoModelParams, ModelConfig, SchedulerPolicy, + TextTask, Toggle, TranscriptionEngineBuilder, TranscriptionInput, TranscriptionTask, + VideoDevice, VideoEngine, VideoGenerationParams, VideoModelConfig, VideoMuxParams, + VideoPartition, }; use crate::abi::Compatibility; use crate::Error; @@ -2493,6 +2628,93 @@ mod tests { assert_eq!(raw.gpu_memory_utilization, 2.0); } + #[test] + fn restricted_task_builders_map_only_their_exposed_configuration() { + let transcription = TranscriptionEngineBuilder::new("transcription-model") + .device(Device::Cuda) + .config; + assert_eq!( + transcription.model_path, + PathBuf::from("transcription-model") + ); + assert_eq!(transcription.device, Some(Device::Cuda)); + assert_eq!(transcription.tokenizer_config_path, None); + assert_eq!(transcription.block_size, None); + assert_eq!(transcription.num_blocks, None); + assert_eq!(transcription.max_model_len, None); + assert_eq!(transcription.max_num_seqs, None); + assert_eq!(transcription.tool_parser, None); + assert_eq!(transcription.reasoning_parser, None); + assert_eq!(transcription.speculative_config, None); + assert_eq!(transcription.prefix_caching, Toggle::Default); + assert_eq!(transcription.max_num_batched_tokens, None); + assert_eq!(transcription.scheduler, None); + assert_eq!(transcription.kv_transfer_config, None); + assert_eq!(transcription.jump_forward, Toggle::Default); + assert_eq!(transcription.gpu_memory_utilization, None); + assert_eq!(transcription.kv_cache_memory_bytes, None); + + let embedding = EmbeddingEngineBuilder::new("embedding-model") + .block_size(16) + .num_blocks(32) + .max_model_len(128) + .max_num_seqs(2) + .max_num_batched_tokens(64) + .prefix_caching(Toggle::Off) + .device(Device::Cpu) + .gpu_memory_utilization(1.25) + .kv_cache_memory_bytes(4096) + .config; + assert_eq!(embedding.model_path, PathBuf::from("embedding-model")); + assert_eq!(embedding.block_size, Some(16)); + assert_eq!(embedding.num_blocks, Some(32)); + assert_eq!(embedding.max_model_len, Some(128)); + assert_eq!(embedding.max_num_seqs, Some(2)); + assert_eq!(embedding.max_num_batched_tokens, Some(64)); + assert_eq!(embedding.prefix_caching, Toggle::Off); + assert_eq!(embedding.device, Some(Device::Cpu)); + assert_eq!(embedding.gpu_memory_utilization, Some(1.25)); + assert_eq!(embedding.kv_cache_memory_bytes, Some(4096)); + assert_eq!(embedding.tokenizer_config_path, None); + assert_eq!(embedding.tool_parser, None); + assert_eq!(embedding.reasoning_parser, None); + assert_eq!(embedding.speculative_config, None); + assert_eq!(embedding.scheduler, None); + assert_eq!(embedding.kv_transfer_config, None); + assert_eq!(embedding.jump_forward, Toggle::Default); + } + + #[test] + fn restricted_task_builder_defaults_preserve_unexposed_native_values() { + for config in [ + TranscriptionEngineBuilder::new("transcription-model").config, + EmbeddingEngineBuilder::new("embedding-model").config, + ] { + let defaults = native_defaults(); + let expected_path = config.model_path.to_string_lossy().into_owned(); + let mut params = MarshaledModelParams::new(config).expect("marshal task builder"); + params.apply_defaults(defaults); + let raw = params.raw(); + assert_eq!(c_string(raw.model_path), expected_path); + assert_eq!(raw.tokenizer_config_path, defaults.tokenizer_config_path); + assert_eq!(raw.block_size, defaults.block_size); + assert_eq!(raw.num_blocks, defaults.num_blocks); + assert_eq!(raw.max_model_len, defaults.max_model_len); + assert_eq!(raw.max_num_seqs, defaults.max_num_seqs); + assert_eq!(raw.tool_parser, defaults.tool_parser); + assert_eq!(raw.reasoning_parser, defaults.reasoning_parser); + assert_eq!(raw.speculative_config, defaults.speculative_config); + assert_eq!(raw.enable_prefix_caching, defaults.enable_prefix_caching); + assert_eq!(raw.max_num_batched_tokens, defaults.max_num_batched_tokens); + assert_eq!(raw.scheduling_policy, defaults.scheduling_policy); + assert_eq!(raw.kv_transfer_config, defaults.kv_transfer_config); + assert_eq!(raw.enable_jump_forward, defaults.enable_jump_forward); + assert_eq!(raw.device, defaults.device); + assert_eq!(raw.gpu_memory_utilization, defaults.gpu_memory_utilization); + assert_eq!(raw.kv_cache_memory_bytes, defaults.kv_cache_memory_bytes); + } + } + fn assert_shared_load_order() { let calls = RefCell::new(Vec::new()); let loaded = load_engine_with::( diff --git a/vllm-cpp/src/lib.rs b/vllm-cpp/src/lib.rs index b5f850d..bb7faef 100644 --- a/vllm-cpp/src/lib.rs +++ b/vllm-cpp/src/lib.rs @@ -4,8 +4,10 @@ //! //! Resolve a Hub model with [`HuggingFaceModel`] (default `main`, or an explicit //! revision), then create a text [`Engine`] with [`Engine::load`] or configure -//! native model settings through [`EngineBuilder`]. [`TranscriptionEngine`] and -//! [`EmbeddingEngine`] provide blocking task-specific operations. A separate +//! native model settings through [`EngineBuilder`]. [`TranscriptionEngineBuilder`] +//! exposes only device selection, while [`EmbeddingEngineBuilder`] exposes the +//! native capacity, prefix-cache, device, and memory controls applicable to +//! embeddings. Their task-specific engines provide blocking operations. A separate //! [`VideoEngine`] loads MiniMax-H3 checkpoint sets and performs exclusive, //! blocking generation; [`compose_video_mux_argv`] composes owned ffmpeg argument //! boundaries without executing a process. [`SamplingParams`] owns sampling, @@ -35,9 +37,9 @@ //! Asynchronous callbacks run on a native delivery thread, must be `Send + 'static`, and surface panic through //! [`Error::CallbackPanicked`]. ABI version 17 forbids waiting for or freeing a //! request from its callback thread; callback-thread drop delegates ownership to -//! a cleanup reaper instead. ABI 17 exposes no task-introspection API, so loading -//! cannot prove or infer a checkpoint's task. Native task selection and future -//! wrong-task diagnostics remain authoritative. Video model format, partition, +//! a cleanup reaper instead. ABI 17 exposes no task-introspection API, so native +//! selects the task at load time and a successful load does not prove compatibility +//! with a Rust task owner. Wrong-task operations remain native errors. Video model format, partition, //! checkpoint capability, and reference-media checks are likewise native //! authority; Rust performs only structural validation. //! @@ -89,10 +91,10 @@ mod request; pub use callback::{StreamControl, StreamEvent, StreamOutcome}; pub use engine::{ - compose_video_mux_argv, Completion, EmbeddingEngine, EmbeddingResult, Engine, EngineBuilder, - FinishReason, TokenCompletion, Transcription, TranscriptionEngine, TranscriptionInput, - VideoDevice, VideoEngine, VideoEngineBuilder, VideoGenerationParams, VideoMuxArgv, - VideoMuxParams, VideoPartition, VideoResult, + compose_video_mux_argv, Completion, EmbeddingEngine, EmbeddingEngineBuilder, EmbeddingResult, + Engine, EngineBuilder, FinishReason, TokenCompletion, Transcription, TranscriptionEngine, + TranscriptionEngineBuilder, TranscriptionInput, VideoDevice, VideoEngine, VideoEngineBuilder, + VideoGenerationParams, VideoMuxArgv, VideoMuxParams, VideoPartition, VideoResult, }; pub use error::{Error, HuggingFaceError}; pub use hf::HuggingFaceModel; diff --git a/vllm-cpp/tests/qwen3.rs b/vllm-cpp/tests/qwen3.rs index 8fd2f70..83ae2ea 100644 --- a/vllm-cpp/tests/qwen3.rs +++ b/vllm-cpp/tests/qwen3.rs @@ -5,9 +5,9 @@ use std::thread; use std::time::{Duration, Instant}; use vllm_cpp::{ - compose_video_mux_argv, EmbeddingEngine, Engine, Error, FinishReason, Request, RequestOutcome, - SamplingParams, StreamControl, StructuredOutput, TranscriptionEngine, TranscriptionInput, - VideoEngine, VideoMuxParams, + compose_video_mux_argv, Device, EmbeddingEngine, Engine, Error, FinishReason, Request, + RequestOutcome, SamplingParams, StreamControl, StructuredOutput, TranscriptionEngine, + TranscriptionInput, VideoEngine, VideoMuxParams, }; fn model_path() -> Option { @@ -72,24 +72,317 @@ fn native_fixture(relative: &str) -> Option { } } -fn read_pcm16_mono_wav(path: &Path) -> Vec { - let bytes = std::fs::read(path).expect("read fixture WAV"); - assert_eq!(&bytes[0..4], b"RIFF"); - assert_eq!(&bytes[8..12], b"WAVE"); - let mut offset = 12; - while offset + 8 <= bytes.len() { - let name = &bytes[offset..offset + 4]; - let size = u32::from_le_bytes(bytes[offset + 4..offset + 8].try_into().unwrap()) as usize; - let start = offset + 8; - if name == b"data" { - return bytes[start..start + size] - .chunks_exact(2) - .map(|sample| i16::from_le_bytes([sample[0], sample[1]]) as f32 / 32768.0) - .collect(); +#[derive(Debug, PartialEq)] +struct FixtureWav { + samples: Vec, + sample_rate: u32, +} + +fn decode_pcm16_mono_wav(bytes: &[u8]) -> Result { + if bytes.len() < 12 { + return Err("truncated RIFF/WAVE header".to_owned()); + } + if &bytes[..4] != b"RIFF" { + return Err("missing RIFF identifier".to_owned()); + } + if &bytes[8..12] != b"WAVE" { + return Err("missing WAVE identifier".to_owned()); + } + + let riff_size = usize::try_from(u32::from_le_bytes( + bytes[4..8] + .try_into() + .map_err(|_| "truncated RIFF size".to_owned())?, + )) + .map_err(|_| "RIFF size exceeds address space".to_owned())?; + let riff_end = 8usize + .checked_add(riff_size) + .ok_or_else(|| "RIFF size overflows address space".to_owned())?; + if riff_end < 12 { + return Err("RIFF size does not include the WAVE identifier".to_owned()); + } + if riff_end > bytes.len() { + return Err("declared RIFF end exceeds input length".to_owned()); + } + if riff_end != bytes.len() { + return Err("bytes remain after the declared RIFF end".to_owned()); + } + + let mut fmt = None; + let mut data = None; + let mut offset = 12usize; + while offset < riff_end { + let header_end = offset + .checked_add(8) + .ok_or_else(|| "chunk header offset overflows address space".to_owned())?; + if header_end > riff_end { + return Err(format!("truncated chunk header at byte {offset}")); + } + let name: [u8; 4] = bytes[offset..offset + 4] + .try_into() + .map_err(|_| format!("truncated chunk identifier at byte {offset}"))?; + let size = usize::try_from(u32::from_le_bytes( + bytes[offset + 4..header_end] + .try_into() + .map_err(|_| format!("truncated chunk size at byte {offset}"))?, + )) + .map_err(|_| format!("chunk length exceeds address space at byte {offset}"))?; + let body_end = header_end + .checked_add(size) + .ok_or_else(|| format!("chunk length overflows address space at byte {offset}"))?; + if body_end > riff_end { + return Err(format!("chunk length exceeds RIFF bounds at byte {offset}")); + } + let padded_end = body_end + .checked_add(size & 1) + .ok_or_else(|| format!("chunk padding overflows address space at byte {offset}"))?; + if padded_end > riff_end { + return Err(format!("missing padding byte for chunk at byte {offset}")); + } + + match &name { + b"fmt " => { + if fmt.is_some() { + return Err("duplicate fmt chunk".to_owned()); + } + if size < 16 { + return Err(format!("fmt chunk is too short: {size} bytes")); + } + let body = &bytes[header_end..body_end]; + let format = u16::from_le_bytes([body[0], body[1]]); + let channels = u16::from_le_bytes([body[2], body[3]]); + let sample_rate = u32::from_le_bytes(body[4..8].try_into().unwrap()); + let byte_rate = u32::from_le_bytes(body[8..12].try_into().unwrap()); + let block_align = u16::from_le_bytes([body[12], body[13]]); + let bits_per_sample = u16::from_le_bytes([body[14], body[15]]); + if format != 1 { + return Err(format!("WAV format code must be PCM 1, found {format}")); + } + if channels != 1 { + return Err(format!("WAV must be mono, found {channels} channels")); + } + if sample_rate != 16_000 { + return Err(format!( + "WAV sample rate must be 16000, found {sample_rate}" + )); + } + if bits_per_sample != 16 { + return Err(format!( + "WAV sample width must be 16 bits, found {bits_per_sample}" + )); + } + if block_align != 2 { + return Err(format!( + "WAV block alignment must be 2, found {block_align}" + )); + } + if byte_rate != 32_000 { + return Err(format!("WAV byte rate must be 32000, found {byte_rate}")); + } + fmt = Some(sample_rate); + } + b"data" => { + if data.is_some() { + return Err("duplicate data chunk".to_owned()); + } + if size == 0 { + return Err("WAV data chunk is empty".to_owned()); + } + if size % 2 != 0 { + return Err(format!("WAV data chunk has odd size {size}")); + } + data = Some((header_end, body_end)); + } + _ => {} + } + offset = padded_end; + } + + let sample_rate = fmt.ok_or_else(|| "missing fmt chunk".to_owned())?; + let (data_start, data_end) = data.ok_or_else(|| "missing data chunk".to_owned())?; + let samples = bytes[data_start..data_end] + .chunks_exact(2) + .map(|sample| i16::from_le_bytes([sample[0], sample[1]]) as f32 / 32768.0) + .collect(); + Ok(FixtureWav { + samples, + sample_rate, + }) +} + +fn read_pcm16_mono_wav(path: &Path) -> Result { + let bytes = std::fs::read(path) + .map_err(|error| format!("failed to read fixture WAV {}: {error}", path.display()))?; + decode_pcm16_mono_wav(&bytes) +} + +#[cfg(test)] +fn fixture_fmt_chunk() -> Vec { + let mut fmt = Vec::new(); + fmt.extend_from_slice(&1u16.to_le_bytes()); + fmt.extend_from_slice(&1u16.to_le_bytes()); + fmt.extend_from_slice(&16_000u32.to_le_bytes()); + fmt.extend_from_slice(&32_000u32.to_le_bytes()); + fmt.extend_from_slice(&2u16.to_le_bytes()); + fmt.extend_from_slice(&16u16.to_le_bytes()); + fmt +} + +#[cfg(test)] +fn fixture_wav_bytes(chunks: &[([u8; 4], Vec)]) -> Vec { + let mut body = b"WAVE".to_vec(); + for (name, chunk) in chunks { + body.extend_from_slice(name); + body.extend_from_slice(&(chunk.len() as u32).to_le_bytes()); + body.extend_from_slice(chunk); + if chunk.len() % 2 != 0 { + body.push(0); } - offset = start + size + (size % 2); } - panic!("fixture WAV has no data chunk"); + let mut bytes = b"RIFF".to_vec(); + bytes.extend_from_slice(&(body.len() as u32).to_le_bytes()); + bytes.extend_from_slice(&body); + bytes +} + +#[cfg(test)] +fn assert_wav_error(bytes: &[u8], expected: &str) { + let error = decode_pcm16_mono_wav(bytes).expect_err("malformed WAV must fail"); + assert!( + error.contains(expected), + "expected {expected:?} in WAV error {error:?}" + ); +} + +#[test] +fn wav_parser_accepts_unknown_padding_and_data_before_fmt() { + let data = [i16::MIN.to_le_bytes(), 0i16.to_le_bytes()].concat(); + let bytes = fixture_wav_bytes(&[ + (*b"JUNK", vec![7]), + (*b"data", data), + (*b"fmt ", fixture_fmt_chunk()), + ]); + let wav = decode_pcm16_mono_wav(&bytes).expect("valid PCM fixture WAV"); + assert_eq!(wav.sample_rate, 16_000); + assert_eq!(wav.samples, [-1.0, 0.0]); +} + +#[test] +fn wav_parser_rejects_identifiers_and_riff_bounds() { + assert_wav_error(&[], "truncated RIFF/WAVE header"); + + let valid = fixture_wav_bytes(&[(*b"fmt ", fixture_fmt_chunk()), (*b"data", vec![0, 0])]); + let mut wrong_riff = valid.clone(); + wrong_riff[..4].copy_from_slice(b"RIFX"); + assert_wav_error(&wrong_riff, "missing RIFF identifier"); + let mut wrong_wave = valid.clone(); + wrong_wave[8..12].copy_from_slice(b"WVAE"); + assert_wav_error(&wrong_wave, "missing WAVE identifier"); + + let mut short_riff = valid.clone(); + short_riff[4..8].copy_from_slice(&3u32.to_le_bytes()); + assert_wav_error( + &short_riff, + "RIFF size does not include the WAVE identifier", + ); + + let mut truncated = valid.clone(); + let declared = u32::from_le_bytes(truncated[4..8].try_into().unwrap()); + truncated[4..8].copy_from_slice(&(declared + 1).to_le_bytes()); + assert_wav_error(&truncated, "declared RIFF end exceeds input length"); + + let mut trailing = valid; + trailing.push(0); + assert_wav_error(&trailing, "bytes remain after the declared RIFF end"); +} + +#[test] +fn wav_parser_rejects_truncated_chunks_and_padding() { + let mut header = b"RIFF".to_vec(); + header.extend_from_slice(&7u32.to_le_bytes()); + header.extend_from_slice(b"WAVEabc"); + assert_wav_error(&header, "truncated chunk header"); + + let mut body = b"WAVEdata".to_vec(); + body.extend_from_slice(&4u32.to_le_bytes()); + body.extend_from_slice(&[0, 0]); + let mut truncated_body = b"RIFF".to_vec(); + truncated_body.extend_from_slice(&(body.len() as u32).to_le_bytes()); + truncated_body.extend_from_slice(&body); + assert_wav_error(&truncated_body, "chunk length exceeds RIFF bounds"); + + let mut odd_body = b"WAVEJUNK".to_vec(); + odd_body.extend_from_slice(&1u32.to_le_bytes()); + odd_body.push(7); + let mut missing_padding = b"RIFF".to_vec(); + missing_padding.extend_from_slice(&(odd_body.len() as u32).to_le_bytes()); + missing_padding.extend_from_slice(&odd_body); + assert_wav_error(&missing_padding, "missing padding byte"); + + let mut huge_body = b"WAVEdata".to_vec(); + huge_body.extend_from_slice(&u32::MAX.to_le_bytes()); + let mut huge = b"RIFF".to_vec(); + huge.extend_from_slice(&(huge_body.len() as u32).to_le_bytes()); + huge.extend_from_slice(&huge_body); + assert_wav_error(&huge, "chunk length exceeds RIFF bounds"); +} + +#[test] +fn wav_parser_rejects_invalid_format_metadata() { + let cases = [ + (0usize, 3u16.to_le_bytes().to_vec(), "format code"), + (2, 2u16.to_le_bytes().to_vec(), "mono"), + (4, 8_000u32.to_le_bytes().to_vec(), "sample rate"), + (8, 16_000u32.to_le_bytes().to_vec(), "byte rate"), + (12, 4u16.to_le_bytes().to_vec(), "block alignment"), + (14, 8u16.to_le_bytes().to_vec(), "sample width"), + ]; + for (offset, replacement, expected) in cases { + let mut fmt = fixture_fmt_chunk(); + fmt[offset..offset + replacement.len()].copy_from_slice(&replacement); + let bytes = fixture_wav_bytes(&[(*b"fmt ", fmt), (*b"data", vec![0, 0])]); + assert_wav_error(&bytes, expected); + } + + let short = fixture_wav_bytes(&[(*b"fmt ", vec![0; 15]), (*b"data", vec![0, 0])]); + assert_wav_error(&short, "fmt chunk is too short"); +} + +#[test] +fn wav_parser_rejects_missing_duplicate_and_invalid_data_chunks() { + let fmt = fixture_fmt_chunk(); + assert_wav_error( + &fixture_wav_bytes(&[(*b"data", vec![0, 0])]), + "missing fmt chunk", + ); + assert_wav_error( + &fixture_wav_bytes(&[(*b"fmt ", fmt.clone())]), + "missing data chunk", + ); + assert_wav_error( + &fixture_wav_bytes(&[ + (*b"fmt ", fmt.clone()), + (*b"fmt ", fmt.clone()), + (*b"data", vec![0, 0]), + ]), + "duplicate fmt chunk", + ); + assert_wav_error( + &fixture_wav_bytes(&[ + (*b"fmt ", fmt.clone()), + (*b"data", vec![0, 0]), + (*b"data", vec![0, 0]), + ]), + "duplicate data chunk", + ); + assert_wav_error( + &fixture_wav_bytes(&[(*b"fmt ", fmt.clone()), (*b"data", Vec::new())]), + "data chunk is empty", + ); + assert_wav_error( + &fixture_wav_bytes(&[(*b"fmt ", fmt), (*b"data", vec![0])]), + "data chunk has odd size", + ); } #[test] @@ -163,18 +456,30 @@ fn committed_transcription_fixture_supports_path_pcm_and_wrong_task() { }; let model = root.join("ctc"); let wav = root.join("audio.wav"); - let mut engine = TranscriptionEngine::load(&model).expect("load CTC fixture"); + let mut engine = TranscriptionEngine::builder(&model) + .device(Device::Cpu) + .load() + .expect("load CTC fixture on CPU"); + let cuda_error = TranscriptionEngine::builder(&model) + .device(Device::Cuda) + .load() + .err() + .expect("transcription fixture must refuse explicit CUDA"); + assert!( + matches!(cuda_error, Error::InvalidArgument { .. }), + "{cuda_error:?}" + ); let from_path = engine .transcribe(TranscriptionInput::WavFile(&wav)) .expect("transcribe fixture path"); assert_eq!(from_path.token_ids, [3, 4, 3]); assert_eq!(from_path.text.as_deref(), Some("atheat")); - let samples = read_pcm16_mono_wav(&wav); + let fixture = read_pcm16_mono_wav(&wav).expect("parse fixture WAV"); let from_pcm = engine .transcribe(TranscriptionInput::Pcm { - samples: &samples, - sample_rate: 16_000, + samples: &fixture.samples, + sample_rate: fixture.sample_rate, }) .expect("transcribe fixture PCM"); assert_eq!(from_pcm, from_path); @@ -200,7 +505,18 @@ fn committed_embedding_fixture_preserves_shape_order_ownership_and_wrong_task() let Some(model) = native_fixture("llama_embed_e2e") else { return; }; - let mut engine = EmbeddingEngine::load(&model).expect("load embedding fixture"); + let mut engine = EmbeddingEngine::builder(&model) + .block_size(16) + .num_blocks(32) + .max_model_len(128) + .max_num_seqs(2) + .max_num_batched_tokens(128) + .prefix_caching(vllm_cpp::Toggle::Off) + .device(Device::Cpu) + .gpu_memory_utilization(1.25) + .kv_cache_memory_bytes(4096) + .load() + .expect("load configured embedding fixture"); let result = engine .embed(["the quick brown fox", "the lazy dog"]) .expect("embed fixture inputs"); diff --git a/vllm-cpp/tests/safe_api.rs b/vllm-cpp/tests/safe_api.rs index 14c4ac1..23a5762 100644 --- a/vllm-cpp/tests/safe_api.rs +++ b/vllm-cpp/tests/safe_api.rs @@ -3,16 +3,18 @@ use std::path::Path; use static_assertions::{assert_impl_all, assert_not_impl_any}; use vllm_cpp::{ - compose_video_mux_argv, Device, EmbeddingEngine, EmbeddingResult, Engine, EngineBuilder, Error, - HuggingFaceError, HuggingFaceModel, Request, SchedulerPolicy, Toggle, TokenCompletion, - Transcription, TranscriptionEngine, TranscriptionInput, VideoDevice, VideoEngine, - VideoEngineBuilder, VideoGenerationParams, VideoMuxArgv, VideoMuxParams, VideoPartition, - VideoResult, + compose_video_mux_argv, Device, EmbeddingEngine, EmbeddingEngineBuilder, EmbeddingResult, + Engine, EngineBuilder, Error, HuggingFaceError, HuggingFaceModel, Request, SchedulerPolicy, + Toggle, TokenCompletion, Transcription, TranscriptionEngine, TranscriptionEngineBuilder, + TranscriptionInput, VideoDevice, VideoEngine, VideoEngineBuilder, VideoGenerationParams, + VideoMuxArgv, VideoMuxParams, VideoPartition, VideoResult, }; assert_impl_all!(Device: Clone, Copy, std::fmt::Debug, Default, Eq, PartialEq, Send, Sync); assert_impl_all!(Engine: Send, Sync, Clone); assert_impl_all!(EngineBuilder: Clone, std::fmt::Debug, Send, Sync); +assert_impl_all!(TranscriptionEngineBuilder: Clone, std::fmt::Debug, Send, Sync); +assert_impl_all!(EmbeddingEngineBuilder: Clone, std::fmt::Debug, Send, Sync); assert_impl_all!(HuggingFaceError: Clone, std::fmt::Debug, Eq, PartialEq); assert_impl_all!(HuggingFaceModel: Clone, std::fmt::Debug); assert_impl_all!(Request: Send); @@ -29,8 +31,8 @@ assert_impl_all!(VideoMuxParams: Clone, std::fmt::Debug, Send, Sync); assert_impl_all!(VideoResult: Clone, std::fmt::Debug, Eq, PartialEq, Send, Sync); assert_impl_all!(VideoMuxArgv: Clone, std::fmt::Debug, Eq, PartialEq, Send, Sync); assert_not_impl_any!(Request: Sync); -assert_not_impl_any!(TranscriptionEngine: Send, Sync); -assert_not_impl_any!(EmbeddingEngine: Send, Sync); +assert_not_impl_any!(TranscriptionEngine: Send, Sync, Clone); +assert_not_impl_any!(EmbeddingEngine: Send, Sync, Clone); assert_not_impl_any!(VideoEngine: Send, Sync, Clone); fn missing_model() -> &'static str { @@ -82,15 +84,33 @@ fn reports_expected_abi() { } #[test] -fn missing_model_is_typed_for_every_task_owner() { +fn missing_model_is_typed_for_every_task_owner_and_builder() { let errors = [ Engine::load(missing_model()).unwrap_err(), TranscriptionEngine::load(missing_model()) .err() .expect("missing transcription model error"), + TranscriptionEngine::builder(missing_model()) + .device(Device::Cpu) + .load() + .err() + .expect("configured missing transcription model error"), EmbeddingEngine::load(missing_model()) .err() .expect("missing embedding model error"), + EmbeddingEngine::builder(missing_model()) + .block_size(16) + .num_blocks(32) + .max_model_len(128) + .max_num_seqs(2) + .max_num_batched_tokens(64) + .prefix_caching(Toggle::Off) + .device(Device::Cpu) + .gpu_memory_utilization(1.25) + .kv_cache_memory_bytes(4096) + .load() + .err() + .expect("configured missing embedding model error"), ]; for error in errors { assert!(matches!(error, Error::ModelLoad { .. }), "{error:?}"); @@ -109,13 +129,25 @@ fn malformed_engine_json_is_invalid_argument_before_loading() { #[test] fn interior_nul_fails_before_ffi() { - let error = Engine::builder("bad\0model").load().unwrap_err(); - assert_eq!( - error, - Error::InteriorNul { - field: "model path" - } - ); + let errors = [ + Engine::builder("bad\0model").load().unwrap_err(), + TranscriptionEngineBuilder::new("bad\0model") + .load() + .err() + .expect("transcription interior NUL"), + EmbeddingEngineBuilder::new("bad\0model") + .load() + .err() + .expect("embedding interior NUL"), + ]; + for error in errors { + assert_eq!( + error, + Error::InteriorNul { + field: "model path" + } + ); + } } #[test] @@ -149,28 +181,44 @@ fn engine_builder_accepts_all_safe_options() { #[test] fn rejects_invalid_gpu_memory_utilization() { for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY, 0.0, -0.0, -1.0] { - let error = Engine::builder(missing_model()) - .gpu_memory_utilization(value) - .load() - .unwrap_err(); - assert!( - matches!(error, Error::InvalidConfiguration { .. }), - "{value:?}: {error:?}" - ); + for error in [ + Engine::builder(missing_model()) + .gpu_memory_utilization(value) + .load() + .unwrap_err(), + EmbeddingEngine::builder(missing_model()) + .gpu_memory_utilization(value) + .load() + .err() + .expect("invalid embedding utilization"), + ] { + assert!( + matches!(error, Error::InvalidConfiguration { .. }), + "{value:?}: {error:?}" + ); + } } } #[test] fn rejects_invalid_kv_cache_memory_bytes() { for value in [0, i64::MAX as u64 + 1, u64::MAX] { - let error = Engine::builder(missing_model()) - .kv_cache_memory_bytes(value) - .load() - .unwrap_err(); - assert!( - matches!(error, Error::InvalidConfiguration { .. }), - "{value}: {error:?}" - ); + for error in [ + Engine::builder(missing_model()) + .kv_cache_memory_bytes(value) + .load() + .unwrap_err(), + EmbeddingEngine::builder(missing_model()) + .kv_cache_memory_bytes(value) + .load() + .err() + .expect("invalid embedding KV bytes"), + ] { + assert!( + matches!(error, Error::InvalidConfiguration { .. }), + "{value}: {error:?}" + ); + } } }