diff --git a/.github/workflows/platforms.yml b/.github/workflows/platforms.yml index 6f81931..f68afdc 100644 --- a/.github/workflows/platforms.yml +++ b/.github/workflows/platforms.yml @@ -13,7 +13,7 @@ concurrency: env: CMAKE_BUILD_PARALLEL_LEVEL: "2" CMAKE_GENERATOR: Ninja - NATIVE_SHA: 34aedfbe8ed9779697905541a62e2160ccfd9c05 + NATIVE_SHA: 7020de93652ca920424a10ac5255b34810dd2f24 jobs: msrv: diff --git a/Justfile b/Justfile index d841c30..fa4c63a 100644 --- a/Justfile +++ b/Justfile @@ -164,7 +164,10 @@ link-modes: vllm_chat_stream vllm_complete vllm_complete_stream + vllm_complete_tokens vllm_completion_free + vllm_embed + vllm_embedding_result_free vllm_engine_free vllm_engine_load vllm_last_error @@ -176,8 +179,21 @@ link-modes: vllm_request_submit vllm_request_wait vllm_sampling_params_default + vllm_server_main vllm_string_free + vllm_transcribe + vllm_transcription_free + vllm_transcription_params_default vllm_version + vllm_video_engine_free + vllm_video_engine_load + vllm_video_generate + vllm_video_model_params_default + vllm_video_mux_argv + vllm_video_mux_argv_free + vllm_video_mux_params_default + vllm_video_params_default + vllm_video_result_free EOF diff -u "$expected" "$actual" } @@ -701,8 +717,8 @@ package-test: use vllm_cpp_sys as ffi; fn main() { - assert_eq!(ffi::VLLM_ABI_VERSION, 10); - assert_eq!(unsafe { ffi::vllm_abi_version() }, 10); + assert_eq!(ffi::VLLM_ABI_VERSION, 17); + assert_eq!(unsafe { ffi::vllm_abi_version() }, 17); let version = unsafe { CStr::from_ptr(ffi::vllm_version()) }; assert!(!version.to_bytes().is_empty()); @@ -772,8 +788,8 @@ package-test: }; fn main() { - assert_eq!(expected_abi_version(), 10); - assert_eq!(abi_version(), 10); + assert_eq!(expected_abi_version(), 17); + assert_eq!(abi_version(), 17); assert!(!version().expect("native version").is_empty()); let _params = SamplingParams::greedy() .max_tokens(1) diff --git a/vllm-cpp-sys/README.md b/vllm-cpp-sys/README.md index cbc3649..2693b60 100644 --- a/vllm-cpp-sys/README.md +++ b/vllm-cpp-sys/README.md @@ -2,7 +2,7 @@ Raw Rust bindings and native linking for the stable C API of [vllm.cpp](https://github.com/mudler/vllm.cpp). -This crate exposes checked-in generated unsafe declarations for the 19 exported C symbols in ABI version 10. Callers are responsible for pointer validity, lifetimes, callback threading, status/error handling, and matching every native allocation with its documented free function. Applications should prefer the current safe [`vllm-cpp`](https://docs.rs/vllm-cpp) crate unless they require direct ABI access. Ordinary consumers do not need Just, bindgen, or libclang. +This crate exposes checked-in generated unsafe declarations for the 35 exported C symbols in ABI version 17. Callers are responsible for pointer validity, lifetimes, callback threading, status/error handling, and matching every native allocation with its documented free function. Applications should prefer the current safe [`vllm-cpp`](https://docs.rs/vllm-cpp) crate unless they require direct ABI access. Ordinary consumers do not need Just, bindgen, or libclang. The package contains Rust declarations and conformance tests, native build/link integration, the pinned native source inputs required by the supported feature set, and their licenses/notices. It excludes upstream tests, fixtures, models, examples, benchmarks, agent records, fetched SDKs, external CUTLASS trees, and build output. See the repository [changelog](https://github.com/querymt/vllm-cpp-rs/blob/main/CHANGELOG.md) and [release process](https://github.com/querymt/vllm-cpp-rs/blob/main/RELEASING.md) for the coordinated crate boundary. @@ -14,7 +14,7 @@ The package contains Rust declarations and conformance tests, native build/link `bundled` and `system` are mutually exclusive. CPU runtime support is limited to native Linux x86_64; Linux aarch64 and Apple ARM64 have model-free build/test surfaces. Accelerator features below are experimental build configuration. -System mode requires `VLLM_CPP_ROOT`, whose prefix must contain `include/vllm.h` plus a `lib` or `lib64` directory. `VLLM_CPP_LIB_DIR` can override the vllm library directory. Consumer builds validate that the selected system header exists but do not compare its layout. The maintainer integration test compiles its C probe at test runtime against that header, compares its C layouts with the generated Rust declarations, and the runtime test requires ABI version 10. +System mode requires `VLLM_CPP_ROOT`, whose prefix must contain `include/vllm.h` plus a `lib` or `lib64` directory. `VLLM_CPP_LIB_DIR` can override the vllm library directory. Consumer builds validate that the selected system header exists but do not compare its layout. The maintainer integration test compiles its C probe at test runtime against that header, compares its C layouts with the generated Rust declarations, and the runtime test requires ABI version 17. Upstream's normal CMake install provides `libvllm` and `vllm.h` but does not install the private `libblake3_vendored.a` target. A stock install therefore works directly with `system,dynamic-link`. System static mode requires callers to provision the matching `libblake3_vendored.a` separately and set `VLLM_CPP_BLAKE3_LIB_DIR`; when the variable is unset, the build script checks the selected vllm library directory for backward compatibility. It does not search arbitrary build trees. @@ -41,7 +41,7 @@ These features do not claim runtime support. Known native blockers remain: a CUD ## Generated Bindings -The bundled source is pinned to commit `34aedfbe8ed9779697905541a62e2160ccfd9c05` and exposes C ABI version 10. Bindings are generated with bindgen 0.72.1 from `wrapper.h`, which includes `vllm.cpp/include/vllm.h`, and committed to `src/bindings.rs`. The exported stable C boundary is narrower than the broader native C++ implementation; these declarations do not promise access to undocumented internals. Maintainers use Just 1.40 or newer from the repository root: +The bundled source is pinned to vllm.cpp `v0.0.2` at commit `7020de93652ca920424a10ac5255b34810dd2f24` and exposes C ABI version 17. Bindings are generated with bindgen 0.72.1 from `wrapper.h`, which includes `vllm.cpp/include/vllm.h`, and committed to `src/bindings.rs`. The exported stable C boundary is narrower than the broader native C++ implementation; these declarations do not promise access to undocumented internals. Maintainers use Just 1.40 or newer from the repository root: ```console just bindings @@ -49,7 +49,7 @@ just sys just link-modes ``` -The conformance gate verifies the generated output, C and C++ header compatibility, C/Rust layout, the exact 19-symbol export set, pure backend plans/cache parsing, Apple/MLX validation and link ordering, pinned CUDA architecture mappings, Triton AOT drift, and compile-time/runtime ABI 10. CI separately runs bundled static/dynamic and fixture-backed system static/dynamic CPU tests; dynamic tests set the required loader path. `build.rs` only performs consumer native build/link integration and does not compile or execute the layout probe. `tests/layout.rs` compiles and executes `tests/layout.c` with the bundled header or `VLLM_CPP_ROOT/include/vllm.h` at test runtime using the Rust standard library. Native Linux CPU is the supported runtime target; cross-compiling that integration test is unsupported. +The conformance gate verifies the generated output, C and C++ header compatibility, C/Rust layout, the exact 35-symbol export set, pure backend plans/cache parsing, Apple/MLX validation and link ordering, pinned CUDA architecture mappings, Triton AOT drift, and compile-time/runtime ABI 17. CI separately runs bundled static/dynamic and fixture-backed system static/dynamic CPU tests; dynamic tests set the required loader path. `build.rs` only performs consumer native build/link integration and does not compile or execute the layout probe. `tests/layout.rs` compiles and executes `tests/layout.c` with the bundled header or `VLLM_CPP_ROOT/include/vllm.h` at test runtime using the Rust standard library. Native Linux CPU is the supported runtime target; cross-compiling that integration test is unsupported. The Rust crate is dual-licensed under MIT or Apache-2.0. The bundled vllm.cpp source retains its upstream Apache-2.0 license and notices. diff --git a/vllm-cpp-sys/THIRD_PARTY.md b/vllm-cpp-sys/THIRD_PARTY.md index aea57ab..574ad7b 100644 --- a/vllm-cpp-sys/THIRD_PARTY.md +++ b/vllm-cpp-sys/THIRD_PARTY.md @@ -4,7 +4,7 @@ | Component | Packaged source | Provenance | License text | |---|---|---|---| -| vllm.cpp | `vllm.cpp/**` | vllm.cpp commit `34aedfbe8ed9779697905541a62e2160ccfd9c05` | `vllm.cpp/LICENSE`, `vllm.cpp/NOTICE` | +| vllm.cpp | `vllm.cpp/**` | vllm.cpp `v0.0.2`, commit `7020de93652ca920424a10ac5255b34810dd2f24` | `vllm.cpp/LICENSE`, `vllm.cpp/NOTICE` | | BLAKE3 C reference | `vllm.cpp/third_party/blake3/**` | BLAKE3 1.5.5, commit `81f772a` | `vllm.cpp/third_party/blake3/LICENSE_A2`, `vllm.cpp/third_party/blake3/LICENSE_CC0` | | google/minja | `vllm.cpp/third_party/minja/**` | minja commit `021c229` | `vllm.cpp/third_party/minja/LICENSE` | | Vulkan-Headers | `vllm.cpp/third_party/vulkan/**` | Vulkan SDK 1.4.328.1, generated Khronos headers | Apache-2.0 in each generated header and `vllm.cpp/LICENSE` | diff --git a/vllm-cpp-sys/src/bindings.rs b/vllm-cpp-sys/src/bindings.rs index 9cd44c6..b78ab9c 100644 --- a/vllm-cpp-sys/src/bindings.rs +++ b/vllm-cpp-sys/src/bindings.rs @@ -1,6 +1,6 @@ /* automatically generated by rust-bindgen 0.72.1 */ -pub const VLLM_ABI_VERSION: u32 = 10; +pub const VLLM_ABI_VERSION: u32 = 17; pub const vllm_status_VLLM_OK: vllm_status = 0; pub const vllm_status_VLLM_ERR_INVALID_ARGUMENT: vllm_status = 1; pub const vllm_status_VLLM_ERR_MODEL_LOAD: vllm_status = 2; @@ -34,6 +34,9 @@ pub struct vllm_model_params { pub scheduling_policy: *const ::std::os::raw::c_char, pub kv_transfer_config: *const ::std::os::raw::c_char, pub enable_jump_forward: i32, + pub device: i32, + pub gpu_memory_utilization: f64, + pub kv_cache_memory_bytes: i64, } pub type vllm_logits_processor = ::std::option::Option< unsafe extern "C" fn( @@ -101,6 +104,18 @@ unsafe extern "C" { out: *mut vllm_completion, ) -> vllm_status; } +unsafe extern "C" { + pub fn vllm_complete_tokens( + engine: *mut vllm_engine, + prompt_tokens: *const i32, + n_prompt_tokens: i32, + params: *const vllm_sampling_params, + out_tokens: *mut i32, + max_out_tokens: i32, + n_out_tokens: *mut i32, + out: *mut vllm_completion, + ) -> vllm_status; +} pub type vllm_token_callback = ::std::option::Option< unsafe extern "C" fn( delta_text: *const ::std::os::raw::c_char, @@ -157,6 +172,153 @@ unsafe extern "C" { user_data: *mut ::std::os::raw::c_void, ) -> vllm_status; } +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_transcription_params { + pub audio_path: *const ::std::os::raw::c_char, + pub pcm: *const f32, + pub n_samples: i64, + pub sample_rate: i32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_transcription { + pub text: *mut ::std::os::raw::c_char, + pub token_ids: *mut i32, + pub n_token_ids: i32, + pub has_text: i32, +} +unsafe extern "C" { + pub fn vllm_transcription_params_default() -> vllm_transcription_params; +} +unsafe extern "C" { + pub fn vllm_transcribe( + engine: *mut vllm_engine, + params: *const vllm_transcription_params, + out: *mut vllm_transcription, + ) -> vllm_status; +} +unsafe extern "C" { + pub fn vllm_transcription_free(out: *mut vllm_transcription); +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_embedding_result { + pub values: *mut f32, + pub n_embeddings: i32, + pub dim: i32, + pub prompt_tokens: i32, +} +unsafe extern "C" { + pub fn vllm_embed( + engine: *mut vllm_engine, + texts: *const *const ::std::os::raw::c_char, + n_texts: i32, + out: *mut vllm_embedding_result, + ) -> vllm_status; +} +unsafe extern "C" { + pub fn vllm_embedding_result_free(out: *mut vllm_embedding_result); +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_video_engine { + _unused: [u8; 0], +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_video_model_params { + pub dit_path: *const ::std::os::raw::c_char, + pub encoder_path: *const ::std::os::raw::c_char, + pub tokenizer_path: *const ::std::os::raw::c_char, + pub video_vae_path: *const ::std::os::raw::c_char, + pub video_vae_config_path: *const ::std::os::raw::c_char, + pub audio_vae_path: *const ::std::os::raw::c_char, + pub audio_vae_config_path: *const ::std::os::raw::c_char, + pub prompt_embeds_path: *const ::std::os::raw::c_char, + pub partition: *const ::std::os::raw::c_char, + pub device: i32, + pub dequant_bf16: i32, + pub fp4_resident: i32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_video_params { + pub prompt: *const ::std::os::raw::c_char, + pub width: i32, + pub height: i32, + pub num_frames: i32, + pub steps: i32, + pub seed: u64, + pub has_seed: i32, + pub first_frame: *const ::std::os::raw::c_char, + pub last_frame: *const ::std::os::raw::c_char, + pub ref_image: *const ::std::os::raw::c_char, + pub ref_video: *const ::std::os::raw::c_char, + pub ref_audio: *const ::std::os::raw::c_char, + pub noise_aug: f32, + pub output_dir: *const ::std::os::raw::c_char, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_video_result { + pub frame_dir: *mut ::std::os::raw::c_char, + pub audio_path: *mut ::std::os::raw::c_char, + pub frame_count: i32, + pub width: i32, + pub height: i32, + pub fps: i32, + pub sample_rate: i32, + pub mux_argv: *mut *mut ::std::os::raw::c_char, + pub mux_argc: i32, +} +unsafe extern "C" { + pub fn vllm_video_model_params_default() -> vllm_video_model_params; +} +unsafe extern "C" { + pub fn vllm_video_params_default() -> vllm_video_params; +} +unsafe extern "C" { + pub fn vllm_video_engine_load( + params: *const vllm_video_model_params, + out: *mut *mut vllm_video_engine, + ) -> vllm_status; +} +unsafe extern "C" { + pub fn vllm_video_engine_free(engine: *mut vllm_video_engine); +} +unsafe extern "C" { + pub fn vllm_video_generate( + engine: *mut vllm_video_engine, + params: *const vllm_video_params, + out: *mut vllm_video_result, + ) -> vllm_status; +} +unsafe extern "C" { + pub fn vllm_video_result_free(out: *mut vllm_video_result); +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vllm_video_mux_params { + pub frames: *const ::std::os::raw::c_char, + pub audio_path: *const ::std::os::raw::c_char, + pub output_path: *const ::std::os::raw::c_char, + pub fps: i32, + pub crf: i32, +} +unsafe extern "C" { + pub fn vllm_video_mux_params_default() -> vllm_video_mux_params; +} +unsafe extern "C" { + pub fn vllm_video_mux_argv( + params: *const vllm_video_mux_params, + out_argv: *mut *mut *mut ::std::os::raw::c_char, + out_argc: *mut i32, + ) -> vllm_status; +} +unsafe extern "C" { + pub fn vllm_video_mux_argv_free(argv: *mut *mut ::std::os::raw::c_char, argc: i32); +} unsafe extern "C" { pub fn vllm_string_free(s: *mut ::std::os::raw::c_char); } @@ -169,6 +331,9 @@ unsafe extern "C" { unsafe extern "C" { pub fn vllm_version() -> *const ::std::os::raw::c_char; } +unsafe extern "C" { + pub fn vllm_server_main(argc: i32, argv: *mut *mut ::std::os::raw::c_char) -> i32; +} unsafe extern "C" { pub fn vllm_abi_version() -> i32; } diff --git a/vllm-cpp-sys/src/lib.rs b/vllm-cpp-sys/src/lib.rs index 9da0808..885310a 100644 --- a/vllm-cpp-sys/src/lib.rs +++ b/vllm-cpp-sys/src/lib.rs @@ -1,9 +1,9 @@ //! Raw FFI declarations for the stable vllm.cpp C API. //! //! The checked-in bindings are generated from `vllm.cpp/include/vllm.h` and -//! expose that header's 19-symbol C boundary, versioned structs, constants, and -//! callback signatures. They target ABI version 10 from pinned vllm.cpp commit -//! `34aedfbe8ed9779697905541a62e2160ccfd9c05`. This exported ABI is narrower than +//! expose that header's 35-symbol C boundary, versioned structs, constants, and +//! callback signatures. They target ABI version 17 from pinned vllm.cpp commit +//! `7020de93652ca920424a10ac5255b34810dd2f24` (`v0.0.2`). This exported ABI is narrower than //! the broader native C++ implementation and does not expose undocumented //! vllm.cpp internals. //! @@ -14,7 +14,7 @@ //! header. They must check returned status values, copy thread-local error text //! before another API call on that thread, and release engines, requests, //! completions, and allocated strings with their matching `vllm_*_free` -//! functions. In particular, ABI version 10 prohibits waiting for or freeing a +//! functions. In particular, ABI version 17 prohibits waiting for or freeing a //! request from that request's callback thread. Prefer the safe `vllm-cpp` crate //! unless direct ABI access is required. //! diff --git a/vllm-cpp-sys/tests/layout.c b/vllm-cpp-sys/tests/layout.c index 923fdcb..8dcc4b4 100644 --- a/vllm-cpp-sys/tests/layout.c +++ b/vllm-cpp-sys/tests/layout.c @@ -5,11 +5,23 @@ #define PRINT_LAYOUT(type) \ printf(#type " %zu %zu\n", sizeof(type), _Alignof(type)) +#define PRINT_NAMED_LAYOUT(name, type) \ + printf(#name " %zu %zu\n", sizeof(type), _Alignof(type)) #define PRINT_OFFSET(type, field) \ printf(#type "." #field " %zu\n", offsetof(type, field)) +#define PRINT_VALUE(name) printf(#name " %d\n", (int)(name)) int main(void) { PRINT_LAYOUT(vllm_status); + PRINT_VALUE(VLLM_OK); + PRINT_VALUE(VLLM_ERR_INVALID_ARGUMENT); + PRINT_VALUE(VLLM_ERR_MODEL_LOAD); + PRINT_VALUE(VLLM_ERR_RUNTIME); + PRINT_VALUE(VLLM_ERR_UNKNOWN); + + PRINT_NAMED_LAYOUT(vllm_engine_ptr, vllm_engine*); + PRINT_NAMED_LAYOUT(vllm_request_ptr, vllm_request*); + PRINT_NAMED_LAYOUT(vllm_video_engine_ptr, vllm_video_engine*); PRINT_LAYOUT(vllm_model_params); PRINT_OFFSET(vllm_model_params, model_path); @@ -26,6 +38,9 @@ int main(void) { PRINT_OFFSET(vllm_model_params, scheduling_policy); PRINT_OFFSET(vllm_model_params, kv_transfer_config); PRINT_OFFSET(vllm_model_params, enable_jump_forward); + PRINT_OFFSET(vllm_model_params, device); + PRINT_OFFSET(vllm_model_params, gpu_memory_utilization); + PRINT_OFFSET(vllm_model_params, kv_cache_memory_bytes); PRINT_LAYOUT(vllm_sampling_params); PRINT_OFFSET(vllm_sampling_params, temperature); @@ -57,6 +72,72 @@ int main(void) { PRINT_OFFSET(vllm_completion, prompt_tokens); PRINT_OFFSET(vllm_completion, completion_tokens); + PRINT_LAYOUT(vllm_transcription_params); + PRINT_OFFSET(vllm_transcription_params, audio_path); + PRINT_OFFSET(vllm_transcription_params, pcm); + PRINT_OFFSET(vllm_transcription_params, n_samples); + PRINT_OFFSET(vllm_transcription_params, sample_rate); + + PRINT_LAYOUT(vllm_transcription); + PRINT_OFFSET(vllm_transcription, text); + PRINT_OFFSET(vllm_transcription, token_ids); + PRINT_OFFSET(vllm_transcription, n_token_ids); + PRINT_OFFSET(vllm_transcription, has_text); + + PRINT_LAYOUT(vllm_embedding_result); + PRINT_OFFSET(vllm_embedding_result, values); + PRINT_OFFSET(vllm_embedding_result, n_embeddings); + PRINT_OFFSET(vllm_embedding_result, dim); + PRINT_OFFSET(vllm_embedding_result, prompt_tokens); + + PRINT_LAYOUT(vllm_video_model_params); + PRINT_OFFSET(vllm_video_model_params, dit_path); + PRINT_OFFSET(vllm_video_model_params, encoder_path); + PRINT_OFFSET(vllm_video_model_params, tokenizer_path); + PRINT_OFFSET(vllm_video_model_params, video_vae_path); + PRINT_OFFSET(vllm_video_model_params, video_vae_config_path); + PRINT_OFFSET(vllm_video_model_params, audio_vae_path); + PRINT_OFFSET(vllm_video_model_params, audio_vae_config_path); + PRINT_OFFSET(vllm_video_model_params, prompt_embeds_path); + PRINT_OFFSET(vllm_video_model_params, partition); + PRINT_OFFSET(vllm_video_model_params, device); + PRINT_OFFSET(vllm_video_model_params, dequant_bf16); + PRINT_OFFSET(vllm_video_model_params, fp4_resident); + + PRINT_LAYOUT(vllm_video_params); + PRINT_OFFSET(vllm_video_params, prompt); + PRINT_OFFSET(vllm_video_params, width); + PRINT_OFFSET(vllm_video_params, height); + PRINT_OFFSET(vllm_video_params, num_frames); + PRINT_OFFSET(vllm_video_params, steps); + PRINT_OFFSET(vllm_video_params, seed); + PRINT_OFFSET(vllm_video_params, has_seed); + PRINT_OFFSET(vllm_video_params, first_frame); + PRINT_OFFSET(vllm_video_params, last_frame); + PRINT_OFFSET(vllm_video_params, ref_image); + PRINT_OFFSET(vllm_video_params, ref_video); + PRINT_OFFSET(vllm_video_params, ref_audio); + PRINT_OFFSET(vllm_video_params, noise_aug); + PRINT_OFFSET(vllm_video_params, output_dir); + + PRINT_LAYOUT(vllm_video_result); + PRINT_OFFSET(vllm_video_result, frame_dir); + PRINT_OFFSET(vllm_video_result, audio_path); + PRINT_OFFSET(vllm_video_result, frame_count); + PRINT_OFFSET(vllm_video_result, width); + PRINT_OFFSET(vllm_video_result, height); + PRINT_OFFSET(vllm_video_result, fps); + PRINT_OFFSET(vllm_video_result, sample_rate); + PRINT_OFFSET(vllm_video_result, mux_argv); + PRINT_OFFSET(vllm_video_result, mux_argc); + + PRINT_LAYOUT(vllm_video_mux_params); + PRINT_OFFSET(vllm_video_mux_params, frames); + PRINT_OFFSET(vllm_video_mux_params, audio_path); + PRINT_OFFSET(vllm_video_mux_params, output_path); + PRINT_OFFSET(vllm_video_mux_params, fps); + PRINT_OFFSET(vllm_video_mux_params, crf); + PRINT_LAYOUT(vllm_token_callback); PRINT_LAYOUT(vllm_logits_processor); return 0; diff --git a/vllm-cpp-sys/tests/layout.rs b/vllm-cpp-sys/tests/layout.rs index 8d2ce4c..b222883 100644 --- a/vllm-cpp-sys/tests/layout.rs +++ b/vllm-cpp-sys/tests/layout.rs @@ -7,8 +7,12 @@ use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; use vllm_cpp_sys::{ - vllm_completion, vllm_logits_processor, vllm_model_params, vllm_sampling_params, vllm_status, - vllm_token_callback, + vllm_completion, vllm_embedding_result, vllm_engine, vllm_logits_processor, vllm_model_params, + vllm_request, vllm_sampling_params, vllm_status, vllm_status_VLLM_ERR_INVALID_ARGUMENT, + vllm_status_VLLM_ERR_MODEL_LOAD, vllm_status_VLLM_ERR_RUNTIME, vllm_status_VLLM_ERR_UNKNOWN, + vllm_status_VLLM_OK, vllm_token_callback, vllm_transcription, vllm_transcription_params, + vllm_video_engine, vllm_video_model_params, vllm_video_mux_params, vllm_video_params, + vllm_video_result, }; #[test] @@ -47,7 +51,39 @@ fn generated_bindings_match_c_layout() { }; } + macro_rules! expect_named_layout { + ($name:literal, $type:ty) => { + assert_eq!( + actual[$name], + (size_of::<$type>(), align_of::<$type>()), + "layout mismatch for {}", + $name + ); + }; + } + macro_rules! expect_value { + ($name:literal, $value:expr) => { + assert_eq!( + actual[$name].0, $value as usize, + "value mismatch for {}", + $name + ); + }; + } + expect_layout!(vllm_status); + expect_value!("VLLM_OK", vllm_status_VLLM_OK); + expect_value!( + "VLLM_ERR_INVALID_ARGUMENT", + vllm_status_VLLM_ERR_INVALID_ARGUMENT + ); + expect_value!("VLLM_ERR_MODEL_LOAD", vllm_status_VLLM_ERR_MODEL_LOAD); + expect_value!("VLLM_ERR_RUNTIME", vllm_status_VLLM_ERR_RUNTIME); + expect_value!("VLLM_ERR_UNKNOWN", vllm_status_VLLM_ERR_UNKNOWN); + + expect_named_layout!("vllm_engine_ptr", *mut vllm_engine); + expect_named_layout!("vllm_request_ptr", *mut vllm_request); + expect_named_layout!("vllm_video_engine_ptr", *mut vllm_video_engine); expect_layout!(vllm_model_params); expect_offset!(vllm_model_params, model_path); @@ -64,6 +100,9 @@ fn generated_bindings_match_c_layout() { expect_offset!(vllm_model_params, scheduling_policy); expect_offset!(vllm_model_params, kv_transfer_config); expect_offset!(vllm_model_params, enable_jump_forward); + expect_offset!(vllm_model_params, device); + expect_offset!(vllm_model_params, gpu_memory_utilization); + expect_offset!(vllm_model_params, kv_cache_memory_bytes); expect_layout!(vllm_sampling_params); expect_offset!(vllm_sampling_params, temperature); @@ -95,6 +134,72 @@ fn generated_bindings_match_c_layout() { expect_offset!(vllm_completion, prompt_tokens); expect_offset!(vllm_completion, completion_tokens); + expect_layout!(vllm_transcription_params); + expect_offset!(vllm_transcription_params, audio_path); + expect_offset!(vllm_transcription_params, pcm); + expect_offset!(vllm_transcription_params, n_samples); + expect_offset!(vllm_transcription_params, sample_rate); + + expect_layout!(vllm_transcription); + expect_offset!(vllm_transcription, text); + expect_offset!(vllm_transcription, token_ids); + expect_offset!(vllm_transcription, n_token_ids); + expect_offset!(vllm_transcription, has_text); + + expect_layout!(vllm_embedding_result); + expect_offset!(vllm_embedding_result, values); + expect_offset!(vllm_embedding_result, n_embeddings); + expect_offset!(vllm_embedding_result, dim); + expect_offset!(vllm_embedding_result, prompt_tokens); + + expect_layout!(vllm_video_model_params); + expect_offset!(vllm_video_model_params, dit_path); + expect_offset!(vllm_video_model_params, encoder_path); + expect_offset!(vllm_video_model_params, tokenizer_path); + expect_offset!(vllm_video_model_params, video_vae_path); + expect_offset!(vllm_video_model_params, video_vae_config_path); + expect_offset!(vllm_video_model_params, audio_vae_path); + expect_offset!(vllm_video_model_params, audio_vae_config_path); + expect_offset!(vllm_video_model_params, prompt_embeds_path); + expect_offset!(vllm_video_model_params, partition); + expect_offset!(vllm_video_model_params, device); + expect_offset!(vllm_video_model_params, dequant_bf16); + expect_offset!(vllm_video_model_params, fp4_resident); + + expect_layout!(vllm_video_params); + expect_offset!(vllm_video_params, prompt); + expect_offset!(vllm_video_params, width); + expect_offset!(vllm_video_params, height); + expect_offset!(vllm_video_params, num_frames); + expect_offset!(vllm_video_params, steps); + expect_offset!(vllm_video_params, seed); + expect_offset!(vllm_video_params, has_seed); + expect_offset!(vllm_video_params, first_frame); + expect_offset!(vllm_video_params, last_frame); + expect_offset!(vllm_video_params, ref_image); + expect_offset!(vllm_video_params, ref_video); + expect_offset!(vllm_video_params, ref_audio); + expect_offset!(vllm_video_params, noise_aug); + expect_offset!(vllm_video_params, output_dir); + + expect_layout!(vllm_video_result); + expect_offset!(vllm_video_result, frame_dir); + expect_offset!(vllm_video_result, audio_path); + expect_offset!(vllm_video_result, frame_count); + expect_offset!(vllm_video_result, width); + expect_offset!(vllm_video_result, height); + expect_offset!(vllm_video_result, fps); + expect_offset!(vllm_video_result, sample_rate); + expect_offset!(vllm_video_result, mux_argv); + expect_offset!(vllm_video_result, mux_argc); + + expect_layout!(vllm_video_mux_params); + expect_offset!(vllm_video_mux_params, frames); + expect_offset!(vllm_video_mux_params, audio_path); + expect_offset!(vllm_video_mux_params, output_path); + expect_offset!(vllm_video_mux_params, fps); + expect_offset!(vllm_video_mux_params, crf); + expect_layout!(vllm_token_callback); expect_layout!(vllm_logits_processor); } diff --git a/vllm-cpp-sys/tests/symbols.rs b/vllm-cpp-sys/tests/symbols.rs index 34d1d79..2a4ac5d 100644 --- a/vllm-cpp-sys/tests/symbols.rs +++ b/vllm-cpp-sys/tests/symbols.rs @@ -1,8 +1,9 @@ -use std::ffi::{c_char, c_void}; +use std::collections::BTreeSet; +use std::ffi::{c_char, c_void, CStr, CString}; use vllm_cpp_sys as ffi; -const _: [(); 10] = [(); ffi::VLLM_ABI_VERSION as usize]; +const _: [(); 17] = [(); ffi::VLLM_ABI_VERSION as usize]; unsafe extern "C" fn token_callback( _delta_text: *const c_char, @@ -12,33 +13,82 @@ unsafe extern "C" fn token_callback( true } -#[test] -fn every_c_api_symbol_links() { - assert_ne!(ffi::vllm_model_params_default as *const () as usize, 0); - assert_ne!(ffi::vllm_sampling_params_default as *const () as usize, 0); - assert_ne!(ffi::vllm_engine_load as *const () as usize, 0); - assert_ne!(ffi::vllm_engine_free as *const () as usize, 0); - assert_ne!(ffi::vllm_complete as *const () as usize, 0); - assert_ne!(ffi::vllm_complete_stream as *const () as usize, 0); - assert_ne!(ffi::vllm_request_submit as *const () as usize, 0); - assert_ne!(ffi::vllm_request_cancel as *const () as usize, 0); - assert_ne!(ffi::vllm_request_wait as *const () as usize, 0); - assert_ne!(ffi::vllm_request_done as *const () as usize, 0); - assert_ne!(ffi::vllm_request_error as *const () as usize, 0); - assert_ne!(ffi::vllm_request_free as *const () as usize, 0); - assert_ne!(ffi::vllm_chat as *const () as usize, 0); - assert_ne!(ffi::vllm_chat_stream as *const () as usize, 0); - assert_ne!(ffi::vllm_string_free as *const () as usize, 0); - assert_ne!(ffi::vllm_completion_free as *const () as usize, 0); - assert_ne!(ffi::vllm_last_error as *const () as usize, 0); - assert_ne!(ffi::vllm_version as *const () as usize, 0); - assert_ne!(ffi::vllm_abi_version as *const () as usize, 0); +unsafe extern "C" fn logits_processor( + _token_ids: *const i32, + _n_token_ids: i32, + _logits: *mut f32, + _vocab_size: i32, + _user_data: *mut c_void, +) { +} + +macro_rules! c_api_inventory { + ($($name:ident: $signature:ty;)+) => { + const C_API_SYMBOL_NAMES: &[&str] = &[$(stringify!($name)),+]; + + #[test] + fn every_c_api_symbol_links_with_the_header_signature() { + let mut addresses = Vec::new(); + $( + let function: $signature = ffi::$name; + addresses.push(function as *const ()); + )+ + assert_eq!(addresses.len(), 35); + assert!(addresses.iter().all(|address| !address.is_null())); + assert_eq!(C_API_SYMBOL_NAMES.len(), 35); + assert_eq!( + C_API_SYMBOL_NAMES.iter().copied().collect::>().len(), + 35 + ); + } + }; +} + +c_api_inventory! { + vllm_model_params_default: unsafe extern "C" fn() -> ffi::vllm_model_params; + vllm_sampling_params_default: unsafe extern "C" fn() -> ffi::vllm_sampling_params; + vllm_engine_load: unsafe extern "C" fn(*const ffi::vllm_model_params, *mut *mut ffi::vllm_engine) -> ffi::vllm_status; + vllm_engine_free: unsafe extern "C" fn(*mut ffi::vllm_engine); + vllm_complete: unsafe extern "C" fn(*mut ffi::vllm_engine, *const c_char, *const ffi::vllm_sampling_params, *mut ffi::vllm_completion) -> ffi::vllm_status; + vllm_complete_tokens: unsafe extern "C" fn(*mut ffi::vllm_engine, *const i32, i32, *const ffi::vllm_sampling_params, *mut i32, i32, *mut i32, *mut ffi::vllm_completion) -> ffi::vllm_status; + vllm_complete_stream: unsafe extern "C" fn(*mut ffi::vllm_engine, *const c_char, *const ffi::vllm_sampling_params, ffi::vllm_token_callback, *mut c_void) -> ffi::vllm_status; + vllm_request_submit: unsafe extern "C" fn(*mut ffi::vllm_engine, *const c_char, *const ffi::vllm_sampling_params, ffi::vllm_token_callback, *mut c_void, *mut *mut ffi::vllm_request) -> ffi::vllm_status; + vllm_request_cancel: unsafe extern "C" fn(*mut ffi::vllm_request) -> ffi::vllm_status; + vllm_request_wait: unsafe extern "C" fn(*mut ffi::vllm_request) -> ffi::vllm_status; + vllm_request_done: unsafe extern "C" fn(*const ffi::vllm_request) -> bool; + vllm_request_error: unsafe extern "C" fn(*const ffi::vllm_request) -> *const c_char; + vllm_request_free: unsafe extern "C" fn(*mut ffi::vllm_request); + vllm_chat: unsafe extern "C" fn(*mut ffi::vllm_engine, *const c_char, *mut *mut c_char) -> ffi::vllm_status; + vllm_chat_stream: unsafe extern "C" fn(*mut ffi::vllm_engine, *const c_char, ffi::vllm_token_callback, *mut c_void) -> ffi::vllm_status; + vllm_transcription_params_default: unsafe extern "C" fn() -> ffi::vllm_transcription_params; + vllm_transcribe: unsafe extern "C" fn(*mut ffi::vllm_engine, *const ffi::vllm_transcription_params, *mut ffi::vllm_transcription) -> ffi::vllm_status; + vllm_transcription_free: unsafe extern "C" fn(*mut ffi::vllm_transcription); + vllm_embed: unsafe extern "C" fn(*mut ffi::vllm_engine, *const *const c_char, i32, *mut ffi::vllm_embedding_result) -> ffi::vllm_status; + vllm_embedding_result_free: unsafe extern "C" fn(*mut ffi::vllm_embedding_result); + vllm_video_model_params_default: unsafe extern "C" fn() -> ffi::vllm_video_model_params; + vllm_video_params_default: unsafe extern "C" fn() -> ffi::vllm_video_params; + vllm_video_engine_load: unsafe extern "C" fn(*const ffi::vllm_video_model_params, *mut *mut ffi::vllm_video_engine) -> ffi::vllm_status; + vllm_video_engine_free: unsafe extern "C" fn(*mut ffi::vllm_video_engine); + vllm_video_generate: unsafe extern "C" fn(*mut ffi::vllm_video_engine, *const ffi::vllm_video_params, *mut ffi::vllm_video_result) -> ffi::vllm_status; + vllm_video_result_free: unsafe extern "C" fn(*mut ffi::vllm_video_result); + vllm_video_mux_params_default: unsafe extern "C" fn() -> ffi::vllm_video_mux_params; + vllm_video_mux_argv: unsafe extern "C" fn(*const ffi::vllm_video_mux_params, *mut *mut *mut c_char, *mut i32) -> ffi::vllm_status; + vllm_video_mux_argv_free: unsafe extern "C" fn(*mut *mut c_char, i32); + vllm_string_free: unsafe extern "C" fn(*mut c_char); + vllm_completion_free: unsafe extern "C" fn(*mut ffi::vllm_completion); + vllm_last_error: unsafe extern "C" fn() -> *const c_char; + vllm_version: unsafe extern "C" fn() -> *const c_char; + vllm_server_main: unsafe extern "C" fn(i32, *mut *mut c_char) -> i32; + vllm_abi_version: unsafe extern "C" fn() -> i32; } #[test] -fn reports_expected_abi_and_handles_invalid_model_path() { - assert_eq!(ffi::VLLM_ABI_VERSION, 10); - assert_eq!(unsafe { ffi::vllm_abi_version() }, 10); +fn reports_target_identity_and_handles_invalid_model_path() { + assert_eq!(ffi::VLLM_ABI_VERSION, 17); + assert_eq!(unsafe { ffi::vllm_abi_version() }, 17); + + let version = unsafe { CStr::from_ptr(ffi::vllm_version()) }; + assert!(version.to_bytes().starts_with(b"0.0.2"), "{version:?}"); let mut params = unsafe { ffi::vllm_model_params_default() }; params.model_path = c"/nonexistent/vllm-cpp-rs-sys-model".as_ptr(); @@ -47,12 +97,39 @@ fn reports_expected_abi_and_handles_invalid_model_path() { assert_eq!(status, ffi::vllm_status_VLLM_ERR_MODEL_LOAD); assert!(engine.is_null()); - let error = unsafe { std::ffi::CStr::from_ptr(ffi::vllm_last_error()) }; + let error = unsafe { CStr::from_ptr(ffi::vllm_last_error()) }; assert!(!error.to_bytes().is_empty()); } #[test] -fn callback_type_matches_header_contract() { - let callback: ffi::vllm_token_callback = Some(token_callback); - assert!(callback.is_some()); +fn server_entry_point_is_present_and_nonblocking_for_help_or_server_off() { + let mut arguments = [ + CString::new("vllm-server").unwrap(), + CString::new("--help").unwrap(), + ]; + let mut argv = arguments + .iter_mut() + .map(|argument| argument.as_ptr().cast_mut()) + .collect::>(); + let status = unsafe { ffi::vllm_server_main(argv.len() as i32, argv.as_mut_ptr()) }; + assert!(matches!(status, 0 | 1)); + if status == 1 { + let error = unsafe { CStr::from_ptr(ffi::vllm_last_error()) }; + assert!(error + .to_bytes() + .windows(13) + .any(|part| part == b"without VLLM_")); + } +} + +#[test] +fn callback_types_match_header_contract_and_are_nullable() { + let token: ffi::vllm_token_callback = Some(token_callback); + let no_token: ffi::vllm_token_callback = None; + let logits: ffi::vllm_logits_processor = Some(logits_processor); + let no_logits: ffi::vllm_logits_processor = None; + assert!(token.is_some()); + assert!(no_token.is_none()); + assert!(logits.is_some()); + assert!(no_logits.is_none()); } diff --git a/vllm-cpp-sys/vllm.cpp b/vllm-cpp-sys/vllm.cpp index 34aedfb..7020de9 160000 --- a/vllm-cpp-sys/vllm.cpp +++ b/vllm-cpp-sys/vllm.cpp @@ -1 +1 @@ -Subproject commit 34aedfbe8ed9779697905541a62e2160ccfd9c05 +Subproject commit 7020de93652ca920424a10ac5255b34810dd2f24 diff --git a/vllm-cpp/src/engine.rs b/vllm-cpp/src/engine.rs index 4cbfd64..4ab3d90 100644 --- a/vllm-cpp/src/engine.rs +++ b/vllm-cpp/src/engine.rs @@ -342,7 +342,6 @@ impl EngineBuilder { } pub fn load(self) -> Result { - ensure_abi()?; let model_path = path_to_cstring(&self.model_path, "model path")?; let tokenizer_config_path = self .tokenizer_config_path @@ -362,8 +361,7 @@ impl EngineBuilder { "KV transfer configuration", )?; - // ABI equality is checked before this struct-returning native call. - let mut raw = unsafe { ffi::vllm_model_params_default() }; + let mut raw = checked_model_params_default()?; raw.model_path = model_path.as_ptr(); raw.tokenizer_config_path = optional_pointer(tokenizer_config_path.as_ref()); raw.block_size = optional_u32_to_i32(self.block_size, "block_size")?; @@ -394,15 +392,30 @@ impl EngineBuilder { } } -fn ensure_abi() -> Result<(), Error> { - // SAFETY: this base ABI function takes no pointers or versioned structs. - let actual = unsafe { ffi::vllm_abi_version() }; +fn checked_model_params_default() -> Result { + checked_model_params_default_with( + || { + // SAFETY: this base ABI function takes no pointers or versioned structs. + unsafe { ffi::vllm_abi_version() } + }, + || { + // SAFETY: exact ABI equality was established immediately before this + // by-value return of a versioned struct. + unsafe { ffi::vllm_model_params_default() } + }, + ) +} + +fn checked_model_params_default_with( + abi_version: impl FnOnce() -> i32, + model_params_default: impl FnOnce() -> ffi::vllm_model_params, +) -> Result { + let actual = abi_version(); let expected = ffi::VLLM_ABI_VERSION as i32; - if actual == expected { - Ok(()) - } else { - Err(Error::AbiMismatch { expected, actual }) + if actual != expected { + return Err(Error::AbiMismatch { expected, actual }); } + Ok(model_params_default()) } fn completion_from_raw(raw: &ffi::vllm_completion) -> Result { @@ -507,3 +520,34 @@ impl Drop for NativeStringGuard { unsafe { ffi::vllm_string_free(self.0.as_ptr()) }; } } + +#[cfg(test)] +mod tests { + use super::checked_model_params_default_with; + use crate::Error; + use std::cell::RefCell; + + #[test] + fn abi_mismatch_prevents_model_params_default_call() { + let calls = RefCell::new(Vec::new()); + let result = checked_model_params_default_with( + || { + calls.borrow_mut().push("abi"); + 10 + }, + || { + calls.borrow_mut().push("default"); + unreachable!("default helper must not run after an ABI mismatch") + }, + ); + + assert!(matches!( + result, + Err(Error::AbiMismatch { + expected: 17, + actual: 10 + }) + )); + assert_eq!(*calls.borrow(), ["abi"]); + } +} diff --git a/vllm-cpp/src/lib.rs b/vllm-cpp/src/lib.rs index abf514c..794782c 100644 --- a/vllm-cpp/src/lib.rs +++ b/vllm-cpp/src/lib.rs @@ -24,7 +24,7 @@ //! free/join completes. Requests are `Send` but intentionally not `Sync`, while //! engines are `Send + Sync`. Asynchronous callbacks run on a native delivery //! thread, must be `Send + 'static`, and surface panic through -//! [`Error::CallbackPanicked`]. ABI version 10 forbids waiting for or freeing a +//! [`Error::CallbackPanicked`]. ABI version 17 forbids waiting for or freeing a //! request from its callback thread; callback-thread drop delegates ownership to //! a cleanup reaper instead. //! diff --git a/vllm-cpp/tests/safe_api.rs b/vllm-cpp/tests/safe_api.rs index 8b9d1c9..13c0e06 100644 --- a/vllm-cpp/tests/safe_api.rs +++ b/vllm-cpp/tests/safe_api.rs @@ -24,9 +24,11 @@ fn hugging_face_constructors_accept_default_and_explicit_revisions() { #[test] fn reports_expected_abi() { - assert_eq!(vllm_cpp::expected_abi_version(), 10); - assert_eq!(vllm_cpp::abi_version(), 10); - assert!(!vllm_cpp::version().expect("native version").is_empty()); + assert_eq!(vllm_cpp::expected_abi_version(), 17); + assert_eq!(vllm_cpp::abi_version(), 17); + assert!(vllm_cpp::version() + .expect("native version") + .starts_with("0.0.2")); } #[test]