diff --git a/Cargo.toml b/Cargo.toml index f526e32..9d38938 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -rdma-mummy-sys = "0.2.3" +rdma-mummy-sys = "0.2.4" tabled = "0.18" libc = "0.2" os_socketaddr = "0.2" diff --git a/src/ibverbs/memory_region.rs b/src/ibverbs/memory_region.rs index 73ad003..c015255 100644 --- a/src/ibverbs/memory_region.rs +++ b/src/ibverbs/memory_region.rs @@ -1,6 +1,6 @@ //! Users need to register memory they allocated as memory region for accessing it later. -use rdma_mummy_sys::{ibv_dereg_mr, ibv_mr, ibv_reg_mr}; -use std::{io, ptr::NonNull, sync::Arc}; +use rdma_mummy_sys::{ibv_dereg_mr, ibv_mr, ibv_reg_dmabuf_mr, ibv_reg_mr}; +use std::{io, os::fd::RawFd, ptr::NonNull, sync::Arc}; use super::protection_domain::ProtectionDomain; use super::AccessFlags; @@ -89,6 +89,29 @@ impl MemoryRegion { _pd: pd, }) } + + /// Register a dma-buf-backed memory region. + /// + /// # Safety + /// + /// The caller must ensure that `fd` refers to a valid dma-buf whose exported + /// memory remains compatible with the target device for the specified + /// `offset`, `length`, and `iova` mapping, and that the requested `access` + /// flags satisfy the libibverbs requirements for the underlying provider. + pub(crate) unsafe fn reg_dmabuf_mr( + pd: Arc, offset: u64, length: usize, iova: u64, fd: RawFd, access: AccessFlags, + ) -> Result { + let mr = unsafe { ibv_reg_dmabuf_mr(pd.pd.as_ptr(), offset, length, iova, fd, access.into()) }; + + if mr.is_null() { + return Err(RegisterMemoryRegionErrorKind::Ibverbs(io::Error::last_os_error()).into()); + } + + Ok(Self { + mr: NonNull::new(mr).unwrap(), + _pd: pd, + }) + } } unsafe impl Send for MemoryRegion {} diff --git a/src/ibverbs/mod.rs b/src/ibverbs/mod.rs index 563079b..4911d69 100644 --- a/src/ibverbs/mod.rs +++ b/src/ibverbs/mod.rs @@ -63,7 +63,7 @@ use rdma_mummy_sys::ibv_access_flags; /// # Reference /// /// - RDMAmojo: [`ibv_modify_qp`](https://www.rdmamojo.com/2013/01/12/ibv_modify_qp/), [`ibv_reg_mr`](https://www.rdmamojo.com/2012/09/07/ibv_reg_mr/) -/// - rdma-core manpages: [`ibv_reg_mr`](https://man7.org/linux/man-pages/man3/ibv_reg_mr.3.html) +/// - rdma-core manpages: [`ibv_reg_mr`](https://man7.org/linux/man-pages/man3/ibv_reg_mr.3.html), [`ibv_reg_dmabuf_mr`](https://man7.org/linux/man-pages/man3/ibv_reg_mr.3.html) /// /// [`QueuePair`]: queue_pair::QueuePair /// [`MemoryRegion`]: memory_region::MemoryRegion diff --git a/src/ibverbs/protection_domain.rs b/src/ibverbs/protection_domain.rs index cc16671..98ebc8f 100644 --- a/src/ibverbs/protection_domain.rs +++ b/src/ibverbs/protection_domain.rs @@ -4,6 +4,7 @@ //! [`QueuePair`]: crate::ibverbs::queue_pair::QueuePair //! use rdma_mummy_sys::{ibv_dealloc_pd, ibv_pd}; +use std::os::fd::RawFd; use std::ptr::NonNull; use std::sync::Arc; @@ -54,6 +55,49 @@ impl ProtectionDomain { Ok(Arc::new(MemoryRegion::reg_mr(Arc::clone(self), ptr, len, access)?)) } + /// Register a dma-buf-backed memory region. + /// + /// # Safety + /// + /// The caller must ensure that `fd` refers to a valid dma-buf file descriptor, + /// that the exported memory remains valid for `offset..offset + length`, and + /// that `iova` and `access` satisfy the requirements of the target provider. + /// + /// ```no_run + /// use std::os::fd::RawFd; + /// + /// use sideway::ibverbs::device::DeviceList; + /// use sideway::ibverbs::AccessFlags; + /// + /// let device_list = DeviceList::new().unwrap(); + /// let device = device_list.get(0).unwrap(); + /// let context = device.open().unwrap(); + /// let pd = context.alloc_pd().unwrap(); + /// let dmabuf_fd: RawFd = 0; // replace with a real dma-buf FD from your allocator/exporter + /// + /// let _mr = unsafe { + /// pd.reg_dmabuf_mr( + /// 0, + /// 4096, + /// 0, + /// dmabuf_fd, + /// AccessFlags::LocalWrite | AccessFlags::RemoteWrite, + /// ) + /// }; + /// ``` + pub unsafe fn reg_dmabuf_mr( + self: &Arc, offset: u64, length: usize, iova: u64, fd: RawFd, access: AccessFlags, + ) -> Result, RegisterMemoryRegionError> { + Ok(Arc::new(MemoryRegion::reg_dmabuf_mr( + Arc::clone(self), + offset, + length, + iova, + fd, + access, + )?)) + } + /// Create a [`QueuePairBuilder`] for building QPs on this protection domain /// later. pub fn create_qp_builder(self: &Arc) -> QueuePairBuilder { diff --git a/tests/test_mr_and_cq.rs b/tests/test_mr_and_cq.rs index 9fc15f7..b93acef 100644 --- a/tests/test_mr_and_cq.rs +++ b/tests/test_mr_and_cq.rs @@ -1,4 +1,6 @@ use sideway::ibverbs::{device, AccessFlags}; +use std::fs::File; +use std::os::fd::AsRawFd; #[test] fn main() -> Result<(), Box> { @@ -16,6 +18,17 @@ fn main() -> Result<(), Box> { ) .unwrap() }; + let dmabuf = File::open("/dev/null")?; + let dmabuf_result = unsafe { + pd.reg_dmabuf_mr( + 0, + data.len(), + 0, + dmabuf.as_raw_fd(), + AccessFlags::LocalWrite | AccessFlags::RemoteWrite, + ) + }; + assert!(dmabuf_result.is_err(), "expected /dev/null registration to fail"); let mut builder = ctx.create_cq_builder(); let cq = builder.setup_cqe(32).build().unwrap();