fxcp_core/operations/

napi.rs

// SPDX-License-Identifier: GPL-2.0-or-later
// Copyright (C) 2025 Joel Wirāmu Pauling <aenertia@aenertia.net>
//
//! io_uring NAPI busy-poll registration for NFS-target rings.
//!
//! NAPI busy-polling reduces network completion latency by having the kernel
//! poll NIC receive queues directly from `io_uring_enter()`, avoiding the
//! interrupt->softirq->wakeup path. This is particularly beneficial for NFS
//! targets where each copy operation involves multiple RPC round-trips.
//!
//! **Kernel requirements:** Linux 6.9+, `CONFIG_NET_RX_BUSY_POLL=y`.
//! The wrapper degrades gracefully on older kernels (returns `Err(ENOSYS)`).
//!
//! Uses the `io-uring` 0.7.13 crate's native `Submitter::register_napi` API
//! (`io_uring::types::Napi`), which wraps `IORING_REGISTER_NAPI` (opcode 27).

use std::io;

use io_uring::IoUring;
use io_uring::types::Napi;
use tracing::{debug, warn};

/// Default busy-poll timeout in microseconds.
///
/// 50us is long enough to absorb NFS completion jitter without burning
/// excessive CPU on idle rings. Empirically a good balance for 1GbE-25GbE
/// NFS targets.
pub const DEFAULT_NAPI_BUSY_POLL_TO_US: u32 = 50;

/// Register NAPI busy-poll parameters with an io_uring ring.
///
/// Configures the kernel to busy-poll NIC NAPI queues when waiting for
/// completions on this ring, reducing tail latency for network I/O (NFS).
///
/// # Arguments
///
/// * `ring`  --  The io_uring instance to configure.
/// * `busy_poll_to_us`  --  Busy-poll timeout in microseconds. Use
///   [`DEFAULT_NAPI_BUSY_POLL_TO_US`] (50us) for NFS targets.
/// * `prefer_busy_poll`  --  If `true`, the kernel prefers busy-polling over
///   interrupt-driven completion. Set `true` for dedicated NFS copy rings.
///
/// # Errors
///
/// * `ENOSYS`  --  Kernel does not support io_uring NAPI (< 6.9 or
///   `CONFIG_NET_RX_BUSY_POLL` disabled). Callers should treat this as a
///   non-fatal degradation.
/// * `EINVAL`  --  Invalid parameters (should not occur with valid inputs).
pub fn register_napi_with_ring(
    ring: &IoUring,
    busy_poll_to_us: u32,
    prefer_busy_poll: bool,
) -> io::Result<()> {
    let mut napi = Napi::new()
        .set_busy_poll_timeout(busy_poll_to_us)
        .set_prefer_busy_poll(prefer_busy_poll);

    let result = ring.submitter().register_napi(&mut napi);

    match &result {
        Ok(()) => {
            debug!(
                busy_poll_to_us,
                prefer_busy_poll,
                "io_uring NAPI busy-poll registered"
            );
        }
        Err(e) if e.raw_os_error() == Some(libc::ENOSYS) => {
            debug!("io_uring NAPI not supported by kernel (ENOSYS), continuing without busy-poll");
        }
        Err(e) => {
            warn!(error = %e, "io_uring NAPI registration failed");
        }
    }

    result
}

/// Unregister NAPI busy-poll from an io_uring ring.
///
/// Disables busy-polling previously configured via [`register_napi_with_ring`].
/// The kernel writes the previous settings back into the `Napi` struct before
/// clearing them.
///
/// # Errors
///
/// * `ENOSYS`  --  Kernel does not support NAPI (non-fatal).
/// * `EINVAL`  --  NAPI was not registered on this ring.
pub fn unregister_napi_from_ring(ring: &IoUring) -> io::Result<()> {
    let mut napi = Napi::new();
    let result = ring.submitter().unregister_napi(&mut napi);

    match &result {
        Ok(()) => {
            debug!(
                prev_timeout_us = napi.busy_poll_timeout(),
                prev_prefer = napi.prefer_busy_poll(),
                "io_uring NAPI busy-poll unregistered"
            );
        }
        Err(e) if e.raw_os_error() == Some(libc::ENOSYS) => {
            debug!("io_uring NAPI unregister: not supported (ENOSYS)");
        }
        Err(e) => {
            warn!(error = %e, "io_uring NAPI unregister failed");
        }
    }

    result
}

/// Attempt to register NAPI busy-poll, treating `ENOSYS` as success.
///
/// Convenience wrapper for callers that want fire-and-forget semantics:
/// returns `Ok(true)` if NAPI was registered, `Ok(false)` if the kernel
/// doesn't support it, and `Err` only on unexpected failures.
pub fn try_register_napi(
    ring: &IoUring,
    busy_poll_to_us: u32,
    prefer_busy_poll: bool,
) -> io::Result<bool> {
    match register_napi_with_ring(ring, busy_poll_to_us, prefer_busy_poll) {
        Ok(()) => Ok(true),
        Err(e) if e.raw_os_error() == Some(libc::ENOSYS) => Ok(false),
        Err(e) => Err(e),
    }
}

#[cfg(test)]
mod tests {
    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
    use super::*;

    #[test]
    fn napi_default_timeout_is_50us() {
        assert_eq!(DEFAULT_NAPI_BUSY_POLL_TO_US, 50);
    }

    #[test]
    fn napi_builder_roundtrip() {
        // Verify the Napi builder from io-uring 0.7.13 works as expected.
        let napi = Napi::new()
            .set_busy_poll_timeout(100)
            .set_prefer_busy_poll(true);
        assert_eq!(napi.busy_poll_timeout(), 100);
        assert!(napi.prefer_busy_poll());
    }

    #[test]
    fn napi_builder_defaults() {
        let napi = Napi::new();
        assert_eq!(napi.busy_poll_timeout(), 0);
        assert!(!napi.prefer_busy_poll());
    }

    #[test]
    fn try_register_handles_enosys_gracefully() {
        // On kernels without NAPI support, try_register_napi should return Ok(false).
        // On kernels with NAPI, it returns Ok(true).
        // Either way, it should not panic.
        let ring = IoUring::new(8);
        if let Ok(ring) = ring {
            match try_register_napi(&ring, DEFAULT_NAPI_BUSY_POLL_TO_US, true) {
                Ok(registered) => {
                    if registered {
                        // Clean up: unregister.
                        let _ = unregister_napi_from_ring(&ring);
                    }
                }
                Err(e) => {
                    // EPERM is expected in unprivileged test environments
                    // where io_uring_register is restricted.
                    assert!(
                        e.raw_os_error() == Some(libc::EPERM)
                            || e.raw_os_error() == Some(libc::ENOMEM),
                        "unexpected error: {e}"
                    );
                }
            }
        }
        // If IoUring::new fails (e.g., no io_uring support), that's fine  --  skip.
    }
}