1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
//! The `basic` API flavor provides barebones UTF-8 checking at the highest speed.
//!
//! It is fastest on valid UTF-8, but only checks for errors after processing the whole byte sequence
//! and does not provide detailed information if the data is not valid UTF-8. [`Utf8Error`] is a zero-sized error struct.
//!
//! If you need detailed error information use the functions from the [`crate::compat`] module instead.
use core::str::{from_utf8_unchecked, from_utf8_unchecked_mut};
use crate::implementation::validate_utf8_basic;
/// Simple zero-sized UTF-8 error.
///
/// No information is provided where the error occured or how long the invalid byte
/// byte sequence is.
#[derive(Copy, Eq, PartialEq, Clone, Debug)]
pub struct Utf8Error;
impl core::fmt::Display for Utf8Error {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.write_str("invalid utf-8 sequence")
}
}
#[cfg(feature = "std")]
impl std::error::Error for Utf8Error {}
/// Analogue to [`std::str::from_utf8()`].
///
/// Checks if the passed byte sequence is valid UTF-8 and returns an
/// [`std::str`] reference to the passed byte slice wrapped in `Ok()` if it is.
///
/// # Errors
/// Will return the zero-sized Err([`Utf8Error`]) on if the input contains invalid UTF-8.
#[inline]
pub fn from_utf8(input: &[u8]) -> Result<&str, Utf8Error> {
unsafe {
validate_utf8_basic(input)?;
Ok(from_utf8_unchecked(input))
}
}
/// Analogue to [`std::str::from_utf8_mut()`].
///
/// Checks if the passed mutable byte sequence is valid UTF-8 and returns a mutable
/// [`std::str`] reference to the passed byte slice wrapped in `Ok()` if it is.
///
/// # Errors
/// Will return the zero-sized Err([`Utf8Error`]) on if the input contains invalid UTF-8.
#[inline]
pub fn from_utf8_mut(input: &mut [u8]) -> Result<&mut str, Utf8Error> {
unsafe {
validate_utf8_basic(input)?;
Ok(from_utf8_unchecked_mut(input))
}
}
/// Allows direct access to the platform-specific unsafe validation implementations.
#[cfg(feature = "public_imp")]
pub mod imp {
use crate::basic;
/// A low-level interfacne for streaming validation of UTF-8 data. It is meant to be integrated
/// in high-performance data processing pipelines.
///
/// Data can be streamed in arbitrarily-sized chunks using the [`Self::update()`] method. There is
/// no way to find out if the input so far was valid UTF-8 during the validation. Only when
/// the validation is completed with the [`Self::finalize()`] method the result of the validation is
/// returned. Use [`ChunkedUtf8Validator`] if possible for highest performance.
///
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to use it if the required CPU features are not available which
/// is why all trait methods are `unsafe`.
///
/// General usage:
/// ```rust
/// use simdutf8::basic::imp::Utf8Validator;
/// use std::io::{stdin, Read, Result};
///
/// # #[cfg(target_arch = "x86_64")]
/// fn main() -> Result<()> {
/// unsafe {
/// if !std::is_x86_feature_detected!("avx2") {
/// panic!("This example only works with CPUs supporting AVX 2");
/// }
///
/// let mut validator = simdutf8::basic::imp::x86::avx2::Utf8ValidatorImp::new();
/// let mut buf = vec![0; 8192];
/// loop {
/// let bytes_read = stdin().read(buf.as_mut())?;
/// if bytes_read == 0 {
/// break;
/// }
/// validator.update(&buf);
/// }
///
/// if validator.finalize().is_ok() {
/// println!("Input is valid UTF-8");
/// } else {
/// println!("Input is not valid UTF-8");
/// }
/// }
///
/// Ok(())
/// }
///
/// # #[cfg(not(target_arch = "x86_64"))]
/// # fn main() { }
/// ```
///
pub trait Utf8Validator {
/// Creates a new validator.
///
/// # Safety
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to call it if the required CPU features are not available.
#[must_use]
unsafe fn new() -> Self
where
Self: Sized;
/// Updates the validator with `input`.
///
/// # Safety
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to call it if the required CPU features are not available.
unsafe fn update(&mut self, input: &[u8]);
/// Finishes the validation and returns `Ok(())` if the input was valid UTF-8.
///
/// # Errors
/// A [`basic::Utf8Error`] is returned if the input was not valid UTF-8. No
/// further information about the location of the error is provided.
///
/// # Safety
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to call it if the required CPU features are not available.
unsafe fn finalize(self) -> core::result::Result<(), basic::Utf8Error>;
}
/// Like [`Utf8Validator`] this low-level API is for streaming validation of UTF-8 data.
/// It has additional restrictions imposed on how the input is passed in to allow
/// validation with as little overhead as possible.
///
/// To feed it data you need to call the [`Self::update_from_chunks()`] method which takes slices which
/// have to be a multiple of 64 bytes long. The method will panic otherwise. There is
/// no way to find out if the input so far was valid UTF-8 during the validation. Only when
/// the validation is completed with the [`Self::finalize()`] method the result of the validation is
/// returned.
///
/// The `Self::finalize()` method can be fed the rest of the data. There is no restriction on the
/// data passed to it.
///
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to use it if the required CPU features are not available which
/// is why all trait methods are `unsafe`.
pub trait ChunkedUtf8Validator {
/// Creates a new validator.
///
/// # Safety
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to call it if the required CPU features are not available.
#[must_use]
unsafe fn new() -> Self
where
Self: Sized;
/// Updates the validator with `input`.
///
/// # Panics
/// If `input.len()` is not a multiple of 64.
///
/// # Safety
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to call it if the required CPU features are not available.
unsafe fn update_from_chunks(&mut self, input: &[u8]);
/// Updates the validator with remaining input if any. There is no restriction on the
/// data provided.
///
/// Finishes the validation and returns `Ok(())` if the input was valid UTF-8.
///
/// # Errors
/// A [`basic::Utf8Error`] is returned if the input was not valid UTF-8. No
/// further information about the location of the error is provided.
///
/// # Safety
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to call it if the required CPU features are not available.
unsafe fn finalize(
self,
remaining_input: core::option::Option<&[u8]>,
) -> core::result::Result<(), basic::Utf8Error>;
}
/// Includes the x86/x86-64 SIMD implementations.
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64")))]
pub mod x86 {
/// Includes the validation implementation for AVX 2-compatible CPUs.
///
/// Using the provided functionality on CPUs which do not support AVX 2 is undefined
/// behavior and will very likely cause a crash.
pub mod avx2 {
pub use crate::implementation::x86::avx2::validate_utf8_basic as validate_utf8;
pub use crate::implementation::x86::avx2::ChunkedUtf8ValidatorImp;
pub use crate::implementation::x86::avx2::Utf8ValidatorImp;
}
/// Includes the validation implementation for SSE 4.2-compatible CPUs.
///
/// Using the provided functionality on CPUs which do not support SSE 4.2 is undefined
/// behavior and will very likely cause a crash.
pub mod sse42 {
pub use crate::implementation::x86::sse42::validate_utf8_basic as validate_utf8;
pub use crate::implementation::x86::sse42::ChunkedUtf8ValidatorImp;
pub use crate::implementation::x86::sse42::Utf8ValidatorImp;
}
}
/// Includes the aarch64 SIMD implementations.
#[cfg(all(feature = "aarch64_neon", target_arch = "aarch64"))]
pub mod aarch64 {
/// Includes the Neon-based validation implementation for aarch64 CPUs.
///
/// Should be supported on all ARM64 CPUSs. If it is not supported by the operating
/// system using it is undefined behavior and will likely cause a crash.
pub mod neon {
pub use crate::implementation::aarch64::neon::validate_utf8_basic as validate_utf8;
pub use crate::implementation::aarch64::neon::ChunkedUtf8ValidatorImp;
pub use crate::implementation::aarch64::neon::Utf8ValidatorImp;
}
}
/// Includes the wasm32 SIMD implementations.
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
pub mod wasm32 {
/// Includes the simd128-based validation implementation for WASM runtimes.
///
/// Using the provided functionality on WASM runtimes that do not support SIMD
/// instructions will likely cause a crash.
pub mod simd128 {
pub use crate::implementation::wasm32::simd128::validate_utf8_basic as validate_utf8;
pub use crate::implementation::wasm32::simd128::ChunkedUtf8ValidatorImp;
pub use crate::implementation::wasm32::simd128::Utf8ValidatorImp;
}
}
}