wide/
u16x8_.rs

1use super::*;
2
3pick! {
4  if #[cfg(target_feature="sse2")] {
5    #[derive(Default, Clone, Copy, PartialEq, Eq)]
6    #[repr(C, align(16))]
7    pub struct u16x8 { pub(crate) sse: m128i }
8  } else if #[cfg(target_feature="simd128")] {
9    use core::arch::wasm32::*;
10
11    #[derive(Clone, Copy)]
12    #[repr(transparent)]
13    pub struct u16x8 { pub(crate) simd: v128 }
14
15    impl Default for u16x8 {
16      fn default() -> Self {
17        Self::splat(0)
18      }
19    }
20
21    impl PartialEq for u16x8 {
22      fn eq(&self, other: &Self) -> bool {
23        u16x8_all_true(u16x8_eq(self.simd, other.simd))
24      }
25    }
26
27    impl Eq for u16x8 { }
28  } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
29      use core::arch::aarch64::*;
30      #[repr(C)]
31      #[derive(Copy, Clone)]
32      pub struct u16x8 { pub(crate) neon : uint16x8_t }
33
34      impl Default for u16x8 {
35        #[inline]
36        fn default() -> Self {
37          Self::splat(0)
38        }
39      }
40
41      impl PartialEq for u16x8 {
42        #[inline]
43        fn eq(&self, other: &Self) -> bool {
44          unsafe { vminvq_u16(vceqq_u16(self.neon, other.neon))==u16::MAX }
45        }
46      }
47
48      impl Eq for u16x8 { }
49  } else {
50    #[derive(Default, Clone, Copy, PartialEq, Eq)]
51    #[repr(C, align(16))]
52    pub struct u16x8 { pub(crate) arr: [u16;8] }
53  }
54}
55
56int_uint_consts!(u16, 8, u16x8, 128);
57
58unsafe impl Zeroable for u16x8 {}
59unsafe impl Pod for u16x8 {}
60
61impl AlignTo for u16x8 {
62  type Elem = u16;
63}
64
65impl Add for u16x8 {
66  type Output = Self;
67  #[inline]
68  fn add(self, rhs: Self) -> Self::Output {
69    pick! {
70      if #[cfg(target_feature="sse2")] {
71        Self { sse: add_i16_m128i(self.sse, rhs.sse) }
72      } else if #[cfg(target_feature="simd128")] {
73        Self { simd: u16x8_add(self.simd, rhs.simd) }
74      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
75        unsafe { Self { neon: vaddq_u16(self.neon, rhs.neon) } }
76      } else {
77        Self { arr: [
78          self.arr[0].wrapping_add(rhs.arr[0]),
79          self.arr[1].wrapping_add(rhs.arr[1]),
80          self.arr[2].wrapping_add(rhs.arr[2]),
81          self.arr[3].wrapping_add(rhs.arr[3]),
82          self.arr[4].wrapping_add(rhs.arr[4]),
83          self.arr[5].wrapping_add(rhs.arr[5]),
84          self.arr[6].wrapping_add(rhs.arr[6]),
85          self.arr[7].wrapping_add(rhs.arr[7]),
86        ]}
87      }
88    }
89  }
90}
91
92impl Sub for u16x8 {
93  type Output = Self;
94  #[inline]
95  fn sub(self, rhs: Self) -> Self::Output {
96    pick! {
97      if #[cfg(target_feature="sse2")] {
98        Self { sse: sub_i16_m128i(self.sse, rhs.sse) }
99      } else if #[cfg(target_feature="simd128")] {
100        Self { simd: u16x8_sub(self.simd, rhs.simd) }
101      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
102        unsafe {Self { neon: vsubq_u16(self.neon, rhs.neon) }}
103      } else {
104        Self { arr: [
105          self.arr[0].wrapping_sub(rhs.arr[0]),
106          self.arr[1].wrapping_sub(rhs.arr[1]),
107          self.arr[2].wrapping_sub(rhs.arr[2]),
108          self.arr[3].wrapping_sub(rhs.arr[3]),
109          self.arr[4].wrapping_sub(rhs.arr[4]),
110          self.arr[5].wrapping_sub(rhs.arr[5]),
111          self.arr[6].wrapping_sub(rhs.arr[6]),
112          self.arr[7].wrapping_sub(rhs.arr[7]),
113        ]}
114      }
115    }
116  }
117}
118
119impl Mul for u16x8 {
120  type Output = Self;
121  #[inline]
122  fn mul(self, rhs: Self) -> Self::Output {
123    pick! {
124      if #[cfg(target_feature="sse2")] {
125        Self { sse: mul_i16_keep_low_m128i(self.sse, rhs.sse) }
126      } else if #[cfg(target_feature="simd128")] {
127        Self { simd: u16x8_mul(self.simd, rhs.simd) }
128      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
129        unsafe {Self { neon: vmulq_u16(self.neon, rhs.neon) }}
130      } else {
131        Self { arr: [
132          self.arr[0].wrapping_mul(rhs.arr[0]),
133          self.arr[1].wrapping_mul(rhs.arr[1]),
134          self.arr[2].wrapping_mul(rhs.arr[2]),
135          self.arr[3].wrapping_mul(rhs.arr[3]),
136          self.arr[4].wrapping_mul(rhs.arr[4]),
137          self.arr[5].wrapping_mul(rhs.arr[5]),
138          self.arr[6].wrapping_mul(rhs.arr[6]),
139          self.arr[7].wrapping_mul(rhs.arr[7]),
140        ]}
141      }
142    }
143  }
144}
145
146integer_impl_div_rem!(u16, u16x8, [0, 1, 2, 3, 4, 5, 6, 7]);
147
148impl Shl for u16x8 {
149  type Output = Self;
150
151  /// Shifts lanes by the corresponding lane.
152  ///
153  /// Bitwise shift-left; yields `self << mask(rhs)`, where mask removes any
154  /// high-order bits of `rhs` that would cause the shift to exceed the bitwidth
155  /// of the type. (same as `wrapping_shl`)
156  #[inline]
157  fn shl(self, rhs: Self) -> Self::Output {
158    pick! {
159      if #[cfg(all(target_feature="avx512bw", target_feature="avx512vl"))] {
160        #[cfg(target_arch = "x86")]
161        use core::arch::x86::_mm_sllv_epi16;
162        #[cfg(target_arch = "x86_64")]
163        use core::arch::x86_64::_mm_sllv_epi16;
164
165        // Mask `rhs` to 15 to match `wrapping_shl`.
166        let rhs = bitand_m128i(rhs.sse, set_splat_i16_m128i(15));
167        // TODO(safe_arch): Add `_mm_sllv_epi16`.
168        cast(unsafe { _mm_sllv_epi16(self.sse.0, rhs.0) })
169      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))] {
170        unsafe {
171          // Mask `rhs` to 15 to match `wrapping_shl`.
172          let rhs = vreinterpretq_s16_u16(vandq_u16(rhs.neon, vmovq_n_u16(15)));
173          Self { neon: vshlq_u16(self.neon, rhs) }
174        }
175      } else {
176        let self_array = self.to_array();
177        let rhs_array = rhs.to_array();
178
179        Self::new([
180          self_array[0].wrapping_shl(rhs_array[0] as u32),
181          self_array[1].wrapping_shl(rhs_array[1] as u32),
182          self_array[2].wrapping_shl(rhs_array[2] as u32),
183          self_array[3].wrapping_shl(rhs_array[3] as u32),
184          self_array[4].wrapping_shl(rhs_array[4] as u32),
185          self_array[5].wrapping_shl(rhs_array[5] as u32),
186          self_array[6].wrapping_shl(rhs_array[6] as u32),
187          self_array[7].wrapping_shl(rhs_array[7] as u32),
188        ])
189      }
190    }
191  }
192}
193
194impl Shr for u16x8 {
195  type Output = Self;
196
197  /// Shifts lanes by the corresponding lane.
198  ///
199  /// Bitwise shift-right; yields `self >> mask(rhs)`, where mask removes any
200  /// high-order bits of `rhs` that would cause the shift to exceed the bitwidth
201  /// of the type. (same as `wrapping_shr`)
202  #[inline]
203  fn shr(self, rhs: Self) -> Self::Output {
204    pick! {
205      if #[cfg(all(target_feature="avx512bw", target_feature="avx512vl"))] {
206        #[cfg(target_arch = "x86")]
207        use core::arch::x86::_mm_srlv_epi16;
208        #[cfg(target_arch = "x86_64")]
209        use core::arch::x86_64::_mm_srlv_epi16;
210
211        // Mask `rhs` to 15 to match `wrapping_shr`.
212        let rhs = bitand_m128i(rhs.sse, set_splat_i16_m128i(15));
213        // TODO(safe_arch): Add `_mm_srlv_epi16`.
214        cast(unsafe { _mm_srlv_epi16(self.sse.0, rhs.0) })
215      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))] {
216        unsafe {
217          // Mask `rhs` to 15 to match `wrapping_shr`, and negate it because
218          // there is no shift-right intrinsic.
219          let neg_rhs = vnegq_s16(vreinterpretq_s16_u16(vandq_u16(rhs.neon, vmovq_n_u16(15))));
220          Self { neon: vshlq_u16(self.neon, neg_rhs) }
221        }
222      } else {
223        let self_array = self.to_array();
224        let rhs_array = rhs.to_array();
225
226        Self::new([
227          self_array[0].wrapping_shr(rhs_array[0] as u32),
228          self_array[1].wrapping_shr(rhs_array[1] as u32),
229          self_array[2].wrapping_shr(rhs_array[2] as u32),
230          self_array[3].wrapping_shr(rhs_array[3] as u32),
231          self_array[4].wrapping_shr(rhs_array[4] as u32),
232          self_array[5].wrapping_shr(rhs_array[5] as u32),
233          self_array[6].wrapping_shr(rhs_array[6] as u32),
234          self_array[7].wrapping_shr(rhs_array[7] as u32),
235        ])
236      }
237    }
238  }
239}
240
241impl Add<u16> for u16x8 {
242  type Output = Self;
243  #[inline]
244  fn add(self, rhs: u16) -> Self::Output {
245    self.add(Self::splat(rhs))
246  }
247}
248
249impl Sub<u16> for u16x8 {
250  type Output = Self;
251  #[inline]
252  fn sub(self, rhs: u16) -> Self::Output {
253    self.sub(Self::splat(rhs))
254  }
255}
256
257impl Mul<u16> for u16x8 {
258  type Output = Self;
259  #[inline]
260  fn mul(self, rhs: u16) -> Self::Output {
261    self.mul(Self::splat(rhs))
262  }
263}
264
265impl Add<u16x8> for u16 {
266  type Output = u16x8;
267  #[inline]
268  fn add(self, rhs: u16x8) -> Self::Output {
269    u16x8::splat(self).add(rhs)
270  }
271}
272
273impl Sub<u16x8> for u16 {
274  type Output = u16x8;
275  #[inline]
276  fn sub(self, rhs: u16x8) -> Self::Output {
277    u16x8::splat(self).sub(rhs)
278  }
279}
280
281impl Mul<u16x8> for u16 {
282  type Output = u16x8;
283  #[inline]
284  fn mul(self, rhs: u16x8) -> Self::Output {
285    u16x8::splat(self).mul(rhs)
286  }
287}
288
289impl BitAnd for u16x8 {
290  type Output = Self;
291  #[inline]
292  fn bitand(self, rhs: Self) -> Self::Output {
293    pick! {
294      if #[cfg(target_feature="sse2")] {
295        Self { sse: bitand_m128i(self.sse, rhs.sse) }
296      } else if #[cfg(target_feature="simd128")] {
297        Self { simd: v128_and(self.simd, rhs.simd) }
298      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
299        unsafe {Self { neon: vandq_u16(self.neon, rhs.neon) }}
300      } else {
301        Self { arr: [
302          self.arr[0].bitand(rhs.arr[0]),
303          self.arr[1].bitand(rhs.arr[1]),
304          self.arr[2].bitand(rhs.arr[2]),
305          self.arr[3].bitand(rhs.arr[3]),
306          self.arr[4].bitand(rhs.arr[4]),
307          self.arr[5].bitand(rhs.arr[5]),
308          self.arr[6].bitand(rhs.arr[6]),
309          self.arr[7].bitand(rhs.arr[7]),
310        ]}
311      }
312    }
313  }
314}
315
316impl BitOr for u16x8 {
317  type Output = Self;
318  #[inline]
319  fn bitor(self, rhs: Self) -> Self::Output {
320    pick! {
321      if #[cfg(target_feature="sse2")] {
322        Self { sse: bitor_m128i(self.sse, rhs.sse) }
323      } else if #[cfg(target_feature="simd128")] {
324        Self { simd: v128_or(self.simd, rhs.simd) }
325      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
326        unsafe {Self { neon: vorrq_u16(self.neon, rhs.neon) }}
327      } else {
328        Self { arr: [
329          self.arr[0].bitor(rhs.arr[0]),
330          self.arr[1].bitor(rhs.arr[1]),
331          self.arr[2].bitor(rhs.arr[2]),
332          self.arr[3].bitor(rhs.arr[3]),
333          self.arr[4].bitor(rhs.arr[4]),
334          self.arr[5].bitor(rhs.arr[5]),
335          self.arr[6].bitor(rhs.arr[6]),
336          self.arr[7].bitor(rhs.arr[7]),
337        ]}
338      }
339    }
340  }
341}
342
343impl BitXor for u16x8 {
344  type Output = Self;
345  #[inline]
346  fn bitxor(self, rhs: Self) -> Self::Output {
347    pick! {
348      if #[cfg(target_feature="sse2")] {
349        Self { sse: bitxor_m128i(self.sse, rhs.sse) }
350      } else if #[cfg(target_feature="simd128")] {
351        Self { simd: v128_xor(self.simd, rhs.simd) }
352      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
353        unsafe {Self { neon: veorq_u16(self.neon, rhs.neon) }}
354      } else {
355        Self { arr: [
356          self.arr[0].bitxor(rhs.arr[0]),
357          self.arr[1].bitxor(rhs.arr[1]),
358          self.arr[2].bitxor(rhs.arr[2]),
359          self.arr[3].bitxor(rhs.arr[3]),
360          self.arr[4].bitxor(rhs.arr[4]),
361          self.arr[5].bitxor(rhs.arr[5]),
362          self.arr[6].bitxor(rhs.arr[6]),
363          self.arr[7].bitxor(rhs.arr[7]),
364        ]}
365      }
366    }
367  }
368}
369
370macro_rules! impl_shl_t_for_u16x8 {
371  ($($shift_type:ty),+ $(,)?) => {
372    $(impl Shl<$shift_type> for u16x8 {
373      type Output = Self;
374      /// Shifts all lanes by the value given.
375      #[inline]
376      fn shl(self, rhs: $shift_type) -> Self::Output {
377        pick! {
378          if #[cfg(target_feature="sse2")] {
379            let shift = cast([rhs as u64, 0]);
380            Self { sse: shl_all_u16_m128i(self.sse, shift) }
381          } else if #[cfg(target_feature="simd128")] {
382            Self { simd: u16x8_shl(self.simd, rhs as u32) }
383          } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
384            unsafe {Self { neon: vshlq_u16(self.neon, vmovq_n_s16(rhs as i16)) }}
385          } else {
386            let u = rhs as u32;
387            Self { arr: [
388              self.arr[0].wrapping_shl(u),
389              self.arr[1].wrapping_shl(u),
390              self.arr[2].wrapping_shl(u),
391              self.arr[3].wrapping_shl(u),
392              self.arr[4].wrapping_shl(u),
393              self.arr[5].wrapping_shl(u),
394              self.arr[6].wrapping_shl(u),
395              self.arr[7].wrapping_shl(u),
396            ]}
397          }
398        }
399      }
400    })+
401  };
402}
403impl_shl_t_for_u16x8!(i8, u8, i16, u16, i32, u32, i64, u64, i128, u128);
404
405macro_rules! impl_shr_t_for_u16x8 {
406  ($($shift_type:ty),+ $(,)?) => {
407    $(impl Shr<$shift_type> for u16x8 {
408      type Output = Self;
409      /// Shifts all lanes by the value given.
410      #[inline]
411      fn shr(self, rhs: $shift_type) -> Self::Output {
412        pick! {
413          if #[cfg(target_feature="sse2")] {
414            let shift = cast([rhs as u64, 0]);
415            Self { sse: shr_all_u16_m128i(self.sse, shift) }
416          } else if #[cfg(target_feature="simd128")] {
417            Self { simd: u16x8_shr(self.simd, rhs as u32) }
418          } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
419            unsafe {Self { neon: vshlq_u16(self.neon, vmovq_n_s16( -(rhs as i16))) }}
420          } else {
421            let u = rhs as u32;
422            Self { arr: [
423              self.arr[0].wrapping_shr(u),
424              self.arr[1].wrapping_shr(u),
425              self.arr[2].wrapping_shr(u),
426              self.arr[3].wrapping_shr(u),
427              self.arr[4].wrapping_shr(u),
428              self.arr[5].wrapping_shr(u),
429              self.arr[6].wrapping_shr(u),
430              self.arr[7].wrapping_shr(u),
431            ]}
432          }
433        }
434      }
435    })+
436  };
437}
438impl_shr_t_for_u16x8!(i8, u8, i16, u16, i32, u32, i64, u64, i128, u128);
439
440#[expect(deprecated)]
441impl CmpEq for u16x8 {
442  type Output = Self;
443  #[inline]
444  fn simd_eq(self, rhs: Self) -> Self::Output {
445    pick! {
446      if #[cfg(target_feature="sse2")] {
447        Self { sse: cmp_eq_mask_i16_m128i(self.sse, rhs.sse) }
448      } else if #[cfg(target_feature="simd128")] {
449        Self { simd: u16x8_eq(self.simd, rhs.simd) }
450      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
451        unsafe {Self { neon: vceqq_u16(self.neon, rhs.neon) }}
452      } else {
453        Self { arr: [
454          if self.arr[0] == rhs.arr[0] { u16::MAX } else { 0 },
455          if self.arr[1] == rhs.arr[1] { u16::MAX } else { 0 },
456          if self.arr[2] == rhs.arr[2] { u16::MAX } else { 0 },
457          if self.arr[3] == rhs.arr[3] { u16::MAX } else { 0 },
458          if self.arr[4] == rhs.arr[4] { u16::MAX } else { 0 },
459          if self.arr[5] == rhs.arr[5] { u16::MAX } else { 0 },
460          if self.arr[6] == rhs.arr[6] { u16::MAX } else { 0 },
461          if self.arr[7] == rhs.arr[7] { u16::MAX } else { 0 },
462        ]}
463      }
464    }
465  }
466}
467
468#[expect(deprecated)]
469impl CmpGt for u16x8 {
470  type Output = Self;
471  #[inline]
472  fn simd_gt(self, rhs: Self) -> Self::Output {
473    pick! {
474      if #[cfg(target_feature = "sse2")] {
475        use safe_arch::*;
476
477        let bias = m128i::from([0x8000u16; 8]);
478
479        let a_biased = sub_i16_m128i(self.sse, bias);
480        let b_biased = sub_i16_m128i(rhs.sse, bias);
481        let mask = cmp_gt_mask_i16_m128i(a_biased, b_biased);
482
483        Self { sse: mask }
484      } else if #[cfg(target_feature="simd128")] {
485        Self { simd: u16x8_gt(self.simd, rhs.simd) }
486      } else if #[cfg(all(target_feature = "neon", target_arch = "aarch64"))] {
487        unsafe {
488          use core::arch::aarch64::*;
489          Self {
490            neon: vcgtq_u16(self.neon, rhs.neon),
491          }
492        }
493      } else {
494        Self {
495          arr: [
496            if self.arr[0] > rhs.arr[0] { u16::MAX } else { 0 },
497            if self.arr[1] > rhs.arr[1] { u16::MAX } else { 0 },
498            if self.arr[2] > rhs.arr[2] { u16::MAX } else { 0 },
499            if self.arr[3] > rhs.arr[3] { u16::MAX } else { 0 },
500            if self.arr[4] > rhs.arr[4] { u16::MAX } else { 0 },
501            if self.arr[5] > rhs.arr[5] { u16::MAX } else { 0 },
502            if self.arr[6] > rhs.arr[6] { u16::MAX } else { 0 },
503            if self.arr[7] > rhs.arr[7] { u16::MAX } else { 0 },
504          ]
505        }
506      }
507    }
508  }
509}
510
511#[expect(deprecated)]
512impl CmpLt for u16x8 {
513  type Output = Self;
514  #[inline]
515  fn simd_lt(self, rhs: Self) -> Self::Output {
516    // no lt, so reverse gt
517    Self::simd_gt(rhs, self)
518  }
519}
520
521#[expect(deprecated)]
522impl CmpNe for u16x8 {
523  type Output = Self;
524  #[inline]
525  fn simd_ne(self, rhs: Self) -> Self::Output {
526    pick! {
527      if #[cfg(target_feature="sse2")] {
528        !self.simd_eq(rhs)
529      } else if #[cfg(target_feature="simd128")] {
530        Self { simd: u16x8_ne(self.simd, rhs.simd) }
531      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
532        !self.simd_eq(rhs)
533      } else {
534        Self { arr: [
535          if self.arr[0] != rhs.arr[0] { u16::MAX } else { 0 },
536          if self.arr[1] != rhs.arr[1] { u16::MAX } else { 0 },
537          if self.arr[2] != rhs.arr[2] { u16::MAX } else { 0 },
538          if self.arr[3] != rhs.arr[3] { u16::MAX } else { 0 },
539          if self.arr[4] != rhs.arr[4] { u16::MAX } else { 0 },
540          if self.arr[5] != rhs.arr[5] { u16::MAX } else { 0 },
541          if self.arr[6] != rhs.arr[6] { u16::MAX } else { 0 },
542          if self.arr[7] != rhs.arr[7] { u16::MAX } else { 0 },
543        ]}
544      }
545    }
546  }
547}
548
549#[expect(deprecated)]
550impl CmpLe for u16x8 {
551  type Output = Self;
552  #[inline]
553  fn simd_le(self, rhs: Self) -> Self::Output {
554    pick! {
555      if #[cfg(target_feature="sse2")] {
556        !self.simd_gt(rhs)
557      } else if #[cfg(target_feature="simd128")] {
558        Self { simd: u16x8_le(self.simd, rhs.simd) }
559      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
560        !self.simd_gt(rhs)
561      } else {
562        Self { arr: [
563          if self.arr[0] <= rhs.arr[0] { u16::MAX } else { 0 },
564          if self.arr[1] <= rhs.arr[1] { u16::MAX } else { 0 },
565          if self.arr[2] <= rhs.arr[2] { u16::MAX } else { 0 },
566          if self.arr[3] <= rhs.arr[3] { u16::MAX } else { 0 },
567          if self.arr[4] <= rhs.arr[4] { u16::MAX } else { 0 },
568          if self.arr[5] <= rhs.arr[5] { u16::MAX } else { 0 },
569          if self.arr[6] <= rhs.arr[6] { u16::MAX } else { 0 },
570          if self.arr[7] <= rhs.arr[7] { u16::MAX } else { 0 },
571        ]}
572      }
573    }
574  }
575}
576
577#[expect(deprecated)]
578impl CmpGe for u16x8 {
579  type Output = Self;
580  #[inline]
581  fn simd_ge(self, rhs: Self) -> Self::Output {
582    pick! {
583      if #[cfg(target_feature="sse2")] {
584        !self.simd_lt(rhs)
585      } else if #[cfg(target_feature="simd128")] {
586        Self { simd: u16x8_ge(self.simd, rhs.simd) }
587      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
588        !self.simd_lt(rhs)
589      } else {
590        Self { arr: [
591          if self.arr[0] >= rhs.arr[0] { u16::MAX } else { 0 },
592          if self.arr[1] >= rhs.arr[1] { u16::MAX } else { 0 },
593          if self.arr[2] >= rhs.arr[2] { u16::MAX } else { 0 },
594          if self.arr[3] >= rhs.arr[3] { u16::MAX } else { 0 },
595          if self.arr[4] >= rhs.arr[4] { u16::MAX } else { 0 },
596          if self.arr[5] >= rhs.arr[5] { u16::MAX } else { 0 },
597          if self.arr[6] >= rhs.arr[6] { u16::MAX } else { 0 },
598          if self.arr[7] >= rhs.arr[7] { u16::MAX } else { 0 },
599        ]}
600      }
601    }
602  }
603}
604
605impl u16x8 {
606  #[inline]
607  #[must_use]
608  pub const fn new(array: [u16; 8]) -> Self {
609    unsafe { core::mem::transmute(array) }
610  }
611
612  simd_comparison_fns!();
613
614  #[inline]
615  #[must_use]
616  pub fn blend(self, t: Self, f: Self) -> Self {
617    pick! {
618      if #[cfg(target_feature="sse4.1")] {
619        Self { sse: blend_varying_i8_m128i(f.sse, t.sse, self.sse) }
620      } else if #[cfg(target_feature="simd128")] {
621        Self { simd: v128_bitselect(t.simd, f.simd, self.simd) }
622      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
623        unsafe {Self { neon: vbslq_u16(self.neon, t.neon, f.neon) }}
624      } else {
625        generic_bit_blend(self, t, f)
626      }
627    }
628  }
629
630  #[inline]
631  #[must_use]
632  pub fn reduce_add(self) -> u16 {
633    cast(i16x8::reduce_add(cast(self)))
634  }
635
636  #[inline]
637  #[must_use]
638  pub fn reduce_min(self) -> u16 {
639    pick! {
640      if #[cfg(all(target_feature="ssse3", target_feature="sse4.1"))] {
641        let hi64 = shuffle_ai_f32_all_m128i::<0b01_00_11_10>(self.sse);
642        let sum64 = min_u16_m128i(self.sse, hi64);
643        let hi32 = shuffle_ai_f32_all_m128i::<0b11_10_00_01>(sum64);
644        let sum32 = min_u16_m128i(sum64, hi32);
645        let lo16 = shr_imm_u32_m128i::<16>(sum32);
646        let sum16 = min_u16_m128i(sum32, lo16);
647        extract_i16_as_i32_m128i::<0>(sum16) as u16
648      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
649        unsafe { vminvq_u16(self.neon) }
650      } else {
651        let arr: [u16; 8] = cast(self);
652
653        // most boring implementation possible so optimizer doesn't overthink this
654        let mut r = arr[0];
655        r = r.min(arr[1]);
656        r = r.min(arr[2]);
657        r = r.min(arr[3]);
658        r = r.min(arr[4]);
659        r = r.min(arr[5]);
660        r = r.min(arr[6]);
661        r.min(arr[7])
662      }
663    }
664  }
665
666  #[inline]
667  #[must_use]
668  pub fn reduce_max(self) -> u16 {
669    pick! {
670      if #[cfg(all(target_feature="ssse3", target_feature="sse4.1"))] {
671        let hi64 = shuffle_ai_f32_all_m128i::<0b01_00_11_10>(self.sse);
672        let sum64 = max_u16_m128i(self.sse, hi64);
673        let hi32 = shuffle_ai_f32_all_m128i::<0b11_10_00_01>(sum64);
674        let sum32 = max_u16_m128i(sum64, hi32);
675        let lo16 = shr_imm_u32_m128i::<16>(sum32);
676        let sum16 = max_u16_m128i(sum32, lo16);
677        extract_i16_as_i32_m128i::<0>(sum16) as u16
678      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
679        unsafe { vmaxvq_u16(self.neon) }
680      } else {
681        let arr: [u16; 8] = cast(self);
682
683        // most boring implementation possible so optimizer doesn't overthink this
684        let mut r = arr[0];
685        r = r.max(arr[1]);
686        r = r.max(arr[2]);
687        r = r.max(arr[3]);
688        r = r.max(arr[4]);
689        r = r.max(arr[5]);
690        r = r.max(arr[6]);
691        r.max(arr[7])
692      }
693    }
694  }
695
696  #[inline]
697  #[must_use]
698  pub fn max(self, rhs: Self) -> Self {
699    pick! {
700      if #[cfg(target_feature="sse4.1")] {
701        Self { sse: max_u16_m128i(self.sse, rhs.sse) }
702      } else if #[cfg(target_feature="simd128")] {
703        Self { simd: u16x8_max(self.simd, rhs.simd) }
704      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
705        unsafe {Self { neon: vmaxq_u16(self.neon, rhs.neon) }}
706      } else {
707        let arr: [u16; 8] = cast(self);
708        let rhs: [u16; 8] = cast(rhs);
709        cast([
710          arr[0].max(rhs[0]),
711          arr[1].max(rhs[1]),
712          arr[2].max(rhs[2]),
713          arr[3].max(rhs[3]),
714          arr[4].max(rhs[4]),
715          arr[5].max(rhs[5]),
716          arr[6].max(rhs[6]),
717          arr[7].max(rhs[7]),
718        ])
719      }
720    }
721  }
722  #[inline]
723  #[must_use]
724  pub fn min(self, rhs: Self) -> Self {
725    pick! {
726      if #[cfg(target_feature="sse4.1")] {
727        Self { sse: min_u16_m128i(self.sse, rhs.sse) }
728      } else if #[cfg(target_feature="simd128")] {
729        Self { simd: u16x8_min(self.simd, rhs.simd) }
730      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
731        unsafe {Self { neon: vminq_u16(self.neon, rhs.neon) }}
732      } else {
733        let arr: [u16; 8] = cast(self);
734        let rhs: [u16; 8] = cast(rhs);
735        cast([
736          arr[0].min(rhs[0]),
737          arr[1].min(rhs[1]),
738          arr[2].min(rhs[2]),
739          arr[3].min(rhs[3]),
740          arr[4].min(rhs[4]),
741          arr[5].min(rhs[5]),
742          arr[6].min(rhs[6]),
743          arr[7].min(rhs[7]),
744        ])
745      }
746    }
747  }
748
749  integer_fn_clamp!();
750
751  #[inline]
752  #[must_use]
753  pub fn saturating_add(self, rhs: Self) -> Self {
754    pick! {
755      if #[cfg(target_feature="sse2")] {
756        Self { sse: add_saturating_u16_m128i(self.sse, rhs.sse) }
757      } else if #[cfg(target_feature="simd128")] {
758        Self { simd: u16x8_add_sat(self.simd, rhs.simd) }
759      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
760        unsafe {Self { neon: vqaddq_u16(self.neon, rhs.neon) }}
761      } else {
762        Self { arr: [
763          self.arr[0].saturating_add(rhs.arr[0]),
764          self.arr[1].saturating_add(rhs.arr[1]),
765          self.arr[2].saturating_add(rhs.arr[2]),
766          self.arr[3].saturating_add(rhs.arr[3]),
767          self.arr[4].saturating_add(rhs.arr[4]),
768          self.arr[5].saturating_add(rhs.arr[5]),
769          self.arr[6].saturating_add(rhs.arr[6]),
770          self.arr[7].saturating_add(rhs.arr[7]),
771        ]}
772      }
773    }
774  }
775  #[inline]
776  #[must_use]
777  pub fn saturating_sub(self, rhs: Self) -> Self {
778    pick! {
779      if #[cfg(target_feature="sse2")] {
780        Self { sse: sub_saturating_u16_m128i(self.sse, rhs.sse) }
781      } else if #[cfg(target_feature="simd128")] {
782        Self { simd: u16x8_sub_sat(self.simd, rhs.simd) }
783      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
784        unsafe {Self { neon: vqsubq_u16(self.neon, rhs.neon) }}
785      } else {
786        Self { arr: [
787          self.arr[0].saturating_sub(rhs.arr[0]),
788          self.arr[1].saturating_sub(rhs.arr[1]),
789          self.arr[2].saturating_sub(rhs.arr[2]),
790          self.arr[3].saturating_sub(rhs.arr[3]),
791          self.arr[4].saturating_sub(rhs.arr[4]),
792          self.arr[5].saturating_sub(rhs.arr[5]),
793          self.arr[6].saturating_sub(rhs.arr[6]),
794          self.arr[7].saturating_sub(rhs.arr[7]),
795        ]}
796      }
797    }
798  }
799
800  /// Lanewise saturating multiply.
801  #[inline]
802  #[must_use]
803  pub fn saturating_mul(self, rhs: Self) -> Self {
804    pick! {
805      if #[cfg(target_feature="simd128")] {
806        let low_wide_mul = u32x4_extmul_low_u16x8(self.simd, rhs.simd);
807        let high_wide_mul = u32x4_extmul_high_u16x8(self.simd, rhs.simd);
808        let low = Self { simd: u16x8_shuffle::<0, 2, 4, 6, 8, 10, 12, 14>(low_wide_mul, high_wide_mul) };
809        let high = Self { simd: u16x8_shuffle::<1, 3, 5, 7, 9, 11, 13, 15>(low_wide_mul, high_wide_mul) };
810
811        let no_overflow = high.simd_eq(Self::ZERO);
812        no_overflow.blend(low, Self::MAX)
813      } else if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
814        unsafe {
815          let low_wide_mul = vreinterpretq_u16_u32(
816            vmull_u16(vget_low_u16(self.neon), vget_low_u16(rhs.neon)),
817          );
818          let high_wide_mul = vreinterpretq_u16_u32(
819            vmull_u16(vget_high_u16(self.neon), vget_high_u16(rhs.neon)),
820          );
821          let low_high = vuzpq_u16(low_wide_mul, high_wide_mul);
822          let low = Self { neon: low_high.0 };
823          let high = Self { neon: low_high.1 };
824
825          let no_overflow = high.simd_eq(Self::ZERO);
826          no_overflow.blend(low, Self::MAX)
827        }
828      } else {
829        let self_array = self.to_array();
830        let rhs_array = rhs.to_array();
831
832        Self::new([
833          self_array[0].saturating_mul(rhs_array[0]),
834          self_array[1].saturating_mul(rhs_array[1]),
835          self_array[2].saturating_mul(rhs_array[2]),
836          self_array[3].saturating_mul(rhs_array[3]),
837          self_array[4].saturating_mul(rhs_array[4]),
838          self_array[5].saturating_mul(rhs_array[5]),
839          self_array[6].saturating_mul(rhs_array[6]),
840          self_array[7].saturating_mul(rhs_array[7]),
841        ])
842      }
843    }
844  }
845
846  integer_fn_saturating_div!([0, 1, 2, 3, 4, 5, 6, 7]);
847
848  /// Unpack the lower half of the input and zero expand it to `u16` values.
849  #[inline]
850  #[must_use]
851  pub fn from_u8x16_low(u: u8x16) -> Self {
852    pick! {
853      if #[cfg(target_feature="sse2")] {
854        Self{ sse: unpack_low_i8_m128i(u.sse, m128i::zeroed()) }
855      } else {
856        let u_arr: [u8; 16] = cast(u);
857        cast([
858          u_arr[0] as u16,
859          u_arr[1] as u16,
860          u_arr[2] as u16,
861          u_arr[3] as u16,
862          u_arr[4] as u16,
863          u_arr[5] as u16,
864          u_arr[6] as u16,
865          u_arr[7] as u16,
866        ])
867      }
868    }
869  }
870
871  /// Unpack the upper half of the input and zero expand it to `u16` values.
872  #[inline]
873  #[must_use]
874  pub fn from_u8x16_high(u: u8x16) -> Self {
875    pick! {
876      if #[cfg(target_feature="sse2")] {
877        Self{ sse: unpack_high_i8_m128i(u.sse, m128i::zeroed()) }
878      } else {
879        let u_arr: [u8; 16] = cast(u);
880        cast([
881          u_arr[8] as u16,
882          u_arr[9] as u16,
883          u_arr[10] as u16,
884          u_arr[11] as u16,
885          u_arr[12] as u16,
886          u_arr[13] as u16,
887          u_arr[14] as u16,
888          u_arr[15] as u16,
889        ])
890      }
891    }
892  }
893
894  /// multiplies two u16x8 and returns the result as a widened u32x8
895  #[inline]
896  #[must_use]
897  pub fn mul_widen(self, rhs: Self) -> u32x8 {
898    pick! {
899      if #[cfg(target_feature="avx2")] {
900        let a = convert_to_i32_m256i_from_u16_m128i(self.sse);
901        let b = convert_to_i32_m256i_from_u16_m128i(rhs.sse);
902        u32x8 { avx2: mul_i32_keep_low_m256i(a,b) }
903      } else if #[cfg(target_feature="sse2")] {
904         let low = mul_i16_keep_low_m128i(self.sse, rhs.sse);
905         let high = mul_u16_keep_high_m128i(self.sse, rhs.sse);
906         u32x8 {
907          a: u32x4 { sse:unpack_low_i16_m128i(low, high) },
908          b: u32x4 { sse:unpack_high_i16_m128i(low, high) }
909        }
910      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))] {
911         let lhs_low = unsafe { vget_low_u16(self.neon) };
912         let rhs_low = unsafe { vget_low_u16(rhs.neon) };
913
914         let lhs_high = unsafe { vget_high_u16(self.neon) };
915         let rhs_high = unsafe { vget_high_u16(rhs.neon) };
916
917         let low = unsafe { vmull_u16(lhs_low, rhs_low) };
918         let high = unsafe { vmull_u16(lhs_high, rhs_high) };
919
920         u32x8 { a: u32x4 { neon: low }, b: u32x4 {neon: high } }
921       } else {
922        let a = self.as_array();
923        let b = rhs.as_array();
924         u32x8::new([
925           u32::from(a[0]) * u32::from(b[0]),
926           u32::from(a[1]) * u32::from(b[1]),
927           u32::from(a[2]) * u32::from(b[2]),
928           u32::from(a[3]) * u32::from(b[3]),
929           u32::from(a[4]) * u32::from(b[4]),
930           u32::from(a[5]) * u32::from(b[5]),
931           u32::from(a[6]) * u32::from(b[6]),
932           u32::from(a[7]) * u32::from(b[7]),
933         ])
934       }
935    }
936  }
937
938  /// Multiples two `u16x8` and return the high part of intermediate `u32x8`
939  #[inline]
940  #[must_use]
941  pub fn mul_keep_high(self, rhs: Self) -> Self {
942    pick! {
943      if #[cfg(target_feature="sse2")] {
944        Self { sse: mul_u16_keep_high_m128i(self.sse, rhs.sse) }
945      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))] {
946        let lhs_low = unsafe { vget_low_u16(self.neon) };
947        let rhs_low = unsafe { vget_low_u16(rhs.neon) };
948
949        let lhs_high = unsafe { vget_high_u16(self.neon) };
950        let rhs_high = unsafe { vget_high_u16(rhs.neon) };
951
952        let low = unsafe { vmull_u16(lhs_low, rhs_low) };
953        let high = unsafe { vmull_u16(lhs_high, rhs_high) };
954
955        u16x8 { neon: unsafe { vuzpq_u16(vreinterpretq_u16_u32(low), vreinterpretq_u16_u32(high)).1 } }
956      } else if #[cfg(target_feature="simd128")] {
957        let low =  u32x4_extmul_low_u16x8(self.simd, rhs.simd);
958        let high = u32x4_extmul_high_u16x8(self.simd, rhs.simd);
959
960        Self { simd: u16x8_shuffle::<1, 3, 5, 7, 9, 11, 13, 15>(low, high) }
961      } else {
962        u16x8::new([
963          ((u32::from(rhs.as_array()[0]) * u32::from(self.as_array()[0])) >> 16) as u16,
964          ((u32::from(rhs.as_array()[1]) * u32::from(self.as_array()[1])) >> 16) as u16,
965          ((u32::from(rhs.as_array()[2]) * u32::from(self.as_array()[2])) >> 16) as u16,
966          ((u32::from(rhs.as_array()[3]) * u32::from(self.as_array()[3])) >> 16) as u16,
967          ((u32::from(rhs.as_array()[4]) * u32::from(self.as_array()[4])) >> 16) as u16,
968          ((u32::from(rhs.as_array()[5]) * u32::from(self.as_array()[5])) >> 16) as u16,
969          ((u32::from(rhs.as_array()[6]) * u32::from(self.as_array()[6])) >> 16) as u16,
970          ((u32::from(rhs.as_array()[7]) * u32::from(self.as_array()[7])) >> 16) as u16,
971        ])
972      }
973    }
974  }
975
976  #[inline]
977  #[must_use]
978  #[doc(alias("movemask", "move_mask"))]
979  pub fn to_bitmask(self) -> u32 {
980    i16x8::to_bitmask(cast(self))
981  }
982
983  #[inline]
984  #[must_use]
985  pub fn any(self) -> bool {
986    i16x8::any(cast(self))
987  }
988
989  #[inline]
990  #[must_use]
991  pub fn all(self) -> bool {
992    i16x8::all(cast(self))
993  }
994
995  #[inline]
996  #[must_use]
997  pub fn none(self) -> bool {
998    !self.any()
999  }
1000
1001  /// Transpose matrix of 8x8 `u16` matrix.
1002  #[must_use]
1003  #[inline]
1004  pub fn transpose(data: [u16x8; 8]) -> [u16x8; 8] {
1005    cast(i16x8::transpose(cast(data)))
1006  }
1007
1008  #[inline]
1009  pub fn to_array(self) -> [u16; 8] {
1010    cast(self)
1011  }
1012
1013  #[inline]
1014  pub fn as_array(&self) -> &[u16; 8] {
1015    cast_ref(self)
1016  }
1017
1018  #[inline]
1019  pub fn as_mut_array(&mut self) -> &mut [u16; 8] {
1020    cast_mut(self)
1021  }
1022}
wide/u16x8_.rs

wide/
u16x8_.rs
