wide/
u16x32_.rs

1use super::*;
2
3pick! {
4  if #[cfg(target_feature="avx512bw")] {
5    #[derive(Default, Clone, Copy, PartialEq, Eq)]
6    #[repr(C, align(64))]
7    pub struct u16x32 { pub(crate) avx512: m512i }
8  } else {
9    #[derive(Default, Clone, Copy, PartialEq, Eq)]
10    #[repr(C, align(64))]
11    pub struct u16x32 { pub(crate) a : u16x16, pub(crate) b : u16x16 }
12  }
13}
14
15int_uint_consts!(u16, 32, u16x32, 512);
16
17unsafe impl Zeroable for u16x32 {}
18unsafe impl Pod for u16x32 {}
19
20impl AlignTo for u16x32 {
21  type Elem = u16;
22}
23
24impl Add for u16x32 {
25  type Output = Self;
26  #[inline]
27  fn add(self, rhs: Self) -> Self::Output {
28    pick! {
29      if #[cfg(target_feature="avx512bw")] {
30        Self { avx512: add_i16_m512i(self.avx512, rhs.avx512) }
31      } else {
32        Self {
33          a : self.a.add(rhs.a),
34          b : self.b.add(rhs.b),
35        }
36      }
37    }
38  }
39}
40
41impl Sub for u16x32 {
42  type Output = Self;
43  #[inline]
44  fn sub(self, rhs: Self) -> Self::Output {
45    pick! {
46      if #[cfg(target_feature="avx512bw")] {
47        Self { avx512: sub_i16_m512i(self.avx512, rhs.avx512) }
48      } else {
49        Self {
50          a : self.a.sub(rhs.a),
51          b : self.b.sub(rhs.b),
52        }
53      }
54    }
55  }
56}
57
58impl Mul for u16x32 {
59  type Output = Self;
60  #[inline]
61  fn mul(self, rhs: Self) -> Self::Output {
62    pick! {
63      if #[cfg(target_feature="avx512bw")] {
64        Self { avx512: mul_i16_keep_low_m512i(self.avx512, rhs.avx512) }
65      } else {
66        Self { a: self.a.mul(rhs.a), b: self.b.mul(rhs.b) }
67      }
68    }
69  }
70}
71
72integer_impl_div_rem!(
73  u16,
74  u16x32,
75  [
76    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
77    21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
78  ],
79);
80
81impl Shl for u16x32 {
82  type Output = Self;
83
84  /// Shifts lanes by the corresponding lane.
85  ///
86  /// Bitwise shift-left; yields `self << mask(rhs)`, where mask removes any
87  /// high-order bits of `rhs` that would cause the shift to exceed the bitwidth
88  /// of the type. (same as `wrapping_shl`)
89  #[inline]
90  fn shl(self, rhs: Self) -> Self::Output {
91    pick! {
92      if #[cfg(target_feature="avx512bw")] {
93        // Mask `rhs` to 15 to match `wrapping_shl`.
94        let rhs = bitand_m512i(rhs.avx512, set_splat_i16_m512i(15));
95        Self { avx512: shl_each_u16_m512i(self.avx512, rhs) }
96      } else {
97        let [self_a, self_b]: [u16x16; 2] = cast(self);
98        let [rhs_a, rhs_b]: [u16x16; 2] = cast(rhs);
99
100        cast([self_a << rhs_a, self_b << rhs_b])
101      }
102    }
103  }
104}
105
106impl Shr for u16x32 {
107  type Output = Self;
108
109  /// Shifts lanes by the corresponding lane.
110  ///
111  /// Bitwise shift-right; yields `self >> mask(rhs)`, where mask removes any
112  /// high-order bits of `rhs` that would cause the shift to exceed the bitwidth
113  /// of the type. (same as `wrapping_shr`)
114  #[inline]
115  fn shr(self, rhs: Self) -> Self::Output {
116    pick! {
117      if #[cfg(target_feature="avx512bw")] {
118        // Mask `rhs` to 15 to match `wrapping_shr`.
119        let rhs = bitand_m512i(rhs.avx512, set_splat_i16_m512i(15));
120        Self { avx512: shr_each_u16_m512i(self.avx512, rhs) }
121      } else {
122        let [self_a, self_b]: [u16x16; 2] = cast(self);
123        let [rhs_a, rhs_b]: [u16x16; 2] = cast(rhs);
124
125        cast([self_a >> rhs_a, self_b >> rhs_b])
126      }
127    }
128  }
129}
130
131impl Add<u16> for u16x32 {
132  type Output = Self;
133  #[inline]
134  fn add(self, rhs: u16) -> Self::Output {
135    self.add(Self::splat(rhs))
136  }
137}
138
139impl Sub<u16> for u16x32 {
140  type Output = Self;
141  #[inline]
142  fn sub(self, rhs: u16) -> Self::Output {
143    self.sub(Self::splat(rhs))
144  }
145}
146
147impl Mul<u16> for u16x32 {
148  type Output = Self;
149  #[inline]
150  fn mul(self, rhs: u16) -> Self::Output {
151    self.mul(Self::splat(rhs))
152  }
153}
154
155impl Add<u16x32> for u16 {
156  type Output = u16x32;
157  #[inline]
158  fn add(self, rhs: u16x32) -> Self::Output {
159    u16x32::splat(self).add(rhs)
160  }
161}
162
163impl Sub<u16x32> for u16 {
164  type Output = u16x32;
165  #[inline]
166  fn sub(self, rhs: u16x32) -> Self::Output {
167    u16x32::splat(self).sub(rhs)
168  }
169}
170
171impl Mul<u16x32> for u16 {
172  type Output = u16x32;
173  #[inline]
174  fn mul(self, rhs: u16x32) -> Self::Output {
175    u16x32::splat(self).mul(rhs)
176  }
177}
178
179impl BitAnd for u16x32 {
180  type Output = Self;
181  #[inline]
182  fn bitand(self, rhs: Self) -> Self::Output {
183    pick! {
184      if #[cfg(target_feature="avx512bw")] {
185        Self { avx512: bitand_m512i(self.avx512, rhs.avx512) }
186      } else {
187        Self {
188          a : self.a.bitand(rhs.a),
189          b : self.b.bitand(rhs.b),
190        }
191      }
192    }
193  }
194}
195
196impl BitOr for u16x32 {
197  type Output = Self;
198  #[inline]
199  fn bitor(self, rhs: Self) -> Self::Output {
200    pick! {
201    if #[cfg(target_feature="avx512bw")] {
202        Self { avx512: bitor_m512i(self.avx512, rhs.avx512) }
203      } else {
204        Self {
205          a : self.a.bitor(rhs.a),
206          b : self.b.bitor(rhs.b),
207        }
208      }
209    }
210  }
211}
212
213impl BitXor for u16x32 {
214  type Output = Self;
215  #[inline]
216  fn bitxor(self, rhs: Self) -> Self::Output {
217    pick! {
218      if #[cfg(target_feature="avx512bw")] {
219        Self { avx512: bitxor_m512i(self.avx512, rhs.avx512) }
220      } else {
221        Self {
222          a : self.a.bitxor(rhs.a),
223          b : self.b.bitxor(rhs.b),
224        }
225      }
226    }
227  }
228}
229
230macro_rules! impl_shl_t_for_u16x32 {
231  ($($shift_type:ty),+ $(,)?) => {
232    $(impl Shl<$shift_type> for u16x32 {
233      type Output = Self;
234      /// Shifts all lanes by the value given.
235      #[inline]
236      fn shl(self, rhs: $shift_type) -> Self::Output {
237        pick! {
238          if #[cfg(target_feature="avx512bw")] {
239            let shift = cast(rhs as u16);
240            Self { avx512: shl_all_u16_m512i(self.avx512, shift) }
241          } else {
242            Self {
243              a : self.a.shl(rhs),
244              b : self.b.shl(rhs),
245            }
246          }
247        }
248      }
249    })+
250  };
251}
252impl_shl_t_for_u16x32!(i8, u8, i16, u16, i32, u32, i64, u64, i128, u128);
253
254macro_rules! impl_shr_t_for_u16x32 {
255  ($($shift_type:ty),+ $(,)?) => {
256    $(impl Shr<$shift_type> for u16x32 {
257      type Output = Self;
258      /// Shifts all lanes by the value given.
259      #[inline]
260      fn shr(self, rhs: $shift_type) -> Self::Output {
261        pick! {
262          if #[cfg(target_feature="avx512bw")] {
263            let shift = cast(rhs as u16);
264            Self { avx512: shr_all_u16_m512i(self.avx512, shift) }
265          } else {
266            Self {
267              a : self.a.shr(rhs),
268              b : self.b.shr(rhs),
269            }
270          }
271        }
272      }
273    })+
274  };
275}
276impl_shr_t_for_u16x32!(i8, u8, i16, u16, i32, u32, i64, u64, i128, u128);
277
278#[expect(deprecated)]
279impl CmpEq for u16x32 {
280  type Output = Self;
281  #[inline]
282  fn simd_eq(self, rhs: Self) -> Self::Output {
283    pick! {
284      if #[cfg(target_feature="avx512bw")] {
285        Self { avx512: cmp_op_mask_u16_m512i::<{cmp_int_op!(Eq)}>(self.avx512, rhs.avx512) }
286      } else {
287        Self {
288          a : self.a.simd_eq(rhs.a),
289          b : self.b.simd_eq(rhs.b),
290        }
291      }
292    }
293  }
294}
295
296#[expect(deprecated)]
297impl CmpLt for u16x32 {
298  type Output = Self;
299  #[inline]
300  fn simd_lt(self, rhs: Self) -> Self::Output {
301    pick! {
302      if #[cfg(target_feature="avx512bw")] {
303        Self { avx512: cmp_op_mask_u16_m512i::<{cmp_int_op!(Lt)}>(self.avx512, rhs.avx512) }
304      } else {
305        Self {
306          a : rhs.a.simd_gt(self.a),
307          b : rhs.b.simd_gt(self.b),
308        }
309      }
310    }
311  }
312}
313
314#[expect(deprecated)]
315impl CmpGt for u16x32 {
316  type Output = Self;
317  #[inline]
318  fn simd_gt(self, rhs: Self) -> Self::Output {
319    pick! {
320      if #[cfg(target_feature="avx512bw")] {
321        Self { avx512: cmp_op_mask_u16_m512i::<{cmp_int_op!(Nle)}>(self.avx512, rhs.avx512) }
322      } else {
323        Self {
324          a : self.a.simd_gt(rhs.a),
325          b : self.b.simd_gt(rhs.b),
326        }
327      }
328    }
329  }
330}
331
332#[expect(deprecated)]
333impl CmpNe for u16x32 {
334  type Output = Self;
335  #[inline]
336  fn simd_ne(self, rhs: Self) -> Self::Output {
337    pick! {
338      if #[cfg(target_feature="avx512bw")] {
339        Self { avx512: cmp_op_mask_u16_m512i::<{cmp_int_op!(Ne)}>(self.avx512, rhs.avx512) }
340      } else {
341        Self {
342          a : self.a.simd_ne(rhs.a),
343          b : self.b.simd_ne(rhs.b),
344        }
345      }
346    }
347  }
348}
349
350#[expect(deprecated)]
351impl CmpLe for u16x32 {
352  type Output = Self;
353  #[inline]
354  fn simd_le(self, rhs: Self) -> Self::Output {
355    pick! {
356      if #[cfg(target_feature="avx512bw")] {
357        Self { avx512: cmp_op_mask_u16_m512i::<{cmp_int_op!(Le)}>(self.avx512, rhs.avx512) }
358      } else {
359        Self {
360          a : self.a.simd_le(rhs.a),
361          b : self.b.simd_le(rhs.b),
362        }
363      }
364    }
365  }
366}
367
368#[expect(deprecated)]
369impl CmpGe for u16x32 {
370  type Output = Self;
371  #[inline]
372  fn simd_ge(self, rhs: Self) -> Self::Output {
373    pick! {
374      if #[cfg(target_feature="avx512bw")] {
375        Self { avx512: cmp_op_mask_u16_m512i::<{cmp_int_op!(Nlt)}>(self.avx512, rhs.avx512) }
376      } else {
377        Self {
378          a : self.a.simd_ge(rhs.a),
379          b : self.b.simd_ge(rhs.b),
380        }
381      }
382    }
383  }
384}
385
386impl Not for u16x32 {
387  type Output = Self;
388  #[inline]
389  fn not(self) -> Self::Output {
390    pick! {
391      if #[cfg(target_feature="avx512bw")] {
392        Self { avx512: bitxor_m512i(self.avx512, set_splat_i16_m512i(-1)) }
393      } else {
394        Self {
395          a : self.a.not(),
396          b : self.b.not(),
397        }
398      }
399    }
400  }
401}
402
403impl u16x32 {
404  #[inline]
405  #[must_use]
406  pub const fn new(array: [u16; 32]) -> Self {
407    unsafe { core::mem::transmute(array) }
408  }
409
410  simd_comparison_fns!();
411
412  #[inline]
413  #[must_use]
414  pub fn blend(self, t: Self, f: Self) -> Self {
415    pick! {
416      if #[cfg(target_feature="avx512bw")] {
417        Self { avx512: blend_varying_i8_m512i(f.avx512,t.avx512,movepi8_mask_m512i(self.avx512)) }
418      } else {
419        Self {
420          a : self.a.blend(t.a, f.a),
421          b : self.b.blend(t.b, f.b),
422        }
423      }
424    }
425  }
426
427  #[inline]
428  #[must_use]
429  pub fn reduce_add(self) -> u16 {
430    cast(i16x32::reduce_add(cast(self)))
431  }
432
433  #[inline]
434  #[must_use]
435  pub fn reduce_max(self) -> u16 {
436    let array: [u16x16; 2] = cast(self);
437    array[0].max(array[1]).reduce_max()
438  }
439
440  #[inline]
441  #[must_use]
442  pub fn reduce_min(self) -> u16 {
443    let array: [u16x16; 2] = cast(self);
444    array[0].min(array[1]).reduce_min()
445  }
446
447  #[inline]
448  #[must_use]
449  pub fn min(self, rhs: Self) -> Self {
450    pick! {
451      if #[cfg(target_feature="avx512bw")] {
452        Self { avx512: min_u16_m512i(self.avx512, rhs.avx512) }
453      } else {
454        Self {
455          a: self.a.min(rhs.a),
456          b: self.b.min(rhs.b),
457        }
458      }
459    }
460  }
461
462  #[inline]
463  #[must_use]
464  pub fn max(self, rhs: Self) -> Self {
465    pick! {
466      if #[cfg(target_feature="avx512bw")] {
467        Self { avx512: max_u16_m512i(self.avx512, rhs.avx512) }
468      } else {
469        Self {
470          a: self.a.max(rhs.a),
471          b: self.b.max(rhs.b),
472        }
473      }
474    }
475  }
476
477  integer_fn_clamp!();
478
479  #[inline]
480  #[must_use]
481  pub fn saturating_add(self, rhs: Self) -> Self {
482    pick! {
483      if #[cfg(target_feature="avx512bw")] {
484        Self { avx512: add_saturating_u16_m512i(self.avx512, rhs.avx512) }
485      } else {
486        Self {
487          a: self.a.saturating_add(rhs.a),
488          b: self.b.saturating_add(rhs.b),
489        }
490      }
491    }
492  }
493
494  #[inline]
495  #[must_use]
496  pub fn saturating_sub(self, rhs: Self) -> Self {
497    pick! {
498      if #[cfg(target_feature="avx512bw")] {
499        Self { avx512: sub_saturating_u16_m512i(self.avx512, rhs.avx512) }
500      } else {
501        Self {
502          a: self.a.saturating_sub(rhs.a),
503          b: self.b.saturating_sub(rhs.b),
504        }
505      }
506    }
507  }
508
509  /// Lanewise saturating multiply.
510  #[inline]
511  #[must_use]
512  pub fn saturating_mul(self, rhs: Self) -> Self {
513    let [self_a, self_b]: [u16x16; 2] = cast(self);
514    let [rhs_a, rhs_b]: [u16x16; 2] = cast(rhs);
515    cast([self_a.saturating_mul(rhs_a), self_b.saturating_mul(rhs_b)])
516  }
517
518  integer_fn_saturating_div!([
519    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
520    21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
521  ]);
522
523  #[inline]
524  #[must_use]
525  #[doc(alias("movemask", "move_mask"))]
526  pub fn to_bitmask(self) -> u32 {
527    i16x32::to_bitmask(cast(self))
528  }
529
530  #[inline]
531  #[must_use]
532  pub fn any(self) -> bool {
533    i16x32::any(cast(self))
534  }
535
536  #[inline]
537  #[must_use]
538  pub fn all(self) -> bool {
539    i16x32::all(cast(self))
540  }
541
542  #[inline]
543  #[must_use]
544  pub fn none(self) -> bool {
545    !self.any()
546  }
547
548  /// Transpose matrix of 32x32 `u16` matrix. Currently not accelerated.
549  #[must_use]
550  #[inline]
551  pub fn transpose(data: [u16x32; 32]) -> [u16x32; 32] {
552    cast(i16x32::transpose(cast(data)))
553  }
554
555  #[inline]
556  pub fn to_array(self) -> [u16; 32] {
557    cast(self)
558  }
559
560  #[inline]
561  pub fn as_array(&self) -> &[u16; 32] {
562    cast_ref(self)
563  }
564
565  #[inline]
566  pub fn as_mut_array(&mut self) -> &mut [u16; 32] {
567    cast_mut(self)
568  }
569}
wide/u16x32_.rs

wide/
u16x32_.rs
