Add SIMD funnel shift and round-to-even intrinsics

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -495,7 +495,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         | sym::simd_flog
         | sym::simd_flog10
         | sym::simd_flog2
-        | sym::simd_round => {
+        | sym::simd_round
+        | sym::simd_round_ties_even => {
             intrinsic_args!(fx, args => (a); intrinsic);
 
             if !a.layout().ty.is_simd() {
@@ -526,6 +527,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                     (sym::simd_flog2, types::F64) => "log2",
                     (sym::simd_round, types::F32) => "roundf",
                     (sym::simd_round, types::F64) => "round",
+                    (sym::simd_round_ties_even, types::F32) => "rintf",
+                    (sym::simd_round_ties_even, types::F64) => "rint",
                     _ => unreachable!("{:?}", intrinsic),
                 };
                 fx.lib_call(

compiler/rustc_codegen_gcc/src/intrinsic/simd.rs

@@ -779,6 +779,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
             sym::simd_fsin => "sin",
             sym::simd_fsqrt => "sqrt",
             sym::simd_round => "round",
+            sym::simd_round_ties_even => "rint",
             sym::simd_trunc => "trunc",
             _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
         };
@@ -826,6 +827,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
             | sym::simd_fsin
             | sym::simd_fsqrt
             | sym::simd_round
+            | sym::simd_round_ties_even
             | sym::simd_trunc
     ) {
         return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -1532,6 +1532,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             sym::simd_fsin => "llvm.sin",
             sym::simd_fsqrt => "llvm.sqrt",
             sym::simd_round => "llvm.round",
+            sym::simd_round_ties_even => "llvm.rint",
             sym::simd_trunc => "llvm.trunc",
             _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
         };
@@ -1558,6 +1559,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             | sym::simd_fsqrt
             | sym::simd_relaxed_fma
             | sym::simd_round
+            | sym::simd_round_ties_even
             | sym::simd_trunc
     ) {
         return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
@@ -2304,7 +2306,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     // Unary integer intrinsics
     if matches!(
         name,
-        sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop | sym::simd_cttz
+        sym::simd_bswap
+            | sym::simd_bitreverse
+            | sym::simd_ctlz
+            | sym::simd_ctpop
+            | sym::simd_cttz
+            | sym::simd_funnel_shl
+            | sym::simd_funnel_shr
     ) {
         let vec_ty = bx.cx.type_vector(
             match *in_elem.kind() {
@@ -2325,6 +2333,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             sym::simd_ctlz => "llvm.ctlz",
             sym::simd_ctpop => "llvm.ctpop",
             sym::simd_cttz => "llvm.cttz",
+            sym::simd_funnel_shl => "llvm.fshl",
+            sym::simd_funnel_shr => "llvm.fshr",
             _ => unreachable!(),
         };
         let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
@@ -2345,6 +2355,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                 // simple unary argument cases
                 Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[args[0].immediate()]))
             }
+            sym::simd_funnel_shl | sym::simd_funnel_shr => Ok(bx.call_intrinsic(
+                llvm_intrinsic,
+                &[vec_ty],
+                &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
+            )),
             _ => unreachable!(),
         };
     }

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -594,8 +594,9 @@ pub(crate) fn check_intrinsic_type(
         | sym::simd_ceil
         | sym::simd_floor
         | sym::simd_round
+        | sym::simd_round_ties_even
         | sym::simd_trunc => (1, 0, vec![param(0)], param(0)),
-        sym::simd_fma | sym::simd_relaxed_fma => {
+        sym::simd_fma | sym::simd_relaxed_fma | sym::simd_funnel_shl | sym::simd_funnel_shr => {
             (1, 0, vec![param(0), param(0), param(0)], param(0))
         }
         sym::simd_gather => (3, 0, vec![param(0), param(1), param(2)], param(0)),

compiler/rustc_span/src/symbol.rs

@@ -1967,6 +1967,8 @@ symbols! {
         simd_fmin,
         simd_fsin,
         simd_fsqrt,
+        simd_funnel_shl,
+        simd_funnel_shr,
         simd_gather,
         simd_ge,
         simd_gt,
@@ -1994,6 +1996,7 @@ symbols! {
         simd_relaxed_fma,
         simd_rem,
         simd_round,
+        simd_round_ties_even,
         simd_saturating_add,
         simd_saturating_sub,
         simd_scatter,

library/core/src/intrinsics/simd.rs

@@ -126,6 +126,40 @@ pub unsafe fn simd_shl<T>(lhs: T, rhs: T) -> T;
 #[rustc_nounwind]
 pub unsafe fn simd_shr<T>(lhs: T, rhs: T) -> T;
 
+/// Funnel Shifts vector left elementwise, with UB on overflow.
+///
+/// Concatenates `a` and `b` elementwise (with `a` in the most significant half),
+/// creating a vector of the same length, but with each element being twice as
+/// wide. Then shift this vector left elementwise by `shift`, shifting in zeros,
+/// and extract the most significant half of each of the elements. If `a` and `b`
+/// are the same, this is equivalent to an elementwise rotate left operation.
+///
+/// `T` must be a vector of integers.
+///
+/// # Safety
+///
+/// Each element of `shift` must be less than `<int>::BITS`.
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub unsafe fn simd_funnel_shl<T>(a: T, b: T, shift: T) -> T;
+
+/// Funnel Shifts vector right elementwise, with UB on overflow.
+///
+/// Concatenates `a` and `b` elementwise (with `a` in the most significant half),
+/// creating a vector of the same length, but with each element being twice as
+/// wide. Then shift this vector right elementwise by `shift`, shifting in zeros,
+/// and extract the least significant half of each of the elements. If `a` and `b`
+/// are the same, this is equivalent to an elementwise rotate right operation.
+///
+/// `T` must be a vector of integers.
+///
+/// # Safety
+///
+/// Each element of `shift` must be less than `<int>::BITS`.
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub unsafe fn simd_funnel_shr<T>(a: T, b: T, shift: T) -> T;
+
 /// "Ands" vectors elementwise.
 ///
 /// `T` must be a vector of integers.
@@ -678,6 +712,14 @@ pub unsafe fn simd_floor<T>(x: T) -> T;
 #[rustc_nounwind]
 pub unsafe fn simd_round<T>(x: T) -> T;
 
+/// Rounds each element to the closest integer-valued float.
+/// Ties are resolved by rounding to the number with an even least significant digit
+///
+/// `T` must be a vector of floats.
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub unsafe fn simd_round_ties_even<T>(x: T) -> T;
+
 /// Returns the integer part of each element as an integer-valued float.
 /// In other words, non-integer values are truncated towards zero.
 ///

src/tools/miri/src/intrinsics/simd.rs

@@ -37,6 +37,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
             | "ceil"
             | "floor"
             | "round"
+            | "round_ties_even"
             | "trunc"
             | "fsqrt"
             | "fsin"
@@ -72,6 +73,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                     "ceil" => Op::Round(rustc_apfloat::Round::TowardPositive),
                     "floor" => Op::Round(rustc_apfloat::Round::TowardNegative),
                     "round" => Op::Round(rustc_apfloat::Round::NearestTiesToAway),
+                    "round_ties_even" => Op::Round(rustc_apfloat::Round::NearestTiesToEven),
                     "trunc" => Op::Round(rustc_apfloat::Round::TowardZero),
                     "ctlz" => Op::Numeric(sym::ctlz),
                     "ctpop" => Op::Numeric(sym::ctpop),

src/tools/miri/tests/pass/intrinsics/portable-simd.rs

@@ -569,6 +569,10 @@ fn simd_round() {
         f32x4::from_array([0.9, 1.001, 2.0, -4.5]).round(),
         f32x4::from_array([1.0, 1.0, 2.0, -5.0])
     );
+    assert_eq!(
+        unsafe { intrinsics::simd_round_ties_even(f32x4::from_array([0.9, 1.001, 2.0, -4.5])) },
+        f32x4::from_array([1.0, 1.0, 2.0, -4.0])
+    );
     assert_eq!(
         f32x4::from_array([0.9, 1.001, 2.0, -4.5]).trunc(),
         f32x4::from_array([0.0, 1.0, 2.0, -4.0])
@@ -586,6 +590,10 @@ fn simd_round() {
         f64x4::from_array([0.9, 1.001, 2.0, -4.5]).round(),
         f64x4::from_array([1.0, 1.0, 2.0, -5.0])
     );
+    assert_eq!(
+        unsafe { intrinsics::simd_round_ties_even(f64x4::from_array([0.9, 1.001, 2.0, -4.5])) },
+        f64x4::from_array([1.0, 1.0, 2.0, -4.0])
+    );
     assert_eq!(
         f64x4::from_array([0.9, 1.001, 2.0, -4.5]).trunc(),
         f64x4::from_array([0.0, 1.0, 2.0, -4.0])

tests/ui/simd/intrinsic/float-math-pass.rs

@@ -85,6 +85,9 @@ fn main() {
         let r = simd_round(h);
         assert_eq!(x, r);
 
+        let r = simd_round_ties_even(h);
+        assert_eq!(z, r);
+
         let r = simd_trunc(h);
         assert_eq!(z, r);
     }

tests/ui/simd/intrinsic/generic-arithmetic-2.rs

@@ -43,6 +43,10 @@ fn main() {
         simd_shl(y, y);
         simd_shr(x, x);
         simd_shr(y, y);
+        simd_funnel_shl(x, x, x);
+        simd_funnel_shl(y, y, y);
+        simd_funnel_shr(x, x, x);
+        simd_funnel_shr(y, y, y);
         simd_and(x, x);
         simd_and(y, y);
         simd_or(x, x);
@@ -73,6 +77,10 @@ fn main() {
         //~^ ERROR expected SIMD input type, found non-SIMD `i32`
         simd_shr(0, 0);
         //~^ ERROR expected SIMD input type, found non-SIMD `i32`
+        simd_funnel_shl(0, 0, 0);
+        //~^ ERROR expected SIMD input type, found non-SIMD `i32`
+        simd_funnel_shr(0, 0, 0);
+        //~^ ERROR expected SIMD input type, found non-SIMD `i32`
         simd_and(0, 0);
         //~^ ERROR expected SIMD input type, found non-SIMD `i32`
         simd_or(0, 0);
@@ -95,6 +103,10 @@ fn main() {
         //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_shr(z, z);
         //~^ ERROR unsupported operation on `f32x4` with element `f32`
+        simd_funnel_shl(z, z, z);
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
+        simd_funnel_shr(z, z, z);
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_and(z, z);
         //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_or(z, z);