From d56fcd968d1334bf0c80205fb805a88999fffb5a Mon Sep 17 00:00:00 2001 From: sayantn Date: Fri, 30 May 2025 21:41:36 +0530 Subject: [PATCH] Simplify implementation of Rust intrinsics by using type parameters in the cache --- compiler/rustc_codegen_gcc/src/builder.rs | 2 +- compiler/rustc_codegen_llvm/src/builder.rs | 183 ++---- compiler/rustc_codegen_llvm/src/context.rs | 462 ++++---------- compiler/rustc_codegen_llvm/src/intrinsic.rs | 591 +++++++----------- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 12 + compiler/rustc_codegen_llvm/src/llvm/mod.rs | 45 +- compiler/rustc_codegen_llvm/src/type_.rs | 9 + .../rustc_codegen_ssa/src/traits/builder.rs | 4 +- 8 files changed, 479 insertions(+), 829 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index d1fb8d8f9d6b8..1bd8921210085 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -897,7 +897,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn checked_binop( &mut self, oop: OverflowOp, - typ: Ty<'_>, + typ: Ty<'tcx>, lhs: Self::Value, rhs: Self::Value, ) -> (Self::Value, Self::Value) { diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index ec006b591929d..09cb74d9dcb00 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -14,7 +14,6 @@ use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; -use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers, @@ -484,73 +483,31 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn checked_binop( &mut self, oop: OverflowOp, - ty: Ty<'_>, + ty: Ty<'tcx>, lhs: Self::Value, rhs: Self::Value, ) -> (Self::Value, Self::Value) { - use rustc_middle::ty::IntTy::*; - use rustc_middle::ty::UintTy::*; - use rustc_middle::ty::{Int, Uint}; - - let new_kind = match ty.kind() { - Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)), - Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)), - t @ (Uint(_) | Int(_)) => *t, - _ => panic!("tried to get overflow intrinsic for op applied to non-int type"), + let (size, signed) = ty.int_size_and_signed(self.tcx); + let width = size.bits(); + + if oop == OverflowOp::Sub && !signed { + // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these + // to be the canonical form. It will attempt to reform llvm.usub.with.overflow + // in the backend if profitable. + let sub = self.sub(lhs, rhs); + let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs); + return (sub, cmp); + } + + let oop_str = match oop { + OverflowOp::Add => "add", + OverflowOp::Sub => "sub", + OverflowOp::Mul => "mul", }; - let name = match oop { - OverflowOp::Add => match new_kind { - Int(I8) => "llvm.sadd.with.overflow.i8", - Int(I16) => "llvm.sadd.with.overflow.i16", - Int(I32) => "llvm.sadd.with.overflow.i32", - Int(I64) => "llvm.sadd.with.overflow.i64", - Int(I128) => "llvm.sadd.with.overflow.i128", - - Uint(U8) => "llvm.uadd.with.overflow.i8", - Uint(U16) => "llvm.uadd.with.overflow.i16", - Uint(U32) => "llvm.uadd.with.overflow.i32", - Uint(U64) => "llvm.uadd.with.overflow.i64", - Uint(U128) => "llvm.uadd.with.overflow.i128", - - _ => unreachable!(), - }, - OverflowOp::Sub => match new_kind { - Int(I8) => "llvm.ssub.with.overflow.i8", - Int(I16) => "llvm.ssub.with.overflow.i16", - Int(I32) => "llvm.ssub.with.overflow.i32", - Int(I64) => "llvm.ssub.with.overflow.i64", - Int(I128) => "llvm.ssub.with.overflow.i128", - - Uint(_) => { - // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these - // to be the canonical form. It will attempt to reform llvm.usub.with.overflow - // in the backend if profitable. - let sub = self.sub(lhs, rhs); - let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs); - return (sub, cmp); - } - - _ => unreachable!(), - }, - OverflowOp::Mul => match new_kind { - Int(I8) => "llvm.smul.with.overflow.i8", - Int(I16) => "llvm.smul.with.overflow.i16", - Int(I32) => "llvm.smul.with.overflow.i32", - Int(I64) => "llvm.smul.with.overflow.i64", - Int(I128) => "llvm.smul.with.overflow.i128", - - Uint(U8) => "llvm.umul.with.overflow.i8", - Uint(U16) => "llvm.umul.with.overflow.i16", - Uint(U32) => "llvm.umul.with.overflow.i32", - Uint(U64) => "llvm.umul.with.overflow.i64", - Uint(U128) => "llvm.umul.with.overflow.i128", - - _ => unreachable!(), - }, - }; + let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' }); - let res = self.call_intrinsic(name, &[lhs, rhs]); + let res = self.call_intrinsic(&name, &[self.type_ix(width)], &[lhs, rhs]); (self.extract_value(res, 0), self.extract_value(res, 1)) } @@ -954,11 +911,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - self.fptoint_sat(false, val, dest_ty) + self.call_intrinsic("llvm.fptoui.sat", &[dest_ty, self.val_ty(val)], &[val]) } fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - self.fptoint_sat(true, val, dest_ty) + self.call_intrinsic("llvm.fptosi.sat", &[dest_ty, self.val_ty(val)], &[val]) } fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { @@ -981,15 +938,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { if self.cx.type_kind(src_ty) != TypeKind::Vector { let float_width = self.cx.float_width(src_ty); let int_width = self.cx.int_width(dest_ty); - let name = match (int_width, float_width) { - (32, 32) => Some("llvm.wasm.trunc.unsigned.i32.f32"), - (32, 64) => Some("llvm.wasm.trunc.unsigned.i32.f64"), - (64, 32) => Some("llvm.wasm.trunc.unsigned.i64.f32"), - (64, 64) => Some("llvm.wasm.trunc.unsigned.i64.f64"), - _ => None, - }; - if let Some(name) = name { - return self.call_intrinsic(name, &[val]); + if matches!((int_width, float_width), (32 | 64, 32 | 64)) { + return self.call_intrinsic( + "llvm.wasm.trunc.unsigned", + &[dest_ty, src_ty], + &[val], + ); } } } @@ -1003,15 +957,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { if self.cx.type_kind(src_ty) != TypeKind::Vector { let float_width = self.cx.float_width(src_ty); let int_width = self.cx.int_width(dest_ty); - let name = match (int_width, float_width) { - (32, 32) => Some("llvm.wasm.trunc.signed.i32.f32"), - (32, 64) => Some("llvm.wasm.trunc.signed.i32.f64"), - (64, 32) => Some("llvm.wasm.trunc.signed.i64.f32"), - (64, 64) => Some("llvm.wasm.trunc.signed.i64.f64"), - _ => None, - }; - if let Some(name) = name { - return self.call_intrinsic(name, &[val]); + if matches!((int_width, float_width), (32 | 64, 32 | 64)) { + return self.call_intrinsic( + "llvm.wasm.trunc.signed", + &[dest_ty, src_ty], + &[val], + ); } } } @@ -1084,22 +1035,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { return None; } - let name = match (ty.is_signed(), ty.primitive_size(self.tcx).bits()) { - (true, 8) => "llvm.scmp.i8.i8", - (true, 16) => "llvm.scmp.i8.i16", - (true, 32) => "llvm.scmp.i8.i32", - (true, 64) => "llvm.scmp.i8.i64", - (true, 128) => "llvm.scmp.i8.i128", - - (false, 8) => "llvm.ucmp.i8.i8", - (false, 16) => "llvm.ucmp.i8.i16", - (false, 32) => "llvm.ucmp.i8.i32", - (false, 64) => "llvm.ucmp.i8.i64", - (false, 128) => "llvm.ucmp.i8.i128", + let size = ty.primitive_size(self.tcx); + let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" }; - _ => bug!("three-way compare unsupported for type {ty:?}"), - }; - Some(self.call_intrinsic(name, &[lhs, rhs])) + Some(self.call_intrinsic(&name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs])) } /* Miscellaneous instructions */ @@ -1385,11 +1324,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) { - self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size); + self.call_lifetime_intrinsic("llvm.lifetime.start", ptr, size); } fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) { - self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size); + self.call_lifetime_intrinsic("llvm.lifetime.end", ptr, size); } fn call( @@ -1454,7 +1393,7 @@ impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> { // Forward to the `get_static` method of `CodegenCx` let global = self.cx().get_static(def_id); if self.cx().tcx.is_thread_local_static(def_id) { - let pointer = self.call_intrinsic("llvm.threadlocal.address", &[global]); + let pointer = self.call_intrinsic("llvm.threadlocal.address", &[], &[global]); // Cast to default address space if globals are in a different addrspace self.pointercast(pointer, self.type_ptr()) } else { @@ -1649,8 +1588,13 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { } impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { - pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { - let (ty, f) = self.cx.get_intrinsic(intrinsic); + pub(crate) fn call_intrinsic( + &mut self, + base_name: &str, + type_params: &[&'ll Type], + args: &[&'ll Value], + ) -> &'ll Value { + let (ty, f) = self.cx.get_intrinsic(base_name, type_params); self.call(ty, None, None, f, args, None, None) } @@ -1664,7 +1608,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { return; } - self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]); + self.call_intrinsic(intrinsic, &[self.type_ptr()], &[self.cx.const_u64(size), ptr]); } } impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { @@ -1689,31 +1633,6 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { } } impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { - fn fptoint_sat(&mut self, signed: bool, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - let src_ty = self.cx.val_ty(val); - let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector { - assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); - ( - self.cx.element_type(src_ty), - self.cx.element_type(dest_ty), - Some(self.cx.vector_length(src_ty)), - ) - } else { - (src_ty, dest_ty, None) - }; - let float_width = self.cx.float_width(float_ty); - let int_width = self.cx.int_width(int_ty); - - let instr = if signed { "fptosi" } else { "fptoui" }; - let name = if let Some(vector_length) = vector_length { - format!("llvm.{instr}.sat.v{vector_length}i{int_width}.v{vector_length}f{float_width}") - } else { - format!("llvm.{instr}.sat.i{int_width}.f{float_width}") - }; - let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); - self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None, None) - } - pub(crate) fn landing_pad( &mut self, ty: &'ll Type, @@ -1819,7 +1738,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { // llvm.type.test intrinsic. The LowerTypeTests link-time optimization pass replaces // calls to this intrinsic with code to test type membership. let typeid = self.get_metadata_value(typeid_metadata); - let cond = self.call_intrinsic("llvm.type.test", &[llfn, typeid]); + let cond = self.call_intrinsic("llvm.type.test", &[], &[llfn, typeid]); let bb_pass = self.append_sibling_block("type_test.pass"); let bb_fail = self.append_sibling_block("type_test.fail"); self.cond_br(cond, bb_pass, bb_fail); @@ -1887,7 +1806,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { num_counters: &'ll Value, index: &'ll Value, ) { - self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]); + self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]); } /// Emits a call to `llvm.instrprof.mcdc.parameters`. @@ -1906,7 +1825,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { hash: &'ll Value, bitmap_bits: &'ll Value, ) { - self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[fn_name, hash, bitmap_bits]); + self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[], &[fn_name, hash, bitmap_bits]); } #[instrument(level = "debug", skip(self))] @@ -1918,7 +1837,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { mcdc_temp: &'ll Value, ) { let args = &[fn_name, hash, bitmap_index, mcdc_temp]; - self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", args); + self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", &[], args); } #[instrument(level = "debug", skip(self))] diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 8d6e1d8941b72..f36f3b2b16b83 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -137,7 +137,8 @@ pub(crate) struct FullCx<'ll, 'tcx> { eh_catch_typeinfo: Cell>, pub rust_try_fn: Cell>, - intrinsics: RefCell>, + intrinsics: + RefCell), (&'ll Type, &'ll Value)>>, /// A counter that is used for generating local symbol names local_gen_sym_counter: Cell, @@ -842,17 +843,24 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } impl<'ll> CodegenCx<'ll, '_> { - pub(crate) fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) { - if let Some(v) = self.intrinsics.borrow().get(key).cloned() { - return v; + pub(crate) fn get_intrinsic( + &self, + base_name: &str, + type_params: &[&'ll Type], + ) -> (&'ll Type, &'ll Value) { + if let Some(v) = + self.intrinsics.borrow().get(&(base_name, SmallVec::from_slice(type_params))) + { + return *v; } - self.declare_intrinsic(key).unwrap_or_else(|| bug!("unknown intrinsic '{}'", key)) + self.declare_intrinsic(base_name, type_params) } fn insert_intrinsic( &self, - name: &'static str, + base_name: &'static str, + type_params: &[&'ll Type], args: Option<&[&'ll llvm::Type]>, ret: &'ll llvm::Type, ) -> (&'ll llvm::Type, &'ll llvm::Value) { @@ -861,372 +869,153 @@ impl<'ll> CodegenCx<'ll, '_> { } else { self.type_variadic_func(&[], ret) }; - let f = self.declare_cfn(name, llvm::UnnamedAddr::No, fn_ty); - self.intrinsics.borrow_mut().insert(name, (fn_ty, f)); + + let intrinsic = llvm::Intrinsic::lookup(base_name.as_bytes()) + .expect("Unknown LLVM intrinsic `{base_name}`"); + + let full_name = if intrinsic.is_overloaded() { + &intrinsic.overloaded_name(self.llmod, type_params) + } else { + base_name + }; + + let f = self.declare_cfn(full_name, llvm::UnnamedAddr::No, fn_ty); + self.intrinsics + .borrow_mut() + .insert((base_name, SmallVec::from_slice(type_params)), (fn_ty, f)); (fn_ty, f) } - fn declare_intrinsic(&self, key: &str) -> Option<(&'ll Type, &'ll Value)> { + fn declare_intrinsic( + &self, + base_name: &str, + type_params: &[&'ll Type], + ) -> (&'ll Type, &'ll Value) { + macro_rules! param { + ($index:literal) => { + type_params[$index] + }; + ($other:expr) => { + $other + }; + } macro_rules! ifn { - ($name:expr, fn() -> $ret:expr) => ( - if key == $name { - return Some(self.insert_intrinsic($name, Some(&[]), $ret)); - } - ); ($name:expr, fn(...) -> $ret:expr) => ( - if key == $name { - return Some(self.insert_intrinsic($name, None, $ret)); + if base_name == $name { + return self.insert_intrinsic($name, type_params, None, param!($ret)); } ); ($name:expr, fn($($arg:expr),*) -> $ret:expr) => ( - if key == $name { - return Some(self.insert_intrinsic($name, Some(&[$($arg),*]), $ret)); + if base_name == $name { + return self.insert_intrinsic($name, type_params, Some(&[$(param!($arg)),*]), param!($ret)); } ); } macro_rules! mk_struct { - ($($field_ty:expr),*) => (self.type_struct( &[$($field_ty),*], false)) + ($($field_ty:expr),*) => (self.type_struct( &[$(param!($field_ty)),*], false)) } + let same_width_vector = |index, element_ty| { + self.type_vector(element_ty, self.vector_length(type_params[index]) as u64) + }; + let ptr = self.type_ptr(); let void = self.type_void(); let i1 = self.type_i1(); - let t_i8 = self.type_i8(); - let t_i16 = self.type_i16(); let t_i32 = self.type_i32(); let t_i64 = self.type_i64(); - let t_i128 = self.type_i128(); let t_isize = self.type_isize(); - let t_f16 = self.type_f16(); - let t_f32 = self.type_f32(); - let t_f64 = self.type_f64(); - let t_f128 = self.type_f128(); let t_metadata = self.type_metadata(); let t_token = self.type_token(); ifn!("llvm.wasm.get.exception", fn(t_token) -> ptr); ifn!("llvm.wasm.get.ehselector", fn(t_token) -> t_i32); - ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32); - ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32); - ifn!("llvm.wasm.trunc.unsigned.i64.f32", fn(t_f32) -> t_i64); - ifn!("llvm.wasm.trunc.unsigned.i64.f64", fn(t_f64) -> t_i64); - ifn!("llvm.wasm.trunc.signed.i32.f32", fn(t_f32) -> t_i32); - ifn!("llvm.wasm.trunc.signed.i32.f64", fn(t_f64) -> t_i32); - ifn!("llvm.wasm.trunc.signed.i64.f32", fn(t_f32) -> t_i64); - ifn!("llvm.wasm.trunc.signed.i64.f64", fn(t_f64) -> t_i64); - - ifn!("llvm.fptosi.sat.i8.f32", fn(t_f32) -> t_i8); - ifn!("llvm.fptosi.sat.i16.f32", fn(t_f32) -> t_i16); - ifn!("llvm.fptosi.sat.i32.f32", fn(t_f32) -> t_i32); - ifn!("llvm.fptosi.sat.i64.f32", fn(t_f32) -> t_i64); - ifn!("llvm.fptosi.sat.i128.f32", fn(t_f32) -> t_i128); - ifn!("llvm.fptosi.sat.i8.f64", fn(t_f64) -> t_i8); - ifn!("llvm.fptosi.sat.i16.f64", fn(t_f64) -> t_i16); - ifn!("llvm.fptosi.sat.i32.f64", fn(t_f64) -> t_i32); - ifn!("llvm.fptosi.sat.i64.f64", fn(t_f64) -> t_i64); - ifn!("llvm.fptosi.sat.i128.f64", fn(t_f64) -> t_i128); - - ifn!("llvm.fptoui.sat.i8.f32", fn(t_f32) -> t_i8); - ifn!("llvm.fptoui.sat.i16.f32", fn(t_f32) -> t_i16); - ifn!("llvm.fptoui.sat.i32.f32", fn(t_f32) -> t_i32); - ifn!("llvm.fptoui.sat.i64.f32", fn(t_f32) -> t_i64); - ifn!("llvm.fptoui.sat.i128.f32", fn(t_f32) -> t_i128); - ifn!("llvm.fptoui.sat.i8.f64", fn(t_f64) -> t_i8); - ifn!("llvm.fptoui.sat.i16.f64", fn(t_f64) -> t_i16); - ifn!("llvm.fptoui.sat.i32.f64", fn(t_f64) -> t_i32); - ifn!("llvm.fptoui.sat.i64.f64", fn(t_f64) -> t_i64); - ifn!("llvm.fptoui.sat.i128.f64", fn(t_f64) -> t_i128); + ifn!("llvm.wasm.trunc.unsigned", fn(1) -> 0); + ifn!("llvm.wasm.trunc.signed", fn(1) -> 0); + ifn!("llvm.fptosi.sat", fn(1) -> 0); + ifn!("llvm.fptoui.sat", fn(1) -> 0); ifn!("llvm.trap", fn() -> void); ifn!("llvm.debugtrap", fn() -> void); ifn!("llvm.frameaddress", fn(t_i32) -> ptr); - ifn!("llvm.powi.f16.i32", fn(t_f16, t_i32) -> t_f16); - ifn!("llvm.powi.f32.i32", fn(t_f32, t_i32) -> t_f32); - ifn!("llvm.powi.f64.i32", fn(t_f64, t_i32) -> t_f64); - ifn!("llvm.powi.f128.i32", fn(t_f128, t_i32) -> t_f128); - - ifn!("llvm.pow.f16", fn(t_f16, t_f16) -> t_f16); - ifn!("llvm.pow.f32", fn(t_f32, t_f32) -> t_f32); - ifn!("llvm.pow.f64", fn(t_f64, t_f64) -> t_f64); - ifn!("llvm.pow.f128", fn(t_f128, t_f128) -> t_f128); - - ifn!("llvm.sqrt.f16", fn(t_f16) -> t_f16); - ifn!("llvm.sqrt.f32", fn(t_f32) -> t_f32); - ifn!("llvm.sqrt.f64", fn(t_f64) -> t_f64); - ifn!("llvm.sqrt.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.sin.f16", fn(t_f16) -> t_f16); - ifn!("llvm.sin.f32", fn(t_f32) -> t_f32); - ifn!("llvm.sin.f64", fn(t_f64) -> t_f64); - ifn!("llvm.sin.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.cos.f16", fn(t_f16) -> t_f16); - ifn!("llvm.cos.f32", fn(t_f32) -> t_f32); - ifn!("llvm.cos.f64", fn(t_f64) -> t_f64); - ifn!("llvm.cos.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.exp.f16", fn(t_f16) -> t_f16); - ifn!("llvm.exp.f32", fn(t_f32) -> t_f32); - ifn!("llvm.exp.f64", fn(t_f64) -> t_f64); - ifn!("llvm.exp.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.exp2.f16", fn(t_f16) -> t_f16); - ifn!("llvm.exp2.f32", fn(t_f32) -> t_f32); - ifn!("llvm.exp2.f64", fn(t_f64) -> t_f64); - ifn!("llvm.exp2.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.log.f16", fn(t_f16) -> t_f16); - ifn!("llvm.log.f32", fn(t_f32) -> t_f32); - ifn!("llvm.log.f64", fn(t_f64) -> t_f64); - ifn!("llvm.log.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.log10.f16", fn(t_f16) -> t_f16); - ifn!("llvm.log10.f32", fn(t_f32) -> t_f32); - ifn!("llvm.log10.f64", fn(t_f64) -> t_f64); - ifn!("llvm.log10.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.log2.f16", fn(t_f16) -> t_f16); - ifn!("llvm.log2.f32", fn(t_f32) -> t_f32); - ifn!("llvm.log2.f64", fn(t_f64) -> t_f64); - ifn!("llvm.log2.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.fma.f16", fn(t_f16, t_f16, t_f16) -> t_f16); - ifn!("llvm.fma.f32", fn(t_f32, t_f32, t_f32) -> t_f32); - ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64); - ifn!("llvm.fma.f128", fn(t_f128, t_f128, t_f128) -> t_f128); - - ifn!("llvm.fmuladd.f16", fn(t_f16, t_f16, t_f16) -> t_f16); - ifn!("llvm.fmuladd.f32", fn(t_f32, t_f32, t_f32) -> t_f32); - ifn!("llvm.fmuladd.f64", fn(t_f64, t_f64, t_f64) -> t_f64); - ifn!("llvm.fmuladd.f128", fn(t_f128, t_f128, t_f128) -> t_f128); - - ifn!("llvm.fabs.f16", fn(t_f16) -> t_f16); - ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32); - ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64); - ifn!("llvm.fabs.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.minnum.f16", fn(t_f16, t_f16) -> t_f16); - ifn!("llvm.minnum.f32", fn(t_f32, t_f32) -> t_f32); - ifn!("llvm.minnum.f64", fn(t_f64, t_f64) -> t_f64); - ifn!("llvm.minnum.f128", fn(t_f128, t_f128) -> t_f128); - - ifn!("llvm.minimum.f16", fn(t_f16, t_f16) -> t_f16); - ifn!("llvm.minimum.f32", fn(t_f32, t_f32) -> t_f32); - ifn!("llvm.minimum.f64", fn(t_f64, t_f64) -> t_f64); - // There are issues on x86_64 and aarch64 with the f128 variant. - // - https://github.com/llvm/llvm-project/issues/139380 - // - https://github.com/llvm/llvm-project/issues/139381 - // ifn!("llvm.minimum.f128", fn(t_f128, t_f128) -> t_f128); - - ifn!("llvm.maxnum.f16", fn(t_f16, t_f16) -> t_f16); - ifn!("llvm.maxnum.f32", fn(t_f32, t_f32) -> t_f32); - ifn!("llvm.maxnum.f64", fn(t_f64, t_f64) -> t_f64); - ifn!("llvm.maxnum.f128", fn(t_f128, t_f128) -> t_f128); - - ifn!("llvm.maximum.f16", fn(t_f16, t_f16) -> t_f16); - ifn!("llvm.maximum.f32", fn(t_f32, t_f32) -> t_f32); - ifn!("llvm.maximum.f64", fn(t_f64, t_f64) -> t_f64); - // There are issues on x86_64 and aarch64 with the f128 variant. - // - https://github.com/llvm/llvm-project/issues/139380 - // - https://github.com/llvm/llvm-project/issues/139381 - // ifn!("llvm.maximum.f128", fn(t_f128, t_f128) -> t_f128); - - ifn!("llvm.floor.f16", fn(t_f16) -> t_f16); - ifn!("llvm.floor.f32", fn(t_f32) -> t_f32); - ifn!("llvm.floor.f64", fn(t_f64) -> t_f64); - ifn!("llvm.floor.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.ceil.f16", fn(t_f16) -> t_f16); - ifn!("llvm.ceil.f32", fn(t_f32) -> t_f32); - ifn!("llvm.ceil.f64", fn(t_f64) -> t_f64); - ifn!("llvm.ceil.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.trunc.f16", fn(t_f16) -> t_f16); - ifn!("llvm.trunc.f32", fn(t_f32) -> t_f32); - ifn!("llvm.trunc.f64", fn(t_f64) -> t_f64); - ifn!("llvm.trunc.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.copysign.f16", fn(t_f16, t_f16) -> t_f16); - ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32); - ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64); - ifn!("llvm.copysign.f128", fn(t_f128, t_f128) -> t_f128); - - ifn!("llvm.round.f16", fn(t_f16) -> t_f16); - ifn!("llvm.round.f32", fn(t_f32) -> t_f32); - ifn!("llvm.round.f64", fn(t_f64) -> t_f64); - ifn!("llvm.round.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.roundeven.f16", fn(t_f16) -> t_f16); - ifn!("llvm.roundeven.f32", fn(t_f32) -> t_f32); - ifn!("llvm.roundeven.f64", fn(t_f64) -> t_f64); - ifn!("llvm.roundeven.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.rint.f16", fn(t_f16) -> t_f16); - ifn!("llvm.rint.f32", fn(t_f32) -> t_f32); - ifn!("llvm.rint.f64", fn(t_f64) -> t_f64); - ifn!("llvm.rint.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.nearbyint.f16", fn(t_f16) -> t_f16); - ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32); - ifn!("llvm.nearbyint.f64", fn(t_f64) -> t_f64); - ifn!("llvm.nearbyint.f128", fn(t_f128) -> t_f128); - - ifn!("llvm.ctpop.i8", fn(t_i8) -> t_i8); - ifn!("llvm.ctpop.i16", fn(t_i16) -> t_i16); - ifn!("llvm.ctpop.i32", fn(t_i32) -> t_i32); - ifn!("llvm.ctpop.i64", fn(t_i64) -> t_i64); - ifn!("llvm.ctpop.i128", fn(t_i128) -> t_i128); - - ifn!("llvm.ctlz.i8", fn(t_i8, i1) -> t_i8); - ifn!("llvm.ctlz.i16", fn(t_i16, i1) -> t_i16); - ifn!("llvm.ctlz.i32", fn(t_i32, i1) -> t_i32); - ifn!("llvm.ctlz.i64", fn(t_i64, i1) -> t_i64); - ifn!("llvm.ctlz.i128", fn(t_i128, i1) -> t_i128); - - ifn!("llvm.cttz.i8", fn(t_i8, i1) -> t_i8); - ifn!("llvm.cttz.i16", fn(t_i16, i1) -> t_i16); - ifn!("llvm.cttz.i32", fn(t_i32, i1) -> t_i32); - ifn!("llvm.cttz.i64", fn(t_i64, i1) -> t_i64); - ifn!("llvm.cttz.i128", fn(t_i128, i1) -> t_i128); - - ifn!("llvm.bswap.i16", fn(t_i16) -> t_i16); - ifn!("llvm.bswap.i32", fn(t_i32) -> t_i32); - ifn!("llvm.bswap.i64", fn(t_i64) -> t_i64); - ifn!("llvm.bswap.i128", fn(t_i128) -> t_i128); - - ifn!("llvm.bitreverse.i8", fn(t_i8) -> t_i8); - ifn!("llvm.bitreverse.i16", fn(t_i16) -> t_i16); - ifn!("llvm.bitreverse.i32", fn(t_i32) -> t_i32); - ifn!("llvm.bitreverse.i64", fn(t_i64) -> t_i64); - ifn!("llvm.bitreverse.i128", fn(t_i128) -> t_i128); - - ifn!("llvm.fshl.i8", fn(t_i8, t_i8, t_i8) -> t_i8); - ifn!("llvm.fshl.i16", fn(t_i16, t_i16, t_i16) -> t_i16); - ifn!("llvm.fshl.i32", fn(t_i32, t_i32, t_i32) -> t_i32); - ifn!("llvm.fshl.i64", fn(t_i64, t_i64, t_i64) -> t_i64); - ifn!("llvm.fshl.i128", fn(t_i128, t_i128, t_i128) -> t_i128); - - ifn!("llvm.fshr.i8", fn(t_i8, t_i8, t_i8) -> t_i8); - ifn!("llvm.fshr.i16", fn(t_i16, t_i16, t_i16) -> t_i16); - ifn!("llvm.fshr.i32", fn(t_i32, t_i32, t_i32) -> t_i32); - ifn!("llvm.fshr.i64", fn(t_i64, t_i64, t_i64) -> t_i64); - ifn!("llvm.fshr.i128", fn(t_i128, t_i128, t_i128) -> t_i128); - - ifn!("llvm.sadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); - ifn!("llvm.sadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); - ifn!("llvm.sadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); - ifn!("llvm.sadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); - ifn!("llvm.sadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); - - ifn!("llvm.uadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); - ifn!("llvm.uadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); - ifn!("llvm.uadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); - ifn!("llvm.uadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); - ifn!("llvm.uadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); - - ifn!("llvm.ssub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); - ifn!("llvm.ssub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); - ifn!("llvm.ssub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); - ifn!("llvm.ssub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); - ifn!("llvm.ssub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); - - ifn!("llvm.usub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); - ifn!("llvm.usub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); - ifn!("llvm.usub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); - ifn!("llvm.usub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); - ifn!("llvm.usub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); - - ifn!("llvm.smul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); - ifn!("llvm.smul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); - ifn!("llvm.smul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); - ifn!("llvm.smul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); - ifn!("llvm.smul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); - - ifn!("llvm.umul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); - ifn!("llvm.umul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); - ifn!("llvm.umul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); - ifn!("llvm.umul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); - ifn!("llvm.umul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); - - ifn!("llvm.sadd.sat.i8", fn(t_i8, t_i8) -> t_i8); - ifn!("llvm.sadd.sat.i16", fn(t_i16, t_i16) -> t_i16); - ifn!("llvm.sadd.sat.i32", fn(t_i32, t_i32) -> t_i32); - ifn!("llvm.sadd.sat.i64", fn(t_i64, t_i64) -> t_i64); - ifn!("llvm.sadd.sat.i128", fn(t_i128, t_i128) -> t_i128); - - ifn!("llvm.uadd.sat.i8", fn(t_i8, t_i8) -> t_i8); - ifn!("llvm.uadd.sat.i16", fn(t_i16, t_i16) -> t_i16); - ifn!("llvm.uadd.sat.i32", fn(t_i32, t_i32) -> t_i32); - ifn!("llvm.uadd.sat.i64", fn(t_i64, t_i64) -> t_i64); - ifn!("llvm.uadd.sat.i128", fn(t_i128, t_i128) -> t_i128); - - ifn!("llvm.ssub.sat.i8", fn(t_i8, t_i8) -> t_i8); - ifn!("llvm.ssub.sat.i16", fn(t_i16, t_i16) -> t_i16); - ifn!("llvm.ssub.sat.i32", fn(t_i32, t_i32) -> t_i32); - ifn!("llvm.ssub.sat.i64", fn(t_i64, t_i64) -> t_i64); - ifn!("llvm.ssub.sat.i128", fn(t_i128, t_i128) -> t_i128); - - ifn!("llvm.usub.sat.i8", fn(t_i8, t_i8) -> t_i8); - ifn!("llvm.usub.sat.i16", fn(t_i16, t_i16) -> t_i16); - ifn!("llvm.usub.sat.i32", fn(t_i32, t_i32) -> t_i32); - ifn!("llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64); - ifn!("llvm.usub.sat.i128", fn(t_i128, t_i128) -> t_i128); - - ifn!("llvm.scmp.i8.i8", fn(t_i8, t_i8) -> t_i8); - ifn!("llvm.scmp.i8.i16", fn(t_i16, t_i16) -> t_i8); - ifn!("llvm.scmp.i8.i32", fn(t_i32, t_i32) -> t_i8); - ifn!("llvm.scmp.i8.i64", fn(t_i64, t_i64) -> t_i8); - ifn!("llvm.scmp.i8.i128", fn(t_i128, t_i128) -> t_i8); - - ifn!("llvm.ucmp.i8.i8", fn(t_i8, t_i8) -> t_i8); - ifn!("llvm.ucmp.i8.i16", fn(t_i16, t_i16) -> t_i8); - ifn!("llvm.ucmp.i8.i32", fn(t_i32, t_i32) -> t_i8); - ifn!("llvm.ucmp.i8.i64", fn(t_i64, t_i64) -> t_i8); - ifn!("llvm.ucmp.i8.i128", fn(t_i128, t_i128) -> t_i8); - - ifn!("llvm.lifetime.start.p0i8", fn(t_i64, ptr) -> void); - ifn!("llvm.lifetime.end.p0i8", fn(t_i64, ptr) -> void); - - // FIXME: This is an infinitesimally small portion of the types you can - // pass to this intrinsic, if we can ever lazily register intrinsics we - // should register these when they're used, that way any type can be - // passed. - ifn!("llvm.is.constant.i1", fn(i1) -> i1); - ifn!("llvm.is.constant.i8", fn(t_i8) -> i1); - ifn!("llvm.is.constant.i16", fn(t_i16) -> i1); - ifn!("llvm.is.constant.i32", fn(t_i32) -> i1); - ifn!("llvm.is.constant.i64", fn(t_i64) -> i1); - ifn!("llvm.is.constant.i128", fn(t_i128) -> i1); - ifn!("llvm.is.constant.isize", fn(t_isize) -> i1); - ifn!("llvm.is.constant.f16", fn(t_f16) -> i1); - ifn!("llvm.is.constant.f32", fn(t_f32) -> i1); - ifn!("llvm.is.constant.f64", fn(t_f64) -> i1); - ifn!("llvm.is.constant.f128", fn(t_f128) -> i1); - ifn!("llvm.is.constant.ptr", fn(ptr) -> i1); - - ifn!("llvm.expect.i1", fn(i1, i1) -> i1); - ifn!("llvm.eh.typeid.for", fn(ptr) -> t_i32); + ifn!("llvm.powi", fn(0, 1) -> 0); + ifn!("llvm.pow", fn(0, 0) -> 0); + ifn!("llvm.sqrt", fn(0) -> 0); + ifn!("llvm.sin", fn(0) -> 0); + ifn!("llvm.cos", fn(0) -> 0); + ifn!("llvm.exp", fn(0) -> 0); + ifn!("llvm.exp2", fn(0) -> 0); + ifn!("llvm.log", fn(0) -> 0); + ifn!("llvm.log10", fn(0) -> 0); + ifn!("llvm.log2", fn(0) -> 0); + ifn!("llvm.fma", fn(0, 0, 0) -> 0); + ifn!("llvm.fmuladd", fn(0, 0, 0) -> 0); + ifn!("llvm.fabs", fn(0) -> 0); + ifn!("llvm.minnum", fn(0, 0) -> 0); + ifn!("llvm.minimum", fn(0, 0) -> 0); + ifn!("llvm.maxnum", fn(0, 0) -> 0); + ifn!("llvm.maximum", fn(0, 0) -> 0); + ifn!("llvm.floor", fn(0) -> 0); + ifn!("llvm.ceil", fn(0) -> 0); + ifn!("llvm.trunc", fn(0) -> 0); + ifn!("llvm.copysign", fn(0, 0) -> 0); + ifn!("llvm.round", fn(0) -> 0); + ifn!("llvm.rint", fn(0) -> 0); + ifn!("llvm.nearbyint", fn(0) -> 0); + + ifn!("llvm.ctpop", fn(0) -> 0); + ifn!("llvm.ctlz", fn(0, i1) -> 0); + ifn!("llvm.cttz", fn(0, i1) -> 0); + ifn!("llvm.bswap", fn(0) -> 0); + ifn!("llvm.bitreverse", fn(0) -> 0); + ifn!("llvm.fshl", fn(0, 0, 0) -> 0); + ifn!("llvm.fshr", fn(0, 0, 0) -> 0); + + ifn!("llvm.sadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1}); + ifn!("llvm.uadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1}); + ifn!("llvm.ssub.with.overflow", fn(0, 0) -> mk_struct! {0, i1}); + ifn!("llvm.usub.with.overflow", fn(0, 0) -> mk_struct! {0, i1}); + ifn!("llvm.smul.with.overflow", fn(0, 0) -> mk_struct! {0, i1}); + ifn!("llvm.umul.with.overflow", fn(0, 0) -> mk_struct! {0, i1}); + + ifn!("llvm.sadd.sat", fn(0, 0) -> 0); + ifn!("llvm.uadd.sat", fn(0, 0) -> 0); + ifn!("llvm.ssub.sat", fn(0, 0) -> 0); + ifn!("llvm.usub.sat", fn(0, 0) -> 0); + + ifn!("llvm.scmp", fn(1, 1) -> 0); + ifn!("llvm.ucmp", fn(1, 1) -> 0); + + ifn!("llvm.lifetime.start", fn(t_i64, 0) -> void); + ifn!("llvm.lifetime.end", fn(t_i64, 0) -> void); + + ifn!("llvm.is.constant", fn(0) -> i1); + ifn!("llvm.expect", fn(0, 0) -> 0); + + ifn!("llvm.eh.typeid.for", fn(0) -> t_i32); ifn!("llvm.localescape", fn(...) -> void); ifn!("llvm.localrecover", fn(ptr, ptr, t_i32) -> ptr); - ifn!("llvm.x86.seh.recoverfp", fn(ptr, ptr) -> ptr); ifn!("llvm.assume", fn(i1) -> void); - ifn!("llvm.prefetch", fn(ptr, t_i32, t_i32, t_i32) -> void); + ifn!("llvm.prefetch", fn(0, t_i32, t_i32, t_i32) -> void); // This isn't an "LLVM intrinsic", but LLVM's optimization passes // recognize it like one (including turning it into `bcmp` sometimes) // and we use it to implement intrinsics like `raw_eq` and `compare_bytes` - match self.sess().target.arch.as_ref() { - "avr" | "msp430" => ifn!("memcmp", fn(ptr, ptr, t_isize) -> t_i16), - _ => ifn!("memcmp", fn(ptr, ptr, t_isize) -> t_i32), + if base_name == "memcmp" { + let fn_ty = self.type_func(&[ptr, ptr, t_isize], self.type_int()); + let f = self.declare_cfn("memcmp", llvm::UnnamedAddr::No, fn_ty); + self.intrinsics.borrow_mut().insert(("memcmp", SmallVec::new()), (fn_ty, f)); + + return (fn_ty, f); } // variadic intrinsics - ifn!("llvm.va_start", fn(ptr) -> void); - ifn!("llvm.va_end", fn(ptr) -> void); - ifn!("llvm.va_copy", fn(ptr, ptr) -> void); + ifn!("llvm.va_start", fn(0) -> void); + ifn!("llvm.va_end", fn(0) -> void); + ifn!("llvm.va_copy", fn(0, 0) -> void); if self.sess().instrument_coverage() { ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void); @@ -1238,14 +1027,19 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.type.checked.load", fn(ptr, t_i32, t_metadata) -> mk_struct! {ptr, i1}); if self.sess().opts.debuginfo != DebugInfo::None { - ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); - ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); + ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata, t_metadata) -> void); + ifn!("llvm.dbg.value", fn(t_metadata, t_metadata, t_metadata) -> void); } - ifn!("llvm.ptrmask", fn(ptr, t_isize) -> ptr); + ifn!("llvm.ptrmask", fn(0, 1) -> 0); ifn!("llvm.threadlocal.address", fn(ptr) -> ptr); - None + ifn!("llvm.masked.load", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0); + ifn!("llvm.masked.store", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void); + ifn!("llvm.masked.gather", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0); + ifn!("llvm.masked.scatter", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void); + + bug!("Unknown intrinsic: `{base_name}`") } pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value { diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 10697b9a71f9c..497c31706ec8b 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -15,7 +15,7 @@ use rustc_middle::ty::{self, GenericArgsRef, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{Span, Symbol, sym}; use rustc_symbol_mangling::mangle_internal_symbol; -use rustc_target::spec::{HasTargetSpec, PanicStrategy}; +use rustc_target::spec::PanicStrategy; use tracing::debug; use crate::abi::FnAbiLlvmExt; @@ -27,137 +27,142 @@ use crate::type_of::LayoutLlvmExt; use crate::va_arg::emit_va_arg; use crate::value::Value; -fn get_simple_intrinsic<'ll>( - cx: &CodegenCx<'ll, '_>, +fn call_simple_intrinsic<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, name: Symbol, -) -> Option<(&'ll Type, &'ll Value)> { - let llvm_name = match name { - sym::sqrtf16 => "llvm.sqrt.f16", - sym::sqrtf32 => "llvm.sqrt.f32", - sym::sqrtf64 => "llvm.sqrt.f64", - sym::sqrtf128 => "llvm.sqrt.f128", - - sym::powif16 => "llvm.powi.f16.i32", - sym::powif32 => "llvm.powi.f32.i32", - sym::powif64 => "llvm.powi.f64.i32", - sym::powif128 => "llvm.powi.f128.i32", - - sym::sinf16 => "llvm.sin.f16", - sym::sinf32 => "llvm.sin.f32", - sym::sinf64 => "llvm.sin.f64", - sym::sinf128 => "llvm.sin.f128", - - sym::cosf16 => "llvm.cos.f16", - sym::cosf32 => "llvm.cos.f32", - sym::cosf64 => "llvm.cos.f64", - sym::cosf128 => "llvm.cos.f128", - - sym::powf16 => "llvm.pow.f16", - sym::powf32 => "llvm.pow.f32", - sym::powf64 => "llvm.pow.f64", - sym::powf128 => "llvm.pow.f128", - - sym::expf16 => "llvm.exp.f16", - sym::expf32 => "llvm.exp.f32", - sym::expf64 => "llvm.exp.f64", - sym::expf128 => "llvm.exp.f128", - - sym::exp2f16 => "llvm.exp2.f16", - sym::exp2f32 => "llvm.exp2.f32", - sym::exp2f64 => "llvm.exp2.f64", - sym::exp2f128 => "llvm.exp2.f128", - - sym::logf16 => "llvm.log.f16", - sym::logf32 => "llvm.log.f32", - sym::logf64 => "llvm.log.f64", - sym::logf128 => "llvm.log.f128", - - sym::log10f16 => "llvm.log10.f16", - sym::log10f32 => "llvm.log10.f32", - sym::log10f64 => "llvm.log10.f64", - sym::log10f128 => "llvm.log10.f128", - - sym::log2f16 => "llvm.log2.f16", - sym::log2f32 => "llvm.log2.f32", - sym::log2f64 => "llvm.log2.f64", - sym::log2f128 => "llvm.log2.f128", - - sym::fmaf16 => "llvm.fma.f16", - sym::fmaf32 => "llvm.fma.f32", - sym::fmaf64 => "llvm.fma.f64", - sym::fmaf128 => "llvm.fma.f128", - - sym::fmuladdf16 => "llvm.fmuladd.f16", - sym::fmuladdf32 => "llvm.fmuladd.f32", - sym::fmuladdf64 => "llvm.fmuladd.f64", - sym::fmuladdf128 => "llvm.fmuladd.f128", - - sym::fabsf16 => "llvm.fabs.f16", - sym::fabsf32 => "llvm.fabs.f32", - sym::fabsf64 => "llvm.fabs.f64", - sym::fabsf128 => "llvm.fabs.f128", - - sym::minnumf16 => "llvm.minnum.f16", - sym::minnumf32 => "llvm.minnum.f32", - sym::minnumf64 => "llvm.minnum.f64", - sym::minnumf128 => "llvm.minnum.f128", - - sym::minimumf16 => "llvm.minimum.f16", - sym::minimumf32 => "llvm.minimum.f32", - sym::minimumf64 => "llvm.minimum.f64", + args: &[OperandRef<'tcx, &'ll Value>], +) -> Option<&'ll Value> { + let (base_name, type_params): (&'static str, &[&'ll Type]) = match name { + sym::sqrtf16 => ("llvm.sqrt", &[bx.type_f16()]), + sym::sqrtf32 => ("llvm.sqrt", &[bx.type_f32()]), + sym::sqrtf64 => ("llvm.sqrt", &[bx.type_f64()]), + sym::sqrtf128 => ("llvm.sqrt", &[bx.type_f128()]), + + sym::powif16 => ("llvm.powi", &[bx.type_f16(), bx.type_i32()]), + sym::powif32 => ("llvm.powi", &[bx.type_f32(), bx.type_i32()]), + sym::powif64 => ("llvm.powi", &[bx.type_f64(), bx.type_i32()]), + sym::powif128 => ("llvm.powi", &[bx.type_f128(), bx.type_i32()]), + + sym::sinf16 => ("llvm.sin", &[bx.type_f16()]), + sym::sinf32 => ("llvm.sin", &[bx.type_f32()]), + sym::sinf64 => ("llvm.sin", &[bx.type_f64()]), + sym::sinf128 => ("llvm.sin", &[bx.type_f128()]), + + sym::cosf16 => ("llvm.cos", &[bx.type_f16()]), + sym::cosf32 => ("llvm.cos", &[bx.type_f32()]), + sym::cosf64 => ("llvm.cos", &[bx.type_f64()]), + sym::cosf128 => ("llvm.cos", &[bx.type_f128()]), + + sym::powf16 => ("llvm.pow", &[bx.type_f16()]), + sym::powf32 => ("llvm.pow", &[bx.type_f32()]), + sym::powf64 => ("llvm.pow", &[bx.type_f64()]), + sym::powf128 => ("llvm.pow", &[bx.type_f128()]), + + sym::expf16 => ("llvm.exp", &[bx.type_f16()]), + sym::expf32 => ("llvm.exp", &[bx.type_f32()]), + sym::expf64 => ("llvm.exp", &[bx.type_f64()]), + sym::expf128 => ("llvm.exp", &[bx.type_f128()]), + + sym::exp2f16 => ("llvm.exp2", &[bx.type_f16()]), + sym::exp2f32 => ("llvm.exp2", &[bx.type_f32()]), + sym::exp2f64 => ("llvm.exp2", &[bx.type_f64()]), + sym::exp2f128 => ("llvm.exp2", &[bx.type_f128()]), + + sym::logf16 => ("llvm.log", &[bx.type_f16()]), + sym::logf32 => ("llvm.log", &[bx.type_f32()]), + sym::logf64 => ("llvm.log", &[bx.type_f64()]), + sym::logf128 => ("llvm.log", &[bx.type_f128()]), + + sym::log10f16 => ("llvm.log10", &[bx.type_f16()]), + sym::log10f32 => ("llvm.log10", &[bx.type_f32()]), + sym::log10f64 => ("llvm.log10", &[bx.type_f64()]), + sym::log10f128 => ("llvm.log10", &[bx.type_f128()]), + + sym::log2f16 => ("llvm.log2", &[bx.type_f16()]), + sym::log2f32 => ("llvm.log2", &[bx.type_f32()]), + sym::log2f64 => ("llvm.log2", &[bx.type_f64()]), + sym::log2f128 => ("llvm.log2", &[bx.type_f128()]), + + sym::fmaf16 => ("llvm.fma", &[bx.type_f16()]), + sym::fmaf32 => ("llvm.fma", &[bx.type_f32()]), + sym::fmaf64 => ("llvm.fma", &[bx.type_f64()]), + sym::fmaf128 => ("llvm.fma", &[bx.type_f128()]), + + sym::fmuladdf16 => ("llvm.fmuladd", &[bx.type_f16()]), + sym::fmuladdf32 => ("llvm.fmuladd", &[bx.type_f32()]), + sym::fmuladdf64 => ("llvm.fmuladd", &[bx.type_f64()]), + sym::fmuladdf128 => ("llvm.fmuladd", &[bx.type_f128()]), + + sym::fabsf16 => ("llvm.fabs", &[bx.type_f16()]), + sym::fabsf32 => ("llvm.fabs", &[bx.type_f32()]), + sym::fabsf64 => ("llvm.fabs", &[bx.type_f64()]), + sym::fabsf128 => ("llvm.fabs", &[bx.type_f128()]), + + sym::minnumf16 => ("llvm.minnum", &[bx.type_f16()]), + sym::minnumf32 => ("llvm.minnum", &[bx.type_f32()]), + sym::minnumf64 => ("llvm.minnum", &[bx.type_f64()]), + sym::minnumf128 => ("llvm.minnum", &[bx.type_f128()]), + + sym::minimumf16 => ("llvm.minimum", &[bx.type_f16()]), + sym::minimumf32 => ("llvm.minimum", &[bx.type_f32()]), + sym::minimumf64 => ("llvm.minimum", &[bx.type_f64()]), // There are issues on x86_64 and aarch64 with the f128 variant, // let's instead use the instrinsic fallback body. - // sym::minimumf128 => "llvm.minimum.f128", - sym::maxnumf16 => "llvm.maxnum.f16", - sym::maxnumf32 => "llvm.maxnum.f32", - sym::maxnumf64 => "llvm.maxnum.f64", - sym::maxnumf128 => "llvm.maxnum.f128", - - sym::maximumf16 => "llvm.maximum.f16", - sym::maximumf32 => "llvm.maximum.f32", - sym::maximumf64 => "llvm.maximum.f64", + // sym::minimumf128 => ("llvm.minimum", &[cx.type_f128()]), + sym::maxnumf16 => ("llvm.maxnum", &[bx.type_f16()]), + sym::maxnumf32 => ("llvm.maxnum", &[bx.type_f32()]), + sym::maxnumf64 => ("llvm.maxnum", &[bx.type_f64()]), + sym::maxnumf128 => ("llvm.maxnum", &[bx.type_f128()]), + + sym::maximumf16 => ("llvm.maximum", &[bx.type_f16()]), + sym::maximumf32 => ("llvm.maximum", &[bx.type_f32()]), + sym::maximumf64 => ("llvm.maximum", &[bx.type_f64()]), // There are issues on x86_64 and aarch64 with the f128 variant, // let's instead use the instrinsic fallback body. - // sym::maximumf128 => "llvm.maximum.f128", - sym::copysignf16 => "llvm.copysign.f16", - sym::copysignf32 => "llvm.copysign.f32", - sym::copysignf64 => "llvm.copysign.f64", - sym::copysignf128 => "llvm.copysign.f128", - - sym::floorf16 => "llvm.floor.f16", - sym::floorf32 => "llvm.floor.f32", - sym::floorf64 => "llvm.floor.f64", - sym::floorf128 => "llvm.floor.f128", - - sym::ceilf16 => "llvm.ceil.f16", - sym::ceilf32 => "llvm.ceil.f32", - sym::ceilf64 => "llvm.ceil.f64", - sym::ceilf128 => "llvm.ceil.f128", - - sym::truncf16 => "llvm.trunc.f16", - sym::truncf32 => "llvm.trunc.f32", - sym::truncf64 => "llvm.trunc.f64", - sym::truncf128 => "llvm.trunc.f128", + // sym::maximumf128 => ("llvm.maximum", &[cx.type_f128()]), + sym::copysignf16 => ("llvm.copysign", &[bx.type_f16()]), + sym::copysignf32 => ("llvm.copysign", &[bx.type_f32()]), + sym::copysignf64 => ("llvm.copysign", &[bx.type_f64()]), + sym::copysignf128 => ("llvm.copysign", &[bx.type_f128()]), + + sym::floorf16 => ("llvm.floor", &[bx.type_f16()]), + sym::floorf32 => ("llvm.floor", &[bx.type_f32()]), + sym::floorf64 => ("llvm.floor", &[bx.type_f64()]), + sym::floorf128 => ("llvm.floor", &[bx.type_f128()]), + + sym::ceilf16 => ("llvm.ceil", &[bx.type_f16()]), + sym::ceilf32 => ("llvm.ceil", &[bx.type_f32()]), + sym::ceilf64 => ("llvm.ceil", &[bx.type_f64()]), + sym::ceilf128 => ("llvm.ceil", &[bx.type_f128()]), + + sym::truncf16 => ("llvm.trunc", &[bx.type_f16()]), + sym::truncf32 => ("llvm.trunc", &[bx.type_f32()]), + sym::truncf64 => ("llvm.trunc", &[bx.type_f64()]), + sym::truncf128 => ("llvm.trunc", &[bx.type_f128()]), // We could use any of `rint`, `nearbyint`, or `roundeven` // for this -- they are all identical in semantics when // assuming the default FP environment. // `rint` is what we used for $forever. - sym::round_ties_even_f16 => "llvm.rint.f16", - sym::round_ties_even_f32 => "llvm.rint.f32", - sym::round_ties_even_f64 => "llvm.rint.f64", - sym::round_ties_even_f128 => "llvm.rint.f128", + sym::round_ties_even_f16 => ("llvm.rint", &[bx.type_f16()]), + sym::round_ties_even_f32 => ("llvm.rint", &[bx.type_f32()]), + sym::round_ties_even_f64 => ("llvm.rint", &[bx.type_f64()]), + sym::round_ties_even_f128 => ("llvm.rint", &[bx.type_f128()]), - sym::roundf16 => "llvm.round.f16", - sym::roundf32 => "llvm.round.f32", - sym::roundf64 => "llvm.round.f64", - sym::roundf128 => "llvm.round.f128", + sym::roundf16 => ("llvm.round", &[bx.type_f16()]), + sym::roundf32 => ("llvm.round", &[bx.type_f32()]), + sym::roundf64 => ("llvm.round", &[bx.type_f64()]), + sym::roundf128 => ("llvm.round", &[bx.type_f128()]), - sym::ptr_mask => "llvm.ptrmask", + sym::ptr_mask => ("llvm.ptrmask", &[bx.type_ptr(), bx.type_isize()]), _ => return None, }; - Some(cx.get_intrinsic(llvm_name)) + Some(bx.call_intrinsic( + base_name, + type_params, + &args.iter().map(|arg| arg.immediate()).collect::>(), + )) } impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { @@ -173,36 +178,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let name = tcx.item_name(instance.def_id()); let fn_args = instance.args; - let simple = get_simple_intrinsic(self, name); + let simple = call_simple_intrinsic(self, name, args); let llval = match name { - _ if simple.is_some() => { - let (simple_ty, simple_fn) = simple.unwrap(); - self.call( - simple_ty, - None, - None, - simple_fn, - &args.iter().map(|arg| arg.immediate()).collect::>(), - None, - Some(instance), - ) - } + _ if simple.is_some() => simple.unwrap(), sym::is_val_statically_known => { - let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx); - let kind = self.type_kind(intrinsic_type); - let intrinsic_name = match kind { - TypeKind::Pointer | TypeKind::Integer => { - Some(format!("llvm.is.constant.{intrinsic_type:?}")) - } - // LLVM float types' intrinsic names differ from their type names. - TypeKind::Half => Some(format!("llvm.is.constant.f16")), - TypeKind::Float => Some(format!("llvm.is.constant.f32")), - TypeKind::Double => Some(format!("llvm.is.constant.f64")), - TypeKind::FP128 => Some(format!("llvm.is.constant.f128")), - _ => None, - }; - if let Some(intrinsic_name) = intrinsic_name { - self.call_intrinsic(&intrinsic_name, &[args[0].immediate()]) + if let OperandValue::Immediate(imm) = args[0].val { + self.call_intrinsic( + "llvm.is.constant", + &[args[0].layout.immediate_llvm_type(self.cx)], + &[imm], + ) } else { self.const_bool(false) } @@ -246,10 +231,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { ); return Ok(()); } - sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]), - sym::va_copy => { - self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()]) - } + sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[], &[]), + sym::va_copy => self.call_intrinsic( + "llvm.va_copy", + &[self.type_ptr()], + &[args[0].immediate(), args[1].immediate()], + ), sym::va_arg => { match result.layout.backend_repr { BackendRepr::Scalar(scalar) => { @@ -324,6 +311,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { }; self.call_intrinsic( "llvm.prefetch", + &[self.type_ptr()], &[ args[0].immediate(), self.const_i32(rw), @@ -385,11 +373,13 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } let (size, signed) = ty.int_size_and_signed(self.tcx); let width = size.bits(); + let llty = self.type_ix(width); match name { sym::ctlz | sym::cttz => { let y = self.const_bool(false); let ret = self.call_intrinsic( - &format!("llvm.{name}.i{width}"), + &format!("llvm.{name}"), + &[llty], &[args[0].immediate(), y], ); @@ -397,62 +387,54 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } sym::ctlz_nonzero => { let y = self.const_bool(true); - let llvm_name = &format!("llvm.ctlz.i{width}"); - let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); + let ret = + self.call_intrinsic("llvm.ctlz", &[llty], &[args[0].immediate(), y]); self.intcast(ret, result.layout.llvm_type(self), false) } sym::cttz_nonzero => { let y = self.const_bool(true); - let llvm_name = &format!("llvm.cttz.i{width}"); - let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); + let ret = + self.call_intrinsic("llvm.cttz", &[llty], &[args[0].immediate(), y]); self.intcast(ret, result.layout.llvm_type(self), false) } sym::ctpop => { - let ret = self.call_intrinsic( - &format!("llvm.ctpop.i{width}"), - &[args[0].immediate()], - ); + let ret = + self.call_intrinsic("llvm.ctpop", &[llty], &[args[0].immediate()]); self.intcast(ret, result.layout.llvm_type(self), false) } sym::bswap => { if width == 8 { args[0].immediate() // byte swap a u8/i8 is just a no-op } else { - self.call_intrinsic( - &format!("llvm.bswap.i{width}"), - &[args[0].immediate()], - ) + self.call_intrinsic("llvm.bswap", &[llty], &[args[0].immediate()]) } } - sym::bitreverse => self.call_intrinsic( - &format!("llvm.bitreverse.i{width}"), - &[args[0].immediate()], - ), + sym::bitreverse => { + self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()]) + } sym::rotate_left | sym::rotate_right => { let is_left = name == sym::rotate_left; let val = args[0].immediate(); let raw_shift = args[1].immediate(); // rotate = funnel shift with first two args the same - let llvm_name = - &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); + let llvm_name = format!("llvm.fsh{}", if is_left { 'l' } else { 'r' }); // llvm expects shift to be the same type as the values, but rust // always uses `u32`. let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); - self.call_intrinsic(llvm_name, &[val, val, raw_shift]) + self.call_intrinsic(&llvm_name, &[llty], &[val, val, raw_shift]) } sym::saturating_add | sym::saturating_sub => { let is_add = name == sym::saturating_add; let lhs = args[0].immediate(); let rhs = args[1].immediate(); - let llvm_name = &format!( - "llvm.{}{}.sat.i{}", + let llvm_name = format!( + "llvm.{}{}.sat", if signed { 's' } else { 'u' }, if is_add { "add" } else { "sub" }, - width ); - self.call_intrinsic(llvm_name, &[lhs, rhs]) + self.call_intrinsic(&llvm_name, &[llty], &[lhs, rhs]) } _ => bug!(), } @@ -484,11 +466,8 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.icmp(IntPredicate::IntEQ, a_val, b_val) } else { let n = self.const_usize(layout.size().bytes()); - let cmp = self.call_intrinsic("memcmp", &[a, b, n]); - match self.cx.sess().target.arch.as_ref() { - "avr" | "msp430" => self.icmp(IntPredicate::IntEQ, cmp, self.const_i16(0)), - _ => self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)), - } + let cmp = self.call_intrinsic("memcmp", &[], &[a, b, n]); + self.icmp(IntPredicate::IntEQ, cmp, self.const_int(self.type_int(), 0)) } } @@ -496,6 +475,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // Here we assume that the `memcmp` provided by the target is a NOP for size 0. let cmp = self.call_intrinsic( "memcmp", + &[], &[args[0].immediate(), args[1].immediate(), args[2].immediate()], ); // Some targets have `memcmp` returning `i16`, but the intrinsic is always `i32`. @@ -619,18 +599,22 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } fn abort(&mut self) { - self.call_intrinsic("llvm.trap", &[]); + self.call_intrinsic("llvm.trap", &[], &[]); } fn assume(&mut self, val: Self::Value) { if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No { - self.call_intrinsic("llvm.assume", &[val]); + self.call_intrinsic("llvm.assume", &[], &[val]); } } fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value { if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No { - self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)]) + self.call_intrinsic( + "llvm.expect", + &[self.type_i1()], + &[cond, self.const_bool(expected)], + ) } else { cond } @@ -644,17 +628,20 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { ) -> Self::Value { let typeid = self.get_metadata_value(typeid); let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32); - let type_checked_load = - self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]); + let type_checked_load = self.call_intrinsic( + "llvm.type.checked.load", + &[], + &[llvtable, vtable_byte_offset, typeid], + ); self.extract_value(type_checked_load, 0) } fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { - self.call_intrinsic("llvm.va_start", &[va_list]) + self.call_intrinsic("llvm.va_start", &[self.type_ptr()], &[va_list]) } fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value { - self.call_intrinsic("llvm.va_end", &[va_list]) + self.call_intrinsic("llvm.va_end", &[self.type_ptr()], &[va_list]) } } @@ -893,8 +880,8 @@ fn codegen_wasm_try<'ll, 'tcx>( let null = bx.const_null(bx.type_ptr()); let funclet = bx.catch_pad(cs, &[null]); - let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[funclet.cleanuppad()]); - let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[funclet.cleanuppad()]); + let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[], &[funclet.cleanuppad()]); + let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[], &[funclet.cleanuppad()]); let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None); @@ -1031,7 +1018,7 @@ fn codegen_emcc_try<'ll, 'tcx>( let selector = bx.extract_value(vals, 1); // Check if the typeid we got is the one for a Rust panic. - let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[tydesc]); + let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[bx.type_ptr()], &[tydesc]); let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid); let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool()); @@ -1522,56 +1509,37 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }}; } - let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() { - let elem_ty = bx.cx.type_float_from_ty(*f); - match f.bit_width() { - 16 => ("f16", elem_ty), - 32 => ("f32", elem_ty), - 64 => ("f64", elem_ty), - 128 => ("f128", elem_ty), - _ => return_error!(InvalidMonomorphization::FloatingPointVector { - span, - name, - f_ty: *f, - in_ty, - }), - } + let elem_ty = if let ty::Float(f) = in_elem.kind() { + bx.cx.type_float_from_ty(*f) } else { return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty }); }; let vec_ty = bx.type_vector(elem_ty, in_len); - let (intr_name, fn_ty) = match name { - sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), - sym::simd_relaxed_fma => ("fmuladd", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), - sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)), + let intr_name = match name { + sym::simd_ceil => "llvm.ceil", + sym::simd_fabs => "llvm.fabs", + sym::simd_fcos => "llvm.cos", + sym::simd_fexp2 => "llvm.exp2", + sym::simd_fexp => "llvm.exp", + sym::simd_flog10 => "llvm.log10", + sym::simd_flog2 => "llvm.log2", + sym::simd_flog => "llvm.log", + sym::simd_floor => "llvm.floor", + sym::simd_fma => "llvm.fma", + sym::simd_relaxed_fma => "llvm.fmuladd", + sym::simd_fsin => "llvm.sin", + sym::simd_fsqrt => "llvm.sqrt", + sym::simd_round => "llvm.round", + sym::simd_trunc => "llvm.trunc", _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }), }; - let llvm_name = &format!("llvm.{intr_name}.v{in_len}{elem_ty_str}"); - let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty); - let c = bx.call( - fn_ty, - None, - None, - f, + Ok(bx.call_intrinsic( + intr_name, + &[vec_ty], &args.iter().map(|arg| arg.immediate()).collect::>(), - None, - None, - ); - Ok(c) + )) } if std::matches!( @@ -1595,29 +1563,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>( return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args); } - // FIXME: use: - // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182 - // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81 - fn llvm_vector_str(bx: &Builder<'_, '_, '_>, elem_ty: Ty<'_>, vec_len: u64) -> String { - match *elem_ty.kind() { - ty::Int(v) => format!( - "v{}i{}", - vec_len, - // Normalize to prevent crash if v: IntTy::Isize - v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() - ), - ty::Uint(v) => format!( - "v{}i{}", - vec_len, - // Normalize to prevent crash if v: UIntTy::Usize - v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() - ), - ty::Float(v) => format!("v{}f{}", vec_len, v.bit_width()), - ty::RawPtr(_, _) => format!("v{}p0", vec_len), - _ => unreachable!(), - } - } - fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type { let elem_ty = match *elem_ty.kind() { ty::Int(v) => cx.type_int_from_ty(v), @@ -1698,38 +1643,22 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ); // Alignment of T, must be a constant integer value: - let alignment_ty = bx.type_i32(); let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); // Truncate the mask vector to a vector of i1s: let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len); - let mask_ty = bx.type_vector(bx.type_i1(), in_len); // Type of the vector of pointers: let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len); - let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len); // Type of the vector of elements: let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len); - let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len); - let llvm_intrinsic = - format!("llvm.masked.gather.{llvm_elem_vec_str}.{llvm_pointer_vec_str}"); - let fn_ty = bx.type_func( - &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty], - llvm_elem_vec_ty, - ); - let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = bx.call( - fn_ty, - None, - None, - f, + return Ok(bx.call_intrinsic( + "llvm.masked.gather", + &[llvm_elem_vec_ty, llvm_pointer_vec_ty], &[args[1].immediate(), alignment, mask, args[0].immediate()], - None, - None, - ); - return Ok(v); + )); } if name == sym::simd_masked_load { @@ -1795,32 +1724,20 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ); let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len); - let mask_ty = bx.type_vector(bx.type_i1(), mask_len); // Alignment of T, must be a constant integer value: - let alignment_ty = bx.type_i32(); let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32); let llvm_pointer = bx.type_ptr(); // Type of the vector of elements: let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len); - let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len); - - let llvm_intrinsic = format!("llvm.masked.load.{llvm_elem_vec_str}.p0"); - let fn_ty = bx - .type_func(&[llvm_pointer, alignment_ty, mask_ty, llvm_elem_vec_ty], llvm_elem_vec_ty); - let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = bx.call( - fn_ty, - None, - None, - f, + + return Ok(bx.call_intrinsic( + "llvm.masked.load", + &[llvm_elem_vec_ty, llvm_pointer], &[args[1].immediate(), alignment, mask, args[2].immediate()], - None, - None, - ); - return Ok(v); + )); } if name == sym::simd_masked_store { @@ -1880,33 +1797,20 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ); let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len); - let mask_ty = bx.type_vector(bx.type_i1(), mask_len); // Alignment of T, must be a constant integer value: - let alignment_ty = bx.type_i32(); let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32); - let ret_t = bx.type_void(); - let llvm_pointer = bx.type_ptr(); // Type of the vector of elements: let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len); - let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len); - - let llvm_intrinsic = format!("llvm.masked.store.{llvm_elem_vec_str}.p0"); - let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer, alignment_ty, mask_ty], ret_t); - let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = bx.call( - fn_ty, - None, - None, - f, + + return Ok(bx.call_intrinsic( + "llvm.masked.store", + &[llvm_elem_vec_ty, llvm_pointer], &[args[2].immediate(), args[1].immediate(), alignment, mask], - None, - None, - ); - return Ok(v); + )); } if name == sym::simd_scatter { @@ -1971,38 +1875,22 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ); // Alignment of T, must be a constant integer value: - let alignment_ty = bx.type_i32(); let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); // Truncate the mask vector to a vector of i1s: let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len); - let mask_ty = bx.type_vector(bx.type_i1(), in_len); - - let ret_t = bx.type_void(); // Type of the vector of pointers: let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len); - let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len); // Type of the vector of elements: let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len); - let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len); - - let llvm_intrinsic = - format!("llvm.masked.scatter.{llvm_elem_vec_str}.{llvm_pointer_vec_str}"); - let fn_ty = - bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t); - let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = bx.call( - fn_ty, - None, - None, - f, + + return Ok(bx.call_intrinsic( + "llvm.masked.scatter", + &[llvm_elem_vec_ty, llvm_pointer_vec_ty], &[args[0].immediate(), args[1].immediate(), alignment, mask], - None, - None, - ); - return Ok(v); + )); } macro_rules! arith_red { @@ -2431,40 +2319,31 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }, in_len as u64, ); - let intrinsic_name = match name { - sym::simd_bswap => "bswap", - sym::simd_bitreverse => "bitreverse", - sym::simd_ctlz => "ctlz", - sym::simd_ctpop => "ctpop", - sym::simd_cttz => "cttz", + let llvm_intrinsic = match name { + sym::simd_bswap => "llvm.bswap", + sym::simd_bitreverse => "llvm.bitreverse", + sym::simd_ctlz => "llvm.ctlz", + sym::simd_ctpop => "llvm.ctpop", + sym::simd_cttz => "llvm.cttz", _ => unreachable!(), }; let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits(); - let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,); return match name { // byte swap is no-op for i8/u8 sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()), sym::simd_ctlz | sym::simd_cttz => { // for the (int, i1 immediate) pair, the second arg adds `(0, true) => poison` - let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty); let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0); - let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - Ok(bx.call( - fn_ty, - None, - None, - f, + Ok(bx.call_intrinsic( + llvm_intrinsic, + &[vec_ty], &[args[0].immediate(), dont_poison_on_zero], - None, - None, )) } sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => { // simple unary argument cases - let fn_ty = bx.type_func(&[vec_ty], vec_ty); - let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None)) + Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[args[0].immediate()])) } _ => unreachable!(), }; @@ -2495,10 +2374,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let lhs = args[0].immediate(); let rhs = args[1].immediate(); let is_add = name == sym::simd_saturating_add; - let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _; - let (signed, elem_width, elem_ty) = match *in_elem.kind() { - ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)), - ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)), + let (signed, elem_ty) = match *in_elem.kind() { + ty::Int(i) => (true, bx.cx.type_int_from_ty(i)), + ty::Uint(i) => (false, bx.cx.type_uint_from_ty(i)), _ => { return_error!(InvalidMonomorphization::ExpectedVectorElementType { span, @@ -2508,19 +2386,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }); } }; - let llvm_intrinsic = &format!( - "llvm.{}{}.sat.v{}i{}", + let llvm_intrinsic = format!( + "llvm.{}{}.sat", if signed { 's' } else { 'u' }, if is_add { "add" } else { "sub" }, - in_len, - elem_width ); let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64); - let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); - let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = bx.call(fn_ty, None, None, f, &[lhs, rhs], None, None); - return Ok(v); + return Ok(bx.call_intrinsic(&llvm_intrinsic, &[vec_ty], &[lhs, rhs])); } span_bug!(span, "unknown SIMD intrinsic"); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index e27fbf94f341d..59c61db5fcdcb 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -15,6 +15,7 @@ use std::fmt::Debug; use std::marker::PhantomData; +use std::num::NonZero; use std::ptr; use bitflags::bitflags; @@ -1195,6 +1196,17 @@ unsafe extern "C" { // Operations on functions pub(crate) fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint); + // Operations about llvm intrinsics + pub(crate) fn LLVMLookupIntrinsicID(Name: *const c_char, NameLen: size_t) -> c_uint; + pub(crate) fn LLVMIntrinsicIsOverloaded(ID: NonZero) -> Bool; + pub(crate) fn LLVMIntrinsicCopyOverloadedName2<'a>( + Mod: &'a Module, + ID: NonZero, + ParamTypes: *const &'a Type, + ParamCount: size_t, + NameLength: *mut size_t, + ) -> *mut c_char; + // Operations on parameters pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; pub(crate) safe fn LLVMCountParams(Fn: &Value) -> c_uint; diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index ed23f91193013..bc3538c768dee 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -1,9 +1,10 @@ #![allow(non_snake_case)] use std::ffi::{CStr, CString}; -use std::ptr; +use std::num::NonZero; use std::str::FromStr; use std::string::FromUtf8Error; +use std::{ptr, slice}; use libc::c_uint; use rustc_abi::{Align, Size, WrappingRange}; @@ -327,6 +328,48 @@ pub(crate) fn get_value_name(value: &Value) -> &[u8] { } } +#[derive(Debug, Copy, Clone)] +pub(crate) struct Intrinsic { + id: NonZero, +} + +impl Intrinsic { + pub(crate) fn lookup(name: &[u8]) -> Option { + let id = unsafe { LLVMLookupIntrinsicID(name.as_c_char_ptr(), name.len()) }; + NonZero::new(id).map(|id| Self { id }) + } + + pub(crate) fn is_overloaded(self) -> bool { + unsafe { LLVMIntrinsicIsOverloaded(self.id) == True } + } + + pub(crate) fn overloaded_name<'ll>( + self, + llmod: &'ll Module, + type_params: &[&'ll Type], + ) -> String { + let mut len = 0; + let ptr = unsafe { + LLVMIntrinsicCopyOverloadedName2( + llmod, + self.id, + type_params.as_ptr(), + type_params.len(), + &mut len, + ) + }; + + let slice = unsafe { slice::from_raw_parts_mut(ptr.cast(), len) }; + let copied = str::from_utf8(slice).expect("Non-UTF8 intrinsic name").to_string(); + + unsafe { + libc::free(ptr.cast()); + } + + copied + } +} + /// Safe wrapper for `LLVMSetValueName2` from a byte slice pub(crate) fn set_value_name(value: &Value, name: &[u8]) { unsafe { diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 169036f515298..2364ad0c9d237 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -1,4 +1,5 @@ use std::borrow::Borrow; +use std::hash::{Hash, Hasher}; use std::{fmt, ptr}; use libc::{c_char, c_uint}; @@ -25,6 +26,14 @@ impl PartialEq for Type { } } +impl Eq for Type {} + +impl Hash for Type { + fn hash(&self, state: &mut H) { + ptr::hash(self, state); + } +} + impl fmt::Debug for Type { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str( diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 7f78bc7569568..f35f551d5906a 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -23,7 +23,7 @@ use crate::common::{AtomicRmwBinOp, IntPredicate, RealPredicate, Synchronization use crate::mir::operand::{OperandRef, OperandValue}; use crate::mir::place::{PlaceRef, PlaceValue}; -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum OverflowOp { Add, Sub, @@ -215,7 +215,7 @@ pub trait BuilderMethods<'a, 'tcx>: fn checked_binop( &mut self, oop: OverflowOp, - ty: Ty<'_>, + ty: Ty<'tcx>, lhs: Self::Value, rhs: Self::Value, ) -> (Self::Value, Self::Value);