Skip to content

Commit 4351488

Browse files
dylanmckaydylanmckay
committed
Require CodegenCx::type_ptr_to callers to specify an address space at all times
Most of the time, the callers will be able to simply pass through the address space of the pointer being operated on.
1 parent a43ac52 commit 4351488

File tree

17 files changed

+220
-78
lines changed

17 files changed

+220
-78
lines changed

‎src/librustc_codegen_llvm/abi.rs

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA};
1414
use rustc_middle::ty::Ty;
1515
use rustc_target::abi::call::ArgAbi;
1616
pub use rustc_target::abi::call::*;
17-
use rustc_target::abi::{self, HasDataLayout, Int, LayoutOf};
17+
use rustc_target::abi::{self, AddressSpace, HasDataLayout, Int, LayoutOf};
1818
pub use rustc_target::spec::abi::Abi;
1919

2020
use libc::c_uint;
@@ -207,7 +207,8 @@ impl ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
207207
// uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}.
208208
let can_store_through_cast_ptr = false;
209209
if can_store_through_cast_ptr {
210-
let cast_ptr_llty = bx.type_ptr_to(cast.llvm_type(bx));
210+
let cast_ptr_llty =
211+
bx.type_ptr_to(cast.llvm_type(bx), bx.cx().address_space_of_value(dst.llval));
211212
let cast_dst = bx.pointercast(dst.llval, cast_ptr_llty);
212213
bx.store(val, cast_dst, self.layout.align.abi);
213214
} else {
@@ -323,7 +324,8 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
323324
PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
324325
PassMode::Cast(cast) => cast.llvm_type(cx),
325326
PassMode::Indirect(..) => {
326-
llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx)));
327+
llargument_tys
328+
.push(cx.type_ptr_to(self.ret.memory_ty(cx), AddressSpace::default()));
327329
cx.type_void()
328330
}
329331
};
@@ -350,7 +352,9 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
350352
continue;
351353
}
352354
PassMode::Cast(cast) => cast.llvm_type(cx),
353-
PassMode::Indirect(_, None) => cx.type_ptr_to(arg.memory_ty(cx)),
355+
PassMode::Indirect(_, None) => {
356+
cx.type_ptr_to(arg.memory_ty(cx), AddressSpace::default())
357+
}
354358
};
355359
llargument_tys.push(llarg_ty);
356360
}

‎src/librustc_codegen_llvm/builder.rs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -719,7 +719,10 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
719719
if flags.contains(MemFlags::NONTEMPORAL) {
720720
// HACK(nox): This is inefficient but there is no nontemporal memcpy.
721721
let val = self.load(src, src_align);
722-
let ptr = self.pointercast(dst, self.type_ptr_to(self.val_ty(val)));
722+
let ptr = self.pointercast(
723+
dst,
724+
self.type_ptr_to(self.val_ty(val), self.cx.address_space_of_value(dst)),
725+
);
723726
self.store_with_flags(val, ptr, dst_align, flags);
724727
return;
725728
}
@@ -752,7 +755,10 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
752755
if flags.contains(MemFlags::NONTEMPORAL) {
753756
// HACK(nox): This is inefficient but there is no nontemporal memmove.
754757
let val = self.load(src, src_align);
755-
let ptr = self.pointercast(dst, self.type_ptr_to(self.val_ty(val)));
758+
let ptr = self.pointercast(
759+
dst,
760+
self.type_ptr_to(self.val_ty(val), self.cx.address_space_of_value(dst)),
761+
);
756762
self.store_with_flags(val, ptr, dst_align, flags);
757763
return;
758764
}
@@ -1191,7 +1197,7 @@ impl Builder<'a, 'll, 'tcx> {
11911197
fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value {
11921198
let dest_ptr_ty = self.cx.val_ty(ptr);
11931199
let stored_ty = self.cx.val_ty(val);
1194-
let stored_ptr_ty = self.cx.type_ptr_to(stored_ty);
1200+
let stored_ptr_ty = self.cx.type_ptr_to(stored_ty, self.cx.address_space_of_value(ptr));
11951201

11961202
assert_eq!(self.cx.type_kind(dest_ptr_ty), TypeKind::Pointer);
11971203

‎src/librustc_codegen_llvm/common.rs

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,10 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
206206
let len = s.as_str().len();
207207
let cs = consts::ptrcast(
208208
self.const_cstr(s, false),
209-
self.type_ptr_to(self.layout_of(self.tcx.types.str_).llvm_type(self)),
209+
self.type_ptr_to(
210+
self.layout_of(self.tcx.types.str_).llvm_type(self),
211+
AddressSpace::default(),
212+
),
210213
);
211214
(cs, self.const_usize(len as u64))
212215
}
@@ -246,7 +249,8 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
246249
Scalar::Ptr(ptr) => {
247250
let base_addr = match self.tcx.global_alloc(ptr.alloc_id) {
248251
GlobalAlloc::Memory(alloc) => {
249-
let init = const_alloc_to_llvm(self, alloc);
252+
let init =
253+
const_alloc_to_llvm(self, alloc, self.address_space_of_type(llty));
250254
let value = match alloc.mutability {
251255
Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
252256
_ => self.static_addr_of(init, alloc.align, None),
@@ -265,7 +269,10 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
265269
};
266270
let llval = unsafe {
267271
llvm::LLVMConstInBoundsGEP(
268-
self.const_bitcast(base_addr, self.type_i8p(AddressSpace::default())),
272+
self.const_bitcast(
273+
base_addr,
274+
self.type_i8p(self.address_space_of_value(base_addr)),
275+
),
269276
&self.const_usize(ptr.offset.bytes()),
270277
1,
271278
)
@@ -286,17 +293,22 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
286293
offset: Size,
287294
) -> PlaceRef<'tcx, &'ll Value> {
288295
assert_eq!(alloc.align, layout.align.abi);
289-
let llty = self.type_ptr_to(layout.llvm_type(self));
296+
let backend_type = layout.llvm_type(self);
297+
let address_space = self.default_address_space_of_type(layout.ty);
298+
let llty = self.type_ptr_to(backend_type, address_space);
290299
let llval = if layout.size == Size::ZERO {
291300
let llval = self.const_usize(alloc.align.bytes());
292301
unsafe { llvm::LLVMConstIntToPtr(llval, llty) }
293302
} else {
294-
let init = const_alloc_to_llvm(self, alloc);
303+
let init = const_alloc_to_llvm(self, alloc, address_space);
295304
let base_addr = self.static_addr_of(init, alloc.align, None);
296305

297306
let llval = unsafe {
298307
llvm::LLVMConstInBoundsGEP(
299-
self.const_bitcast(base_addr, self.type_i8p(AddressSpace::default())),
308+
self.const_bitcast(
309+
base_addr,
310+
self.type_i8p(self.address_space_of_value(base_addr)),
311+
),
300312
&self.const_usize(offset.bytes()),
301313
1,
302314
)

‎src/librustc_codegen_llvm/consts.rs

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,11 @@ use rustc_target::abi::{AddressSpace, Align, HasDataLayout, LayoutOf, Primitive,
2424

2525
use std::ffi::CStr;
2626

27-
pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll Value {
27+
pub fn const_alloc_to_llvm(
28+
cx: &CodegenCx<'ll, '_>,
29+
alloc: &Allocation,
30+
address_space: AddressSpace,
31+
) -> &'ll Value {
2832
let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1);
2933
let dl = cx.data_layout();
3034
let pointer_size = dl.pointer_size.bytes() as usize;
@@ -56,7 +60,7 @@ pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll
5660
llvals.push(cx.scalar_to_backend(
5761
Pointer::new(alloc_id, Size::from_bytes(ptr_offset)).into(),
5862
&Scalar { value: Primitive::Pointer, valid_range: 0..=!0 },
59-
cx.type_i8p(AddressSpace::default()),
63+
cx.type_i8p(address_space),
6064
));
6165
next_offset = offset + pointer_size;
6266
}
@@ -79,11 +83,17 @@ pub fn codegen_static_initializer(
7983
cx: &CodegenCx<'ll, 'tcx>,
8084
def_id: DefId,
8185
) -> Result<(&'ll Value, &'tcx Allocation), ErrorHandled> {
86+
let address_space = if cx.tcx.type_of(def_id).is_fn() {
87+
cx.data_layout().instruction_address_space
88+
} else {
89+
AddressSpace::default()
90+
};
91+
8292
let alloc = match cx.tcx.const_eval_poly(def_id)? {
8393
ConstValue::ByRef { alloc, offset } if offset.bytes() == 0 => alloc,
8494
val => bug!("static const eval returned {:#?}", val),
8595
};
86-
Ok((const_alloc_to_llvm(cx, alloc), alloc))
96+
Ok((const_alloc_to_llvm(cx, alloc, address_space), alloc))
8797
}
8898

8999
fn set_global_alignment(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) {
@@ -212,12 +222,15 @@ impl CodegenCx<'ll, 'tcx> {
212222
let g = if let Some(def_id) = def_id.as_local() {
213223
let id = self.tcx.hir().as_local_hir_id(def_id);
214224
let llty = self.layout_of(ty).llvm_type(self);
225+
215226
// FIXME: refactor this to work without accessing the HIR
216227
let (g, attrs) = match self.tcx.hir().get(id) {
217228
Node::Item(&hir::Item { attrs, span, kind: hir::ItemKind::Static(..), .. }) => {
218229
let sym_str = sym.as_str();
219230
if let Some(g) = self.get_declared_value(&sym_str) {
220-
if self.val_ty(g) != self.type_ptr_to(llty) {
231+
if self.val_ty(g)
232+
!= self.type_ptr_to(llty, self.address_space_of_type(llty))
233+
{
221234
span_bug!(span, "Conflicting types for static");
222235
}
223236
}
@@ -496,7 +509,8 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> {
496509

497510
if attrs.flags.contains(CodegenFnAttrFlags::USED) {
498511
// This static will be stored in the llvm.used variable which is an array of i8*
499-
let cast = llvm::LLVMConstPointerCast(g, self.type_i8p(AddressSpace::default()));
512+
let cast =
513+
llvm::LLVMConstPointerCast(g, self.type_i8p(self.address_space_of_value(g)));
500514
self.used_statics.borrow_mut().push(cast);
501515
}
502516
}

‎src/librustc_codegen_llvm/context.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -394,8 +394,10 @@ impl MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> {
394394
fn create_used_variable(&self) {
395395
let name = const_cstr!("llvm.used");
396396
let section = const_cstr!("llvm.metadata");
397-
let array =
398-
self.const_array(&self.type_ptr_to(self.type_i8()), &*self.used_statics.borrow());
397+
let array = self.const_array(
398+
&self.type_ptr_to(self.type_i8(), AddressSpace::default()),
399+
&*self.used_statics.borrow(),
400+
);
399401

400402
unsafe {
401403
let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr());

‎src/librustc_codegen_llvm/intrinsic.rs

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,10 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
310310
let tp_ty = substs.type_at(0);
311311
let mut ptr = args[0].immediate();
312312
if let PassMode::Cast(ty) = fn_abi.ret.mode {
313-
ptr = self.pointercast(ptr, self.type_ptr_to(ty.llvm_type(self)));
313+
ptr = self.pointercast(
314+
ptr,
315+
self.type_ptr_to(ty.llvm_type(self), self.cx().address_space_of_value(ptr)),
316+
);
314317
}
315318
let load = self.volatile_load(ptr);
316319
let align = if name == "unaligned_volatile_load" {
@@ -753,7 +756,10 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
753756

754757
if !fn_abi.ret.is_ignore() {
755758
if let PassMode::Cast(ty) = fn_abi.ret.mode {
756-
let ptr_llty = self.type_ptr_to(ty.llvm_type(self));
759+
let ptr_llty = self.type_ptr_to(
760+
ty.llvm_type(self),
761+
self.cx().address_space_of_value(result.llval),
762+
);
757763
let ptr = self.pointercast(result.llval, ptr_llty);
758764
self.store(llval, ptr, result.align);
759765
} else {
@@ -1040,7 +1046,7 @@ fn codegen_gnu_try(
10401046
let tydesc = match bx.tcx().lang_items().eh_catch_typeinfo() {
10411047
Some(tydesc) => {
10421048
let tydesc = bx.get_static(tydesc);
1043-
bx.bitcast(tydesc, bx.type_i8p(AddressSpace::default()))
1049+
bx.bitcast(tydesc, bx.type_i8p(bx.cx().address_space_of_value(tydesc)))
10441050
}
10451051
None => bx.const_null(bx.type_i8p(AddressSpace::default())),
10461052
};
@@ -1534,7 +1540,7 @@ fn generic_simd_intrinsic(
15341540
_ => unreachable!(),
15351541
};
15361542
while no_pointers > 0 {
1537-
elem_ty = cx.type_ptr_to(elem_ty);
1543+
elem_ty = cx.type_ptr_to(elem_ty, AddressSpace::default());
15381544
no_pointers -= 1;
15391545
}
15401546
cx.type_vector(elem_ty, vec_len)

‎src/librustc_codegen_llvm/llvm/ffi.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1920,7 +1920,10 @@ extern "C" {
19201920
pub fn LLVMRustDIBuilderCreateOpDeref() -> i64;
19211921
pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> i64;
19221922

1923-
pub fn LLVMRustGetPointerTypeAddressSpace(ty: &Type) -> u32;
1923+
/// Returns `-1` if the value is not a pointer, otherwise returns the actual address space.
1924+
pub fn LLVMRustGetPointerTypeAddressSpace(ty: &Type) -> i64;
1925+
/// Returns `-1` if the value is not a pointer, otherwise returns the actual address space.
1926+
pub fn LLVMRustGetPointerAddressSpace(val: &Value) -> i64;
19241927

19251928
#[allow(improper_ctypes)]
19261929
pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString);

‎src/librustc_codegen_llvm/type_.rs

Lines changed: 27 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ use rustc_middle::bug;
1515
use rustc_middle::ty::layout::TyAndLayout;
1616
use rustc_middle::ty::Ty;
1717
use rustc_target::abi::call::{CastTarget, FnAbi, Reg};
18-
use rustc_target::abi::{AddressSpace, Align, Integer, Size};
18+
use rustc_target::abi::{AddressSpace, Align, HasDataLayout, Integer, Size};
1919

2020
use std::fmt;
2121
use std::ptr;
@@ -194,21 +194,36 @@ impl BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
194194
unsafe { llvm::LLVMRustGetTypeKind(ty).to_generic() }
195195
}
196196

197-
fn type_ptr_to(&self, ty: &'ll Type) -> &'ll Type {
198-
assert_ne!(
199-
self.type_kind(ty),
200-
TypeKind::Function,
201-
"don't call ptr_to on function types, use ptr_to_llvm_type on FnAbi instead or explicitly specify an address space if it makes sense"
202-
);
203-
ty.ptr_to(AddressSpace::default())
204-
}
205-
206-
fn type_ptr_to_ext(&self, ty: &'ll Type, address_space: AddressSpace) -> &'ll Type {
197+
fn type_ptr_to(&self, ty: &'ll Type, address_space: AddressSpace) -> &'ll Type {
198+
if self.type_kind(ty) == TypeKind::Function {
199+
assert!(
200+
address_space == self.data_layout().instruction_address_space,
201+
"attempted to create a function pointer to the wrong address space: {:?} ({:?})",
202+
ty,
203+
self.type_kind(ty)
204+
);
205+
}
207206
ty.ptr_to(address_space)
208207
}
209208

210209
fn address_space_of_type(&self, ty: &'ll Type) -> AddressSpace {
211-
ty.address_space()
210+
let space_number = unsafe { llvm::LLVMRustGetPointerTypeAddressSpace(ty) };
211+
assert_ne!(
212+
space_number, -1,
213+
"attempted to take the address space of a non-pointer type: {:?}",
214+
ty
215+
);
216+
AddressSpace(space_number as u32)
217+
}
218+
219+
fn address_space_of_value(&self, value: &'ll Value) -> AddressSpace {
220+
let space_number = unsafe { llvm::LLVMRustGetPointerAddressSpace(value) };
221+
assert_ne!(
222+
space_number, -1,
223+
"attempted to take the address space of a non-pointer value: {:?}",
224+
value
225+
);
226+
AddressSpace(space_number as u32)
212227
}
213228

214229
fn element_type(&self, ty: &'ll Type) -> &'ll Type {
@@ -252,11 +267,6 @@ impl Type {
252267
Type::i8_llcx(llcx).ptr_to(AddressSpace::default())
253268
}
254269

255-
pub fn address_space(&self) -> AddressSpace {
256-
let space_number = unsafe { llvm::LLVMRustGetPointerTypeAddressSpace(self) };
257-
AddressSpace(space_number)
258-
}
259-
260270
fn ptr_to(&self, address_space: AddressSpace) -> &Type {
261271
unsafe { llvm::LLVMPointerType(&self, address_space.0) }
262272
}

‎src/librustc_codegen_llvm/type_of.rs

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use rustc_middle::bug;
77
use rustc_middle::ty::layout::{FnAbiExt, TyAndLayout};
88
use rustc_middle::ty::print::obsolete::DefPathBasedNames;
99
use rustc_middle::ty::{self, Ty, TypeFoldable};
10-
use rustc_target::abi::{Abi, Align, FieldsShape};
10+
use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape};
1111
use rustc_target::abi::{Int, Pointer, F32, F64};
1212
use rustc_target::abi::{LayoutOf, PointeeInfo, Scalar, Size, TyAndLayoutMethods, Variants};
1313

@@ -238,11 +238,12 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
238238
}
239239
let llty = match self.ty.kind {
240240
ty::Ref(_, ty, _) | ty::RawPtr(ty::TypeAndMut { ty, .. }) => {
241-
cx.type_ptr_to(cx.layout_of(ty).llvm_type(cx))
242-
}
243-
ty::Adt(def, _) if def.is_box() => {
244-
cx.type_ptr_to(cx.layout_of(self.ty.boxed_ty()).llvm_type(cx))
241+
cx.type_ptr_to(cx.layout_of(ty).llvm_type(cx), AddressSpace::default())
245242
}
243+
ty::Adt(def, _) if def.is_box() => cx.type_ptr_to(
244+
cx.layout_of(self.ty.boxed_ty()).llvm_type(cx),
245+
AddressSpace::default(),
246+
),
246247
ty::FnPtr(sig) => cx.fn_ptr_backend_type(&FnAbi::of_fn_ptr(cx, sig, &[])),
247248
_ => self.scalar_llvm_type_at(cx, scalar, Size::ZERO),
248249
};
@@ -315,7 +316,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
315316
} else {
316317
cx.type_i8()
317318
};
318-
cx.type_ptr_to(pointee)
319+
cx.type_ptr_to(pointee, AddressSpace::default())
319320
}
320321
}
321322
}

0 commit comments

Comments
 (0)