From c196097e588b05e86b5ce6de992b2a6e6a7027bd Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 12 Feb 2019 10:20:34 -0800 Subject: [PATCH 1/2] rustc: Update LLVM, remove dead wasm code This commit updates the LLVM branch to the rebased version of the upstream release/8.x branch. This includes a wasm patch which means that the `rewrite_imports` pass in rustc is no longer needed (yay!) and we can instead rely on `wasm-import-module`, an attribute we're already emitting, to take care of all the work. --- src/librustc_codegen_llvm/back/link.rs | 1 - src/librustc_codegen_llvm/back/wasm.rs | 126 ------------------------- src/librustc_codegen_ssa/base.rs | 24 +---- src/librustc_codegen_ssa/lib.rs | 1 - src/llvm-project | 2 +- 5 files changed, 2 insertions(+), 152 deletions(-) diff --git a/src/librustc_codegen_llvm/back/link.rs b/src/librustc_codegen_llvm/back/link.rs index 819f7f9410257..6c175ff4247ca 100644 --- a/src/librustc_codegen_llvm/back/link.rs +++ b/src/librustc_codegen_llvm/back/link.rs @@ -698,7 +698,6 @@ fn link_natively(sess: &Session, } if sess.opts.target_triple.triple() == "wasm32-unknown-unknown" { - wasm::rewrite_imports(&out_filename, &codegen_results.crate_info.wasm_imports); wasm::add_producer_section( &out_filename, &sess.edition().to_string(), diff --git a/src/librustc_codegen_llvm/back/wasm.rs b/src/librustc_codegen_llvm/back/wasm.rs index e1d3351d3deca..f90bb89fbe87d 100644 --- a/src/librustc_codegen_llvm/back/wasm.rs +++ b/src/librustc_codegen_llvm/back/wasm.rs @@ -2,116 +2,11 @@ use std::fs; use std::path::Path; use std::str; -use rustc_data_structures::fx::FxHashMap; use serialize::leb128; // https://webassembly.github.io/spec/core/binary/modules.html#binary-importsec -const WASM_IMPORT_SECTION_ID: u8 = 2; const WASM_CUSTOM_SECTION_ID: u8 = 0; -const WASM_EXTERNAL_KIND_FUNCTION: u8 = 0; -const WASM_EXTERNAL_KIND_TABLE: u8 = 1; -const WASM_EXTERNAL_KIND_MEMORY: u8 = 2; -const WASM_EXTERNAL_KIND_GLOBAL: u8 = 3; - -/// Rewrite the module imports are listed from in a wasm module given the field -/// name to module name mapping in `import_map`. -/// -/// LLVM 6 which we're using right now doesn't have the ability to configure the -/// module a wasm symbol is import from. Rather all imported symbols come from -/// the bland `"env"` module unconditionally. Furthermore we'd *also* need -/// support in LLD for preserving these import modules, which it unfortunately -/// currently does not. -/// -/// This function is intended as a hack for now where we manually rewrite the -/// wasm output by LLVM to have the correct import modules listed. The -/// `#[link(wasm_import_module = "...")]` attribute in Rust translates to the -/// module that each symbol is imported from, so here we manually go through the -/// wasm file, decode it, rewrite imports, and then rewrite the wasm module. -/// -/// Support for this was added to LLVM in -/// https://github.com/llvm-mirror/llvm/commit/0f32e1365, although support still -/// needs to be added, tracked at https://bugs.llvm.org/show_bug.cgi?id=37168 -pub fn rewrite_imports(path: &Path, import_map: &FxHashMap) { - if import_map.is_empty() { - return - } - - let wasm = fs::read(path).expect("failed to read wasm output"); - let mut ret = WasmEncoder::new(); - ret.data.extend(&wasm[..8]); - - // skip the 8 byte wasm/version header - for (id, raw) in WasmSections(WasmDecoder::new(&wasm[8..])) { - ret.byte(id); - if id == WASM_IMPORT_SECTION_ID { - info!("rewriting import section"); - let data = rewrite_import_section( - &mut WasmDecoder::new(raw), - import_map, - ); - ret.bytes(&data); - } else { - info!("carry forward section {}, {} bytes long", id, raw.len()); - ret.bytes(raw); - } - } - - fs::write(path, &ret.data).expect("failed to write wasm output"); - - fn rewrite_import_section( - wasm: &mut WasmDecoder<'_>, - import_map: &FxHashMap, - ) - -> Vec - { - let mut dst = WasmEncoder::new(); - let n = wasm.u32(); - dst.u32(n); - info!("rewriting {} imports", n); - for _ in 0..n { - rewrite_import_entry(wasm, &mut dst, import_map); - } - return dst.data - } - - fn rewrite_import_entry(wasm: &mut WasmDecoder<'_>, - dst: &mut WasmEncoder, - import_map: &FxHashMap) { - // More info about the binary format here is available at: - // https://webassembly.github.io/spec/core/binary/modules.html#import-section - // - // Note that you can also find the whole point of existence of this - // function here, where we map the `module` name to a different one if - // we've got one listed. - let module = wasm.str(); - let field = wasm.str(); - let new_module = if module == "env" { - import_map.get(field).map(|s| &**s).unwrap_or(module) - } else { - module - }; - info!("import rewrite ({} => {}) / {}", module, new_module, field); - dst.str(new_module); - dst.str(field); - let kind = wasm.byte(); - dst.byte(kind); - match kind { - WASM_EXTERNAL_KIND_FUNCTION => dst.u32(wasm.u32()), - WASM_EXTERNAL_KIND_TABLE => { - dst.byte(wasm.byte()); // element_type - dst.limits(wasm.limits()); - } - WASM_EXTERNAL_KIND_MEMORY => dst.limits(wasm.limits()), - WASM_EXTERNAL_KIND_GLOBAL => { - dst.byte(wasm.byte()); // content_type - dst.bool(wasm.bool()); // mutable - } - b => panic!("unknown kind: {}", b), - } - } -} - /// Adds or augment the existing `producers` section to encode information about /// the Rust compiler used to produce the wasm file. pub fn add_producer_section( @@ -266,15 +161,6 @@ impl<'a> WasmDecoder<'a> { let len = self.u32(); str::from_utf8(self.skip(len as usize)).unwrap() } - - fn bool(&mut self) -> bool { - self.byte() == 1 - } - - fn limits(&mut self) -> (u32, Option) { - let has_max = self.bool(); - (self.u32(), if has_max { Some(self.u32()) } else { None }) - } } struct WasmEncoder { @@ -302,16 +188,4 @@ impl WasmEncoder { fn str(&mut self, val: &str) { self.bytes(val.as_bytes()) } - - fn bool(&mut self, b: bool) { - self.byte(b as u8); - } - - fn limits(&mut self, limits: (u32, Option)) { - self.bool(limits.1.is_some()); - self.u32(limits.0); - if let Some(c) = limits.1 { - self.u32(c); - } - } } diff --git a/src/librustc_codegen_ssa/base.rs b/src/librustc_codegen_ssa/base.rs index fad41a11a82ba..39ce15e477296 100644 --- a/src/librustc_codegen_ssa/base.rs +++ b/src/librustc_codegen_ssa/base.rs @@ -16,7 +16,7 @@ use crate::{ModuleCodegen, ModuleKind, CachedModuleCodegen}; use rustc::dep_graph::cgu_reuse_tracker::CguReuse; -use rustc::hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; +use rustc::hir::def_id::{DefId, LOCAL_CRATE}; use rustc::middle::lang_items::StartFnLangItem; use rustc::middle::weak_lang_items; use rustc::mir::mono::{Stats, CodegenUnitNameBuilder}; @@ -816,21 +816,11 @@ impl CrateInfo { used_crates_dynamic: cstore::used_crates(tcx, LinkagePreference::RequireDynamic), used_crates_static: cstore::used_crates(tcx, LinkagePreference::RequireStatic), used_crate_source: Default::default(), - wasm_imports: Default::default(), lang_item_to_crate: Default::default(), missing_lang_items: Default::default(), }; let lang_items = tcx.lang_items(); - let load_wasm_items = tcx.sess.crate_types.borrow() - .iter() - .any(|c| *c != config::CrateType::Rlib) && - tcx.sess.opts.target_triple.triple() == "wasm32-unknown-unknown"; - - if load_wasm_items { - info.load_wasm_imports(tcx, LOCAL_CRATE); - } - let crates = tcx.crates(); let n_crates = crates.len(); @@ -858,9 +848,6 @@ impl CrateInfo { if tcx.is_no_builtins(cnum) { info.is_no_builtins.insert(cnum); } - if load_wasm_items { - info.load_wasm_imports(tcx, cnum); - } let missing = tcx.missing_lang_items(cnum); for &item in missing.iter() { if let Ok(id) = lang_items.require(item) { @@ -879,15 +866,6 @@ impl CrateInfo { return info } - - fn load_wasm_imports(&mut self, tcx: TyCtxt<'_, '_, '_>, cnum: CrateNum) { - self.wasm_imports.extend(tcx.wasm_import_module_map(cnum).iter().map(|(&id, module)| { - let instance = Instance::mono(tcx, id); - let import_name = tcx.symbol_name(instance); - - (import_name.to_string(), module.clone()) - })); - } } fn is_codegened_item(tcx: TyCtxt<'_, '_, '_>, id: DefId) -> bool { diff --git a/src/librustc_codegen_ssa/lib.rs b/src/librustc_codegen_ssa/lib.rs index 92fa2766f8727..f38b22aa04171 100644 --- a/src/librustc_codegen_ssa/lib.rs +++ b/src/librustc_codegen_ssa/lib.rs @@ -138,7 +138,6 @@ pub struct CrateInfo { pub used_crate_source: FxHashMap>, pub used_crates_static: Vec<(CrateNum, LibSource)>, pub used_crates_dynamic: Vec<(CrateNum, LibSource)>, - pub wasm_imports: FxHashMap, pub lang_item_to_crate: FxHashMap, pub missing_lang_items: FxHashMap>, } diff --git a/src/llvm-project b/src/llvm-project index 73a75d35b9776..38ad31bde8ff6 160000 --- a/src/llvm-project +++ b/src/llvm-project @@ -1 +1 @@ -Subproject commit 73a75d35b9776d56160fa3200aca4a970ae49b60 +Subproject commit 38ad31bde8ff681d862dc0f96930a5dd9b7a472e From 320640060f38957028141ea30bc4d5577d1e53b0 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 27 Feb 2019 08:03:54 -0800 Subject: [PATCH 2/2] Whitelist containers that allow older toolchains We'll use this as a temporary measure to get an LLVM update landed, but we'll have to go through and update images later to make sure they've got the right toolchains. --- config.toml.example | 2 ++ src/bootstrap/config.rs | 3 +++ src/bootstrap/native.rs | 4 ++++ src/ci/docker/dist-x86_64-netbsd/Dockerfile | 3 ++- 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/config.toml.example b/config.toml.example index f45db37c33b86..8f6bf03489f06 100644 --- a/config.toml.example +++ b/config.toml.example @@ -104,6 +104,8 @@ # The value specified here will be passed as `-DLLVM_USE_LINKER` to CMake. #use-linker = "lld" +# Whether or not to specify `-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=YES` +#allow-old-toolchain = false # ============================================================================= # General build configuration options diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 7d3e584f1a6fa..d20958854ed6a 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -78,6 +78,7 @@ pub struct Config { pub llvm_link_jobs: Option, pub llvm_version_suffix: Option, pub llvm_use_linker: Option, + pub llvm_allow_old_toolchain: Option, pub lld_enabled: bool, pub lldb_enabled: bool, @@ -263,6 +264,7 @@ struct Llvm { ldflags: Option, use_libcxx: Option, use_linker: Option, + allow_old_toolchain: Option, } #[derive(Deserialize, Default, Clone)] @@ -530,6 +532,7 @@ impl Config { config.llvm_ldflags = llvm.ldflags.clone(); set(&mut config.llvm_use_libcxx, llvm.use_libcxx); config.llvm_use_linker = llvm.use_linker.clone(); + config.llvm_allow_old_toolchain = llvm.allow_old_toolchain.clone(); } if let Some(ref rust) = toml.rust { diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 192b1cd1fbb75..d78670cfe515a 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -238,6 +238,10 @@ impl Step for Llvm { cfg.define("LLVM_USE_LINKER", linker); } + if let Some(true) = builder.config.llvm_allow_old_toolchain { + cfg.define("LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN", "YES"); + } + if let Some(ref python) = builder.config.python { cfg.define("PYTHON_EXECUTABLE", python); } diff --git a/src/ci/docker/dist-x86_64-netbsd/Dockerfile b/src/ci/docker/dist-x86_64-netbsd/Dockerfile index a17a7ebc03dd1..4fe7e2cca2b60 100644 --- a/src/ci/docker/dist-x86_64-netbsd/Dockerfile +++ b/src/ci/docker/dist-x86_64-netbsd/Dockerfile @@ -33,5 +33,6 @@ ENV \ ENV HOSTS=x86_64-unknown-netbsd -ENV RUST_CONFIGURE_ARGS --enable-extended --disable-docs +ENV RUST_CONFIGURE_ARGS --enable-extended --disable-docs \ + --set llvm.allow-old-toolchain ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS