Auto merge of #64000 - Centril:rollup-70s9ar3, r=Centril

Rollup of 4 pull requests

Successful merges:

 - #62734 (Hide trait default methods)
 - #63953 (bootstrap: allow specifying mirror for bootstrap compiler download.)
 - #63956 (rustc: Handle modules in "fat" LTO more robustly)
 - #63979 (std: Remove the `wasm_syscall` feature)

Failed merges:

r? @ghost

Author: bors

config.toml.example

@@ -382,11 +382,6 @@
 # This is the name of the directory in which codegen backends will get installed
 #codegen-backends-dir = "codegen-backends"
 
-# Flag indicating whether `libstd` calls an imported function to handle basic IO
-# when targeting WebAssembly. Enable this to debug tests for the `wasm32-unknown-unknown`
-# target, as without this option the test output will not be captured.
-#wasm-syscall = false
-
 # Indicates whether LLD will be compiled and made available in the sysroot for
 # rustc to execute.
 #lld = false

src/bootstrap/bootstrap.py

@@ -320,7 +320,7 @@ class RustBuild(object):
     def __init__(self):
         self.cargo_channel = ''
         self.date = ''
-        self._download_url = 'https://static.rust-lang.org'
+        self._download_url = ''
         self.rustc_channel = ''
         self.build = ''
         self.build_dir = os.path.join(os.getcwd(), "build")
@@ -733,9 +733,19 @@ def update_submodules(self):
             self.update_submodule(module[0], module[1], recorded_submodules)
         print("Submodules updated in %.2f seconds" % (time() - start_time))
 
+    def set_normal_environment(self):
+        """Set download URL for normal environment"""
+        if 'RUSTUP_DIST_SERVER' in os.environ:
+            self._download_url = os.environ['RUSTUP_DIST_SERVER']
+        else:
+            self._download_url = 'https://static.rust-lang.org'
+
     def set_dev_environment(self):
         """Set download URL for development environment"""
-        self._download_url = 'https://dev-static.rust-lang.org'
+        if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
+            self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
+        else:
+            self._download_url = 'https://dev-static.rust-lang.org'
 
     def check_vendored_status(self):
         """Check that vendoring is configured properly"""
@@ -828,6 +838,8 @@ def bootstrap(help_triggered):
 
     if 'dev' in data:
         build.set_dev_environment()
+    else:
+        build.set_normal_environment()
 
     build.update_submodules()
 

src/bootstrap/config.rs

@@ -122,7 +122,6 @@ pub struct Config {
 
     // libstd features
     pub backtrace: bool, // support for RUST_BACKTRACE
-    pub wasm_syscall: bool,
 
     // misc
     pub low_priority: bool,
@@ -318,7 +317,6 @@ struct Rust {
     save_toolstates: Option<String>,
     codegen_backends: Option<Vec<String>>,
     codegen_backends_dir: Option<String>,
-    wasm_syscall: Option<bool>,
     lld: Option<bool>,
     lldb: Option<bool>,
     llvm_tools: Option<bool>,
@@ -558,7 +556,6 @@ impl Config {
             if let Some(true) = rust.incremental {
                 config.incremental = true;
             }
-            set(&mut config.wasm_syscall, rust.wasm_syscall);
             set(&mut config.lld_enabled, rust.lld);
             set(&mut config.lldb_enabled, rust.lldb);
             set(&mut config.llvm_tools_enabled, rust.llvm_tools);

src/bootstrap/lib.rs

@@ -495,9 +495,6 @@ impl Build {
         if self.config.profiler {
             features.push_str(" profiler");
         }
-        if self.config.wasm_syscall {
-            features.push_str(" wasm_syscall");
-        }
         features
     }
 

src/bootstrap/test.rs

@@ -1811,16 +1811,6 @@ impl Step for Crate {
                     .expect("nodejs not configured"),
             );
         } else if target.starts_with("wasm32") {
-            // Warn about running tests without the `wasm_syscall` feature enabled.
-            // The javascript shim implements the syscall interface so that test
-            // output can be correctly reported.
-            if !builder.config.wasm_syscall {
-                builder.info(
-                    "Libstd was built without `wasm_syscall` feature enabled: \
-                     test output may not be visible."
-                );
-            }
-
             // On the wasm32-unknown-unknown target we're using LTO which is
             // incompatible with `-C prefer-dynamic`, so disable that here
             cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");

src/etc/wasm32-shim.js

@@ -15,113 +15,7 @@ const buffer = fs.readFileSync(process.argv[2]);
 Error.stackTraceLimit = 20;
 
 let m = new WebAssembly.Module(buffer);
-
-let memory = null;
-
-function viewstruct(data, fields) {
-  return new Uint32Array(memory.buffer).subarray(data/4, data/4 + fields);
-}
-
-function copystr(a, b) {
-  let view = new Uint8Array(memory.buffer).subarray(a, a + b);
-  return String.fromCharCode.apply(null, view);
-}
-
-function syscall_write([fd, ptr, len]) {
-  let s = copystr(ptr, len);
-  switch (fd) {
-    case 1: process.stdout.write(s); break;
-    case 2: process.stderr.write(s); break;
-  }
-}
-
-function syscall_exit([code]) {
-  process.exit(code);
-}
-
-function syscall_args(params) {
-  let [ptr, len] = params;
-
-  // Calculate total required buffer size
-  let totalLen = -1;
-  for (let i = 2; i < process.argv.length; ++i) {
-    totalLen += Buffer.byteLength(process.argv[i]) + 1;
-  }
-  if (totalLen < 0) { totalLen = 0; }
-  params[2] = totalLen;
-
-  // If buffer is large enough, copy data
-  if (len >= totalLen) {
-    let view = new Uint8Array(memory.buffer);
-    for (let i = 2; i < process.argv.length; ++i) {
-      let value = process.argv[i];
-      Buffer.from(value).copy(view, ptr);
-      ptr += Buffer.byteLength(process.argv[i]) + 1;
-    }
-  }
-}
-
-function syscall_getenv(params) {
-  let [keyPtr, keyLen, valuePtr, valueLen] = params;
-
-  let key = copystr(keyPtr, keyLen);
-  let value = process.env[key];
-
-  if (value == null) {
-    params[4] = 0xFFFFFFFF;
-  } else {
-    let view = new Uint8Array(memory.buffer);
-    let totalLen = Buffer.byteLength(value);
-    params[4] = totalLen;
-    if (valueLen >= totalLen) {
-      Buffer.from(value).copy(view, valuePtr);
-    }
-  }
-}
-
-function syscall_time(params) {
-  let t = Date.now();
-  let secs = Math.floor(t / 1000);
-  let millis = t % 1000;
-  params[1] = Math.floor(secs / 0x100000000);
-  params[2] = secs % 0x100000000;
-  params[3] = Math.floor(millis * 1000000);
-}
-
-let imports = {};
-imports.env = {
-  // These are generated by LLVM itself for various intrinsic calls. Hopefully
-  // one day this is not necessary and something will automatically do this.
-  fmod: function(x, y) { return x % y; },
-  exp2: function(x) { return Math.pow(2, x); },
-  exp2f: function(x) { return Math.pow(2, x); },
-  ldexp: function(x, y) { return x * Math.pow(2, y); },
-  ldexpf: function(x, y) { return x * Math.pow(2, y); },
-  sin: Math.sin,
-  sinf: Math.sin,
-  cos: Math.cos,
-  cosf: Math.cos,
-  log: Math.log,
-  log2: Math.log2,
-  log10: Math.log10,
-  log10f: Math.log10,
-
-  rust_wasm_syscall: function(index, data) {
-    switch (index) {
-      case 1: syscall_write(viewstruct(data, 3)); return true;
-      case 2: syscall_exit(viewstruct(data, 1)); return true;
-      case 3: syscall_args(viewstruct(data, 3)); return true;
-      case 4: syscall_getenv(viewstruct(data, 5)); return true;
-      case 6: syscall_time(viewstruct(data, 4)); return true;
-      default:
-        console.log("Unsupported syscall: " + index);
-        return false;
-    }
-  }
-};
-
-let instance = new WebAssembly.Instance(m, imports);
-memory = instance.exports.memory;
+let instance = new WebAssembly.Instance(m, {});
 try {
   instance.exports.main();
 } catch (e) {

src/librustc_codegen_llvm/back/lto.rs

@@ -183,14 +183,40 @@ pub(crate) fn prepare_thin(
 
 fn fat_lto(cgcx: &CodegenContext<LlvmCodegenBackend>,
            diag_handler: &Handler,
-           mut modules: Vec<FatLTOInput<LlvmCodegenBackend>>,
+           modules: Vec<FatLTOInput<LlvmCodegenBackend>>,
            cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
            mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
            symbol_white_list: &[*const libc::c_char])
     -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError>
 {
     info!("going for a fat lto");
 
+    // Sort out all our lists of incoming modules into two lists.
+    //
+    // * `serialized_modules` (also and argument to this function) contains all
+    //   modules that are serialized in-memory.
+    // * `in_memory` contains modules which are already parsed and in-memory,
+    //   such as from multi-CGU builds.
+    //
+    // All of `cached_modules` (cached from previous incremental builds) can
+    // immediately go onto the `serialized_modules` modules list and then we can
+    // split the `modules` array into these two lists.
+    let mut in_memory = Vec::new();
+    serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
+        info!("pushing cached module {:?}", wp.cgu_name);
+        (buffer, CString::new(wp.cgu_name).unwrap())
+    }));
+    for module in modules {
+        match module {
+            FatLTOInput::InMemory(m) => in_memory.push(m),
+            FatLTOInput::Serialized { name, buffer } => {
+                info!("pushing serialized module {:?}", name);
+                let buffer = SerializedModule::Local(buffer);
+                serialized_modules.push((buffer, CString::new(name).unwrap()));
+            }
+        }
+    }
+
     // Find the "costliest" module and merge everything into that codegen unit.
     // All the other modules will be serialized and reparsed into the new
     // context, so this hopefully avoids serializing and parsing the largest
@@ -200,14 +226,8 @@ fn fat_lto(cgcx: &CodegenContext<LlvmCodegenBackend>,
     // file copy operations in the backend work correctly. The only other kind
     // of module here should be an allocator one, and if your crate is smaller
     // than the allocator module then the size doesn't really matter anyway.
-    let costliest_module = modules.iter()
+    let costliest_module = in_memory.iter()
         .enumerate()
-        .filter_map(|(i, module)| {
-            match module {
-                FatLTOInput::InMemory(m) => Some((i, m)),
-                FatLTOInput::Serialized { .. } => None,
-            }
-        })
         .filter(|&(_, module)| module.kind == ModuleKind::Regular)
         .map(|(i, module)| {
             let cost = unsafe {
@@ -223,26 +243,14 @@ fn fat_lto(cgcx: &CodegenContext<LlvmCodegenBackend>,
     // re-executing the LTO passes. If that's the case deserialize the first
     // module and create a linker with it.
     let module: ModuleCodegen<ModuleLlvm> = match costliest_module {
-        Some((_cost, i)) => {
-            match modules.remove(i) {
-                FatLTOInput::InMemory(m) => m,
-                FatLTOInput::Serialized { .. } => unreachable!(),
-            }
-        }
+        Some((_cost, i)) => in_memory.remove(i),
         None => {
-            let pos = modules.iter().position(|m| {
-                match m {
-                    FatLTOInput::InMemory(_) => false,
-                    FatLTOInput::Serialized { .. } => true,
-                }
-            }).expect("must have at least one serialized module");
-            let (name, buffer) = match modules.remove(pos) {
-                FatLTOInput::Serialized { name, buffer } => (name, buffer),
-                FatLTOInput::InMemory(_) => unreachable!(),
-            };
+            assert!(serialized_modules.len() > 0, "must have at least one serialized module");
+            let (buffer, name) = serialized_modules.remove(0);
+            info!("no in-memory regular modules to choose from, parsing {:?}", name);
             ModuleCodegen {
-                module_llvm: ModuleLlvm::parse(cgcx, &name, &buffer, diag_handler)?,
-                name,
+                module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), diag_handler)?,
+                name: name.into_string().unwrap(),
                 kind: ModuleKind::Regular,
             }
         }
@@ -265,25 +273,13 @@ fn fat_lto(cgcx: &CodegenContext<LlvmCodegenBackend>,
         // and we want to move everything to the same LLVM context. Currently the
         // way we know of to do that is to serialize them to a string and them parse
         // them later. Not great but hey, that's why it's "fat" LTO, right?
-        let mut new_modules = modules.into_iter().map(|module| {
-            match module {
-                FatLTOInput::InMemory(module) => {
-                    let buffer = ModuleBuffer::new(module.module_llvm.llmod());
-                    let llmod_id = CString::new(&module.name[..]).unwrap();
-                    (SerializedModule::Local(buffer), llmod_id)
-                }
-                FatLTOInput::Serialized { name, buffer } => {
-                    let llmod_id = CString::new(name).unwrap();
-                    (SerializedModule::Local(buffer), llmod_id)
-                }
-            }
-        }).collect::<Vec<_>>();
+        for module in in_memory {
+            let buffer = ModuleBuffer::new(module.module_llvm.llmod());
+            let llmod_id = CString::new(&module.name[..]).unwrap();
+            serialized_modules.push((SerializedModule::Local(buffer), llmod_id));
+        }
         // Sort the modules to ensure we produce deterministic results.
-        new_modules.sort_by(|module1, module2| module1.1.partial_cmp(&module2.1).unwrap());
-        serialized_modules.extend(new_modules);
-        serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
-            (buffer, CString::new(wp.cgu_name).unwrap())
-        }));
+        serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1));
 
         // For all serialized bitcode files we parse them and link them in as we did
         // above, this is all mostly handled in C++. Like above, though, we don't
@@ -850,7 +846,7 @@ fn module_name_to_str(c_str: &CStr) -> &str {
         bug!("Encountered non-utf8 LLVM module name `{}`: {}", c_str.to_string_lossy(), e))
 }
 
-fn parse_module<'a>(
+pub fn parse_module<'a>(
     cx: &'a llvm::Context,
     name: &CStr,
     data: &[u8],

src/librustc_codegen_llvm/lib.rs

@@ -54,6 +54,7 @@ use syntax_pos::symbol::InternedString;
 pub use llvm_util::target_features;
 use std::any::Any;
 use std::sync::{mpsc, Arc};
+use std::ffi::CStr;
 
 use rustc::dep_graph::DepGraph;
 use rustc::middle::cstore::{EncodedMetadata, MetadataLoader};
@@ -386,13 +387,13 @@ impl ModuleLlvm {
 
     fn parse(
         cgcx: &CodegenContext<LlvmCodegenBackend>,
-        name: &str,
-        buffer: &back::lto::ModuleBuffer,
+        name: &CStr,
+        buffer: &[u8],
         handler: &Handler,
     ) -> Result<Self, FatalError> {
         unsafe {
             let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
-            let llmod_raw = buffer.parse(name, llcx, handler)?;
+            let llmod_raw = back::lto::parse_module(llcx, name, buffer, handler)?;
             let tm = match (cgcx.tm_factory.0)() {
                 Ok(m) => m,
                 Err(e) => {