From faee5fd388a564febc56aa642241f221c0ac1cd1 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 26 Sep 2019 14:44:08 -0700 Subject: [PATCH 1/6] [WIP] minimize the rust-std component --- src/bootstrap/builder.rs | 1 + src/bootstrap/dist.rs | 79 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 5d586f0c461db..afdcabc08302c 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -443,6 +443,7 @@ impl<'a> Builder<'a> { dist::Rustc, dist::DebuggerScripts, dist::Std, + dist::StdZ, dist::Analysis, dist::Src, dist::PlainSourceTarball, diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index d9dff77a30e6b..ceb7acb83e749 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -675,6 +675,81 @@ impl Step for Std { return distdir(builder).join(format!("{}-{}.tar.gz", name, target)); } + builder.ensure(compile::Std { compiler, target }); + + let image = tmpdir(builder).join(format!("{}-{}-image", name, target)); + let _ = fs::remove_dir_all(&image); + + + let dst = image.join("lib/rustlib").join(target).join("lib"); + t!(fs::create_dir_all(&dst)); + + let stamp = dbg!(compile::libstd_stamp(builder, compiler, target)); + for (path, host) in builder.read_stamp_file(&stamp) { + if !host { + builder.copy(&path, &dst.join(path.file_name().unwrap())); + } + } + + let mut cmd = rust_installer(builder); + cmd.arg("generate") + .arg("--product-name=Rust") + .arg("--rel-manifest-dir=rustlib") + .arg("--success-message=std-is-standing-at-the-ready.") + .arg("--image-dir").arg(&image) + .arg("--work-dir").arg(&tmpdir(builder)) + .arg("--output-dir").arg(&distdir(builder)) + .arg(format!("--package-name={}-{}", name, target)) + .arg(format!("--component-name=rust-std-{}", target)) + .arg("--legacy-manifest-dirs=rustlib,cargo"); + + builder.info(&format!("Dist std stage{} ({} -> {})", + compiler.stage, &compiler.host, target)); + let _time = timeit(builder); + builder.run(&mut cmd); + builder.remove_dir(&image); + distdir(builder).join(format!("{}-{}.tar.gz", name, target)) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct StdZ { + pub compiler: Compiler, + pub target: Interned, +} + +impl Step for StdZ { + type Output = PathBuf; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/libstdZ") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(StdZ { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let compiler = self.compiler; + let target = self.target; + + let name = pkgname(builder, "rust-stdZ"); + + // The only true set of target libraries came from the build triple, so + // let's reduce redundant work by only producing archives from that host. + if compiler.host != builder.config.build { + builder.info("\tskipping, not a build host"); + return distdir(builder).join(format!("{}-{}.tar.gz", name, target)); + } + // We want to package up as many target libraries as possible // for the `rust-std` package, so if this is a host target we // depend on librustc and otherwise we just depend on libtest. @@ -710,12 +785,12 @@ impl Step for Std { cmd.arg("generate") .arg("--product-name=Rust") .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=std-is-standing-at-the-ready.") + .arg("--success-message=stdZ-is-standing-at-the-ready.") .arg("--image-dir").arg(&image) .arg("--work-dir").arg(&tmpdir(builder)) .arg("--output-dir").arg(&distdir(builder)) .arg(format!("--package-name={}-{}", name, target)) - .arg(format!("--component-name=rust-std-{}", target)) + .arg(format!("--component-name=rust-stdZ-{}", target)) .arg("--legacy-manifest-dirs=rustlib,cargo"); builder.info(&format!("Dist std stage{} ({} -> {})", From dc8ee51c5418ce37b63f1c9bdee98745f5dfe627 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 27 Sep 2019 12:31:00 -0700 Subject: [PATCH 2/6] Use builder.compiler_for() to find the libstd stamp --- src/bootstrap/dist.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index ceb7acb83e749..e1e47d5370a06 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -680,11 +680,11 @@ impl Step for Std { let image = tmpdir(builder).join(format!("{}-{}-image", name, target)); let _ = fs::remove_dir_all(&image); - let dst = image.join("lib/rustlib").join(target).join("lib"); t!(fs::create_dir_all(&dst)); - let stamp = dbg!(compile::libstd_stamp(builder, compiler, target)); + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + let stamp = dbg!(compile::libstd_stamp(builder, compiler_to_use, target)); for (path, host) in builder.read_stamp_file(&stamp) { if !host { builder.copy(&path, &dst.join(path.file_name().unwrap())); From 9175f279b583fa71d8b4f6078af319e9a2155b75 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 3 Oct 2019 10:57:19 -0700 Subject: [PATCH 3/6] add dist::RustcDev for unstable compiler libraries --- src/bootstrap/builder.rs | 2 +- src/bootstrap/dist.rs | 102 +++++++++++++++++---------------------- 2 files changed, 46 insertions(+), 58 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index afdcabc08302c..bfba39961a22e 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -443,7 +443,7 @@ impl<'a> Builder<'a> { dist::Rustc, dist::DebuggerScripts, dist::Std, - dist::StdZ, + dist::RustcDev, dist::Analysis, dist::Src, dist::PlainSourceTarball, diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index e1e47d5370a06..e5a43dcb29f63 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -637,6 +637,28 @@ impl Step for DebuggerScripts { } } +fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool { + // The only true set of target libraries came from the build triple, so + // let's reduce redundant work by only producing archives from that host. + if compiler.host != builder.config.build { + builder.info("\tskipping, not a build host"); + true + } else { + false + } +} + +/// Copy stamped files into an image's `target/lib` directory. +fn copy_target_libs(builder: &Builder<'_>, target: &str, image: &Path, stamp: &Path) { + let dst = image.join("lib/rustlib").join(target).join("lib"); + t!(fs::create_dir_all(&dst)); + for (path, host) in builder.read_stamp_file(stamp) { + if !host || builder.config.build == target { + builder.copy(&path, &dst.join(path.file_name().unwrap())); + } + } +} + #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] pub struct Std { pub compiler: Compiler, @@ -667,12 +689,9 @@ impl Step for Std { let target = self.target; let name = pkgname(builder, "rust-std"); - - // The only true set of target libraries came from the build triple, so - // let's reduce redundant work by only producing archives from that host. - if compiler.host != builder.config.build { - builder.info("\tskipping, not a build host"); - return distdir(builder).join(format!("{}-{}.tar.gz", name, target)); + let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target)); + if skip_host_target_lib(builder, compiler) { + return archive; } builder.ensure(compile::Std { compiler, target }); @@ -680,16 +699,9 @@ impl Step for Std { let image = tmpdir(builder).join(format!("{}-{}-image", name, target)); let _ = fs::remove_dir_all(&image); - let dst = image.join("lib/rustlib").join(target).join("lib"); - t!(fs::create_dir_all(&dst)); - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - let stamp = dbg!(compile::libstd_stamp(builder, compiler_to_use, target)); - for (path, host) in builder.read_stamp_file(&stamp) { - if !host { - builder.copy(&path, &dst.join(path.file_name().unwrap())); - } - } + let stamp = compile::libstd_stamp(builder, compiler_to_use, target); + copy_target_libs(builder, &target, &image, &stamp); let mut cmd = rust_installer(builder); cmd.arg("generate") @@ -708,26 +720,27 @@ impl Step for Std { let _time = timeit(builder); builder.run(&mut cmd); builder.remove_dir(&image); - distdir(builder).join(format!("{}-{}.tar.gz", name, target)) + archive } } #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct StdZ { +pub struct RustcDev { pub compiler: Compiler, pub target: Interned, } -impl Step for StdZ { +impl Step for RustcDev { type Output = PathBuf; const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/libstdZ") + run.path("rustc-dev") } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(StdZ { + run.builder.ensure(RustcDev { compiler: run.builder.compiler_for( run.builder.top_stage, run.builder.config.build, @@ -741,64 +754,39 @@ impl Step for StdZ { let compiler = self.compiler; let target = self.target; - let name = pkgname(builder, "rust-stdZ"); - - // The only true set of target libraries came from the build triple, so - // let's reduce redundant work by only producing archives from that host. - if compiler.host != builder.config.build { - builder.info("\tskipping, not a build host"); - return distdir(builder).join(format!("{}-{}.tar.gz", name, target)); + let name = pkgname(builder, "rustc-dev"); + let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target)); + if skip_host_target_lib(builder, compiler) { + return archive; } - // We want to package up as many target libraries as possible - // for the `rust-std` package, so if this is a host target we - // depend on librustc and otherwise we just depend on libtest. - if builder.hosts.iter().any(|t| t == target) { - builder.ensure(compile::Rustc { compiler, target }); - } else { - builder.ensure(compile::Std { compiler, target }); - } + builder.ensure(compile::Rustc { compiler, target }); let image = tmpdir(builder).join(format!("{}-{}-image", name, target)); let _ = fs::remove_dir_all(&image); - let dst = image.join("lib/rustlib").join(target); - t!(fs::create_dir_all(&dst)); - let mut src = builder.sysroot_libdir(compiler, target).to_path_buf(); - src.pop(); // Remove the trailing /lib folder from the sysroot_libdir - builder.cp_filtered(&src, &dst, &|path| { - if let Some(name) = path.file_name().and_then(|s| s.to_str()) { - if name == builder.config.rust_codegen_backends_dir.as_str() { - return false - } - if name == "bin" { - return false - } - if name.contains("LLVM") { - return false - } - } - true - }); + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + let stamp = compile::librustc_stamp(builder, compiler_to_use, target); + copy_target_libs(builder, &target, &image, &stamp); let mut cmd = rust_installer(builder); cmd.arg("generate") .arg("--product-name=Rust") .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=stdZ-is-standing-at-the-ready.") + .arg("--success-message=Rust-is-ready-to-develop.") .arg("--image-dir").arg(&image) .arg("--work-dir").arg(&tmpdir(builder)) .arg("--output-dir").arg(&distdir(builder)) .arg(format!("--package-name={}-{}", name, target)) - .arg(format!("--component-name=rust-stdZ-{}", target)) + .arg(format!("--component-name=rustc-dev-{}", target)) .arg("--legacy-manifest-dirs=rustlib,cargo"); - builder.info(&format!("Dist std stage{} ({} -> {})", + builder.info(&format!("Dist rustc-dev stage{} ({} -> {})", compiler.stage, &compiler.host, target)); let _time = timeit(builder); builder.run(&mut cmd); builder.remove_dir(&image); - distdir(builder).join(format!("{}-{}.tar.gz", name, target)) + archive } } From bd4e9d5fe146d5475dc35f5f61d5186b15692fdc Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 3 Oct 2019 11:40:39 -0700 Subject: [PATCH 4/6] add rustc-dev to tools/build-manifest --- src/tools/build-manifest/src/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index f41e7dd17ede6..d5933789a6498 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -399,6 +399,7 @@ impl Builder { fn add_packages_to(&mut self, manifest: &mut Manifest) { let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets); package("rustc", HOSTS); + package("rustc-dev", HOSTS); package("cargo", HOSTS); package("rust-mingw", MINGW); package("rust-std", TARGETS); @@ -473,6 +474,7 @@ impl Builder { // and so is rust-mingw if it's available for the target. components.extend(vec![ host_component("rustc"), + host_component("rustc-dev"), host_component("rust-std"), host_component("cargo"), host_component("rust-docs"), @@ -498,6 +500,11 @@ impl Builder { .filter(|&&target| target != host) .map(|target| Component::from_str("rust-std", target)) ); + extensions.extend( + HOSTS.iter() + .filter(|&&target| target != host) + .map(|target| Component::from_str("rustc-dev", target)) + ); extensions.push(Component::from_str("rust-src", "*")); // If the components/extensions don't actually exist for this From 2dcf2f0f7b965d5977dfbafec9128084bdd82a42 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Mon, 7 Oct 2019 09:31:51 -0700 Subject: [PATCH 5/6] Only install rustc-dev by default on nightly --- src/tools/build-manifest/src/main.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index d5933789a6498..97e758f9b8238 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -474,7 +474,6 @@ impl Builder { // and so is rust-mingw if it's available for the target. components.extend(vec![ host_component("rustc"), - host_component("rustc-dev"), host_component("rust-std"), host_component("cargo"), host_component("rust-docs"), @@ -483,6 +482,15 @@ impl Builder { components.push(host_component("rust-mingw")); } + // The compiler libraries are not stable for end users, but `rustc-dev` was only recently + // split out of `rust-std`. We'll include it by default as a transition for nightly users, + // but ship it as an optional component on the beta and stable channels. + if self.rust_release == "nightly" { + components.push(host_component("rustc-dev")); + } else { + extensions.push(host_component("rustc-dev")); + } + // Tools are always present in the manifest, // but might be marked as unavailable if they weren't built. extensions.extend(vec![ From d3052540993b6acf009d39949b79077a49544934 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Mon, 7 Oct 2019 15:49:51 -0700 Subject: [PATCH 6/6] Add rustc-dev to nightly default and complete profiles --- src/tools/build-manifest/src/main.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 97e758f9b8238..c0d2deab2f8ba 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -427,6 +427,13 @@ impl Builder { "rls-preview", "rust-src", "llvm-tools-preview", "lldb-preview", "rust-analysis", "miri-preview" ]); + + // The compiler libraries are not stable for end users, but `rustc-dev` was only recently + // split out of `rust-std`. We'll include it by default as a transition for nightly users. + if self.rust_release == "nightly" { + self.extend_profile("default", &mut manifest.profiles, &["rustc-dev"]); + self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]); + } } fn add_renames_to(&self, manifest: &mut Manifest) { @@ -549,6 +556,14 @@ impl Builder { dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect()); } + fn extend_profile(&mut self, + profile_name: &str, + dst: &mut BTreeMap>, + pkgs: &[&str]) { + dst.get_mut(profile_name).expect("existing profile") + .extend(pkgs.iter().map(|s| (*s).to_owned())); + } + fn package(&mut self, pkgname: &str, dst: &mut BTreeMap,