Skip to content

Remove EMBED_LTO_BITCODE #678

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 8 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ jobs:
- name: Build
run: |
./y.sh prepare --only-libcore
EMBED_LTO_BITCODE=1 ./y.sh build --sysroot --release --release-sysroot
./y.sh build --sysroot --release --release-sysroot
./y.sh test --cargo-tests
./y.sh clean all

Expand All @@ -79,11 +79,11 @@ jobs:
run: |
# FIXME(antoyo): we cannot enable LTO for stdarch tests currently because of some failing LTO tests using proc-macros.
echo -n 'lto = "fat"' >> build_system/build_sysroot/Cargo.toml
EMBED_LTO_BITCODE=1 ./y.sh test --release --clean --release-sysroot --build-sysroot --keep-lto-tests ${{ matrix.commands }}
./y.sh test --release --clean --release-sysroot --build-sysroot --keep-lto-tests ${{ matrix.commands }}

- name: Run y.sh cargo build
run: |
EMBED_LTO_BITCODE=1 CHANNEL="release" ./y.sh cargo build --release --manifest-path tests/hello-world/Cargo.toml
CHANNEL="release" ./y.sh cargo build --release --manifest-path tests/hello-world/Cargo.toml
call_found=$(objdump -dj .text tests/hello-world/target/release/hello_world | grep -c "call .*mylib.*my_func" ) ||:
if [ $call_found -gt 0 ]; then
echo "ERROR: call my_func found in asm"
Expand Down
50 changes: 43 additions & 7 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,18 @@ dependencies = [
"unicode-width",
]

[[package]]
name = "getrandom"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasi",
]

[[package]]
name = "hermit-abi"
version = "0.3.1"
Expand Down Expand Up @@ -111,9 +123,9 @@ checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d"

[[package]]
name = "linux-raw-sys"
version = "0.4.14"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"

[[package]]
name = "memchr"
Expand All @@ -137,6 +149,12 @@ version = "1.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"

[[package]]
name = "r-efi"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"

[[package]]
name = "regex"
version = "1.8.4"
Expand Down Expand Up @@ -166,9 +184,9 @@ dependencies = [

[[package]]
name = "rustix"
version = "0.38.42"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85"
checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
dependencies = [
"bitflags",
"errno",
Expand All @@ -188,12 +206,12 @@ dependencies = [

[[package]]
name = "tempfile"
version = "3.14.0"
version = "3.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c"
checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1"
dependencies = [
"cfg-if",
"fastrand",
"getrandom",
"once_cell",
"rustix",
"windows-sys",
Expand Down Expand Up @@ -242,6 +260,15 @@ dependencies = [
"winapi-util",
]

[[package]]
name = "wasi"
version = "0.14.2+wasi-0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
dependencies = [
"wit-bindgen-rt",
]

[[package]]
name = "winapi"
version = "0.3.9"
Expand Down Expand Up @@ -345,3 +372,12 @@ name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"

[[package]]
name = "wit-bindgen-rt"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
dependencies = [
"bitflags",
]
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ gccjit = "2.7"
[dev-dependencies]
boml = "0.3.1"
lang_tester = "0.8.0"
tempfile = "3.7.1"
tempfile = "3.20.0"

[profile.dev]
# By compiling dependencies with optimizations, performing tests gets much faster.
Expand Down
10 changes: 0 additions & 10 deletions Readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,16 +135,6 @@ $ CHANNEL="release" $CG_GCCJIT_DIR/y.sh cargo run

If you compiled cg_gccjit in debug mode (aka you didn't pass `--release` to `./y.sh test`) you should use `CHANNEL="debug"` instead or omit `CHANNEL="release"` completely.

### LTO

To use LTO, you need to set the variable `EMBED_LTO_BITCODE=1` in addition to setting `lto = "fat"` in the `Cargo.toml`.

Failing to set `EMBED_LTO_BITCODE` will give you the following error:

```
error: failed to copy bitcode to object file: No such file or directory (os error 2)
```

### Rustc

If you want to run `rustc` directly, you can do so with:
Expand Down
2 changes: 1 addition & 1 deletion libgccjit.version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
04ce66d8c918de9273bd7101638ad8724edf5e21
9244ae6e2e96e7d255c73fbd4a9f345e0cd96f35
13 changes: 8 additions & 5 deletions src/back/lto.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ use std::ffi::{CStr, CString};
use std::fs::{self, File};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::Ordering;

use gccjit::{Context, OutputKind};
use object::read::archive::ArchiveFile;
Expand All @@ -41,7 +42,7 @@ use tempfile::{TempDir, tempdir};

use crate::back::write::save_temp_bitcode;
use crate::errors::{DynamicLinkingWithLTO, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib};
use crate::{GccCodegenBackend, GccContext, SyncContext, to_gcc_opt_level};
use crate::{GccCodegenBackend, GccContext, LTO_SUPPORTED, LtoMode, SyncContext, to_gcc_opt_level};

pub fn crate_type_allows_lto(crate_type: CrateType) -> bool {
match crate_type {
Expand Down Expand Up @@ -303,7 +304,7 @@ fn fat_lto(
info!("linking {:?}", name);
match bc_decoded {
SerializedModule::Local(ref module_buffer) => {
module.module_llvm.should_combine_object_files = true;
module.module_llvm.lto_mode = LtoMode::Fat;
module
.module_llvm
.context
Expand Down Expand Up @@ -611,7 +612,7 @@ pub fn optimize_thin_module(
// that LLVM Context and Module.
//let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
//let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &dcx)? as *const _;
let mut should_combine_object_files = false;
let mut lto_mode = LtoMode::None;
let context = match thin_module.shared.thin_buffers.get(thin_module.idx) {
Some(thin_buffer) => Arc::clone(&thin_buffer.context),
None => {
Expand All @@ -622,7 +623,7 @@ pub fn optimize_thin_module(
SerializedModule::Local(ref module_buffer) => {
let path = module_buffer.0.to_str().expect("path");
context.add_driver_option(path);
should_combine_object_files = true;
lto_mode = LtoMode::Thin;
/*module.module_llvm.should_combine_object_files = true;
module
.module_llvm
Expand All @@ -637,11 +638,13 @@ pub fn optimize_thin_module(
Arc::new(SyncContext::new(context))
}
};
let lto_supported = LTO_SUPPORTED.load(Ordering::SeqCst);
let module = ModuleCodegen::new_regular(
thin_module.name().to_string(),
GccContext {
context,
should_combine_object_files,
lto_mode,
lto_supported,
// TODO(antoyo): use the correct relocation model here.
relocation_model: RelocModel::Pic,
temp_dir: None,
Expand Down
101 changes: 34 additions & 67 deletions src/back/write.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use rustc_target::spec::SplitDebuginfo;

use crate::base::add_pic_option;
use crate::errors::CopyBitcode;
use crate::{GccCodegenBackend, GccContext};
use crate::{GccCodegenBackend, GccContext, LtoMode};

pub(crate) fn codegen(
cgcx: &CodegenContext<GccCodegenBackend>,
Expand All @@ -24,12 +24,8 @@ pub(crate) fn codegen(
{
let context = &module.module_llvm.context;

let should_combine_object_files = module.module_llvm.should_combine_object_files;

// NOTE: Only generate object files with GIMPLE when this environment variable is set for
// now because this requires a particular setup (same gcc/lto1/lto-wrapper commit as libgccjit).
// TODO(antoyo): remove this environment variable.
let fat_lto = env::var("EMBED_LTO_BITCODE").as_deref() == Ok("1");
let lto_mode = module.module_llvm.lto_mode;
let lto_supported = module.module_llvm.lto_supported;

let bc_out = cgcx.output_filenames.temp_path_for_cgu(
OutputType::Bitcode,
Expand All @@ -43,80 +39,48 @@ pub(crate) fn codegen(
);

if config.bitcode_needed() {
if fat_lto {
let _timer = cgcx
.prof
.generic_activity_with_arg("GCC_module_codegen_make_bitcode", &*module.name);

// TODO(antoyo)
/*if let Some(bitcode_filename) = bc_out.file_name() {
cgcx.prof.artifact_size(
"llvm_bitcode",
bitcode_filename.to_string_lossy(),
data.len() as u64,
);
}*/

if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
let _timer = cgcx
.prof
.generic_activity_with_arg("GCC_module_codegen_make_bitcode", &*module.name);

// TODO(antoyo)
/*if let Some(bitcode_filename) = bc_out.file_name() {
cgcx.prof.artifact_size(
"llvm_bitcode",
bitcode_filename.to_string_lossy(),
data.len() as u64,
);
}*/

if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
let _timer = cgcx.prof.generic_activity_with_arg(
"GCC_module_codegen_emit_bitcode",
&*module.name,
);
.generic_activity_with_arg("GCC_module_codegen_emit_bitcode", &*module.name);
if lto_supported {
context.add_command_line_option("-flto=auto");
context.add_command_line_option("-flto-partition=one");
// TODO(antoyo): remove since we don't want fat objects when it is for Bitcode only.
context.add_command_line_option("-ffat-lto-objects");
context.compile_to_file(
OutputKind::ObjectFile,
bc_out.to_str().expect("path to str"),
);
}
context
.compile_to_file(OutputKind::ObjectFile, bc_out.to_str().expect("path to str"));
}

if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) {
let _timer = cgcx.prof.generic_activity_with_arg(
"GCC_module_codegen_embed_bitcode",
&*module.name,
);
if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) {
let _timer = cgcx
.prof
.generic_activity_with_arg("GCC_module_codegen_embed_bitcode", &*module.name);
if lto_supported {
// TODO(antoyo): maybe we should call embed_bitcode to have the proper iOS fixes?
//embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, data);

context.add_command_line_option("-flto=auto");
context.add_command_line_option("-flto-partition=one");
context.add_command_line_option("-ffat-lto-objects");
// TODO(antoyo): Send -plugin/usr/lib/gcc/x86_64-pc-linux-gnu/11.1.0/liblto_plugin.so to linker (this should be done when specifying the appropriate rustc cli argument).
context.compile_to_file(
OutputKind::ObjectFile,
bc_out.to_str().expect("path to str"),
);
}
} else {
if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
let _timer = cgcx.prof.generic_activity_with_arg(
"GCC_module_codegen_emit_bitcode",
&*module.name,
);
context.compile_to_file(
OutputKind::ObjectFile,
bc_out.to_str().expect("path to str"),
);
}

if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) {
// TODO(antoyo): we might want to emit to emit an error here, saying to set the
// environment variable EMBED_LTO_BITCODE.
let _timer = cgcx.prof.generic_activity_with_arg(
"GCC_module_codegen_embed_bitcode",
&*module.name,
);
// TODO(antoyo): maybe we should call embed_bitcode to have the proper iOS fixes?
//embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, data);

// TODO(antoyo): Send -plugin/usr/lib/gcc/x86_64-pc-linux-gnu/11.1.0/liblto_plugin.so to linker (this should be done when specifying the appropriate rustc cli argument).
context.compile_to_file(
OutputKind::ObjectFile,
bc_out.to_str().expect("path to str"),
);
}
// TODO(antoyo): Send -plugin/usr/lib/gcc/x86_64-pc-linux-gnu/11.1.0/liblto_plugin.so to linker (this should be done when specifying the appropriate rustc cli argument).
context
.compile_to_file(OutputKind::ObjectFile, bc_out.to_str().expect("path to str"));
}
}

Expand Down Expand Up @@ -165,7 +129,10 @@ pub(crate) fn codegen(
context.set_debug_info(true);
context.dump_to_file(path, true);
}
if should_combine_object_files {
if lto_mode != LtoMode::None {
let fat_lto = lto_mode == LtoMode::Fat;
// We need to check if we're doing LTO since this code is also used for the
// dummy ThinLTO implementation to combine the object files.
if fat_lto {
context.add_command_line_option("-flto=auto");
context.add_command_line_option("-flto-partition=one");
Expand Down
Loading