Auto merge of #84171 - ricobbe:raw-dylib-via-llvm, r=petrochenkov

Partial support for raw-dylib linkage

First cut of functionality for issue #58713: add support for `#[link(kind = "raw-dylib")]` on `extern` blocks in lib crates compiled to .rlib files.  Does not yet support `#[link_name]` attributes on functions, or the `#[link_ordinal]` attribute, or `#[link(kind = "raw-dylib")]` on `extern` blocks in bin crates; I intend to publish subsequent PRs to fill those gaps.  It's also not yet clear whether this works for functions in `extern "stdcall"` blocks; I also intend to investigate that shortly and make any necessary changes as a follow-on PR.

This implementation calls out to an LLVM function to construct the actual `.idata` sections as temporary `.lib` files on disk and then links those into the generated .rlib.

Author: bors

compiler/rustc_codegen_cranelift/src/archive.rs

@@ -254,6 +254,15 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
             }
         }
     }
+
+    fn inject_dll_import_lib(
+        &mut self,
+        _lib_name: &str,
+        _dll_imports: &[rustc_middle::middle::cstore::DllImport],
+        _tmpdir: &rustc_data_structures::temp_dir::MaybeTempDir,
+    ) {
+        bug!("injecting dll imports is not supported");
+    }
 }
 
 impl<'a> ArArchiveBuilder<'a> {

compiler/rustc_codegen_llvm/src/back/archive.rs

@@ -8,9 +8,11 @@ use std::ptr;
 use std::str;
 
 use crate::llvm::archive_ro::{ArchiveRO, Child};
-use crate::llvm::{self, ArchiveKind};
+use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport};
 use rustc_codegen_ssa::back::archive::{find_library, ArchiveBuilder};
 use rustc_codegen_ssa::{looks_like_rust_object_file, METADATA_FILENAME};
+use rustc_data_structures::temp_dir::MaybeTempDir;
+use rustc_middle::middle::cstore::DllImport;
 use rustc_session::Session;
 use rustc_span::symbol::Symbol;
 
@@ -61,6 +63,17 @@ fn archive_config<'a>(sess: &'a Session, output: &Path, input: Option<&Path>) ->
     }
 }
 
+/// Map machine type strings to values of LLVM's MachineTypes enum.
+fn llvm_machine_type(cpu: &str) -> LLVMMachineType {
+    match cpu {
+        "x86_64" => LLVMMachineType::AMD64,
+        "x86" => LLVMMachineType::I386,
+        "aarch64" => LLVMMachineType::ARM64,
+        "arm" => LLVMMachineType::ARM,
+        _ => panic!("unsupported cpu type {}", cpu),
+    }
+}
+
 impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
     /// Creates a new static archive, ready for modifying the archive specified
     /// by `config`.
@@ -175,6 +188,74 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
             self.config.sess.fatal(&format!("failed to build archive: {}", e));
         }
     }
+
+    fn inject_dll_import_lib(
+        &mut self,
+        lib_name: &str,
+        dll_imports: &[DllImport],
+        tmpdir: &MaybeTempDir,
+    ) {
+        let output_path = {
+            let mut output_path: PathBuf = tmpdir.as_ref().to_path_buf();
+            output_path.push(format!("{}_imports", lib_name));
+            output_path.with_extension("lib")
+        };
+
+        // we've checked for \0 characters in the library name already
+        let dll_name_z = CString::new(lib_name).unwrap();
+        // All import names are Rust identifiers and therefore cannot contain \0 characters.
+        // FIXME: when support for #[link_name] implemented, ensure that import.name values don't
+        // have any \0 characters
+        let import_name_vector: Vec<CString> = dll_imports
+            .iter()
+            .map(if self.config.sess.target.arch == "x86" {
+                |import: &DllImport| CString::new(format!("_{}", import.name.to_string())).unwrap()
+            } else {
+                |import: &DllImport| CString::new(import.name.to_string()).unwrap()
+            })
+            .collect();
+
+        let output_path_z = rustc_fs_util::path_to_c_string(&output_path);
+
+        tracing::trace!("invoking LLVMRustWriteImportLibrary");
+        tracing::trace!("  dll_name {:#?}", dll_name_z);
+        tracing::trace!("  output_path {}", output_path.display());
+        tracing::trace!(
+            "  import names: {}",
+            dll_imports.iter().map(|import| import.name.to_string()).collect::<Vec<_>>().join(", "),
+        );
+
+        let ffi_exports: Vec<LLVMRustCOFFShortExport> = import_name_vector
+            .iter()
+            .map(|name_z| LLVMRustCOFFShortExport::from_name(name_z.as_ptr()))
+            .collect();
+        let result = unsafe {
+            crate::llvm::LLVMRustWriteImportLibrary(
+                dll_name_z.as_ptr(),
+                output_path_z.as_ptr(),
+                ffi_exports.as_ptr(),
+                ffi_exports.len(),
+                llvm_machine_type(&self.config.sess.target.arch) as u16,
+                !self.config.sess.target.is_like_msvc,
+            )
+        };
+
+        if result == crate::llvm::LLVMRustResult::Failure {
+            self.config.sess.fatal(&format!(
+                "Error creating import library for {}: {}",
+                lib_name,
+                llvm::last_error().unwrap_or("unknown LLVM error".to_string())
+            ));
+        }
+
+        self.add_archive(&output_path, |_| false).unwrap_or_else(|e| {
+            self.config.sess.fatal(&format!(
+                "failed to add native library {}: {}",
+                output_path.display(),
+                e
+            ));
+        });
+    }
 }
 
 impl<'a> LlvmArchiveBuilder<'a> {

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -29,6 +29,31 @@ pub enum LLVMRustResult {
     Success,
     Failure,
 }
+
+// Rust version of the C struct with the same name in rustc_llvm/llvm-wrapper/RustWrapper.cpp.
+#[repr(C)]
+pub struct LLVMRustCOFFShortExport {
+    pub name: *const c_char,
+}
+
+impl LLVMRustCOFFShortExport {
+    pub fn from_name(name: *const c_char) -> LLVMRustCOFFShortExport {
+        LLVMRustCOFFShortExport { name }
+    }
+}
+
+/// Translation of LLVM's MachineTypes enum, defined in llvm\include\llvm\BinaryFormat\COFF.h.
+///
+/// We include only architectures supported on Windows.
+#[derive(Copy, Clone, PartialEq)]
+#[repr(C)]
+pub enum LLVMMachineType {
+    AMD64 = 0x8664,
+    I386 = 0x14c,
+    ARM64 = 0xaa64,
+    ARM = 0x01c0,
+}
+
 // Consts for the LLVM CallConv type, pre-cast to usize.
 
 /// LLVM CallingConv::ID. Should we wrap this?
@@ -2265,6 +2290,15 @@ extern "C" {
     ) -> &'a mut RustArchiveMember<'a>;
     pub fn LLVMRustArchiveMemberFree(Member: &'a mut RustArchiveMember<'a>);
 
+    pub fn LLVMRustWriteImportLibrary(
+        ImportName: *const c_char,
+        Path: *const c_char,
+        Exports: *const LLVMRustCOFFShortExport,
+        NumExports: usize,
+        Machine: u16,
+        MinGW: bool,
+    ) -> LLVMRustResult;
+
     pub fn LLVMRustSetDataLayoutFromTargetMachine(M: &'a Module, TM: &'a TargetMachine);
 
     pub fn LLVMRustBuildOperandBundleDef(

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -1,3 +1,5 @@
+use rustc_data_structures::temp_dir::MaybeTempDir;
+use rustc_middle::middle::cstore::DllImport;
 use rustc_session::Session;
 use rustc_span::symbol::Symbol;
 
@@ -57,4 +59,11 @@ pub trait ArchiveBuilder<'a> {
     fn update_symbols(&mut self);
 
     fn build(self);
+
+    fn inject_dll_import_lib(
+        &mut self,
+        lib_name: &str,
+        dll_imports: &[DllImport],
+        tmpdir: &MaybeTempDir,
+    );
 }

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -1,9 +1,9 @@
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::temp_dir::MaybeTempDir;
 use rustc_errors::Handler;
 use rustc_fs_util::fix_windows_verbatim_for_gcc;
 use rustc_hir::def_id::CrateNum;
-use rustc_middle::middle::cstore::LibSource;
+use rustc_middle::middle::cstore::{DllImport, LibSource};
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_session::config::{self, CFGuard, CrateType, DebugInfo};
 use rustc_session::config::{OutputFilenames, OutputType, PrintRequest};
@@ -34,6 +34,7 @@ use object::write::Object;
 use object::{Architecture, BinaryFormat, Endianness, FileFlags, SectionFlags, SectionKind};
 use tempfile::Builder as TempFileBuilder;
 
+use std::cmp::Ordering;
 use std::ffi::OsString;
 use std::path::{Path, PathBuf};
 use std::process::{ExitStatus, Output, Stdio};
@@ -343,6 +344,12 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>(
         }
     }
 
+    for (raw_dylib_name, raw_dylib_imports) in
+        collate_raw_dylibs(&codegen_results.crate_info.used_libraries)
+    {
+        ab.inject_dll_import_lib(&raw_dylib_name, &raw_dylib_imports, tmpdir);
+    }
+
     // After adding all files to the archive, we need to update the
     // symbol table of the archive.
     ab.update_symbols();
@@ -524,6 +531,57 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>(
     }
 }
 
+/// Extract all symbols defined in raw-dylib libraries, collated by library name.
+///
+/// If we have multiple extern blocks that specify symbols defined in the same raw-dylib library,
+/// then the CodegenResults value contains one NativeLib instance for each block.  However, the
+/// linker appears to expect only a single import library for each library used, so we need to
+/// collate the symbols together by library name before generating the import libraries.
+fn collate_raw_dylibs(used_libraries: &[NativeLib]) -> Vec<(String, Vec<DllImport>)> {
+    let mut dylib_table: FxHashMap<String, FxHashSet<Symbol>> = FxHashMap::default();
+
+    for lib in used_libraries {
+        if lib.kind == NativeLibKind::RawDylib {
+            let name = lib.name.unwrap_or_else(||
+                bug!("`link` attribute with kind = \"raw-dylib\" and no name should have caused error earlier")
+            );
+            let name = if matches!(lib.verbatim, Some(true)) {
+                name.to_string()
+            } else {
+                format!("{}.dll", name)
+            };
+            dylib_table
+                .entry(name)
+                .or_default()
+                .extend(lib.dll_imports.iter().map(|import| import.name));
+        }
+    }
+
+    // FIXME: when we add support for ordinals, fix this to propagate ordinals.  Also figure out
+    // what we should do if we have two DllImport values with the same name but different
+    // ordinals.
+    let mut result = dylib_table
+        .into_iter()
+        .map(|(lib_name, imported_names)| {
+            let mut names = imported_names
+                .iter()
+                .map(|name| DllImport { name: *name, ordinal: None })
+                .collect::<Vec<_>>();
+            names.sort_unstable_by(|a: &DllImport, b: &DllImport| {
+                match a.name.as_str().cmp(&b.name.as_str()) {
+                    Ordering::Equal => a.ordinal.cmp(&b.ordinal),
+                    x => x,
+                }
+            });
+            (lib_name, names)
+        })
+        .collect::<Vec<_>>();
+    result.sort_unstable_by(|a: &(String, Vec<DllImport>), b: &(String, Vec<DllImport>)| {
+        a.0.cmp(&b.0)
+    });
+    result
+}
+
 /// Create a static archive.
 ///
 /// This is essentially the same thing as an rlib, but it also involves adding all of the upstream
@@ -2303,10 +2361,7 @@ fn add_upstream_native_libraries(
                 // already included them when we included the rust library
                 // previously
                 NativeLibKind::Static { bundle: None | Some(true), .. } => {}
-                NativeLibKind::RawDylib => {
-                    // FIXME(#58713): Proper handling for raw dylibs.
-                    bug!("raw_dylib feature not yet implemented");
-                }
+                NativeLibKind::RawDylib => {}
             }
         }
     }

compiler/rustc_codegen_ssa/src/lib.rs

@@ -110,11 +110,18 @@ pub struct NativeLib {
     pub name: Option<Symbol>,
     pub cfg: Option<ast::MetaItem>,
     pub verbatim: Option<bool>,
+    pub dll_imports: Vec<cstore::DllImport>,
 }
 
 impl From<&cstore::NativeLib> for NativeLib {
     fn from(lib: &cstore::NativeLib) -> Self {
-        NativeLib { kind: lib.kind, name: lib.name, cfg: lib.cfg.clone(), verbatim: lib.verbatim }
+        NativeLib {
+            kind: lib.kind,
+            name: lib.name,
+            cfg: lib.cfg.clone(),
+            verbatim: lib.verbatim,
+            dll_imports: lib.dll_imports.clone(),
+        }
     }
 }
 

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -6,6 +6,7 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/Object/Archive.h"
+#include "llvm/Object/COFFImportFile.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Bitcode/BitcodeWriterPass.h"
 #include "llvm/Support/Signals.h"
@@ -1722,3 +1723,54 @@ extern "C" LLVMValueRef
 LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) {
     return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS),unwrap(RHS)));
 }
+
+// This struct contains all necessary info about a symbol exported from a DLL.
+// At the moment, it's just the symbol's name, but we use a separate struct to
+// make it easier to add other information like ordinal later.
+struct LLVMRustCOFFShortExport {
+  const char* name;
+};
+
+// Machine must be a COFF machine type, as defined in PE specs.
+extern "C" LLVMRustResult LLVMRustWriteImportLibrary(
+  const char* ImportName,
+  const char* Path,
+  const LLVMRustCOFFShortExport* Exports,
+  size_t NumExports,
+  uint16_t Machine,
+  bool MinGW)
+{
+  std::vector<llvm::object::COFFShortExport> ConvertedExports;
+  ConvertedExports.reserve(NumExports);
+
+  for (size_t i = 0; i < NumExports; ++i) {
+    ConvertedExports.push_back(llvm::object::COFFShortExport{
+      Exports[i].name,  // Name
+      std::string{},    // ExtName
+      std::string{},    // SymbolName
+      std::string{},    // AliasTarget
+      0,                // Ordinal
+      false,            // Noname
+      false,            // Data
+      false,            // Private
+      false             // Constant
+    });
+  }
+
+  auto Error = llvm::object::writeImportLibrary(
+    ImportName,
+    Path,
+    ConvertedExports,
+    static_cast<llvm::COFF::MachineTypes>(Machine),
+    MinGW);
+  if (Error) {
+    std::string errorString;
+    llvm::raw_string_ostream stream(errorString);
+    stream << Error;
+    stream.flush();
+    LLVMRustSetLastError(errorString.c_str());
+    return LLVMRustResult::Failure;
+  } else {
+    return LLVMRustResult::Success;
+  }
+}