Load prebuilt macros (#1856)

Originally submitted as
#1812

---------

Co-authored-by: Maksim Zdobnikau <[email protected]>

.github/workflows/ci.yml

@@ -78,6 +78,30 @@ jobs:
       - name: run tests
         run: cargo test -p scarb-metadata
 
+  test-prebuilt-plugins:
+    name: test prebuilt plugins ${{ matrix.platform.name }}
+    runs-on: ${{ matrix.platform.os }}
+    # This is isolated, so it can be run on more platforms.
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - name: linux x86-64
+            os: ubuntu-latest
+          - name: windows x86-64
+            os: windows-latest
+          - name: macos arm64
+            os: macos-latest
+          - name: macos x86-64
+            os: macos-13
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - name: Run prebuilt plugin tests
+        run: |
+          cargo test -p scarb --test proc_macro_prebuilt -- --ignored
+
   check-rust:
     runs-on: ubuntu-latest
     steps:

Cargo.toml

@@ -146,6 +146,7 @@ xshell = "0.2"
 xxhash-rust = { version = "0.8", features = ["xxh3"] }
 zip = { version = "0.6", default-features = false, features = ["deflate"] }
 zstd = "0.13"
+target-triple = "0.1"
 
 # Here we specify real dependency specifications for Cairo crates *if* currently we want to use
 # a particular unreleased commit (which is frequent mid-development).

scarb/Cargo.toml

@@ -90,6 +90,7 @@ windows-sys.workspace = true
 zstd.workspace = true
 cargo_metadata.workspace = true
 flate2.workspace = true
+target-triple.workspace = true
 
 [target.'cfg(not(target_os = "linux"))'.dependencies]
 reqwest = { workspace = true, default-features = true }

scarb/src/bin/scarb/commands/proc_macro_server.rs

@@ -1,4 +1,5 @@
 use anyhow::Result;
+use scarb::ops::CompilationUnitsOpts;
 use scarb::{
     compiler::{plugin::proc_macro::ProcMacroHost, CairoCompilationUnit, CompilationUnit},
     core::{Config, Workspace},
@@ -15,8 +16,11 @@ pub fn run(config: &mut Config) -> Result<()> {
             features: FeaturesSelector::AllFeatures,
             no_default_features: false,
         },
-        true,
         &ws,
+        CompilationUnitsOpts {
+            ignore_cairo_version: true,
+            load_prebuilt_macros: true,
+        },
     )?;
 
     // Compile procedural macros only.
@@ -43,12 +47,12 @@ fn load_plugins(
     ws: &Workspace<'_>,
     proc_macros: &mut ProcMacroHost,
 ) -> Result<()> {
-    for plugin_info in unit
-        .cairo_plugins
-        .into_iter()
-        .filter(|plugin_info| !plugin_info.builtin)
-    {
-        proc_macros.register(plugin_info.package, ws.config())?;
+    for plugin_info in unit.cairo_plugins.into_iter().filter(|p| !p.builtin) {
+        if let Some(prebuilt) = plugin_info.prebuilt {
+            proc_macros.register_instance(prebuilt);
+        } else {
+            proc_macros.register_new(plugin_info.package, ws.config())?;
+        }
     }
 
     Ok(())

scarb/src/compiler/compilation_unit.rs

@@ -1,14 +1,15 @@
-use std::fmt::Write;
-use std::hash::{Hash, Hasher};
-
 use anyhow::{ensure, Result};
 use cairo_lang_filesystem::cfg::CfgSet;
 use cairo_lang_filesystem::db::CrateIdentifier;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 use smol_str::SmolStr;
+use std::fmt::Write;
+use std::hash::{Hash, Hasher};
+use std::sync::Arc;
 use typed_builder::TypedBuilder;
 
+use crate::compiler::plugin::proc_macro::ProcMacroInstance;
 use crate::compiler::Profile;
 use crate::core::{
     ManifestCompilerConfig, Package, PackageId, PackageName, Target, TargetKind, Workspace,
@@ -72,6 +73,9 @@ pub struct ProcMacroCompilationUnit {
 
     /// Rust compiler configuration parameters to use in this unit.
     pub compiler_config: serde_json::Value,
+
+    /// Instance of the proc macro loaded from prebuilt library, if available.
+    pub prebuilt: Option<Arc<ProcMacroInstance>>,
 }
 
 /// Information about a single package that is part of a [`CompilationUnit`].
@@ -96,8 +100,11 @@ pub struct CompilationUnitComponent {
 pub struct CompilationUnitCairoPlugin {
     /// The Scarb plugin [`Package`] to load.
     pub package: Package,
+    /// Indicate whether the plugin is built into Scarb, or compiled from source.
     pub builtin: bool,
     pub prebuilt_allowed: bool,
+    /// Instance of the proc macro loaded from prebuilt library, if available.
+    pub prebuilt: Option<Arc<ProcMacroInstance>>,
 }
 
 /// Unique identifier of the compilation unit component.

scarb/src/compiler/db.rs

@@ -1,3 +1,7 @@
+use crate::compiler::plugin::proc_macro::{ProcMacroHost, ProcMacroHostPlugin};
+use crate::compiler::{CairoCompilationUnit, CompilationUnitAttributes, CompilationUnitComponent};
+use crate::core::Workspace;
+use crate::DEFAULT_MODULE_MAIN_FILE;
 use anyhow::{anyhow, Result};
 use cairo_lang_compiler::db::{RootDatabase, RootDatabaseBuilder};
 use cairo_lang_compiler::project::{AllCratesConfig, ProjectConfig, ProjectConfigContent};
@@ -14,11 +18,6 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use tracing::trace;
 
-use crate::compiler::plugin::proc_macro::{ProcMacroHost, ProcMacroHostPlugin};
-use crate::compiler::{CairoCompilationUnit, CompilationUnitAttributes, CompilationUnitComponent};
-use crate::core::Workspace;
-use crate::DEFAULT_MODULE_MAIN_FILE;
-
 pub struct ScarbDatabase {
     pub db: RootDatabase,
     pub proc_macro_host: Arc<ProcMacroHostPlugin>,
@@ -59,8 +58,10 @@ fn load_plugins(
             let plugin = ws.config().cairo_plugins().fetch(package_id)?;
             let instance = plugin.instantiate()?;
             builder.with_plugin_suite(instance.plugin_suite());
+        } else if let Some(prebuilt) = &plugin_info.prebuilt {
+            proc_macros.register_instance(prebuilt.clone());
         } else {
-            proc_macros.register(plugin_info.package.clone(), ws.config())?;
+            proc_macros.register_new(plugin_info.package.clone(), ws.config())?;
         }
     }
     let macro_host = Arc::new(proc_macros.into_plugin()?);

scarb/src/compiler/plugin/mod.rs

@@ -9,6 +9,7 @@ use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
 use crate::compiler::plugin::builtin::BuiltinCairoRunPlugin;
+use crate::compiler::plugin::proc_macro::compilation::SharedLibraryProvider;
 use crate::core::{Package, PackageId, TargetKind, Workspace};
 
 use self::builtin::{BuiltinStarkNetPlugin, BuiltinTestPlugin};
@@ -29,8 +30,10 @@ pub fn fetch_cairo_plugin(package: &Package, ws: &Workspace<'_>) -> Result<()> {
     assert!(package.is_cairo_plugin());
     let target = package.fetch_target(&TargetKind::CAIRO_PLUGIN)?;
     let props: CairoPluginProps = target.props()?;
-    // No need to fetch for buildin plugins.
-    if !props.builtin {
+    // There is no need to run `cargo fetch` for builtin plugins.
+    // The `fetch` will not be run for a proc macro that contains a prebuilt library file.
+    // Note, that in case the prebuilt lib file is corrupted, it will be later compiled with Cargo anyway.
+    if !props.builtin && package.prebuilt_lib_path().is_none() {
         proc_macro::fetch_crate(package, ws)?;
     }
     Ok(())

scarb/src/compiler/plugin/proc_macro/compilation.rs

@@ -15,12 +15,14 @@ use ra_ap_toolchain::Tool;
 use scarb_ui::{Message, OutputFormat};
 use serde::{Serialize, Serializer};
 use serde_json::value::RawValue;
+use std::env::consts::DLL_SUFFIX;
 use std::fmt::Display;
 use std::fs;
 use std::io::{Seek, SeekFrom};
 use std::ops::Deref;
 use std::process::Command;
 use tar::Archive;
+use target_triple::target;
 use tracing::trace_span;
 
 pub const PROC_MACRO_BUILD_PROFILE: &str = "release";
@@ -31,6 +33,8 @@ pub trait SharedLibraryProvider {
     fn target_path(&self, config: &Config) -> Filesystem;
     /// Location of the shared library for the package.
     fn shared_lib_path(&self, config: &Config) -> Result<Utf8PathBuf>;
+    /// Location of the prebuilt binary for the package, if defined.
+    fn prebuilt_lib_path(&self) -> Option<Utf8PathBuf>;
 }
 
 impl SharedLibraryProvider for Package {
@@ -61,6 +65,27 @@ impl SharedLibraryProvider for Package {
             .path_unchecked()
             .join(lib_name))
     }
+
+    fn prebuilt_lib_path(&self) -> Option<Utf8PathBuf> {
+        let target_triple = target!();
+
+        let prebuilt_name = format!(
+            "{name}_v{version}_{target}{suffix}",
+            name = self.id.name,
+            version = self.id.version,
+            target = target_triple,
+            suffix = DLL_SUFFIX
+        );
+
+        let prebuilt_path = self
+            .root()
+            .join("target")
+            .join("scarb")
+            .join("cairo-plugin")
+            .join(prebuilt_name);
+
+        prebuilt_path.exists().then_some(prebuilt_path)
+    }
 }
 
 pub fn compile_unit(unit: ProcMacroCompilationUnit, ws: &Workspace<'_>) -> Result<()> {

scarb/src/compiler/plugin/proc_macro/ffi.rs

@@ -1,4 +1,4 @@
-use crate::core::{Config, Package, PackageId};
+use crate::core::{Package, PackageId};
 use anyhow::{ensure, Context, Result};
 use cairo_lang_defs::patcher::PatchBuilder;
 use cairo_lang_macro::{
@@ -26,6 +26,7 @@ use libloading::os::unix::Symbol as RawSymbol;
 #[cfg(windows)]
 use libloading::os::windows::Symbol as RawSymbol;
 use smol_str::SmolStr;
+use tracing::trace;
 
 pub trait FromSyntaxNode {
     fn from_syntax_node(db: &dyn SyntaxGroup, node: &impl TypedSyntaxNode) -> Self;
@@ -61,11 +62,22 @@ impl Debug for ProcMacroInstance {
 
 impl ProcMacroInstance {
     /// Load shared library
-    pub fn try_new(package: Package, config: &Config) -> Result<Self> {
-        let lib_path = package
-            .shared_lib_path(config)
-            .context("could not resolve shared library path")?;
-        let plugin = unsafe { Plugin::try_new(lib_path.to_path_buf())? };
+    pub fn try_new(package_id: PackageId, lib_path: Utf8PathBuf) -> Result<Self> {
+        trace!("loading compiled macro for `{}` package", package_id);
+        let plugin = unsafe { Plugin::try_new(lib_path)? };
+        Ok(Self {
+            expansions: unsafe { Self::load_expansions(&plugin, package_id)? },
+            package_id,
+            plugin,
+        })
+    }
+
+    pub fn try_load_prebuilt(package: Package) -> Result<Self> {
+        trace!("loading prebuilt macro for `{}` package", package.id);
+        let prebuilt_path = package
+            .prebuilt_lib_path()
+            .context("could not resolve prebuilt library path")?;
+        let plugin = unsafe { Plugin::try_new(prebuilt_path)? };
         Ok(Self {
             expansions: unsafe { Self::load_expansions(&plugin, package.id)? },
             package_id: package.id,

scarb/src/compiler/plugin/proc_macro/host.rs

@@ -1,8 +1,9 @@
+use crate::compiler::plugin::proc_macro::compilation::SharedLibraryProvider;
 use crate::compiler::plugin::proc_macro::{
     Expansion, ExpansionKind, FromSyntaxNode, ProcMacroInstance,
 };
 use crate::core::{Config, Package, PackageId};
-use anyhow::{ensure, Result};
+use anyhow::{ensure, Context, Result};
 use cairo_lang_defs::ids::{ModuleItemId, TopLevelLanguageElementId};
 use cairo_lang_defs::patcher::{PatchBuilder, RewriteNode};
 use cairo_lang_defs::plugin::{
@@ -1150,9 +1151,16 @@ pub struct ProcMacroHost {
 }
 
 impl ProcMacroHost {
-    pub fn register(&mut self, package: Package, config: &Config) -> Result<()> {
-        let instance = ProcMacroInstance::try_new(package, config)?;
-        self.macros.push(Arc::new(instance));
+    pub fn register_instance(&mut self, instance: Arc<ProcMacroInstance>) {
+        self.macros.push(instance);
+    }
+
+    pub fn register_new(&mut self, package: Package, config: &Config) -> Result<()> {
+        let lib_path = package
+            .shared_lib_path(config)
+            .context("could not resolve shared library path")?;
+        let instance = ProcMacroInstance::try_new(package.id, lib_path)?;
+        self.register_instance(Arc::new(instance));
         Ok(())
     }