Migrate Scarb to per-crate compiler plugins

Cargo.toml

@@ -21,7 +21,7 @@ members = [
 "resolver" = "2"
 
 [workspace.package]
-version = "2.9.1"
+version = "2.9.2"
 edition = "2021"
 
 authors = ["Software Mansion <[email protected]>"]
@@ -86,7 +86,10 @@ expect-test = "1.5"
 flate2 = { version = "1.0.35", default-features = false, features = ["zlib"] }
 fs4 = { version = "0.7", features = ["tokio"] }
 fs_extra = "1"
-futures = { version = "0.3", default-features = false, features = ["std", "async-await"] }
+futures = { version = "0.3", default-features = false, features = [
+    "std",
+    "async-await",
+] }
 gix = ">=0.55"
 gix-path = "0.10"
 glob = "0.3"
@@ -109,7 +112,14 @@ proc-macro2 = "1"
 quote = "1"
 ra_ap_toolchain = "0.0.218"
 redb = "2.3.0"
-reqwest = { version = "0.11", features = ["gzip", "brotli", "deflate", "json", "stream", "multipart"], default-features = false }
+reqwest = { version = "0.11", features = [
+    "gzip",
+    "brotli",
+    "deflate",
+    "json",
+    "stream",
+    "multipart",
+], default-features = false }
 salsa = { package = "rust-analyzer-salsa", version = "0.17.0-pre.6" }
 semver = { version = "1", features = ["serde"] }
 serde = { version = "1", features = ["serde_derive"] }
@@ -130,7 +140,14 @@ tempfile = "3"
 test-case = "3"
 thiserror = "2"
 time = "0.3"
-tokio = { version = "1", features = ["macros", "io-util", "process", "rt", "rt-multi-thread", "sync"] }
+tokio = { version = "1", features = [
+    "macros",
+    "io-util",
+    "process",
+    "rt",
+    "rt-multi-thread",
+    "sync",
+] }
 toml = "0.8"
 toml_edit = { version = "0.22", features = ["serde"] }
 tower-http = { version = "0.4", features = ["fs"] }
@@ -141,7 +158,11 @@ typed-builder = ">=0.17"
 url = { version = "2", features = ["serde"] }
 walkdir = "2"
 which = "7"
-windows-sys = { version = "0.59", features = ["Win32_Foundation", "Win32_System", "Win32_System_Console"] }
+windows-sys = { version = "0.59", features = [
+    "Win32_Foundation",
+    "Win32_System",
+    "Win32_System_Console",
+] }
 xshell = "0.2"
 xxhash-rust = { version = "0.8", features = ["xxh3"] }
 zip = { version = "0.6", default-features = false, features = ["deflate"] }
@@ -153,38 +174,39 @@ zstd = "0.13"
 # on some of them directly.
 # This ensures no duplicate instances of Cairo crates are pulled in by mistake.
 [patch.crates-io]
-cairo-lang-casm = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-compiler = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-debug = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-defs = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-diagnostics = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-doc = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-eq-solver = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-filesystem = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-formatter = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-lowering = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-parser = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-plugins = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-proc-macros = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-project = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-runnable-utils = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-runner = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-semantic = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-sierra = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-sierra-ap-change = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-sierra-gas = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-sierra-generator = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-sierra-to-casm = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-sierra-type-size = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-starknet = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-starknet-classes = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-syntax = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-syntax-codegen = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-test-plugin = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-test-runner = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-test-utils = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-lang-utils = { git = "https://github.com/starkware-libs/cairo", rev = "67c6eff9c276d11bd1cc903d7a3981d8d0eb2fa2" }
-cairo-language-server = { git = "https://github.com/software-mansion/cairols", rev = "94f1543b43caa6355bf492f80cdf46b2ae2bf854" }
+cairo-lang-casm = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-compiler = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-debug = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-defs = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-diagnostics = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-doc = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-eq-solver = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-executable = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-filesystem = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-formatter = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-lowering = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-parser = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-plugins = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-proc-macros = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-project = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-runnable-utils = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-runner = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-semantic = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-sierra = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-sierra-ap-change = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-sierra-gas = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-sierra-generator = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-sierra-to-casm = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-sierra-type-size = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-starknet = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-starknet-classes = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-syntax = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-syntax-codegen = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-test-plugin = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-test-runner = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-test-utils = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-lang-utils = { git = "https://github.com/starkware-libs/cairo", branch = "spr/main/f0989263" }
+cairo-language-server = { git = "https://github.com/software-mansion/cairols", branch = "feature/crate-plugins" }
 
 [profile.release]
 lto = true

extensions/scarb-doc/src/db.rs

@@ -1,5 +1,5 @@
 use cairo_lang_compiler::project::{update_crate_roots_from_project_config, ProjectConfig};
-use cairo_lang_defs::db::{try_ext_as_virtual_impl, DefsDatabase, DefsGroup};
+use cairo_lang_defs::db::{init_defs_group, try_ext_as_virtual_impl, DefsDatabase, DefsGroup};
 use cairo_lang_doc::db::{DocDatabase, DocGroup};
 use cairo_lang_filesystem::cfg::{Cfg, CfgSet};
 use cairo_lang_filesystem::db::{
@@ -8,7 +8,9 @@ use cairo_lang_filesystem::db::{
 use cairo_lang_filesystem::ids::VirtualFile;
 use cairo_lang_lowering::db::{LoweringDatabase, LoweringGroup};
 use cairo_lang_parser::db::{ParserDatabase, ParserGroup};
-use cairo_lang_semantic::db::{SemanticDatabase, SemanticGroup};
+use cairo_lang_semantic::db::{
+    init_semantic_group, PluginSuiteInput, SemanticDatabase, SemanticGroup,
+};
 use cairo_lang_semantic::inline_macros::get_default_plugin_suite;
 use cairo_lang_semantic::plugin::PluginSuite;
 use cairo_lang_starknet::starknet_plugin_suite;
@@ -44,10 +46,13 @@ impl ScarbDocDatabase {
         };
 
         init_files_group(&mut db);
+        init_defs_group(&mut db);
+        init_semantic_group(&mut db);
 
         db.set_cfg_set(Self::initial_cfg_set().into());
 
-        db.apply_plugin_suite(plugin_suite);
+        let interned_plugin_suite = db.intern_plugin_suite(plugin_suite);
+        db.set_default_plugins_from_suite(interned_plugin_suite);
 
         if let Some(config) = project_config {
             db.apply_project_config(config);
@@ -60,12 +65,6 @@ impl ScarbDocDatabase {
         CfgSet::from_iter([Cfg::name("doc")])
     }
 
-    fn apply_plugin_suite(&mut self, plugin_suite: PluginSuite) {
-        self.set_macro_plugins(plugin_suite.plugins);
-        self.set_inline_macro_plugins(plugin_suite.inline_macro_plugins.into());
-        self.set_analyzer_plugins(plugin_suite.analyzer_plugins);
-    }
-
     fn apply_project_config(&mut self, config: ProjectConfig) {
         update_crate_roots_from_project_config(self, &config);
     }

scarb/src/compiler/compilers/lib.rs

@@ -2,6 +2,9 @@ use anyhow::{Context, Result};
 use cairo_lang_compiler::db::RootDatabase;
 use cairo_lang_compiler::CompilerConfig;
 use cairo_lang_defs::db::DefsGroup;
+use cairo_lang_defs::plugin::MacroPlugin;
+use cairo_lang_filesystem::db::FilesGroup;
+use cairo_lang_filesystem::ids::CrateLongId;
 use cairo_lang_sierra::program::VersionedProgram;
 use cairo_lang_sierra_to_casm::compiler::SierraToCasmConfig;
 use cairo_lang_sierra_to_casm::metadata::{calc_metadata, calc_metadata_ap_change_only};
@@ -145,15 +148,22 @@ fn validate_compiler_config(
     unit: &CairoCompilationUnit,
     ws: &Workspace<'_>,
 ) {
+    let main_component = unit.main_component();
+    let main_crate_id = db.intern_crate(CrateLongId::Real {
+        name: main_component.target_name(),
+        discriminator: main_component.id.to_discriminator(),
+    });
+
     // Generally, lib target compilation should be driven by a certain objective (e.g. cairo-run,
     // test framework, etc.), expressed by the plugin set with executables definition.
     // This does not apply to debug build (expressed by `replace_ids` flag),
     // which is a goal by itself.
     // See starkware-libs/cairo#5440 for more context.
-    let executable_plugin = db
-        .macro_plugins()
-        .iter()
-        .any(|plugin| !plugin.executable_attributes().is_empty());
+    let executable_plugin = db.crate_macro_plugins(main_crate_id).iter().any(|&plugin| {
+        !db.lookup_intern_macro_plugin(plugin)
+            .executable_attributes()
+            .is_empty()
+    });
     if !executable_plugin && !compiler_config.replace_ids {
         ws.config().ui().warn(formatdoc! {r#"
             artefacts produced by this build may be hard to utilize due to the build configuration

scarb/src/compiler/db.rs

@@ -58,13 +58,13 @@ fn load_plugins(
             let package_id = plugin_info.package.id;
             let plugin = ws.config().cairo_plugins().fetch(package_id)?;
             let instance = plugin.instantiate()?;
-            builder.with_plugin_suite(instance.plugin_suite());
+            builder.with_default_plugin_suite(instance.plugin_suite());
         } else {
             proc_macros.register(plugin_info.package.clone(), ws.config())?;
         }
     }
     let macro_host = Arc::new(proc_macros.into_plugin()?);
-    builder.with_plugin_suite(ProcMacroHostPlugin::build_plugin_suite(macro_host.clone()));
+    builder.with_default_plugin_suite(ProcMacroHostPlugin::build_plugin_suite(macro_host.clone()));
     Ok(macro_host)
 }
 
@@ -198,10 +198,15 @@ fn build_project_config(unit: &CairoCompilationUnit) -> Result<ProjectConfig> {
     Ok(project_config)
 }
 
-pub(crate) fn has_starknet_plugin(db: &RootDatabase) -> bool {
-    db.macro_plugins()
+pub(crate) fn has_starknet_plugin(db: &RootDatabase, component: &CompilationUnitComponent) -> bool {
+    let crate_id = db.intern_crate(CrateLongId::Real {
+        name: component.target_name(),
+        discriminator: component.id.to_discriminator(),
+    });
+
+    db.crate_macro_plugins(crate_id)
         .iter()
-        .any(|plugin| is_starknet_plugin(&**plugin))
+        .any(|&plugin| is_starknet_plugin(&db.lookup_intern_macro_plugin(plugin)))
 }
 
 fn is_starknet_plugin(plugin: &dyn MacroPlugin) -> bool {

scarb/src/ops/compile.rs

@@ -302,7 +302,7 @@ fn check_starknet_dependency(
     //   I think we can get away with emitting false positives for users who write raw contracts
     //   without using Starknet code generators. Such people shouldn't do what they do 😁
     if unit.main_component().target_kind() == TargetKind::STARKNET_CONTRACT
-        && !has_starknet_plugin(db)
+        && !has_starknet_plugin(db, unit.main_component())
     {
         ws.config().ui().warn(formatdoc! {
             r#"

scarb/src/ops/proc_macro_server/methods/defined_macros.rs

@@ -1,8 +1,10 @@
 use std::sync::Arc;
 
 use anyhow::Result;
+use cairo_lang_utils::ordered_hash_map::OrderedHashMap;
+use itertools::Itertools;
 use scarb_proc_macro_server_types::methods::defined_macros::{
-    DefinedMacros, DefinedMacrosResponse,
+    DefinedMacros, DefinedMacrosCrateInfo, DefinedMacrosResponse,
 };
 
 use super::Handler;
@@ -13,37 +15,52 @@ impl Handler for DefinedMacros {
         proc_macro_host: Arc<ProcMacroHost>,
         _params: Self::Params,
     ) -> Result<Self::Response> {
-        let mut response = proc_macro_host
+        let crate_macro_info = proc_macro_host
             .macros()
             .iter()
-            .map(|e| DefinedMacrosResponse {
-                attributes: e.declared_attributes(),
-                inline_macros: e.inline_macros(),
-                derives: e.declared_derives(),
-                executables: e.executable_attributes(),
-            })
-            .reduce(|mut acc, defined_macros| {
-                acc.attributes.extend(defined_macros.attributes);
-                acc.inline_macros.extend(defined_macros.inline_macros);
-                acc.derives.extend(defined_macros.derives);
-                acc.executables.extend(defined_macros.executables);
+            .map(|macro_instance| {
+                let attributes = macro_instance
+                    .declared_attributes()
+                    .into_iter()
+                    .sorted()
+                    .dedup()
+                    .collect();
 
-                acc
-            })
-            .unwrap_or_default();
+                let inline_macros = macro_instance
+                    .inline_macros()
+                    .into_iter()
+                    .sorted()
+                    .dedup()
+                    .collect();
 
-        response.attributes.sort();
-        response.attributes.dedup();
+                let derives = macro_instance
+                    .declared_derives()
+                    .into_iter()
+                    .sorted()
+                    .dedup()
+                    .collect();
 
-        response.inline_macros.sort();
-        response.inline_macros.dedup();
+                let executables = macro_instance
+                    .executable_attributes()
+                    .into_iter()
+                    .sorted()
+                    .dedup()
+                    .collect();
 
-        response.derives.sort();
-        response.derives.dedup();
+                let package_name = macro_instance.package_id().name.to_smol_str();
 
-        response.executables.sort();
-        response.executables.dedup();
+                (
+                    package_name,
+                    DefinedMacrosCrateInfo {
+                        attributes,
+                        inline_macros,
+                        derives,
+                        executables,
+                    },
+                )
+            })
+            .collect::<OrderedHashMap<_, _>>();
 
-        Ok(response)
+        Ok(DefinedMacrosResponse { crate_macro_info })
     }
 }

scarb/tests/proc_macro_server.rs

@@ -37,11 +37,17 @@ fn defined_macros() {
     let response = proc_macro_server
         .request_and_wait::<DefinedMacros>(DefinedMacrosParams {})
         .unwrap();
+    let response = response.crate_macro_info;
 
-    assert_eq!(response.attributes, vec!["some".to_string()]);
-    assert_eq!(response.derives, vec!["some_derive".to_string()]);
-    assert_eq!(response.inline_macros, vec!["inline_some".to_string()]);
-    assert_eq!(response.executables, vec!["some_executable".to_string()]);
+    let packages = response.keys().collect::<Vec<_>>();
+    assert_eq!(packages.len(), 1);
+    assert_eq!(packages[0], "some");
+
+    let crate_macros = &response["some"];
+    assert_eq!(&crate_macros.attributes, &["some".to_string()]);
+    assert_eq!(&crate_macros.derives, &["some_derive".to_string()]);
+    assert_eq!(&crate_macros.inline_macros, &["inline_some".to_string()]);
+    assert_eq!(&crate_macros.executables, &["some_executable".to_string()]);
 }
 
 #[test]

utils/scarb-proc-macro-server-types/Cargo.toml

@@ -11,5 +11,8 @@ repository.workspace = true
 
 [dependencies]
 cairo-lang-macro = { version = "0.1", features = ["serde"] }
+cairo-lang-semantic.workspace = true
+cairo-lang-utils.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+smol_str.workspace = true