diff --git a/Cargo.lock b/Cargo.lock index 8d478e20d..08c5fa45c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4156,6 +4156,17 @@ dependencies = [ "yansi", ] +[[package]] +name = "priority-queue" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70c501afe3a2e25c9bd219aa56ec1e04cdb3fcdd763055be268778c13fa82c1f" +dependencies = [ + "autocfg", + "equivalent", + "indexmap 2.2.6", +] + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -4184,6 +4195,18 @@ dependencies = [ "human_format", ] +[[package]] +name = "pubgrub" +version = "0.2.1" +source = "git+https://github.com/pubgrub-rs/pubgrub.git?branch=dev#df8395771231be4d3bf6fba80e8fd9658c634c2c" +dependencies = [ + "indexmap 2.2.6", + "log", + "priority-queue", + "rustc-hash", + "thiserror", +] + [[package]] name = "quote" version = "1.0.36" @@ -4623,6 +4646,7 @@ dependencies = [ "pathdiff", "petgraph", "predicates", + "pubgrub", "ra_ap_toolchain", "redb", "reqwest", @@ -4632,6 +4656,7 @@ dependencies = [ "scarb-test-support", "scarb-ui", "semver", + "semver-pubgrub", "serde", "serde-untagged", "serde-value", @@ -4956,6 +4981,15 @@ dependencies = [ "serde", ] +[[package]] +name = "semver-pubgrub" +version = "0.1.0" +source = "git+https://github.com/pubgrub-rs/semver-pubgrub.git#da5538d43c1b718e4d032cf8a63dfc47bd8c2460" +dependencies = [ + "pubgrub", + "semver", +] + [[package]] name = "serde" version = "1.0.204" diff --git a/Cargo.toml b/Cargo.toml index 2d89c7ca2..ea9a63090 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,10 +91,12 @@ ntest = "0.9" num-bigint = { version = "0.4", features = ["rand"] } num-traits = "0.2" once_cell = "1" +once_map = "0.4" pathdiff = { version = "0.2", features = ["camino"] } petgraph = "0.6" predicates = "3" proc-macro2 = "1" +pubgrub = { git = "https://github.com/pubgrub-rs/pubgrub.git", branch = "dev" } quote = "1" ra_ap_toolchain = "0.0.218" rayon = "1.10" @@ -102,6 +104,7 @@ redb = "2.1.1" reqwest = { version = "0.11", features = ["gzip", "brotli", "deflate", "json", "stream"], default-features = false } salsa = "0.16.1" semver = { version = "1", features = ["serde"] } +semver-pubgrub = { git = "https://github.com/pubgrub-rs/semver-pubgrub.git" } serde = { version = "1", features = ["serde_derive"] } serde-untagged = "0.1" serde-value = "0.7" diff --git a/scarb/Cargo.toml b/scarb/Cargo.toml index d7eb49206..20b9d6c4a 100644 --- a/scarb/Cargo.toml +++ b/scarb/Cargo.toml @@ -44,8 +44,8 @@ directories.workspace = true dunce.workspace = true fs4.workspace = true futures.workspace = true -gix.workspace = true gix-path.workspace = true +gix.workspace = true glob.workspace = true ignore.workspace = true include_dir.workspace = true @@ -55,6 +55,7 @@ libloading.workspace = true once_cell.workspace = true pathdiff.workspace = true petgraph.workspace = true +pubgrub.workspace = true ra_ap_toolchain.workspace = true redb.workspace = true reqwest.workspace = true @@ -62,6 +63,7 @@ scarb-build-metadata = { path = "../utils/scarb-build-metadata" } scarb-metadata = { path = "../scarb-metadata", default-features = false, features = ["builder"] } scarb-stable-hash = { path = "../utils/scarb-stable-hash" } scarb-ui = { path = "../utils/scarb-ui" } +semver-pubgrub.workspace = true semver.workspace = true serde-untagged.workspace = true serde-value.workspace = true diff --git a/scarb/src/core/registry/mod.rs b/scarb/src/core/registry/mod.rs index dc1a235f0..32de99053 100644 --- a/scarb/src/core/registry/mod.rs +++ b/scarb/src/core/registry/mod.rs @@ -111,7 +111,7 @@ pub(crate) mod mock { let summary = Summary::builder() .package_id(package_id) .dependencies(dependencies) - .no_core(package_id.is_core()) + .no_core(package_id.name == PackageName::CORE) .build(); let manifest = Box::new( diff --git a/scarb/src/ops/resolve.rs b/scarb/src/ops/resolve.rs index 2edfc044f..bc435c31c 100644 --- a/scarb/src/ops/resolve.rs +++ b/scarb/src/ops/resolve.rs @@ -129,7 +129,13 @@ pub fn resolve_workspace_with_opts( read_lockfile(ws)? }; - let resolve = resolver::resolve(&members_summaries, &patched, lockfile).await?; + let resolve = resolver::resolve( + &members_summaries, + &patched, + lockfile, + ws.config().tokio_handle(), + ) + .await?; write_lockfile(Lockfile::from_resolve(&resolve), ws)?; diff --git a/scarb/src/resolver/algorithm/mod.rs b/scarb/src/resolver/algorithm/mod.rs index 8b1378917..e15e5dfc9 100644 --- a/scarb/src/resolver/algorithm/mod.rs +++ b/scarb/src/resolver/algorithm/mod.rs @@ -1 +1,64 @@ +use crate::core::lockfile::Lockfile; +use crate::core::registry::Registry; +use crate::core::{PackageId, PackageName, Resolve, Summary}; +use crate::resolver::algorithm::provider::{PubGrubDependencyProvider, PubGrubPackage}; +use crate::resolver::algorithm::solution::build_resolve; +use anyhow::bail; +use indoc::indoc; +use pubgrub::type_aliases::SelectedDependencies; +use std::collections::{HashMap, HashSet}; +use tokio::runtime::Handle; +use tokio::task::block_in_place; +mod provider; +mod solution; + +#[allow(clippy::dbg_macro)] +#[allow(dead_code)] +pub async fn resolve<'c>( + summaries: &[Summary], + registry: &dyn Registry, + _lockfile: Lockfile, + handle: &'c Handle, +) -> anyhow::Result { + let main_package_ids: HashSet = + HashSet::from_iter(summaries.iter().map(|sum| sum.package_id)); + block_in_place(|| { + let summary = summaries.iter().next().unwrap(); + let package: PubGrubPackage = summary.into(); + let version = summary.package_id.version.clone(); + let provider = PubGrubDependencyProvider::new(registry, handle, main_package_ids.clone()); + + let solution = pubgrub::solver::resolve(&provider, package, version) + .map_err(|err| anyhow::format_err!("failed to resolve: {:?}", err))?; + + dbg!(&solution); + + validate_solution(&solution)?; + build_resolve(&provider, solution) + }) +} + +fn validate_solution( + solution: &SelectedDependencies>, +) -> anyhow::Result<()> { + // Same package, different sources. + let mut seen: HashMap = Default::default(); + for pkg in solution.keys() { + if let Some(existing) = seen.get(&pkg.name) { + bail!( + indoc! {" + found dependencies on the same package `{}` coming from incompatible \ + sources: + source 1: {} + source 2: {} + "}, + pkg.name, + existing.source_id, + pkg.source_id + ); + } + seen.insert(pkg.name.clone(), pkg.clone()); + } + Ok(()) +} diff --git a/scarb/src/resolver/algorithm/provider.rs b/scarb/src/resolver/algorithm/provider.rs new file mode 100644 index 000000000..fca13f91e --- /dev/null +++ b/scarb/src/resolver/algorithm/provider.rs @@ -0,0 +1,280 @@ +use crate::core::registry::Registry; +use crate::core::{ + DependencyFilter, DependencyVersionReq, ManifestDependency, PackageId, PackageName, SourceId, + Summary, +}; +use itertools::Itertools; +use pubgrub::solver::{Dependencies, DependencyProvider}; +use pubgrub::version_set::VersionSet; +use semver::{Version, VersionReq}; +use semver_pubgrub::SemverPubgrub; +use std::cmp::Reverse; +use std::collections::{HashMap, HashSet}; +use std::fmt::Display; +use std::sync::RwLock; +use thiserror::Error; +use tokio::runtime::Handle; + +#[derive(Eq, PartialEq, Clone, Debug)] +pub struct CustomIncompatibility(String); + +impl Display for CustomIncompatibility { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct PubGrubPackage { + pub name: PackageName, + pub source_id: SourceId, +} + +impl From<&PubGrubPackage> for ManifestDependency { + fn from(package: &PubGrubPackage) -> Self { + ManifestDependency::builder() + .name(package.name.clone()) + .source_id(package.source_id) + .version_req(DependencyVersionReq::Any) + .build() + } +} + +impl From<&ManifestDependency> for PubGrubPackage { + fn from(dependency: &ManifestDependency) -> Self { + Self { + name: dependency.name.clone(), + source_id: dependency.source_id, + } + } +} + +impl From for PubGrubPackage { + fn from(package_id: PackageId) -> Self { + Self { + name: package_id.name.clone(), + source_id: package_id.source_id, + } + } +} + +impl From<&Summary> for PubGrubPackage { + fn from(summary: &Summary) -> Self { + summary.package_id.into() + } +} + +impl Display for PubGrubPackage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name) + } +} + +#[allow(dead_code)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum PubGrubPriority { + /// The package has no specific priority. + /// + /// As such, its priority is based on the order in which the packages were added (FIFO), such + /// that the first package we visit is prioritized over subsequent packages. + /// + /// TODO(charlie): Prefer constrained over unconstrained packages, if they're at the same depth + /// in the dependency graph. + Unspecified(Reverse), + + /// The version range is constrained to a single version (e.g., with the `==` operator). + Singleton(Reverse), + + /// The package was specified via a direct URL. + DirectUrl(Reverse), + + /// The package is the root package. + Root, +} + +pub struct PubGrubDependencyProvider<'a, 'c> { + handle: &'c Handle, + registry: &'a dyn Registry, + priority: RwLock>, + packages: RwLock>, + main_package_ids: HashSet, +} + +impl<'a, 'c> PubGrubDependencyProvider<'a, 'c> { + pub fn new( + registry: &'a dyn Registry, + handle: &'c Handle, + main_package_ids: HashSet, + ) -> Self { + Self { + handle, + registry, + main_package_ids, + priority: RwLock::new(HashMap::new()), + packages: RwLock::new(HashMap::new()), + } + } + + pub fn main_package_ids(&self) -> &HashSet { + &self.main_package_ids + } + + pub fn fetch_summary(&self, package_id: PackageId) -> Result { + let summary = self.packages.read().unwrap().get(&package_id).cloned(); + summary.map(Ok).unwrap_or_else(|| { + let dependency = ManifestDependency::builder() + .name(package_id.name.clone()) + .source_id(package_id.source_id) + .version_req(DependencyVersionReq::exact(&package_id.version)) + .build(); + let summary = self + .handle + .block_on(self.registry.query(&dependency)) + .map_err(DependencyProviderError::PackageQueryFailed)? + .into_iter() + .find_or_first(|summary| summary.package_id == package_id); + if let Some(summary) = summary.as_ref() { + let mut write_lock = self.packages.write().unwrap(); + write_lock.insert(summary.package_id, summary.clone()); + write_lock.insert(package_id, summary.clone()); + } + summary.ok_or_else(|| { + DependencyProviderError::PackageNotFound(dependency.name.clone().to_string()) + }) + }) + } + + fn query( + &self, + dependency: ManifestDependency, + ) -> Result, DependencyProviderError> { + let summaries = self + .handle + .block_on(self.registry.query(&dependency)) + .map_err(DependencyProviderError::PackageQueryFailed)?; + + { + let mut write_lock = self.packages.write().unwrap(); + for summary in summaries.iter() { + write_lock.insert(summary.package_id, summary.clone()); + } + } + + // Sort from highest to lowest. + let summaries = summaries + .into_iter() + .sorted_by_key(|sum| sum.package_id.version.clone()) + .rev() + .collect_vec(); + + Ok(summaries) + } +} + +impl<'a, 'c> DependencyProvider for PubGrubDependencyProvider<'a, 'c> { + type P = PubGrubPackage; + type V = Version; + type VS = SemverPubgrub; + type M = CustomIncompatibility; + + fn prioritize(&self, package: &Self::P, _range: &Self::VS) -> Self::Priority { + // Prioritize by ordering from root. + let priority = self.priority.read().unwrap().get(package).copied(); + if let Some(priority) = priority { + return Some(PubGrubPriority::Unspecified(Reverse(priority))); + } + None + } + + type Priority = Option; + type Err = DependencyProviderError; + + fn choose_version( + &self, + package: &Self::P, + range: &Self::VS, + ) -> Result, Self::Err> { + // Query available versions. + let dependency: ManifestDependency = package.into(); + let summaries = self.query(dependency)?; + + // Choose version. + let summary = summaries + .into_iter() + .find(|summary| range.contains(&summary.package_id.version)); + + // Store retrieved summary for selected version. + if let Some(summary) = summary.as_ref() { + self.packages + .write() + .unwrap() + .insert(summary.package_id, summary.clone()); + } + + Ok(summary.map(|summary| summary.package_id.version.clone())) + } + + fn get_dependencies( + &self, + package: &Self::P, + version: &Self::V, + ) -> Result, Self::Err> { + // Query summary. + let package_id = PackageId::new(package.name.clone(), version.clone(), package.source_id); + let summary = self.fetch_summary(package_id)?; + + // Set priority for dependencies. + let self_priority = self + .priority + .read() + .unwrap() + .get(&PubGrubPackage { + name: package_id.name.clone(), + source_id: package_id.source_id, + }) + .copied(); + if let Some(priority) = self_priority { + let mut write_lock = self.priority.write().unwrap(); + for dependency in summary.full_dependencies() { + let package: PubGrubPackage = dependency.into(); + write_lock.insert(package, priority + 1); + } + } + + // Convert dependencies to constraints. + let dep_filter = + DependencyFilter::propagation(self.main_package_ids.contains(&summary.package_id)); + let deps = summary + .filtered_full_dependencies(dep_filter) + .cloned() + .map(|dependency| { + let req = VersionReq::from(dependency.version_req.clone()); + let dep_name = dependency.name.clone().to_string(); + let summaries = self.query(dependency)?; + summaries + .into_iter() + .find(|summary| req.matches(&summary.package_id.version)) + .map(|summary| (summary, req)) + .ok_or_else(|| DependencyProviderError::PackageNotFound(dep_name)) + }) + .collect::, DependencyProviderError>>()?; + let constraints = deps + .into_iter() + .map(|(summary, req)| (summary.package_id.into(), SemverPubgrub::from(&req))) + .collect(); + + Ok(Dependencies::Available(constraints)) + } +} + +/// Error thrown while trying to execute `scarb` command. +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum DependencyProviderError { + /// Package not found. + #[error("failed to get package `{0}`")] + PackageNotFound(String), + // Package query failed. + #[error("package query failed: {0}")] + PackageQueryFailed(#[from] anyhow::Error), +} diff --git a/scarb/src/resolver/algorithm/solution.rs b/scarb/src/resolver/algorithm/solution.rs new file mode 100644 index 000000000..0066952d9 --- /dev/null +++ b/scarb/src/resolver/algorithm/solution.rs @@ -0,0 +1,51 @@ +use crate::core::resolver::DependencyEdge; +use crate::core::{DepKind, DependencyFilter, PackageId, Resolve, Summary, TargetKind}; +use crate::resolver::algorithm::provider::PubGrubDependencyProvider; +use petgraph::prelude::DiGraphMap; +use pubgrub::type_aliases::SelectedDependencies; +use std::collections::HashMap; + +pub fn build_resolve<'a, 'c>( + provider: &PubGrubDependencyProvider<'a, 'c>, + solution: SelectedDependencies>, +) -> anyhow::Result { + let summaries: HashMap = solution + .into_iter() + .map(|(package, version)| { + let pid = PackageId::new(package.name.clone(), version.clone(), package.source_id); + let sum = provider + .fetch_summary(pid) + .map_err(|err| anyhow::format_err!("failed to get summary: {:?}", err))?; + Ok((sum.package_id, sum)) + }) + .collect::>>()?; + + let mut graph: DiGraphMap = Default::default(); + + for pid in summaries.keys() { + graph.add_node(*pid); + } + + for summary in summaries.values() { + let dep_filter = DependencyFilter::propagation( + provider.main_package_ids().contains(&summary.package_id), + ); + for dep in summary.filtered_full_dependencies(dep_filter) { + let dep_target_kind: Option = match dep.kind.clone() { + DepKind::Normal => None, + DepKind::Target(target_kind) => Some(target_kind), + }; + let Some(dep) = summaries.keys().find(|pid| pid.name == dep.name).copied() else { + continue; + }; + let weight = graph + .edge_weight(summary.package_id, dep) + .cloned() + .unwrap_or_default(); + let weight = weight.extend(dep_target_kind); + graph.add_edge(summary.package_id, dep, weight); + } + } + + Ok(Resolve { graph, summaries }) +} diff --git a/scarb/src/resolver/mod.rs b/scarb/src/resolver/mod.rs index 93b789bc8..f387cff6c 100644 --- a/scarb/src/resolver/mod.rs +++ b/scarb/src/resolver/mod.rs @@ -1,9 +1,9 @@ -use anyhow::Result; - use crate::core::lockfile::Lockfile; use crate::core::registry::Registry; use crate::core::resolver::Resolve; use crate::core::Summary; +use anyhow::Result; +use tokio::runtime::Handle; mod algorithm; mod primitive; @@ -28,12 +28,14 @@ mod primitive; /// /// * `ui` - an [`Ui`] instance used to show warnings to the user. #[tracing::instrument(level = "trace", skip_all)] -pub async fn resolve( +pub async fn resolve<'c>( summaries: &[Summary], registry: &dyn Registry, lockfile: Lockfile, + handle: &'c Handle, ) -> Result { - primitive::resolve(summaries, registry, lockfile).await + // primitive::resolve(summaries, registry, lockfile, handle).await + algorithm::resolve(summaries, registry, lockfile, handle).await } #[cfg(test)] @@ -126,7 +128,12 @@ mod tests { .collect_vec(); let lockfile = Lockfile::new(locks.iter().cloned()); - runtime.block_on(super::resolve(&summaries, ®istry, lockfile)) + runtime.block_on(super::resolve( + &summaries, + ®istry, + lockfile, + runtime.handle(), + )) } fn package_id>(name: S) -> PackageId { @@ -253,20 +260,7 @@ mod tests { ("baz v1.0.0", []), ], &[deps![("foo", "*")]], - // TODO(#2): Expected result is commented out. - // Ok(pkgs![ - // "bar v1.0.0", - // "baz v1.0.0", - // "foo v1.0.0" - // ]), - Err(indoc! {" - Version solving failed: - - bar v2.0.0 cannot use baz v1.0.0, because bar requires baz ^2.0.0 - - Scarb does not have real version solving algorithm yet. - Perhaps in the future this conflict could be resolved, but currently, - please upgrade your dependencies to use latest versions of their dependencies. - "}), + Ok(pkgs!["bar v1.0.0", "baz v1.0.0", "foo v1.0.0"]), ) } @@ -285,20 +279,7 @@ mod tests { ("baz v2.1.0", []), ], &[deps![("bar", "~1.1.0"), ("foo", "~2.7")]], - // TODO(#2): Expected result is commented out. - // Ok(pkgs![ - // "bar v1.1.1", - // "baz v1.7.1", - // "foo v2.7.0" - // ]), - Err(indoc! {" - Version solving failed: - - foo v2.7.0 cannot use baz v2.1.0, because foo requires baz ~1.7.1 - - Scarb does not have real version solving algorithm yet. - Perhaps in the future this conflict could be resolved, but currently, - please upgrade your dependencies to use latest versions of their dependencies. - "}), + Ok(pkgs!["bar v1.1.1", "baz v1.7.1", "foo v2.7.0"]), ) } @@ -807,8 +788,8 @@ mod tests { Err(indoc! {" found dependencies on the same package `baz` coming from \ incompatible sources: - source 1: git+https://example.com/foo.git - source 2: git+https://example.com/bar.git + source 1: git+https://example.com/bar.git + source 2: git+https://example.com/foo.git "}), ) } diff --git a/scarb/src/resolver/primitive.rs b/scarb/src/resolver/primitive.rs index 6e77327ba..cb5291fe2 100644 --- a/scarb/src/resolver/primitive.rs +++ b/scarb/src/resolver/primitive.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] + use crate::core::lockfile::Lockfile; use crate::core::registry::Registry; use crate::core::resolver::DependencyEdge; @@ -9,12 +11,14 @@ use anyhow::bail; use indoc::{formatdoc, indoc}; use petgraph::graphmap::DiGraphMap; use std::collections::{HashMap, HashSet}; +use tokio::runtime::Handle; #[tracing::instrument(level = "trace", skip_all)] -pub async fn resolve( +pub async fn resolve<'c>( summaries: &[Summary], registry: &dyn Registry, lockfile: Lockfile, + _handle: &'c Handle, ) -> anyhow::Result { // TODO(#2): This is very bad, use PubGrub here. let mut graph = DiGraphMap::::new();