diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 27842d983..d99510ae8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -37,10 +37,10 @@ jobs: run: cargo build -p xtask - name: Upgrade Cairo to latest main commit - run: cargo xtask set-dep-version cairo --rev $(git ls-remote --refs "https://github.com/starkware-libs/cairo" main | awk '{print $1}') + run: cargo xtask upgrade cairo --rev $(git ls-remote --refs "https://github.com/starkware-libs/cairo" main | awk '{print $1}') - name: Upgrade CairoLS to latest main commit - run: cargo xtask set-dep-version cairols --rev $(git ls-remote --refs "https://github.com/software-mansion/cairols" main | awk '{print $1}') + run: cargo xtask upgrade cairols --rev $(git ls-remote --refs "https://github.com/software-mansion/cairols" main | awk '{print $1}') - name: Rebuild xtasks after Cargo.toml changes run: cargo build -p xtask @@ -62,7 +62,7 @@ jobs: echo "nightly_branch=$NIGHTLY_BRANCH" >> $GITHUB_OUTPUT - name: Set Scarb version build metadata - run: cargo xtask set-scarb-version --build ${{ env.NIGHTLY_TAG }} --no-pre-release + run: cargo xtask sync-version --build ${{ env.NIGHTLY_TAG }} --no-pre-release - name: Rebuild xtasks after Cargo.toml changes run: cargo build -p xtask diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b64700543..88710f301 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,7 +78,7 @@ We have a script that edits the `Cargo.toml` file to use a local checkout of the To use this tool, run: ```shell -cargo xtask set-dep-version cairo --path ../path/to/cairo +cargo xtask upgrade cairo --path ../path/to/cairo ``` And then you can `cargo build` Scarb with your custom Cairo compiler changes. diff --git a/Cargo.lock b/Cargo.lock index 01c185e56..40727a359 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1215,6 +1215,19 @@ dependencies = [ "which", ] +[[package]] +name = "cairo-toolchain-xtasks" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219b391f6f1220f5a8cc930e42823ce5f6eafb92b3111cb7ea0b5dec4f176abe" +dependencies = [ + "anyhow", + "clap", + "semver", + "toml_edit 0.22.22", + "xshell", +] + [[package]] name = "cairo-vm" version = "1.0.1" @@ -6814,11 +6827,11 @@ name = "xtask" version = "1.0.0" dependencies = [ "anyhow", + "cairo-toolchain-xtasks", "clap", "semver", "serde_json", "time", - "toml_edit 0.22.22", "walkdir", "xshell", ] diff --git a/Cargo.toml b/Cargo.toml index 9646735af..0fbab768b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,7 @@ repository = "https://github.com/software-mansion/scarb" # all tools *always* depend on some crates.io versions of Cairo crates and Scarb uses # [patch.crates.io] table to set final git revision for everything. # -# To keep our Cargo.toml following this contract, always use `cargo xtask set-dep-version` +# To keep our Cargo.toml following this contract, always use `cargo xtask upgrade` # for manipulating these dependencies. [workspace.dependencies] anyhow = "1" diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index b1eba56fb..f11f69e55 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -7,10 +7,10 @@ publish = false # NOTE: Avoid adding new dependencies at all cost. The more complex building of this crate is, the much slower CI is. [dependencies] anyhow.workspace = true +cairo-toolchain-xtasks = "1" clap.workspace = true semver.workspace = true serde_json.workspace = true time.workspace = true -toml_edit.workspace = true walkdir.workspace = true xshell.workspace = true diff --git a/xtask/src/get_nightly_version.rs b/xtask/src/get_nightly_version.rs index b7c78f72a..52be3e77c 100644 --- a/xtask/src/get_nightly_version.rs +++ b/xtask/src/get_nightly_version.rs @@ -1,10 +1,9 @@ use anyhow::Result; +use cairo_toolchain_xtasks::sync_version::expected_version; use clap::Parser; use semver::{BuildMetadata, Prerelease, Version}; use time::OffsetDateTime; -use crate::set_scarb_version::expected_scarb_version; - #[derive(Parser)] pub struct Args { #[arg(short, long)] @@ -23,7 +22,7 @@ pub fn main(args: Args) -> Result<()> { } pub fn nightly_version() -> Result { - let mut version = expected_scarb_version()?; + let mut version = expected_version()?; version.pre = Prerelease::EMPTY; version.build = BuildMetadata::new(&nightly_tag()).unwrap(); Ok(version) @@ -31,5 +30,10 @@ pub fn nightly_version() -> Result { pub fn nightly_tag() -> String { let dt = OffsetDateTime::now_utc(); - format!("nightly-{}-{:0>2}-{:0>2}", dt.year(), u8::from(dt.month()), dt.day()) + format!( + "nightly-{}-{:0>2}-{:0>2}", + dt.year(), + u8::from(dt.month()), + dt.day() + ) } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index c0c827609..2aa06cd73 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -26,8 +26,8 @@ command!(Command( get_nightly_version, list_binaries, nightly_release_notes, - set_dep_version, - set_scarb_version, + sync_version, + upgrade, verify_archive, )); diff --git a/xtask/src/set_dep_version.rs b/xtask/src/set_dep_version.rs deleted file mode 100644 index 2065b7ba1..000000000 --- a/xtask/src/set_dep_version.rs +++ /dev/null @@ -1,279 +0,0 @@ -use crate::set_scarb_version; -use anyhow::Result; -use clap::{Parser, ValueEnum}; -use semver::Version; -use std::mem; -use std::path::PathBuf; -use toml_edit::{DocumentMut, InlineTable, Value}; -use xshell::{cmd, Shell}; - -/// Update toolchain crates properly. -#[derive(Parser)] -pub struct Args { - /// Name of toolchain dependency (group) to update. - dep: DepName, - - #[command(flatten)] - spec: Spec, - - /// Do not edit any files, just inform what would be done. - #[arg(long, default_value_t = false)] - dry_run: bool, -} - -#[derive(ValueEnum, Copy, Clone, Debug)] -enum DepName { - Cairo, - #[value(name = "cairols")] - CairoLS, -} - -#[derive(clap::Args, Clone)] -#[group(required = true, multiple = true)] -struct Spec { - /// Source the dependency from crates.io and use a specific version. - version: Option, - - /// Source the dependency from the GitHub repository and use a specific commit/ref. - #[arg(short, long, conflicts_with = "branch")] - rev: Option, - - /// Source the dependency from the GitHub repository and use a specific branch. - #[arg(short, long)] - branch: Option, - - /// Source the dependency from a local filesystem. - /// - /// This is useful for local development, but avoid commiting this to the repository. - #[arg(short, long, conflicts_with_all = ["rev", "branch"])] - path: Option, -} - -pub fn main(args: Args) -> Result<()> { - let sh = Shell::new()?; - - let mut cargo_toml = sh.read_file("Cargo.toml")?.parse::()?; - - edit_dependencies(&mut cargo_toml, &args); - edit_patch(&mut cargo_toml, &args); - - if !args.dry_run { - sh.write_file("Cargo.toml", cargo_toml.to_string())?; - - cmd!(sh, "cargo fetch").run()?; - - purge_unused_patches(&mut cargo_toml)?; - sh.write_file("Cargo.toml", cargo_toml.to_string())?; - - eprintln!("$ cargo xtask set-scarb-version"); - set_scarb_version::main(Default::default())?; - } - - Ok(()) -} - -fn edit_dependencies(cargo_toml: &mut DocumentMut, args: &Args) { - let deps = cargo_toml["workspace"]["dependencies"] - .as_table_mut() - .unwrap(); - - for (_, dep) in deps.iter_mut().filter(|(key, _)| args.dep.owns(key)) { - let dep = dep.as_value_mut().unwrap(); - - // Always use crates.io requirements so that we can reliably patch them with the - // `[patch.crates-io]` table. - let mut new_dep = InlineTable::from_iter([( - "version", - match &args.spec.version { - Some(version) => Value::from(version.to_string()), - None => Value::from("*"), - }, - )]); - - copy_dependency_features(&mut new_dep, dep); - - *dep = new_dep.into(); - simplify_dependency_table(dep) - } - - deps.fmt(); - deps.sort_values(); - - eprintln!("[workspace.dependencies]"); - for (key, dep) in deps.iter().filter(|(key, _)| args.dep.owns(key)) { - eprintln!("{key} = {dep}"); - } -} - -fn edit_patch(cargo_toml: &mut DocumentMut, args: &Args) { - let patch = cargo_toml["patch"].as_table_mut().unwrap()["crates-io"] - .as_table_mut() - .unwrap(); - - // Clear any existing entries for this dependency. - for crate_name in args.dep.crates() { - patch.remove(crate_name); - } - - // Leave this section as-if if we are requested to just use a specific version. - if args.spec.rev.is_some() || args.spec.branch.is_some() || args.spec.path.is_some() { - // Patch all Cairo crates that exist, even if this project does not directly depend on them, - // to avoid any duplicates in transient dependencies. - for &dep_name in args.dep.crates() { - let mut dep = InlineTable::new(); - - // Add a Git branch or revision reference if requested. - if args.spec.rev.is_some() || args.spec.branch.is_some() { - dep.insert("git", args.dep.repo().into()); - } - - if let Some(branch) = &args.spec.branch { - dep.insert("branch", branch.as_str().into()); - } - - if let Some(rev) = &args.spec.rev { - dep.insert("rev", rev.as_str().into()); - } - - // Add local path reference if requested. - // For local path sources, Cargo is not looking for crates recursively therefore, we - // need to manually provide full paths to Cairo workspace member crates. - if let Some(path) = &args.spec.path { - dep.insert( - "path", - path.join("crates") - .join(dep_name) - .to_string_lossy() - .into_owned() - .into(), - ); - } - - patch.insert(dep_name, dep.into()); - } - } - - patch.fmt(); - patch.sort_values(); - - eprintln!("[patch.crates-io]"); - for (key, dep) in patch.iter() { - eprintln!("{key} = {dep}"); - } -} - -impl DepName { - fn crates(&self) -> &'static [&'static str] { - match self { - DepName::Cairo => { - // List of library crates published from the starkware-libs/cairo repository. - // One can get this list from the `scripts/release_crates.sh` script in that repo. - // Keep this list sorted for better commit diffs. - &[ - "cairo-lang-casm", - "cairo-lang-compiler", - "cairo-lang-debug", - "cairo-lang-defs", - "cairo-lang-diagnostics", - "cairo-lang-doc", - "cairo-lang-eq-solver", - "cairo-lang-executable", - "cairo-lang-filesystem", - "cairo-lang-formatter", - "cairo-lang-lowering", - "cairo-lang-parser", - "cairo-lang-plugins", - "cairo-lang-proc-macros", - "cairo-lang-project", - "cairo-lang-runnable-utils", - "cairo-lang-runner", - "cairo-lang-semantic", - "cairo-lang-sierra", - "cairo-lang-sierra-ap-change", - "cairo-lang-sierra-gas", - "cairo-lang-sierra-generator", - "cairo-lang-sierra-to-casm", - "cairo-lang-sierra-type-size", - "cairo-lang-starknet", - "cairo-lang-starknet-classes", - "cairo-lang-syntax", - "cairo-lang-syntax-codegen", - "cairo-lang-test-plugin", - "cairo-lang-test-runner", - "cairo-lang-test-utils", - "cairo-lang-utils", - ] - } - DepName::CairoLS => &["cairo-language-server"], - } - } - - fn owns(&self, crate_name: &str) -> bool { - self.crates().contains(&crate_name) - } - - fn repo(&self) -> &'static str { - match self { - DepName::Cairo => "https://github.com/starkware-libs/cairo", - DepName::CairoLS => "https://github.com/software-mansion/cairols", - } - } -} - -/// Copies features from source dependency spec to new dependency table, if exists. -fn copy_dependency_features(dest: &mut InlineTable, src: &Value) { - if let Some(dep) = src.as_inline_table() { - if let Some(features) = dep.get("features") { - dest.insert("features", features.clone()); - } - } -} - -/// Simplifies a `{ version = "V" }` dependency spec to shorthand `"V"` if possible. -fn simplify_dependency_table(dep: &mut Value) { - *dep = match mem::replace(dep, false.into()) { - Value::InlineTable(mut table) => { - if table.len() == 1 { - table.remove("version").unwrap_or_else(|| table.into()) - } else { - table.into() - } - } - - dep => dep, - } -} - -/// Remove any unused patches from the `[patch.crates-io]` table. -/// -/// We are adding patch entries for **all** Cairo crates existing, and some may end up being unused. -/// Cargo is emitting warnings about unused patches and keeps a record of them in the `Cargo.lock`. -/// The goal of this function is to resolve these warnings. -fn purge_unused_patches(cargo_toml: &mut DocumentMut) -> Result<()> { - let sh = Shell::new()?; - let cargo_lock = sh.read_file("Cargo.lock")?.parse::()?; - - if let Some(unused_patches) = find_unused_patches(&cargo_lock) { - let patch = cargo_toml["patch"].as_table_mut().unwrap()["crates-io"] - .as_table_mut() - .unwrap(); - - // Remove any patches that are not for Cairo crates. - patch.retain(|key, _| !unused_patches.contains(&key.to_owned())); - } - - Ok(()) -} - -/// Extracts names of unused patches from the `[[patch.unused]]` array from the `Cargo.lock` file. -fn find_unused_patches(cargo_lock: &DocumentMut) -> Option> { - Some( - cargo_lock - .get("patch")? - .get("unused")? - .as_array_of_tables()? - .iter() - .flat_map(|table| Some(table.get("name")?.as_str()?.to_owned())) - .collect(), - ) -} diff --git a/xtask/src/set_scarb_version.rs b/xtask/src/set_scarb_version.rs deleted file mode 100644 index 53fc7e591..000000000 --- a/xtask/src/set_scarb_version.rs +++ /dev/null @@ -1,67 +0,0 @@ -use anyhow::{ensure, Result}; -use clap::Parser; -use semver::{Prerelease, Version}; -use toml_edit::{value, DocumentMut}; -use xshell::{cmd, Shell}; - -pub fn expected_scarb_version() -> Result { - // NOTE: We are reading lockfile manually here, so that we are not dependent on when this - // program was built (that would be the case when using scarb-build-metadata). We are also - // deliberately not using cargo_metadata, to reduce build times of xtasks. - - let sh = Shell::new()?; - let cargo_lock = sh.read_file("Cargo.lock")?.parse::()?; - let packages = cargo_lock["package"].as_array_of_tables().unwrap(); - let compiler = { - let pkgs = packages - .into_iter() - .filter(|pkg| pkg["name"].as_str().unwrap() == "cairo-lang-compiler") - .collect::>(); - ensure!( - pkgs.len() == 1, - "expected exactly one cairo-lang-compiler package in Cargo.lock, found: {}", - pkgs.len() - ); - pkgs.into_iter().next().unwrap() - }; - let compiler_version = compiler["version"].as_str().unwrap(); - Ok(compiler_version.parse()?) -} - -#[derive(Default, Parser)] -pub struct Args { - #[arg(long)] - pub build: Option, - #[arg(long, default_value_t = false)] - pub dry_run: bool, - #[arg(long, default_value_t = false)] - pub no_pre_release: bool, -} - -pub fn main(args: Args) -> Result<()> { - let sh = Shell::new()?; - - let mut cargo_toml = sh.read_file("Cargo.toml")?.parse::()?; - let package = cargo_toml["workspace"]["package"].as_table_mut().unwrap(); - - let mut version = expected_scarb_version()?; - - if let Some(build) = args.build { - version.build = build.parse()?; - } - if args.no_pre_release { - version.pre = Prerelease::EMPTY; - } - - package["version"] = value(version.to_string()); - - eprintln!("[workspace.package]\n{package}"); - - if !args.dry_run { - sh.write_file("Cargo.toml", cargo_toml.to_string())?; - - cmd!(sh, "cargo fetch").run()?; - } - - Ok(()) -} diff --git a/xtask/src/sync_version.rs b/xtask/src/sync_version.rs new file mode 100644 index 000000000..b4aade474 --- /dev/null +++ b/xtask/src/sync_version.rs @@ -0,0 +1 @@ +pub use cairo_toolchain_xtasks::sync_version::{main, Args}; diff --git a/xtask/src/upgrade.rs b/xtask/src/upgrade.rs new file mode 100644 index 000000000..c1fbc1011 --- /dev/null +++ b/xtask/src/upgrade.rs @@ -0,0 +1 @@ +pub use cairo_toolchain_xtasks::upgrade::{main, Args};