Update deps and replace tokio/reqwest with smol/isahc

Signed-off-by: Mohammad AlSaleh <[email protected]>

Author: MoSal

Cargo.toml

@@ -11,16 +11,15 @@ clap = {version = "2" , features = ["yaml"]}
 time = "0.1"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-semver = "0.10"
+semver = "0.11"
 regex = "1"
 lazy_static = "1"
-reqwest = { version = "0.10", features = ["gzip"] }
-bytes = "0.5"
-tokio = { version = "0.2", features = ["full"] }
+isahc = { version = "1", default_features = false, features = ["http2"]}
+smol = "1"
 async-trait = "0.1"
 futures = "0.3"
 fern = "0.6"
 chrono = "0.4"
 log = "0.4"
-crates-index = "0.14"
+crates-index = "0.16"
 once_cell = "1"

src/bin/cargo-esr.rs

@@ -161,9 +161,9 @@ async fn run() {
                 },
                 false => match (search_by_relevance, search_by_total_downloads, search_by_recent_downloads) {
                     // default
-                    (false, false, _) => "per_page=".to_string() + search_limit + "&q=" + &search_str + "&sort=recent-downloads",
-                    (_, true, _)      => "per_page=".to_string() + search_limit + "&q=" + &search_str + "&sort=downloads",
-                    (true, _, _)      => "per_page=".to_string() + search_limit + "&q=" + &search_str,
+                    (false, false, _) => "per_page=".to_string() + search_limit + "&q=" + &*search_str + "&sort=recent-downloads",
+                    (_, true, _)      => "per_page=".to_string() + search_limit + "&q=" + &*search_str + "&sort=downloads",
+                    (true, _, _)      => "per_page=".to_string() + search_limit + "&q=" + &*search_str,
                 },
 
             };
@@ -177,19 +177,7 @@ async fn run() {
                         std::process::exit(1);
                     }
 
-                    let crates_scores_res = match Scores::collect_scores(
-                        crates,
-                        &gh_token,
-                        crate_only,
-                        repo_only)
-                        .await {
-                            Ok(res) => res,
-                            Err(_) => {
-                                EsrPrinter::err("A tokio task or more returned errors.");
-                                std::process::exit(1);
-                            },
-                    };
-
+                    let crates_scores_res = Scores::collect_scores(crates, &gh_token, crate_only, repo_only).await;
                     Scores::search_results(&*crates_scores_res, sort_positive, results_limit_num).println();
                 },
                 Err(ref e) => {
@@ -204,20 +192,5 @@ async fn run() {
 }
 
 fn main() {
-
-    // build runtime
-    let runtime_res = tokio::runtime::Builder::new()
-        .enable_all()
-        //.basic_scheduler()
-        .threaded_scheduler()
-        .core_threads(16)
-        .build();
-
-    match runtime_res {
-        Ok(mut runtime) => runtime.block_on(run()),
-        Err(_) => {
-            EsrPrinter::err("Failed to create tokio runtime.").println();
-            std::process::exit(1);
-        },
-    }
+    smol::block_on(run());
 }

src/esr_crate.rs

@@ -14,7 +14,6 @@ use semver::{Version, VersionReq};
 use serde::Deserialize;
 use async_trait::async_trait;
 use futures::future;
-use tokio::task;
 use once_cell::sync::OnceCell;
 
 use crate::esr_errors::{Result, EsrError};
@@ -107,26 +106,28 @@ struct Dependant {
 impl Dependant {
     async fn dependants_from_id(id: String) -> Result<Vec<Self>> {
         log::debug!("Getting dependats from index for {}", id);
-        let mut ret = Vec::with_capacity(64);
-        for cr in get_index()?.crates() {
-            let latest_version = cr.latest_version();
-            let match_opt = latest_version
-                .dependencies()
-                .iter()
-                .find(|dep| !latest_version.is_yanked() && dep.crate_name() == id);
-
-            if let Some(dep) = match_opt {
-                ret.push(
-                    Self {
-                        crate_name: cr.name().into(),
-                        default_features: dep.has_default_features() && !dep.is_optional(),
-                        optional: dep.is_optional(),
-                        req: dep.requirement().into(),
-                    }
-                );
+        smol::spawn( async move {
+            let mut ret = Vec::with_capacity(64);
+            for cr in get_index()?.crates() {
+                let latest_version = cr.latest_version();
+                let match_opt = latest_version
+                    .dependencies()
+                    .iter()
+                    .find(|dep| !latest_version.is_yanked() && dep.crate_name() == id);
+
+                if let Some(dep) = match_opt {
+                    ret.push(
+                        Self {
+                            crate_name: cr.name().into(),
+                            default_features: dep.has_default_features() && !dep.is_optional(),
+                            optional: dep.is_optional(),
+                            req: dep.requirement().into(),
+                        }
+                    );
+                }
             }
-        }
-        Ok(ret)
+            Ok(ret)
+        }).await
     }
 }
 
@@ -288,8 +289,8 @@ impl CrateScoreInfo {
     async fn from_crate_info(crate_info: &CrateInfo) -> Result<Self> {
         let general_info = &crate_info.general_info;
 
-        let owners_info_fut = task::spawn(CrateOwners::from_id_owned(general_info.id.clone()));
-        let dependants_info_fut = task::spawn(Dependant::dependants_from_id(general_info.id.clone()));
+        let owners_info_fut = smol::spawn(CrateOwners::from_id_owned(general_info.id.clone()));
+        let dependants_info_fut = smol::spawn(Dependant::dependants_from_id(general_info.id.clone()));
 
         let has_desc = general_info.description.is_some() as usize;
         let has_docs = general_info.documentation.is_some() as usize;
@@ -315,8 +316,8 @@ impl CrateScoreInfo {
             None => esr_util::age_in_months(&esr_util::crate_to_iso8601(&general_info.created_at))?,
         };
 
-        let dependants_info = dependants_info_fut.await??;
-        let owners_info = owners_info_fut.await??;
+        let dependants_info = dependants_info_fut.await?;
+        let owners_info = owners_info_fut.await?;
 
         // Reverse dependencies
         let dependants = dependants_info.len();
@@ -347,13 +348,13 @@ impl CrateScoreInfo {
 
         let owners_crates = owners_ids
             .into_iter()
-            .map(|id| task::spawn(CrateSearch::from_id_owned(id)));
+            .map(|id| smol::spawn(CrateSearch::from_id_owned(id)));
 
         let owners_crates = future::join_all(owners_crates)
             .await
             .into_iter()
             .map(|t| t.map_err(|e| e.into()))
-            .collect::<Result<Result<Vec<_>>>>()??;
+            .collect::<Result<Vec<_>>>()?;
 
         let owners_crates_flat: Vec<_> = owners_crates.iter()
             .flat_map(|search| search.crates.iter())

src/esr_errors.rs

@@ -19,9 +19,8 @@ pub enum EsrError {
     TimeParse(time::ParseError),
     SerdeJson(serde_json::Error),
     Regex(regex::Error),
-    Reqwest(reqwest::Error),
+    Isahc(isahc::Error),
     CratesIndex(String),
-    TokioTaskJoin(tokio::task::JoinError),
     Other(String),
 }
 
@@ -32,9 +31,8 @@ impl fmt::Display for EsrError {
             EsrError::TimeParse(ref e) => write!(f, "Time parsing Error: {}", e),
             EsrError::SerdeJson(ref e) => write!(f, "Deserialization Error: {}", e),
             EsrError::Regex(ref e) => write!(f, "Regex Error: {}", e),
-            EsrError::Reqwest(ref e) => write!(f, "Reqwest Error: {}", e),
+            EsrError::Isahc(ref e) => write!(f, "isahc Error: {}", e),
             EsrError::CratesIndex(ref e) => write!(f, "CratesIndex Error: {}", e),
-            EsrError::TokioTaskJoin(ref e) => write!(f, "tokio task join Error: {}", e),
             EsrError::Other(ref e) => write!(f, "Error: {}", e),
         }
     }
@@ -64,15 +62,9 @@ impl From<regex::Error> for EsrError {
     }
 }
 
-impl From<tokio::task::JoinError> for EsrError {
-    fn from(e: tokio::task::JoinError) -> Self {
-        EsrError::TokioTaskJoin(e)
-    }
-}
-
-impl From<reqwest::Error> for EsrError {
-    fn from(e: reqwest::Error) -> Self {
-        EsrError::Reqwest(e)
+impl From<isahc::Error> for EsrError {
+    fn from(e: isahc::Error) -> Self {
+        EsrError::Isahc(e)
     }
 }
 

src/esr_from.rs

@@ -9,14 +9,29 @@
     file, You can obtain one at <http://mozilla.org/MPL/2.0/>.
 */
 
+use once_cell::sync::OnceCell;
 use serde::{Deserialize, de::DeserializeOwned};
-use reqwest::Client;
-use bytes::Bytes;
+use isahc::{HttpClientBuilder, HttpClient, AsyncReadResponseExt};
+use isahc::config::{Configurable, RedirectPolicy};
 use async_trait::async_trait;
 use futures::future;
 
 use crate::esr_errors::Result;
 
+fn get_static_client() -> Result<&'static HttpClient> {
+    static RET: OnceCell<HttpClient> = OnceCell::new();
+    let init = || HttpClientBuilder::new()
+        //.connect_timeout(Duration::from_secs(5))
+        .max_connections(32)
+        .redirect_policy(RedirectPolicy::Limit(5))
+        .auto_referer()
+        .default_header("User-Agent", "cargo-esr/0.1")
+        .build();
+    let ret = RET.get_or_try_init(init)?;
+    Ok(ret)
+
+}
+
 #[derive(Deserialize, Debug, Clone)]
 pub struct Meta {
     pub total: usize,
@@ -36,7 +51,7 @@ pub trait EsrFromMulti: EsrFrom + Sync + Send + 'static {
             let num_pages = (total as f64 / 100.0).ceil() as usize;
 
             let more_pages_iter = (2..=num_pages)
-                .map(|page| url.to_owned() + &format!("&page={}", page))
+                .map(|page| url.to_owned() + &*format!("&page={}", page))
                 .map(|page_url| Self::from_url_owned(page_url));
 
 
@@ -84,30 +99,25 @@ pub trait EsrFrom: Sized + Sync + Send + DeserializeOwned {
         }
     }
 
-    async fn bytes_from_url(url: &str) -> Result<Bytes> {
-        let client = Client::builder().build()?;
-
+    async fn bytes_from_url(url: &str) -> Result<Vec<u8>> {
+        let client = get_static_client()?;
         log::debug!("Getting data from '{}'", url);
 
         // Creating an outgoing request.
-        let ret = client.get(url)
-            .header("user-agent", "cargo-esr/0.1")
-            .send()
-            .await?
-            .error_for_status()?
-            .bytes()
-            .await?;
+        let mut response = client.get_async(url).await?;
+        let mut buf = Vec::with_capacity(64*1024);
+        response.copy_to(&mut buf).await?;
 
-        log::debug!("Got data from '{}' (len={})", url, ret.len());
+        log::debug!("Got data from '{}' (len={})", url, buf.len());
 
-        Ok(ret)
+        Ok(buf)
     }
 
-    async fn bytes_from_id(id: &str) -> Result<Bytes> {
+    async fn bytes_from_id(id: &str) -> Result<Vec<u8>> {
         Self::bytes_from_url(&*Self::url_from_id(id)).await
     }
 
-    async fn bytes_from_id_with_token(id: &str, token: &str) -> Result<Bytes> {
+    async fn bytes_from_id_with_token(id: &str, token: &str) -> Result<Vec<u8>> {
         Self::bytes_from_url(&*Self::url_from_id_and_token(id, token)).await
     }
 

src/esr_github.rs

@@ -10,7 +10,6 @@
 */
 
 use serde::Deserialize;
-use tokio::task;
 
 use crate::esr_from::EsrFrom;
 use crate::esr_util;
@@ -94,16 +93,16 @@ impl RepoInfo {
 
     pub async fn from_id_with_token(id: String, token: String) -> Result<Self> {
         // pulls is slow, so we spawn it first
-        let last_100_pull_requests_fut = task::spawn(RepoPullRequests::from_id_with_token_owned(id.clone(), token.clone()));
-        let last_100_closed_issues_fut = task::spawn(RepoClosedIssues::from_id_with_token_owned(id.clone(), token.clone()));
-        let top_100_contributors_fut = task::spawn(RepoContributors::from_id_with_token_owned(id.clone(), token.clone()));
-        let general_info_fut = task::spawn(RepoGeneralInfo::from_id_with_token_owned(id, token));
+        let last_100_pull_requests_fut = smol::spawn(RepoPullRequests::from_id_with_token_owned(id.clone(), token.clone()));
+        let last_100_closed_issues_fut = smol::spawn(RepoClosedIssues::from_id_with_token_owned(id.clone(), token.clone()));
+        let top_100_contributors_fut = smol::spawn(RepoContributors::from_id_with_token_owned(id.clone(), token.clone()));
+        let general_info_fut = smol::spawn(RepoGeneralInfo::from_id_with_token_owned(id, token));
 
         Ok(Self {
-            general_info: general_info_fut.await??,
-            last_100_closed_issues: last_100_closed_issues_fut.await??,
-            last_100_pull_requests: last_100_pull_requests_fut.await??,
-            top_100_contributors: top_100_contributors_fut.await??,
+            general_info: general_info_fut.await?,
+            last_100_closed_issues: last_100_closed_issues_fut.await?,
+            last_100_pull_requests: last_100_pull_requests_fut.await?,
+            top_100_contributors: top_100_contributors_fut.await?,
         })
     }
 }

src/esr_printer.rs

@@ -131,7 +131,7 @@ impl EsrPrinter {
             } else {
                 score_formatted += sep() + TermString::new(YELLOW_BOLD(), &*line.0) + sep();
                 score_formatted += TermString::new(GREEN_BOLD(), &*line.1) + sep();
-                score_formatted += TermString::new(GREEN_BOLD(), format!("{: ^11}", "+".to_string() + &line.2)) + sep() + "\n";
+                score_formatted += TermString::new(GREEN_BOLD(), format!("{: ^11}", "+".to_string() + &*line.2)) + sep() + "\n";
                 score_formatted += frame_line();
             }
         }

src/esr_score.rs

@@ -17,7 +17,6 @@ use crate::esr_util;
 use crate::esr_errors::Result;
 
 use term_string::TermString;
-use tokio::task;
 
 use std::f64;
 use std::default::Default;
@@ -34,12 +33,12 @@ impl Scores {
 
         let repo_score_res = cr_info.github_id()
             .ok_or("Failed to get GitHub id")
-            .map(|gh_id| task::spawn(RepoInfoWithScore::from_id_with_token(gh_id, gh_token)));
+            .map(|gh_id| smol::spawn(RepoInfoWithScore::from_id_with_token(gh_id, gh_token)));
 
         let cr_score = CrateInfoWithScore::from_info(cr_info).await?;
 
         match repo_score_res {
-            Ok(repo_score) => Ok(Scores::CrateAndRepo(cr_score, repo_score.await?)),
+            Ok(repo_score) => Ok(Scores::CrateAndRepo(cr_score, repo_score.await)),
             Err(_) => Ok(Scores::CrateOnly(cr_score)),
         }
     }
@@ -126,33 +125,30 @@ impl Scores {
 
     pub async fn collect_scores(crates: &[CrateGeneralInfo], token: &str,
                           crate_only: bool,
-                          repo_only: bool) -> Result<Vec<(String, Result<Self>)>> {
+                          repo_only: bool) -> Vec<(String, Result<Self>)> {
 
         let task_iter = if crate_only {
             crates
                 .iter()
                 .map(|cr| String::from(cr.get_id()))
-                .map(|id| task::spawn(async { (id.clone(), Scores::from_id_crate_only(id).await) }))
+                .map(|id| smol::spawn(async { (id.clone(), Scores::from_id_crate_only(id).await) }))
                 .collect::<Vec<_>>()
 
         } else if repo_only {
             crates
                 .iter()
                 .map(|cr| (String::from(cr.get_id()), String::from(token)))
-                .map(|(id, token)| task::spawn(async { (id.clone(), Scores::from_id_with_token_repo_only(id, token).await) }))
+                .map(|(id, token)| smol::spawn(async { (id.clone(), Scores::from_id_with_token_repo_only(id, token).await) }))
                 .collect::<Vec<_>>()
         } else {
             crates
                 .iter()
                 .map(|cr| (String::from(cr.get_id()), String::from(token)))
-                .map(|(id, token)| task::spawn(async { (id.clone(), Scores::from_id_with_token(id, token).await) }))
+                .map(|(id, token)| smol::spawn(async { (id.clone(), Scores::from_id_with_token(id, token).await) }))
                 .collect::<Vec<_>>()
         };
 
         futures::future::join_all(task_iter).await
-            .into_iter()
-            .map(|res| res.map_err(|e| e.into()))
-            .collect::<Result<Vec<_>>>()
     }
 
     fn info_pair(&self, id: &str, sort_positive: bool) -> (f64, TermString) {