chore: Bump hf-hub to 0.4.1 (#142)

feat: bump hf-hub to 0.4.1

.github/workflows/test.yml

@@ -34,7 +34,7 @@ jobs:
           cd ..
 
       - name: Cargo Test With Release Build
-        run: ORT_LIB_LOCATION="$(pwd)/onnxruntime/build/Linux/Release" cargo test --release --no-default-features --features online
+        run: ORT_LIB_LOCATION="$(pwd)/onnxruntime/build/Linux/Release" cargo test --release --no-default-features --features hf-hub-native-tls
 
       - name: Cargo Test Offline
         run: ORT_LIB_LOCATION="$(pwd)/onnxruntime/build/Linux/Release" cargo test --no-default-features

Cargo.toml

@@ -23,21 +23,29 @@ homepage = "https://crates.io/crates/fastembed"
 
 [dependencies]
 anyhow = { version = "1" }
-hf-hub = { version = "0.3", default-features = false }
+hf-hub = { version = "0.4.1", default-features = false, optional = true }
 image = "0.25.2"
 ndarray = { version = "0.16", default-features = false }
 ort = { version = "=2.0.0-rc.9", default-features = false, features = [
   "ndarray",
 ] }
 rayon = { version = "1.10", default-features = false }
 serde_json = { version = "1" }
-tokenizers = { version = "0.19", default-features = false, features = ["onig"] }
+tokenizers = { version = "0.21", default-features = false, features = ["onig"] }
 
 [features]
-default = ["ort-download-binaries", "online"]
-online = ["hf-hub/online"]
+default = ["ort-download-binaries", "hf-hub-native-tls"]
+
+hf-hub = ["dep:hf-hub", "hf-hub?/ureq"]
+hf-hub-native-tls = ["hf-hub", "hf-hub?/native-tls"]
+hf-hub-rustls-tls = ["hf-hub", "hf-hub?/rustls-tls"]
+
 ort-download-binaries = ["ort/download-binaries"]
 ort-load-dynamic = ["ort/load-dynamic"]
+
 # This feature does not change any code, but is used to limit tests if
 # the user does not have `optimum-cli` or even python installed.
 optimum-cli = []
+
+# For compatibility recommend using hf-hub-native-tls
+online = ["hf-hub-native-tls"]

src/common.rs

@@ -1,5 +1,5 @@
 use anyhow::Result;
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use hf_hub::api::sync::ApiRepo;
 use std::io::Read;
 use std::{fs::File, path::PathBuf};
@@ -29,7 +29,7 @@ pub struct TokenizerFiles {
 
 /// The procedure for loading tokenizer files from the hugging face hub is separated
 /// from the main load_tokenizer function (which is expecting bytes, from any source).
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 pub fn load_tokenizer_hf_hub(model_repo: ApiRepo, max_length: usize) -> Result<Tokenizer> {
     let tokenizer_files: TokenizerFiles = TokenizerFiles {
         tokenizer_file: read_file_to_bytes(&model_repo.get("tokenizer.json")?)?,
@@ -49,7 +49,7 @@ pub fn load_tokenizer(tokenizer_files: TokenizerFiles, max_length: usize) -> Res
     let base_error_message =
         "Error building TokenizerFiles for UserDefinedEmbeddingModel. Could not read {} file.";
 
-    // Serialise each tokenizer file
+    // Deserialize each tokenizer file
     let config: serde_json::Value =
         serde_json::from_slice(&tokenizer_files.config_file).map_err(|_| {
             std::io::Error::new(

src/image_embedding/impl.rs

@@ -1,4 +1,4 @@
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use hf_hub::{
     api::sync::{ApiBuilder, ApiRepo},
     Cache,
@@ -8,7 +8,7 @@ use ort::{
     session::{builder::GraphOptimizationLevel, Session},
     value::Value,
 };
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use std::path::PathBuf;
 use std::{path::Path, thread::available_parallelism};
 
@@ -17,10 +17,10 @@ use crate::{
     ModelInfo,
 };
 use anyhow::anyhow;
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use anyhow::Context;
 
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use super::ImageInitOptions;
 use super::{
     init::{ImageInitOptionsUserDefined, UserDefinedImageEmbeddingModel},
@@ -35,7 +35,7 @@ impl ImageEmbedding {
     /// Uses the highest level of Graph optimization
     ///
     /// Uses the total number of CPUs available as the number of intra-threads
-    #[cfg(feature = "online")]
+    #[cfg(feature = "hf-hub")]
     pub fn try_new(options: ImageInitOptions) -> anyhow::Result<Self> {
         let ImageInitOptions {
             model_name,
@@ -104,7 +104,7 @@ impl ImageEmbedding {
     }
 
     /// Return the ImageEmbedding model's directory from cache or remote retrieval
-    #[cfg(feature = "online")]
+    #[cfg(feature = "hf-hub")]
     fn retrieve_model(
         model: ImageEmbeddingModel,
         cache_dir: PathBuf,

src/image_embedding/utils.rs

@@ -2,7 +2,7 @@ use anyhow::{anyhow, Result};
 use image::{imageops::FilterType, DynamicImage, GenericImageView};
 use ndarray::{Array, Array3};
 use std::ops::{Div, Sub};
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use std::{fs::read_to_string, path::Path};
 
 pub enum TransformData {
@@ -171,7 +171,7 @@ impl Compose {
         Self { transforms }
     }
 
-    #[cfg(feature = "online")]
+    #[cfg(feature = "hf-hub")]
     pub fn from_file<P: AsRef<Path>>(file: P) -> anyhow::Result<Self> {
         let content = read_to_string(file)?;
         let config = serde_json::from_str(&content)?;

src/lib.rs

@@ -3,7 +3,7 @@
 //! The library provides the TextEmbedding struct to interface with text embedding models.
 //!
 #![cfg_attr(
-    feature = "online",
+    feature = "hf-hub",
     doc = r#"
  ### Instantiating [TextEmbedding](crate::TextEmbedding)
  ```
@@ -28,7 +28,7 @@
 //! Find more info about the available options in the [InitOptions](crate::InitOptions) documentation.
 //!
 #![cfg_attr(
-    feature = "online",
+    feature = "hf-hub",
     doc = r#"
  ### Embeddings generation
 ```

src/reranking/impl.rs

@@ -1,4 +1,4 @@
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use anyhow::Context;
 use anyhow::Result;
 use ort::{
@@ -7,19 +7,19 @@ use ort::{
 };
 use std::thread::available_parallelism;
 
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use crate::common::load_tokenizer_hf_hub;
 use crate::{
     common::load_tokenizer, models::reranking::reranker_model_list, RerankerModel,
     RerankerModelInfo,
 };
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use hf_hub::{api::sync::ApiBuilder, Cache};
 use ndarray::{s, Array};
 use rayon::{iter::ParallelIterator, slice::ParallelSlice};
 use tokenizers::Tokenizer;
 
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use super::RerankInitOptions;
 use super::{
     OnnxSource, RerankInitOptionsUserDefined, RerankResult, TextRerank, UserDefinedRerankingModel,
@@ -50,7 +50,7 @@ impl TextRerank {
         reranker_model_list()
     }
 
-    #[cfg(feature = "online")]
+    #[cfg(feature = "hf-hub")]
     pub fn try_new(options: RerankInitOptions) -> Result<TextRerank> {
         use super::RerankInitOptions;
 

src/sparse_text_embedding/impl.rs

@@ -1,29 +1,29 @@
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use crate::common::load_tokenizer_hf_hub;
 use crate::{
     models::sparse::{models_list, SparseModel},
     ModelInfo, SparseEmbedding,
 };
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use anyhow::Context;
 use anyhow::Result;
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use hf_hub::{
     api::sync::{ApiBuilder, ApiRepo},
     Cache,
 };
 use ndarray::{Array, ArrayViewD, Axis, CowArray, Dim};
 use ort::{session::Session, value::Value};
-#[cfg_attr(not(feature = "online"), allow(unused_imports))]
+#[cfg_attr(not(feature = "hf-hub"), allow(unused_imports))]
 use rayon::{iter::ParallelIterator, slice::ParallelSlice};
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use std::path::PathBuf;
 use tokenizers::Tokenizer;
 
-#[cfg_attr(not(feature = "online"), allow(unused_imports))]
+#[cfg_attr(not(feature = "hf-hub"), allow(unused_imports))]
 use std::thread::available_parallelism;
 
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use super::SparseInitOptions;
 use super::{SparseTextEmbedding, DEFAULT_BATCH_SIZE};
 
@@ -33,7 +33,7 @@ impl SparseTextEmbedding {
     /// Uses the highest level of Graph optimization
     ///
     /// Uses the total number of CPUs available as the number of intra-threads
-    #[cfg(feature = "online")]
+    #[cfg(feature = "hf-hub")]
     pub fn try_new(options: SparseInitOptions) -> Result<Self> {
         use super::SparseInitOptions;
         use ort::{session::builder::GraphOptimizationLevel, session::Session};
@@ -70,7 +70,7 @@ impl SparseTextEmbedding {
     }
 
     /// Private method to return an instance
-    #[cfg_attr(not(feature = "online"), allow(dead_code))]
+    #[cfg_attr(not(feature = "hf-hub"), allow(dead_code))]
     fn new(tokenizer: Tokenizer, session: Session, model: SparseModel) -> Self {
         let need_token_type_ids = session
             .inputs
@@ -84,7 +84,7 @@ impl SparseTextEmbedding {
         }
     }
     /// Return the SparseTextEmbedding model's directory from cache or remote retrieval
-    #[cfg(feature = "online")]
+    #[cfg(feature = "hf-hub")]
     fn retrieve_model(
         model: SparseModel,
         cache_dir: PathBuf,

src/text_embedding/impl.rs

@@ -1,17 +1,17 @@
 //! The definition of the main struct for text embeddings - [`TextEmbedding`].
 
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use crate::common::load_tokenizer_hf_hub;
 use crate::{
     common::load_tokenizer,
     models::text_embedding::{get_model_info, models_list},
     pooling::Pooling,
     Embedding, EmbeddingModel, EmbeddingOutput, ModelInfo, QuantizationMode, SingleBatchOutput,
 };
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use anyhow::Context;
 use anyhow::Result;
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use hf_hub::{
     api::sync::{ApiBuilder, ApiRepo},
     Cache,
@@ -25,12 +25,12 @@ use rayon::{
     iter::{FromParallelIterator, ParallelIterator},
     slice::ParallelSlice,
 };
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use std::path::PathBuf;
 use std::thread::available_parallelism;
 use tokenizers::Tokenizer;
 
-#[cfg(feature = "online")]
+#[cfg(feature = "hf-hub")]
 use super::InitOptions;
 use super::{
     output, InitOptionsUserDefined, TextEmbedding, UserDefinedEmbeddingModel, DEFAULT_BATCH_SIZE,
@@ -42,7 +42,7 @@ impl TextEmbedding {
     /// Uses the highest level of Graph optimization
     ///
     /// Uses the total number of CPUs available as the number of intra-threads
-    #[cfg(feature = "online")]
+    #[cfg(feature = "hf-hub")]
     pub fn try_new(options: InitOptions) -> Result<Self> {
         let InitOptions {
             model_name,
@@ -142,7 +142,7 @@ impl TextEmbedding {
         }
     }
     /// Return the TextEmbedding model's directory from cache or remote retrieval
-    #[cfg(feature = "online")]
+    #[cfg(feature = "hf-hub")]
     fn retrieve_model(
         model: EmbeddingModel,
         cache_dir: PathBuf,

tests/embeddings.rs

@@ -1,4 +1,4 @@
-#![cfg(feature = "online")]
+#![cfg(feature = "hf-hub")]
 
 use std::fs;
 use std::path::Path;

tests/optimum_cli_export.rs

@@ -1,4 +1,4 @@
-#![cfg(feature = "online")]
+#![cfg(feature = "hf-hub")]
 #![cfg(feature = "optimum-cli")]
 //! Test the use of the ``optimum-cli`` to pull models from the Hugging Face Hub,
 //! and generate embeddings successfully with the pulled model.