Merge pull request #38 from sreedevk/development

Version 0.1.4

Cargo.toml

@@ -1,10 +1,14 @@
 [package]
 name = "deduplicator"
-version = "0.1.3"
+version = "0.1.4"
 edition = "2021"
 description = "find,filter,delete Duplicates"
 license = "MIT"
-authors = ["Sreedev Kodichath <[email protected]>", "Valentin Bersier <[email protected]>", "Dhruva Sagar <[email protected]>"]
+authors = [
+    "Sreedev Kodichath <[email protected]>",
+    "Valentin Bersier <[email protected]>",
+    "Dhruva Sagar <[email protected]>",
+]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -16,7 +20,7 @@ clap = { version = "4.0.32", features = ["derive"] }
 colored = "2.0.0"
 dashmap = { version = "5.4.0", features = ["rayon"] }
 fxhash = "0.2.1"
-glob = "0.3.0"
+globwalk = "0.8.1"
 indicatif = { version = "0.17.2", features = ["rayon", "tokio"] }
 itertools = "0.10.5"
 memmap2 = "0.5.8"

src/output.rs

@@ -5,14 +5,15 @@ use chrono::offset::Utc;
 use chrono::DateTime;
 use colored::Colorize;
 use dashmap::DashMap;
+use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};
 use itertools::Itertools;
 use prettytable::{format, row, Table};
 use std::io::Write;
 use std::{fs, io};
 use unicode_segmentation::UnicodeSegmentation;
 
 fn format_path(path: &str, opts: &Params) -> Result<String> {
-    let display_path = path.replace(&opts.get_directory()?, "");
+    let display_path = path.replace(opts.get_directory()?.to_string_lossy().as_ref(), "");
     let display_range = if display_path.chars().count() > 32 {
         display_path
             .graphemes(true)
@@ -26,7 +27,7 @@ fn format_path(path: &str, opts: &Params) -> Result<String> {
         display_path
     };
 
-    Ok(format!("...{:<32}", display_range))
+    Ok(format!("...{display_range:<32}"))
 }
 
 fn file_size(file: &File) -> Result<String> {
@@ -153,10 +154,18 @@ pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
     }
 
     let mut output_table = Table::new();
+    let progress_bar = ProgressBar::new(duplicates.len() as u64);
+    let progress_style = ProgressStyle::default_bar()
+        .template("{spinner:.green} [generating output] [{wide_bar:.cyan/blue}] {pos}/{len} files")
+        .unwrap();
+
+    progress_bar.set_style(progress_style);
     output_table.set_titles(row!["hash", "duplicates"]);
+
     duplicates
         .into_iter()
         .sorted_unstable_by_key(|(_, f)| f.first().and_then(|ff| ff.size).unwrap_or_default())
+        .progress_with(progress_bar)
         .for_each(|(hash, group)| {
             let mut inner_table = Table::new();
             inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);

src/params.rs

@@ -1,6 +1,8 @@
+use std::{fs, path::PathBuf};
+
 use anyhow::{anyhow, Result};
 use clap::{Parser, ValueHint};
-use std::{fs, path::PathBuf};
+use globwalk::{GlobWalker, GlobWalkerBuilder};
 
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
@@ -30,35 +32,21 @@ impl Params {
         }
     }
 
-    pub fn get_directory(&self) -> Result<String> {
-        let dir_pathbuf: PathBuf = self
-            .dir
-            .as_ref()
-            .unwrap_or(&std::env::current_dir()?)
-            .as_os_str()
-            .into();
-
-        let dir = fs::canonicalize(dir_pathbuf)?
-            .as_os_str()
-            .to_str()
-            .ok_or_else(|| anyhow!("Invalid directory"))?
-            .to_string();
-
+    pub fn get_directory(&self) -> Result<PathBuf> {
+        let current_dir = std::env::current_dir()?;
+        let dir_path = self.dir.as_ref().unwrap_or(&current_dir).as_path();
+        let dir = fs::canonicalize(dir_path)?;
         Ok(dir)
     }
 
-    pub fn get_glob_patterns(&self) -> PathBuf {
-        match self.types.as_ref() {
-            Some(filetypes) => vec![
-                self.get_directory().unwrap(),
-                String::from("**"),
-                format!("{{{}}}", filetypes),
-            ]
-            .iter()
-            .collect::<PathBuf>(),
-            None => vec![self.get_directory().unwrap().as_str(), "**", "*"]
-                .iter()
-                .collect::<PathBuf>(),
-        }
+    pub fn get_glob_walker(&self) -> Result<GlobWalker> {
+        let pattern: String = match self.types.as_ref() {
+            Some(filetypes) => format!("**/*{{{filetypes}}}"),
+            None => "**/*".to_string(),
+        };
+        // TODO: add params for maximum depth and following symlinks, then pass them to this builder
+        GlobWalkerBuilder::from_patterns(self.get_directory()?, &[pattern])
+            .build()
+            .map_err(|e| anyhow!(e))
     }
 }

src/scanner.rs

@@ -2,11 +2,11 @@ use crate::{file_manager::File, filters, params::Params};
 use anyhow::Result;
 use dashmap::DashMap;
 use fxhash::hash64 as hasher;
-use glob::glob;
-use indicatif::{ParallelProgressIterator, ProgressStyle};
+use indicatif::{ParallelProgressIterator, ProgressBar, ProgressIterator, ProgressStyle};
 use memmap2::Mmap;
 use rayon::prelude::*;
 use std::hash::Hasher;
+use std::time::Duration;
 use std::{fs, path::PathBuf};
 
 #[derive(Clone, Copy)]
@@ -40,16 +40,22 @@ pub fn duplicates(app_opts: &Params) -> Result<DashMap<String, Vec<File>>> {
 }
 
 fn scan(app_opts: &Params) -> Result<Vec<File>> {
-    let glob_patterns = app_opts.get_glob_patterns().display().to_string();
-    let glob_iter = glob(&glob_patterns)?;
-    let files = glob_iter
-        .filter(Result::is_ok)
-        .map(|file| file.unwrap())
+    let walker = app_opts.get_glob_walker()?;
+    let progress = ProgressBar::new_spinner();
+    let progress_style =
+        ProgressStyle::with_template("{spinner:.green} [mapping paths] {pos} paths")?;
+    progress.set_style(progress_style);
+    progress.enable_steady_tick(Duration::from_millis(100));
+
+    let files = walker
+        .progress_with(progress)
+        .filter_map(Result::ok)
+        .map(|file| file.into_path())
         .filter(|fpath| fpath.is_file())
         .collect::<Vec<PathBuf>>()
         .into_par_iter()
         .progress_with_style(ProgressStyle::with_template(
-            "{spinner:.green} [processing scan results] [{wide_bar:.cyan/blue}] {pos}/{len} files",
+            "{spinner:.green} [processing mapped paths] [{wide_bar:.cyan/blue}] {pos}/{len} files",
         )?)
         .map(|fpath| fpath.display().to_string())
         .map(|fpath| File {