diff --git a/Cargo.lock b/Cargo.lock index 8fd691fc..544ae8db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,15 +11,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "ansi_term" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" -dependencies = [ - "winapi", -] - [[package]] name = "ansi_term" version = "0.12.1" @@ -69,11 +60,11 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "2.33.3" +version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ - "ansi_term 0.11.0", + "ansi_term", "atty", "bitflags", "strsim", @@ -111,12 +102,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "dtoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" - [[package]] name = "either" version = "1.6.1" @@ -245,6 +230,12 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + [[package]] name = "lazy_static" version = "1.4.0" @@ -253,9 +244,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.105" +version = "0.2.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "869d572136620d55835903746bcb5cdc54cb2851fd0aeec53220b4bb65ef3013" +checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" [[package]] name = "linked-hash-map" @@ -312,9 +303,9 @@ checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "once_cell" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" [[package]] name = "opaque-debug" @@ -395,9 +386,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.30" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc3358ebc67bc8b7fa0c007f945b0b18226f78437d61bec735a9eb96b61ee70" +checksum = "2f84e92c0f7c9d58328b85a78557813e4bd845130db68d7184635344399423b1" dependencies = [ "unicode-xid", ] @@ -507,9 +498,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" [[package]] name = "scopeguard" @@ -519,18 +510,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" -version = "1.0.130" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" +checksum = "8b9875c23cf305cd1fd7eb77234cbb705f21ea6a72c637a5c6db5fe4b8e7f008" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.130" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +checksum = "ecc0db5cb2556c0e558887d9bbdcf6ac4471e83ff66cf696e5419024d1606276" dependencies = [ "proc-macro2", "quote", @@ -539,32 +530,32 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.68" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8" +checksum = "bcbd0344bc6533bc7ec56df11d42fb70f1b912351c0825ccb7211b59d8af7cf5" dependencies = [ - "itoa", + "itoa 1.0.1", "ryu", "serde", ] [[package]] name = "serde_test" -version = "1.0.130" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82178225dbdeae2d5d190e8649287db6a3a32c6d24da22ae3146325aa353e4c" +checksum = "b138b0ce2635ff4052c489bd0a2c9a7e10de455d6b67e4fa29b5455781121300" dependencies = [ "serde", ] [[package]] name = "serde_yaml" -version = "0.8.21" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8c608a35705a5d3cdc9fbe403147647ff34b921f8e833e49306df898f9b20af" +checksum = "a4a521f2940385c165a24ee286aa8599633d162077a54bdcae2a6fd5a7bfa7a0" dependencies = [ - "dtoa", "indexmap", + "ryu", "serde", "yaml-rust", ] @@ -638,9 +629,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.80" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194" +checksum = "8daf5dd0bb60cbd4137b1b587d2fc0ae729bc07cf01cd70b36a1ed5ade3b9d59" dependencies = [ "proc-macro2", "quote", @@ -701,11 +692,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde1cf55178e0293453ba2cca0d5f8392a922e52aa958aee9c28ed02becc6d03" +checksum = "41effe7cfa8af36f439fac33861b66b049edc6f9a32331e2312660529c1c24ad" dependencies = [ - "itoa", + "itoa 0.4.8", "libc", "serde", "time-macros", @@ -760,11 +751,11 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.0" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cf865b5ddc38e503a29c41c4843e616a73028ae18c637bc3eb2afaef4909c84" +checksum = "245da694cc7fc4729f3f418b304cb57789f1bed2a78c575407ab8a23f53cb4d3" dependencies = [ - "ansi_term 0.12.1", + "ansi_term", "lazy_static", "matchers", "regex", diff --git a/Dockerfile b/Dockerfile index 519ff1b7..09847ecf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM rust:alpine ENV RUSTFLAGS="-Zinstrument-coverage" ENV LLVM_PROFILE_FILE="profraw/hoard-python-test-%p-%m.profraw" -ENV CI=true GITHUB_ACTIONS=true HOARD_LOG=debug +ENV CI=true GITHUB_ACTIONS=true HOARD_LOG=trace WORKDIR /hoard-tests RUN apk add python3 tree py3-yaml py3-toml diff --git a/book/src/cli/flags-subcommands.md b/book/src/cli/flags-subcommands.md index 307690da..30607f0c 100644 --- a/book/src/cli/flags-subcommands.md +++ b/book/src/cli/flags-subcommands.md @@ -13,6 +13,8 @@ Flags can be used with any subcommand and must be specified *before* any subcomm - Back up the specified hoard(s). If no `name` is specified, all hoards are backed up. - Restore: `hoard [flags...] restore [name] [name] [...]` - Restore the specified hoard(s). If no `name` is specified, all hoards are restored. +- List Hoards: `hoard list` + - List all configured hoards by name - Validate: `hoard [flags...] validate` - Attempt to parse the default configuration file (or the one provided via `--config-file`) Exits with code `0` if the config is valid. diff --git a/ci-tests/tests/__main__.py b/ci-tests/tests/__main__.py index c8d06803..08882cd4 100644 --- a/ci-tests/tests/__main__.py +++ b/ci-tests/tests/__main__.py @@ -3,13 +3,14 @@ import subprocess import sys from pathlib import Path -from testers.ignore_filter import IgnoreFilterTester -from testers.last_paths import LastPathsTester -from testers.operations import OperationCheckerTester from testers.cleanup import LogCleanupTester -from testers.hoard_tester import HoardFile, Environment from testers.correct_errors import CorrectErrorsTester +from testers.hoard_list import ListHoardsTester +from testers.hoard_tester import HoardFile, Environment +from testers.ignore_filter import IgnoreFilterTester +from testers.last_paths import LastPathsTester from testers.no_config_dir import MissingConfigDirTester +from testers.operations import OperationCheckerTester from testers.yaml_support import YAMLSupportTester @@ -43,43 +44,41 @@ def print_checksums(): print(f"{path}: {hashlib.md5(file.read()).hexdigest()}") +TEST_MAPPING = { + "cleanup": ("cleanup", LogCleanupTester), + "errors": ("expected errors", CorrectErrorsTester), + "ignore": ("ignore filter", IgnoreFilterTester), + "last_paths": ("last paths", LastPathsTester), + "list_hoards": ("list command", ListHoardsTester), + "missing_config": ("missing config dir", MissingConfigDirTester), + "operation": ("operation", OperationCheckerTester), + "yaml": ("YAML compat", YAMLSupportTester), +} + + if __name__ == "__main__": if len(sys.argv) == 1: raise RuntimeError("One argument - the test - is required") try: - if sys.argv[1] == "last_paths": - print("Running last_paths test") - LastPathsTester().run_test() - elif sys.argv[1] == "operation": - print("Running operation test") - OperationCheckerTester().run_test() - elif sys.argv[1] == "ignore": - print("Running ignore filter test") - IgnoreFilterTester().run_test() - elif sys.argv[1] == "cleanup": - print("Running cleanup test") - LogCleanupTester().run_test() - elif sys.argv[1] == "errors": - print("Running errors test") - CorrectErrorsTester().run_test() - elif sys.argv[1] == "missing_config": - print("Running missing config dir test") - MissingConfigDirTester().run_test() - elif sys.argv[1] == "yaml": - print("Running YAML compat tests") - YAMLSupportTester().run_test() - elif sys.argv[1] == "all": + test_arg = sys.argv[1] + if test_arg == "all": print("Running all tests") - YAMLSupportTester().run_test() - MissingConfigDirTester().run_test() - CorrectErrorsTester().run_test() - LastPathsTester().run_test() - IgnoreFilterTester().run_test() - OperationCheckerTester().run_test() - LogCleanupTester().run_test() + successful = [] + for desc, cls in TEST_MAPPING.values(): + print(f"=== Running {desc} test ===") + cls().run_test() + successful.append(desc) + elif test_arg in TEST_MAPPING: + desc, cls = TEST_MAPPING[test_arg] + print(f"Running {desc} test") + cls().run_test() else: - raise RuntimeError(f"Invalid argument {sys.argv[1]}") + raise RuntimeError(f"Invalid argument {test_arg}") except Exception: + if desc: + print(f"=== Error while running {desc} test ===") + if successful and len(successful) > 0: + print(f"=== Successful tests: {', '.join(successful)} ===") data_dir = LastPathsTester.data_dir_path() print("\n### Hoards:") sys.stdout.flush() diff --git a/ci-tests/tests/testers/hoard_list.py b/ci-tests/tests/testers/hoard_list.py new file mode 100644 index 00000000..da7b00ad --- /dev/null +++ b/ci-tests/tests/testers/hoard_list.py @@ -0,0 +1,16 @@ +from .hoard_tester import HoardTester + + +class ListHoardsTester(HoardTester): + def __init__(self): + super().__init__() + self.reset() + + def run_test(self): + expected = b"anon_dir\nanon_file\nnamed\n" + output = self.run_hoard("list", capture_output=True) + assert expected in output.stdout + + self.env["HOARD_LOG"] = "info" + output = self.run_hoard("list", capture_output=True) + assert output.stdout == expected diff --git a/ci-tests/tests/testers/hoard_tester.py b/ci-tests/tests/testers/hoard_tester.py index bb362e6e..43fd420f 100644 --- a/ci-tests/tests/testers/hoard_tester.py +++ b/ci-tests/tests/testers/hoard_tester.py @@ -119,9 +119,13 @@ def reset(self, config_file="config.toml"): for env in list(Environment): for item in list(HoardFile): + path = home.joinpath(f"{env}_{item}") if item is HoardFile.AnonDir or item is HoardFile.NamedDir1 or item is HoardFile.NamedDir2: + try: + shutil.rmtree(path) + except FileNotFoundError: + pass continue - path = home.joinpath(f"{env}_{item}") self.generate_file(path) os.makedirs(config_parent, exist_ok=True) config_file_src = Path.cwd().joinpath("ci-tests", config_file) diff --git a/ci-tests/tests/testers/last_paths.py b/ci-tests/tests/testers/last_paths.py index 76144c32..630b72ba 100644 --- a/ci-tests/tests/testers/last_paths.py +++ b/ci-tests/tests/testers/last_paths.py @@ -1,5 +1,6 @@ -from .hoard_tester import HoardTester +from pathlib import Path import subprocess +from .hoard_tester import HoardTester, Environment, HoardFile, Hoard class LastPathsTester(HoardTester): @@ -27,8 +28,15 @@ def run_test(self): # Doing it again with "first" should still succeed self.env = {"USE_ENV": "1"} self.run_hoard("backup") + + home = Path.home() + ignored = [ + home.joinpath(), + ] + # Make sure the files are consistent with backing up "first" self.assert_first_tree() + # Doing it with "second" but forced should succeed self.env = {"USE_ENV": "2"} self.force = True diff --git a/src/command/mod.rs b/src/command/mod.rs index 456800f6..96ade478 100644 --- a/src/command/mod.rs +++ b/src/command/mod.rs @@ -28,6 +28,8 @@ pub enum Command { /// The name(s) of the hoard(s) to restore. Will restore all hoards if empty. hoards: Vec, }, + /// List configured hoards + List, } impl Default for Command { @@ -41,7 +43,7 @@ mod tests { use super::*; #[test] - fn default_command_is_help() { + fn default_command_is_validate() { // The default command is validate if one is not given assert_eq!(Command::Validate, Command::default()); } diff --git a/src/config/mod.rs b/src/config/mod.rs index 097e8a0b..841a4710 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -111,11 +111,11 @@ impl Config { /// /// The error returned by [`Builder::from_args_then_file`], wrapped in [`Error::Builder`]. pub fn load() -> Result { - tracing::info!("loading configuration..."); + tracing::debug!("loading configuration..."); let config = Builder::from_args_then_file() .map(Builder::build)? .map_err(Error::Builder)?; - tracing::info!("loaded configuration."); + tracing::debug!("loaded configuration."); Ok(config) } @@ -174,6 +174,12 @@ impl Config { Command::Validate => { tracing::info!("configuration is valid"); } + Command::List => { + let mut hoards: Vec<&str> = self.hoards.keys().map(String::as_str).collect(); + hoards.sort_unstable(); + let list = hoards.join("\n"); + tracing::info!("{}", list); + } Command::Cleanup => match crate::checkers::history::operation::cleanup_operations() { Ok(count) => tracing::info!("cleaned up {} log files", count), Err((count, error)) => { @@ -193,7 +199,7 @@ impl Config { for (name, hoard) in hoards { let prefix = self.get_prefix(name); - tracing::info!(hoard = %name, "backing up hoard"); + tracing::info!(hoard = %name, "backing up"); let _span = tracing::info_span!("backup", hoard = %name).entered(); hoard.backup(&prefix).map_err(|error| Error::Backup { name: name.to_string(), diff --git a/src/logging.rs b/src/logging.rs new file mode 100644 index 00000000..b9170beb --- /dev/null +++ b/src/logging.rs @@ -0,0 +1,189 @@ +use std::collections::BTreeMap; +use std::fmt; +use std::str::FromStr; + +use tracing::field::{Field, Visit}; +use tracing::{Event, Level, Subscriber}; +use tracing_subscriber::field::RecordFields; +use tracing_subscriber::fmt::format::Writer; +use tracing_subscriber::fmt::{FmtContext, FormatEvent, FormatFields, FormattedFields}; +use tracing_subscriber::registry::LookupSpan; +use tracing_subscriber::{EnvFilter, FmtSubscriber}; + +const LOG_ENV: &str = "HOARD_LOG"; +const EMPTY_PREFIX: &str = " "; + +struct FormatterVisitor { + message: Option, + fields: BTreeMap, + is_terse: bool, +} + +impl FormatterVisitor { + fn new(is_terse: bool) -> Self { + Self { + message: None, + fields: BTreeMap::new(), + is_terse, + } + } +} + +impl Visit for FormatterVisitor { + fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { + if self.message.is_none() && field.name() == "message" { + self.message = Some(format!("{:?}", value)); + } else { + let value = if self.is_terse { + format!("{:?}", value) + } else { + format!("{:#?}", value) + }; + + self.fields.insert(field.name().to_string(), value); + } + } +} + +#[derive(Clone)] +struct Formatter { + max_level: Level, +} + +impl<'writer> FormatFields<'writer> for Formatter { + fn format_fields( + &self, + mut writer: Writer<'writer>, + fields: R, + ) -> fmt::Result { + let is_terse = matches!(self.max_level, Level::ERROR | Level::WARN | Level::INFO); + let mut visitor = FormatterVisitor::new(is_terse); + fields.record(&mut visitor); + + let empty_prefix_newline = format!("\n{}", EMPTY_PREFIX); + let longest_name_len = visitor + .fields + .iter() + .map(|(name, _)| name.len()) + .max() + .unwrap_or(0); + let fields = visitor + .fields + .iter() + .map(|(name, value)| { + let value = value + .split('\n') + .collect::>() + .join(&empty_prefix_newline); + if is_terse { + format!("{}={}", name, value) + } else { + let padding = " ".repeat(longest_name_len - name.len()); + format!("{}{} = {}", padding, name, value) + } + }) + .collect::>(); + + let fields_output = if is_terse { + fields.join(", ") + } else { + let fields = fields.join(&format!("\n{} ", EMPTY_PREFIX)); + if fields.is_empty() { + fields + } else { + format!("\n{}where {}", EMPTY_PREFIX, fields) + } + }; + + if let Some(msg) = visitor.message { + write!(writer, "{}", msg)?; + if is_terse && !fields_output.is_empty() { + write!(writer, ": ")?; + } + } + + write!(writer, "{}", fields_output) + } +} + +impl FormatEvent for Formatter +where + S: Subscriber + for<'a> LookupSpan<'a>, + N: for<'a> FormatFields<'a> + 'static, +{ + fn format_event( + &self, + ctx: &FmtContext<'_, S, N>, + writer: Writer<'_>, + event: &Event<'_>, + ) -> fmt::Result { + let mut writer = writer; + let metadata = event.metadata(); + + // Only show prefix if debug or higher verbosity, or warning/error + if self.max_level >= Level::DEBUG || metadata.level() != &Level::INFO { + write!(writer, "{} {}: ", metadata.level(), metadata.target())?; + } + + // Write message + // TODO: ANSIfy? + ctx.field_format().format_fields(writer.by_ref(), event)?; + writeln!(writer)?; + + // Format spans only if tracing verbosity + if self.max_level == Level::TRACE { + if let Some(scope) = ctx.event_scope() { + for span in scope.from_root() { + write!(writer, "{}at {}", EMPTY_PREFIX, span.name())?; + let ext = span.extensions(); + let fields = ext + .get::>() + .expect("should never be `None`") + .to_string(); + if !fields.is_empty() { + // join string matches the above write!() + let fields = fields.split('\n').collect::>().join("\n "); + write!(writer, ": {}", fields)?; + } + writeln!(writer)?; + } + } + + // Add extra newline for easier reading + writeln!(writer) + } else { + Ok(()) + } + } +} + +pub fn get_subscriber() -> impl Subscriber { + let max_level = { + let env_str = std::env::var(LOG_ENV).unwrap_or_else(|_| String::new()); + + // Get the last item that is only a level + let level_opt = env_str + .split(',') + .map(str::trim) + .rev() + .map(FromStr::from_str) + .find_map(Result::ok); + + level_opt.unwrap_or_else(|| { + if cfg!(debug_assertions) { + Level::DEBUG + } else { + Level::INFO + } + }) + }; + + let env_filter = EnvFilter::try_from_env(LOG_ENV) + .unwrap_or_else(|_| EnvFilter::new("").add_directive(max_level.into())); + + FmtSubscriber::builder() + .with_env_filter(env_filter) + .event_format(Formatter { max_level }) + .fmt_fields(Formatter { max_level }) + .finish() +} diff --git a/src/main.rs b/src/main.rs index 19d6dd46..e044067e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,52 +1,21 @@ use hoard::Config; -use std::io::Stdout; -use tracing::level_filters::LevelFilter; -use tracing::Level; -use tracing_subscriber::fmt::format::{Format, Pretty}; -use tracing_subscriber::fmt::SubscriberBuilder; -use tracing_subscriber::{EnvFilter, FmtSubscriber}; - -const LOG_ENV: &str = "HOARD_LOG"; +use tracing_subscriber::util::SubscriberInitExt; +mod logging; fn error_and_exit(err: E) -> ! { // Ignore error if default subscriber already exists // This just helps ensure that logging happens and is // consistent. - let _ = get_subscriber().try_init(); + let _guard = logging::get_subscriber().set_default(); tracing::error!("{}", err); std::process::exit(1); } -type Subscriber = SubscriberBuilder, LevelFilter, fn() -> Stdout>; -fn get_subscriber() -> Subscriber { - FmtSubscriber::builder() - .pretty() - .with_ansi(true) - .with_level(true) - .with_target(false) - .without_time() - .with_max_level(if cfg!(debug_assertions) { - Level::DEBUG - } else { - Level::INFO - }) -} - fn main() { // Set up default logging // There is no obvious way to set up a default logging level in case the env // isn't set, so use this match thing instead. - let subscriber = get_subscriber(); - match std::env::var_os(LOG_ENV) { - Some(_) => match EnvFilter::try_from_env(LOG_ENV) { - Err(err) => error_and_exit(err), - Ok(filter) => subscriber - .with_env_filter(filter) - .with_filter_reloading() - .init(), - }, - None => subscriber.init(), - }; + let _guard = logging::get_subscriber().set_default(); // Get configuration let config = match Config::load() {