From c581683bcfbe8e5bbee62b22ee929d6813d9e78c Mon Sep 17 00:00:00 2001 From: Chad Reynolds Date: Wed, 15 Jan 2025 11:33:43 -0800 Subject: [PATCH] Automatically prune the fetch cache after fetches Keeps the fetch cache from growing unbounded. Test: # populate cache if necessary Test: cvd fetch --target_directory=/tmp/cvd/fetch_test --default_build=12924704,12924909,12924919 Test: cvd cache info Test: cvd fetch --target_directory=/tmp/cvd/fetch_test2 --default_build=12924921 --max_cache_size_GB=2 Test: # only prunes when caching is enabled Test: cvd fetch --target_directory=/tmp/cvd/fetch_test3 --default_build=12924578 --max_cache_size_GB=0 --enable_caching=false Test: cvd cache info --- .../cvd/cuttlefish/host/commands/cvd/cache/cache.h | 2 ++ .../host/commands/cvd/cli/commands/cache.cpp | 14 ++++++-------- .../host/commands/cvd/cli/commands/fetch.cpp | 12 +++++++++++- .../host/commands/cvd/fetch/fetch_cvd_parser.cc | 5 +++++ .../host/commands/cvd/fetch/fetch_cvd_parser.h | 3 +++ 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/base/cvd/cuttlefish/host/commands/cvd/cache/cache.h b/base/cvd/cuttlefish/host/commands/cvd/cache/cache.h index 85f027657e..0ede04d416 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cache/cache.h +++ b/base/cvd/cuttlefish/host/commands/cvd/cache/cache.h @@ -23,6 +23,8 @@ namespace cuttlefish { +inline constexpr std::size_t kDefaultCacheSizeGb = 25; + Result EmptyCache(const std::string& cache_directory); Result GetCacheInfo(const std::string& cache_directory, bool json_formatted); diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/cache.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/cache.cpp index fb978d4923..879acfc343 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/cache.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/cache.cpp @@ -39,8 +39,6 @@ namespace cuttlefish { namespace { -constexpr int kDefaultCacheSizeGB = 25; - constexpr char kSummaryHelpText[] = "Manage the files cached by cvd"; enum class Action { @@ -51,7 +49,7 @@ enum class Action { struct CacheArguments { Action action = Action::Info; - std::size_t allowed_size_GB = kDefaultCacheSizeGB; + std::size_t allowed_size_gb = kDefaultCacheSizeGb; bool json_formatted = false; }; @@ -82,7 +80,7 @@ Result ProcessArguments( }; std::vector flags; - flags.emplace_back(GflagsCompatFlag("allowed_size_GB", result.allowed_size_GB) + flags.emplace_back(GflagsCompatFlag("allowed_size_gb", result.allowed_size_gb) .Help("Allowed size of the cache during prune " "operation, in gigabytes.")); flags.emplace_back(GflagsCompatFlag("json", result.json_formatted) @@ -122,9 +120,9 @@ Result CvdCacheCommandHandler::Handle(const CommandRequest& request) { break; case Action::Prune: std::cout << CF_EXPECTF( - PruneCache(cache_directory, arguments.allowed_size_GB), + PruneCache(cache_directory, arguments.allowed_size_gb), "Error pruning cache at {} to {}GB", cache_directory, - arguments.allowed_size_GB); + arguments.allowed_size_gb); break; } @@ -146,13 +144,13 @@ Example usage: cvd cache info --json - the same as above, but in JSON format cvd cache prune - caps the cache at the default size ({}GB) - cvd cache prune --allowed_size_GB= - caps the cache at the given size + cvd cache prune --allowed_size_gb= - caps the cache at the given size **Notes**: - info and prune round the cache size up to the nearest gigabyte - prune uses last modification time to remove oldest files first )", - kDefaultCacheSizeGB); + kDefaultCacheSizeGb); } } // namespace diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/fetch.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/fetch.cpp index bfc4763673..d7f52885b2 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/fetch.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/fetch.cpp @@ -29,10 +29,12 @@ #include "common/libs/utils/result.h" #include "common/libs/utils/subprocess.h" #include "common/libs/utils/tee_logging.h" +#include "host/commands/cvd/cache/cache.h" #include "host/commands/cvd/cli/commands/command_handler.h" #include "host/commands/cvd/cli/types.h" #include "host/commands/cvd/fetch/fetch_cvd.h" #include "host/commands/cvd/fetch/fetch_cvd_parser.h" +#include "host/commands/cvd/utils/common.h" namespace cuttlefish { @@ -60,7 +62,15 @@ Result CvdFetchCommandHandler::Handle(const CommandRequest& request) { ScopedTeeLogger logger( LogToStderrAndFiles({log_file}, "", metadata_level, flags.verbosity)); - CF_EXPECT(FetchCvdMain(flags)); + Result result = FetchCvdMain(flags); + if (flags.build_api_flags.enable_caching) { + const std::string cache_directory = PerUserCacheDir(); + LOG(INFO) << CF_EXPECTF( + PruneCache(cache_directory, flags.build_api_flags.max_cache_size_gb), + "Error pruning cache at {} to {}GB", cache_directory, + flags.build_api_flags.max_cache_size_gb); + } + CF_EXPECT(std::move(result)); return {}; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd_parser.cc b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd_parser.cc index a4da9c442a..f6ca221848 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd_parser.cc +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd_parser.cc @@ -119,6 +119,11 @@ std::vector GetFlagsVector(FetchFlags& fetch_flags, flags.emplace_back( GflagsCompatFlag("enable_caching", build_api_flags.enable_caching) .Help("Whether to enable local fetch file caching or not")); + flags.emplace_back( + GflagsCompatFlag("max_cache_size_gb", build_api_flags.max_cache_size_gb) + .Help("Max allowed size(in gigabytes) of the local fetch file cache. " + " If the cache grows beyond this size it will be pruned after " + "the fetches complete.")); CredentialFlags& credential_flags = build_api_flags.credential_flags; flags.emplace_back( diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd_parser.h b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd_parser.h index ccb5c61fca..d226e471e3 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd_parser.h +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd_parser.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include "common/libs/utils/result.h" +#include "host/commands/cvd/cache/cache.h" #include "host/libs/web/android_build_api.h" #include "host/libs/web/android_build_string.h" #include "host/libs/web/cas/cas_downloader.h" @@ -69,6 +71,7 @@ struct BuildApiFlags { bool external_dns_resolver = kDefaultExternalDnsResolver; std::string api_base_url = kAndroidBuildServiceUrl; bool enable_caching = kDefaultEnableCaching; + std::size_t max_cache_size_gb = kDefaultCacheSizeGb; CasDownloaderFlags cas_downloader_flags; };