Result saving, extends monitoring

janjurca · Mar 11, 2024 · febf2f5 · febf2f5
1 parent 6344cbb
commit febf2f5
Show file tree

Hide file tree

Showing 10 changed files with 151 additions and 35 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -27,6 +27,12 @@ include(cmake/CPM.cmake)
 
 # PackageProject.cmake will be used to make our target installable
 CPMAddPackage("gh:TheLartians/[email protected]")
+CPMAddPackage(
+    NAME nlohmann_json
+    GITHUB_REPOSITORY nlohmann/json
+    VERSION 3.9.1
+    OPTIONS "JSON_BuildTests OFF"
+    )
 
 find_package(fmt REQUIRED)
 find_package(spdlog REQUIRED)
@@ -48,7 +54,7 @@ set_target_properties(${PROJECT_NAME} PROPERTIES CXX_STANDARD 17)
 target_compile_options(${PROJECT_NAME} PUBLIC "$<$<COMPILE_LANG_AND_ID:CXX,MSVC>:/permissive->")
 
 # Link dependencies
-target_link_libraries(${PROJECT_NAME} PRIVATE fmt::fmt spdlog::spdlog)
+target_link_libraries(${PROJECT_NAME} PRIVATE fmt::fmt spdlog::spdlog nlohmann_json::nlohmann_json)
 
 target_include_directories(
   ${PROJECT_NAME} PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
@@ -79,7 +85,7 @@ file(GLOB standalone_sources CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/stand
 add_executable(${PROJECT_NAME}Executable ${standalone_sources})
 set_target_properties(${PROJECT_NAME}Executable PROPERTIES CXX_STANDARD 17 OUTPUT_NAME "filestorm")
 
-target_link_libraries(${PROJECT_NAME}Executable ${PROJECT_NAME} fmt::fmt spdlog::spdlog)
+target_link_libraries(${PROJECT_NAME}Executable ${PROJECT_NAME} fmt::fmt spdlog::spdlog nlohmann_json::nlohmann_json)
 
 # Installation rules for the executable
 install(TARGETS ${PROJECT_NAME}Executable RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}

diff --git a/include/filestorm/filetree.h b/include/filestorm/filetree.h
@@ -21,15 +21,15 @@ class FileTree {
 
   unsigned int _max_depth;
 
-  struct Node {
+  class Node {
   public:
     std::string name;
     Type type;
     Node* parent;
     long size;
     std::map<std::string, std::unique_ptr<Node>> folders;
     std::map<std::string, std::unique_ptr<Node>> files;
-    int extents_count = -1;
+    int extents_count = 0;
 
     Node(const std::string& n, Type t, Node* p, long size = 0) : name(n), type(t), parent(p), size(size) {}
     std::string path(bool include_root = false) const {
@@ -42,23 +42,24 @@ class FileTree {
       }
       return parent->path(include_root) + "/" + name;
     }
-    /*
-        int getExtentsCount(bool update = true) {
-          if (extents_count == -1 || update) {
-            if (type == Type::FILE) {
-              extents_count = get_extents(path().c_str()).size();
-            } else {
-              extents_count = 0;
-            }
-          }
-          return extents_count;
+
+    int getExtentsCount(bool update = true) {
+      if (update) {
+        if (type == Type::FILE) {
+          extents_count = get_extents(path(true).c_str()).size();
+        } else {
+          extents_count = 0;
         }
-    */
+      }
+      return extents_count;
+    }
   };
 
   std::vector<Node*> all_files;
   std::vector<Node*> all_directories;
 
+  int64_t total_extents_count = 0;
+
 private:
   std::unique_ptr<Node> root;
 

diff --git a/include/filestorm/result.h b/include/filestorm/result.h
@@ -3,24 +3,68 @@
 #include <filestorm/data_sizes.h>
 #include <filestorm/filetree.h>
 
+#include <chrono>
+#include <fstream>
+#include <nlohmann/json.hpp>
+#include <vector>
+
 class Result {
 public:
-  enum Operation { WRITE, READ, DELETE_FILE, CREATE_FILE, CREATE_DIR };
+  enum Action { DELETE_FILE, CREATE_FILE, CREATE_DIR, ALTER_SMALLER, ALTER_BIGGER };
+  enum Operation { WRITE, READ };
+  static std::vector<Result> results;
 
 private:
   int _iteration;
+  Action _action;
   Operation _operation;
   std::string _path;
   DataSize<DataUnit::B> _size;
   std::chrono::nanoseconds _duration;
+  int64_t _extents_count;
 
 public:
-  Result(int iteration, Operation operation, std::string path, DataSize<DataUnit::B> size, std::chrono::nanoseconds duration)
-      : _iteration(iteration), _operation(operation), _path(path), _size(size), _duration(duration) {}
+  Result(int iteration, Action action, Operation operation, std::string path, DataSize<DataUnit::B> size, std::chrono::nanoseconds duration, int64_t extents_count)
+      : _iteration(iteration), _action(action), _operation(operation), _path(path), _size(size), _duration(duration), _extents_count(extents_count) {}
+  Result() = default;
 
   int getIteration() const { return _iteration; }
+  Action getAction() const { return _action; }
   Operation getOperation() const { return _operation; }
   std::string getPath() const { return _path; }
   DataSize<DataUnit::B> getSize() const { return _size; }
   std::chrono::nanoseconds getDuration() const { return _duration; }
+  int64_t getExtentsCount() const { return _extents_count; }
+
+  // setters
+  void setIteration(int iteration) { _iteration = iteration; }
+  void setAction(Action action) { _action = action; }
+  void setOperation(Operation operation) { _operation = operation; }
+  void setPath(std::string path) { _path = path; }
+  void setSize(DataSize<DataUnit::B> size) { _size = size; }
+  void setDuration(std::chrono::nanoseconds duration) { _duration = duration; }
+  void setExtentsCount(int64_t extents_count) { _extents_count = extents_count; }
+
+  void commit() { results.push_back(*this); }
+
+  static void save(const std::string& filename) {
+    // Save all results to a JSON file
+    std::ofstream file(filename);
+    if (file.is_open()) {
+      nlohmann::json jsonResults;
+      for (const auto& result : results) {
+        nlohmann::json jsonResult;
+        jsonResult["iteration"] = result.getIteration();
+        jsonResult["action"] = result.getAction();
+        jsonResult["operation"] = result.getOperation();
+        jsonResult["path"] = result.getPath();
+        jsonResult["size"] = result.getSize().get_value();
+        jsonResult["duration"] = result.getDuration().count();
+        jsonResult["extents_count"] = result.getExtentsCount();
+        jsonResults.push_back(jsonResult);
+      }
+      file << jsonResults.dump(2);
+      file.close();
+    }
+  }
 };
diff --git a/include/filestorm/scenarios/scenario.h b/include/filestorm/scenarios/scenario.h
@@ -64,6 +64,7 @@ class Scenario {
   void addParameter(Parameter parameter) { _parameters.push_back(parameter); };
   Parameter getParameter(const std::string& name) const;
   virtual void run();
+  virtual void save();
 };
 
 class BasicScenario : public Scenario {

diff --git a/source/filefrag.cpp b/source/filefrag.cpp
@@ -1,5 +1,5 @@
 #include <filestorm/filefrag.h>
-
+#include <fmt/core.h>
 struct fiemap_extent_data {
   struct fiemap *fiemap_ptr;
 };
@@ -10,7 +10,7 @@ std::vector<extents> get_extents(const char *file_path) {
   int fd = open(file_path, O_RDONLY);
   if (fd < 0) {
     perror("open");
-    throw std::runtime_error("Error opening file, for extents scan.");
+    throw std::runtime_error(fmt::format("Error opening file [{}], for extents scan.", file_path));
   }
 
   struct fiemap_extent_data fed;

diff --git a/source/result.cpp b/source/result.cpp
@@ -0,0 +1,3 @@
+#include <filestorm/result.h>
+
+std::vector<Result> Result::results;
diff --git a/source/scenarios/aging.cpp b/source/scenarios/aging.cpp
@@ -18,8 +18,7 @@
 #include <filesystem>
 #include <fstream>
 #include <iostream>  // for std::cerr
-#include <iostream>
-#include <memory>  // for std::unique_ptr
+#include <memory>    // for std::unique_ptr
 #include <random>
 #include <string_view>
 
@@ -36,6 +35,8 @@ AgingScenario::AgingScenario() {
   addParameter(Parameter("p", "sdist", "File size probabilistic distribution", "uniform"));
   addParameter(Parameter("i", "iterations", "Iterations to run", "1000"));
   addParameter(Parameter("b", "blocksize", "RW operations blocksize", "4k"));
+  addParameter(Parameter("y", "sync", "Sync after each write", "false"));
+  addParameter(Parameter("o", "direct_io", "Use direct IO", "false"));
 }
 
 AgingScenario::~AgingScenario() {}
@@ -81,7 +82,10 @@ void AgingScenario::run() {
   ProbabilisticStateMachine psm(transtions, S);
   std::map<std::string, double> probabilities;
 
+  std::vector<FileTree::Node*> touched_files;
+  Result result;
   while (iteration < getParameter("iterations").get_int()) {
+    result.setIteration(iteration);
     compute_probabilities(probabilities, tree);
     psm.performTransition(probabilities);
     switch (psm.getCurrentState()) {
@@ -101,7 +105,6 @@ void AgingScenario::run() {
         FileTree::Node* file_node = tree.mkfile(tree.newFilePath());
         spdlog::debug(fmt::format("CREATE_FILE {}", file_node->path(true)));
         DataSize<DataUnit::B> file_size = get_file_size();
-        spdlog::debug("CREATE_FILE {} with size {}", file_node->path(true), file_size.get_value());
 
         int fd;
         std::unique_ptr<char[]> line(new char[get_block_size().get_value()]);
@@ -111,12 +114,18 @@ void AgingScenario::run() {
 #  error "Windows is not supported"
 #elif __APPLE__
         fd = open(file_node->path(true).c_str(), O_RDWR | O_CREAT, S_IRWXU);
-        if (fcntl(fd, F_NOCACHE, 1) == -1 && fd != -1) {
-          close(fd);
-          throw std::runtime_error("WriteMonitoredAction::work: error on fcntl F_NOCACHE");
+        if (getParameter("direct_io").get_bool() && fd != -1) {
+          if (fcntl(fd, F_NOCACHE, 1) == -1) {
+            close(fd);
+            throw std::runtime_error("WriteMonitoredAction::work: error on fcntl F_NOCACHE");
+          }
         }
 #elif __linux__ || __unix__ || defined(_POSIX_VERSION)
-        fd = open(file_node->path(true).c_str(), O_DIRECT | O_RDWR | O_CREAT, S_IRWXU);
+        int flags = O_RDWR | O_CREAT;
+        if (getParameter("direct_io").get_bool()) {
+          flags |= O_DIRECT;
+        }
+        fd = open(file_node->path(true).c_str(), flags, S_IRWXU);
 #else
 #  error "Unknown system"
 #endif
@@ -127,7 +136,7 @@ void AgingScenario::run() {
           }
           throw std::runtime_error(fmt::format("Error opening file {}", file_node->path(true)));
         }
-        spdlog::debug("CREATE_FILE {} with size {}", file_node->path(true), file_size.get_value());
+        spdlog::debug("CREATE_FILE {} with size {} MB", file_node->path(true), float(file_size.get_value()) / 1024. / 1024.);
         MeasuredCBAction action([&]() {
           for (uint64_t written_bytes = 0; written_bytes < file_size.get_value(); written_bytes += block_size) {
             write(fd, line.get(), block_size);
@@ -136,17 +145,24 @@ void AgingScenario::run() {
         });
         auto duration = action.exec();
         spdlog::debug(fmt::format("Wrote {} bytes in {} ms", file_size.get_value(), duration.count() / 1000000.0));
+        touched_files.push_back(file_node);
+        result.setAction(Result::Action::CREATE_FILE);
+        result.setOperation(Result::Operation::WRITE);
+        result.setPath(file_node->path(true));
+        result.setSize(file_size);
+        result.setDuration(duration);
         break;
       }
       case CREATE_DIR: {
         auto new_dir_path = tree.newDirectoryPath();
         spdlog::debug("CREATE_DIR {}", new_dir_path);
         FileTree::Node* dir_node = tree.mkdir(new_dir_path);
         auto dir_path = dir_node->path(true);
-        spdlog::debug(fmt::format("CREATE_DIR {}", new_dir_path, dir_path));
 
         MeasuredCBAction action([&]() { std::filesystem::create_directory(dir_path); });
         action.exec();
+        result.setAction(Result::Action::CREATE_DIR);
+        result.setPath(dir_path);
         break;
       }
       case ALTER_SMALLER: {
@@ -156,7 +172,12 @@ void AgingScenario::run() {
         auto new_file_size = get_file_size(0, actual_file_size, false);
         spdlog::debug("ALTER_SMALLER {} from {} kB to {} kB", random_file_path, actual_file_size / 1024, new_file_size.get_value() / 1024);
         MeasuredCBAction action([&]() { truncate(random_file_path.c_str(), new_file_size.convert<DataUnit::B>().get_value()); });
-        action.exec();
+        touched_files.push_back(random_file);
+        auto duration = action.exec();
+        result.setAction(Result::Action::ALTER_SMALLER);
+        result.setPath(random_file_path);
+        result.setSize(new_file_size);
+        result.setDuration(duration);
         break;
       }
       case ALTER_BIGGER: {
@@ -203,6 +224,12 @@ void AgingScenario::run() {
           close(fd);
         });
         auto duration = action.exec();
+        touched_files.push_back(random_file);
+        result.setAction(Result::Action::ALTER_BIGGER);
+        result.setOperation(Result::Operation::WRITE);
+        result.setPath(random_file_path);
+        result.setSize(new_file_size - DataSize<DataUnit::B>(actual_file_size));
+        result.setDuration(duration);
         break;
       }
       case ALTER_METADATA:
@@ -215,16 +242,34 @@ void AgingScenario::run() {
         MeasuredCBAction action([&]() { std::filesystem::remove(random_file_path); });
         action.exec();
         tree.rm(random_file->path(false));
+        result.setAction(Result::Action::DELETE_FILE);
+        result.setPath(random_file_path);
         break;
       }
       case DELETE_DIR: {
         auto random_dir = tree.randomDirectory();
         auto random_dir_path = random_dir->path(true);
         spdlog::debug("DELETE_DIR {}", random_dir_path);
+        // TODO: remove directory Is it necesary and good idea at all?
         break;
       }
       case END:
         spdlog::debug("END");
+        if (getParameter("sync").get_bool()) {
+          spdlog::debug("Syncing...");
+          sync();
+        }
+        for (auto& file : touched_files) {
+          auto original_extents = file->getExtentsCount(false);
+          auto updated_extents = file->getExtentsCount(true);
+          spdlog::debug("File {} extents: {} -> {}", file->path(true), original_extents, updated_extents);
+          tree.total_extents_count += updated_extents - original_extents;
+        }
+        spdlog::debug("Total extents count: {}", tree.total_extents_count);
+        touched_files.clear();
+        result.commit();
+        result = Result();
+
         iteration++;
         break;
 
@@ -236,15 +281,15 @@ void AgingScenario::run() {
   }
 
   // Count files and total extent count
-  /*
+
   int file_count = 0;
   int total_extents = 0;
   for (auto& file : tree.all_files) {
     file_count++;
     total_extents += file->getExtentsCount();
   }
   spdlog::info("File count: {}, total extents: {}", file_count, total_extents);
-*/
+
   // cleanup
   for (auto& file : tree.all_files) {
     std::filesystem::remove(file->path(true));
@@ -318,6 +363,6 @@ DataSize<DataUnit::B> AgingScenario::get_file_size() {
   auto max_size = DataSize<DataUnit::B>::fromString(getParameter("maxfsize").get_string()).convert<DataUnit::B>();
   auto min_size = DataSize<DataUnit::B>::fromString(getParameter("minfsize").get_string()).convert<DataUnit::B>();
   auto fsize = get_file_size(min_size.get_value(), max_size.get_value());
-  spdlog::debug("get_file_size: {} - {} : {} / {}", min_size.get_value(), max_size.get_value(), fsize.get_value());
+  spdlog::debug("get_file_size: {} - {} : {} ", min_size.get_value(), max_size.get_value(), fsize.get_value());
   return fsize;
 }
diff --git a/source/scenarios/scenario.cpp b/source/scenarios/scenario.cpp
@@ -1,10 +1,14 @@
+#include <filestorm/result.h>
 #include <filestorm/scenarios/scenario.h>
 #include <fmt/format.h>
 
 #include <filestorm/cxxopts.hpp>
 #include <iostream>
 
-Scenario::Scenario() { addParameter(Parameter("h", "help", "Show help", "false", false)); }
+Scenario::Scenario() {
+  addParameter(Parameter("h", "help", "Show help", "false", false));
+  addParameter(Parameter("", "save", "Save results to a file", "results.json", true));
+}
 
 Scenario::~Scenario() {}
 
@@ -39,6 +43,12 @@ void Scenario::setup(int argc, char** argv) {
 
 void Scenario::run() {}
 
+void Scenario::save() {
+  filename = getParameter("save").get_string();
+  spdlog::info("Saving results to {}", filename);
+  Result::save(filename);
+}
+
 Parameter Scenario::getParameter(const std::string& name) const {
   for (auto parameter : parameters()) {
     if (parameter.long_name() == name) {
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		#include <filestorm/result.h>

		std::vector<Result> Result::results;