MCOL-5724 Move journal to FDB

storage-manager/src/IOCoordinator.h

@@ -37,7 +37,7 @@
 
 namespace storagemanager
 {
-std::shared_ptr<char[]> seekToEndOfHeader1(int fd, size_t* bytesRead);
+std::shared_ptr<char[]> seekToEndOfHeader1_(const std::string& dataStr, size_t* bytesRead);
 
 class IOCoordinator : public boost::noncopyable
 {
@@ -58,16 +58,12 @@ class IOCoordinator : public boost::noncopyable
   // The shared logic for merging a journal file with its base file.
   // len should be set to the length of the data requested
   std::shared_ptr<uint8_t[]> mergeJournal(const char* objectPath, const char* journalPath, off_t offset,
-                                            size_t len, size_t* sizeRead) const;
+                                           size_t len, size_t* sizeRead) const;
 
   // this version modifies object data in memory, given the journal filename.  Processes the whole object
   // and whole journal file.
   int mergeJournalInMem(std::shared_ptr<uint8_t[]>& objData, size_t len, const char* journalPath,
-                        size_t* sizeRead) const;
-
-  // this version of MJIM has a higher IOPS requirement and lower mem usage.
-  int mergeJournalInMem_bigJ(std::shared_ptr<uint8_t[]>& objData, size_t len, const char* journalPath,
-                             size_t* sizeRead) const;
+                         size_t* sizeRead) const;
 
   // this version takes already-open file descriptors, and an already-allocated buffer as input.
   // file descriptor are positioned, eh, best not to assume anything about their positions

storage-manager/src/KVPrefixes.cpp

@@ -15,10 +15,15 @@
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
    MA 02110-1301, USA. */
 
-#include "KVPrefixes.hpp"
+#include "KVPrefixes.h"
 
 namespace storagemanager
 {
 // FDB recommends keep the key size up to 32 bytes.
-const char* KVPrefixes[2] = {/*OWNERSHIP*/ "SM_O_", /*META*/ "SM_M_"};
+const char* KVPrefixes[3] = {/*OWNERSHIP*/ "SM_O_", /*META*/ "SM_M_", /*JOURNAL*/ "SM_J_"};
+
+std::string getJournalName(const std::string& key)
+{
+  return KVPrefixes[static_cast<uint32_t>(KVPrefixId::SM_JOURNAL)] + key;
+}
 }  // namespace storagemanager

storage-manager/src/KVPrefixes.hpp → storage-manager/src/KVPrefixes.h

@@ -16,14 +16,19 @@
    MA 02110-1301, USA. */
 
 #pragma once
+#include <string>
+#include <cstdint>
 
 namespace storagemanager
 {
 enum class KVPrefixId
 {
   SM_OWNERSHIP = 0,
-  SM_META
+  SM_META,
+  SM_JOURNAL
 };
 
 extern const char* KVPrefixes[];
+
+std::string getJournalName(const std::string &key);
 }  // namespace storagemanager

storage-manager/src/MetadataFile.cpp

@@ -45,7 +45,7 @@
 #include <boost/uuid/random_generator.hpp>
 #include <unistd.h>
 #include "fdbcs.hpp"
-#include "KVPrefixes.hpp"
+#include "KVPrefixes.h"
 
 #define max(x, y) (x > y ? x : y)
 #define min(x, y) (x < y ? x : y)

storage-manager/src/Ownership.cpp

@@ -20,7 +20,7 @@
 #include "Cache.h"
 #include "Synchronizer.h"
 #include "KVStorageInitializer.h"
-#include "KVPrefixes.hpp"
+#include "KVPrefixes.h"
 #include "fdbcs.hpp"
 #include <sys/types.h>
 #include <sys/stat.h>

storage-manager/src/PrefixCache.cpp

@@ -17,6 +17,8 @@
 
 #include "PrefixCache.h"
 #include "Cache.h"
+#include "KVPrefixes.h"
+#include "KVStorageInitializer.h"
 #include "Config.h"
 #include "Downloader.h"
 #include "Synchronizer.h"
@@ -575,7 +577,8 @@ void PrefixCache::rename(const string& oldKey, const string& newKey, ssize_t siz
 int PrefixCache::ifExistsThenDelete(const string& key)
 {
   bf::path cachedPath = cachePrefix / key;
-  bf::path journalPath = journalPrefix / (key + ".journal");
+  const auto journalName = getJournalName((journalPrefix / (key + ".journal")).string());
+  const auto journalSizeName = getJournalName((journalPrefix / (key + "_size.journal")).string());
 
   boost::unique_lock<boost::mutex> s(lru_mutex);
   bool objectExists = false;
@@ -593,16 +596,18 @@ int PrefixCache::ifExistsThenDelete(const string& key)
     else  // let makeSpace() delete it if it's already in progress
       return 0;
   }
-  bool journalExists = bf::exists(journalPath);
-  // assert(objectExists == bf::exists(cachedPath));
+  auto kvStorage = KVStorageInitializer::getStorageInstance();
+  auto tnx = kvStorage->createTransaction();
+  auto journalResult = tnx->get(journalName);
+  auto journalSizeResult = tnx->get(journalSizeName);
 
+  bool journalExists = journalSizeResult.first;
   size_t objectSize = (objectExists ? bf::file_size(cachedPath) : 0);
-  // size_t objectSize = (objectExists ? MetadataFile::getLengthFromKey(key) : 0);
-  size_t journalSize = (journalExists ? bf::file_size(journalPath) : 0);
-  currentCacheSize -= (objectSize + journalSize);
-
-  // assert(!objectExists || objectSize == bf::file_size(cachedPath));
+  size_t journalSize = 0;
+  if (journalExists)
+    journalSize = std::atoi(journalSizeResult.second.c_str());
 
+  currentCacheSize -= (objectSize + journalSize);
   return (objectExists ? 1 : 0) | (journalExists ? 2 : 0);
 }