The draft system

.gitignore

@@ -0,0 +1,14 @@
+*.pb.*
+*.logs
+*.stat
+*.ldbs
+*.vote
+*.snap
+bin/
+build/
+entry_reader
+reconfigure
+raft_client
+raft_server
+config_client
+config_server

CMakeLists.txt

@@ -0,0 +1,15 @@
+#
+# Copyright (c) 2023-present DeepGrace (complex dot invoke at gmail dot com)
+#
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+#
+# Official repository: https://github.com/deepgrace/draft
+#
+
+cmake_minimum_required(VERSION 3.22)
+project(DRAFT)
+
+add_subdirectory(raft)
+add_subdirectory(client)
+add_subdirectory(server)

LICENSE_1_0.txt

@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,141 @@
+# draft [![LICENSE](https://img.shields.io/github/license/deepgrace/draft.svg)](https://github.com/deepgrace/draft/blob/main/LICENSE_1_0.txt) [![Language](https://img.shields.io/badge/language-C%2B%2B20-blue.svg)](https://en.cppreference.com/w/cpp/compiler_support) [![Platform](https://img.shields.io/badge/platform-Linux-lightgrey.svg)](https://github.com/deepgrace/draft)
+
+> **Distributed, Reliable, Replicated, Redundant And Fault Tolerant**
+
+## Introduction
+draft is a C++ library used for distributed system development, which is header-only, extensible and modern C++ oriented.  
+It's built on top off the **arpc** and protobuf, it's based on the **Proactor** design pattern with performance in mind.  
+draft enables you to develop scalable and distributed system in a straightforward, asynchronous and OOP manner.  
+
+draft is mainly consist of five parts:
+- **raft** The implementation of the raft consensus algorithm
+- **config** The configuration programs to manage the cluster
+- **client** The client to connect to a server in the cluster
+- **server** The snapshot based kv storage built on top off raft
+- **cluster** The distributed kv storage system consist of three servers
+
+draft provides the following features:
+- **Data transfer**
+- **Leader election**
+- **Log compaction**
+- **Log replication**
+- **Leadership transfer**
+- **Membership changes**
+- **Persistence**
+
+## Prerequsites
+[arpc](https://github.com/deepgrace/arpc)  
+[protobuf](https://github.com/protocolbuffers/protobuf)  
+
+## Compiler requirements
+The library relies on a C++20 compiler and standard library
+
+More specifically, draft requires a compiler/standard library supporting the following C++20 features (non-exhaustively):
+- concepts
+- lambda templates
+- All the C++20 type traits from the <type_traits> header
+
+## Building
+draft is header-only. To use it just add the necessary `#include` line to your source files, like this:
+```cpp
+#include <draft.hpp>
+```
+
+`git clone https://github.com/deepgrace/arpc.git` and place it with draft under the same directory.  
+To build the project with cmake, `cd` to the root of the project and setup the build directory:
+```bash
+mkdir build
+cd build
+cmake ..
+```
+
+Make and install the executables:
+```
+make -j4
+make install
+```
+
+The executables are now located at the `config`, `client`, `server` and `cluster` directories of the root of the project.  
+The project can also be built with the script `build.sh`, just run it, the executables will be put at the corresponding 
+directories.  
+
+Note: the `config_server` executable will be put at the `/tmp` directory, as a standalone program, it is just a show case.
+
+## Cleaning
+```bash
+./clean.sh
+```
+
+## Running
+`cd cluster`, open three terminals, then run `start.sh` in directory `a`, `b` and `c` respectively (a is default the leader).  
+The `start.sh` script accepts a argument of any kind to indicate whether to start a fresh run or load the last saved states.  
+Default is the former, for a reload start, just run `./start.sh 1`, which will load all the data since the server last shutdown.
+
+## Connecting
+`cd client`, `./connect.sh <a | b | c>`, after the connection has been established, it's time to enter commands.
+
+## Supported commands
+`set key value`  
+`get key`  
+`del key`  
+
+## Management of the cluster
+`cd config`  
+
+show servers in the cluster:  
+`./show_servers.sh <a | b | c>`  
+
+show server states in the cluster:  
+`./show_states.sh <a | b | c>`  
+
+add `b`, `c` to the cluster:  
+`./add_servers.sh a b c`  
+
+transfer leadership from `a` to `b`:  
+`./transfer.sh a b`  
+
+del `a`, `c` from the cluster:  
+`./del_servers.sh b a c`  
+
+append log to the cluster:  
+`./append.sh b "set metaprogramming amazing"`  
+
+read or write the log content of a server:  
+`./read_entry.sh <a | b | c>`  
+
+then enter commands, the `read_entry.sh` script supports the following commands:  
+`exit`  
+`help`  
+`last`  
+`show`  
+`append <log>`  
+`chop <index>`  
+`drop <index>`  
+
+The `read_entry.sh` should only run when the server is stopped.  
+
+## Configuration
+The configure file `raft.conf` under directory `a`, `b` and `c` of `cluster` defined a variety of information needed to 
+bootstrap the server smoothly, it's well commented and self explained.  
+
+To add a new server to the cluster, say `d`, just copy it from `c`, modify the `host` and `port` parts in `raft.conf`, 
+modify the listen address in `start.sh`, then add that address to `address.sh` under directory `config`, that's all.
+
+## Logging
+The server is currently using the Boost.Log as its logger, the logs to print is adjustable by a log threshold, which is 
+defined in the variable `SEVERITY_THRESHOLD` in file `severity_logger.hpp` under the `raft` directory.
+
+## Storage
+The storage service implemented in server is a memory based STL map, the server uses the memory based snapshotting 
+approach to do data compaction.  
+
+A more efficient and incremental approaches to data compaction is to deploy a dedicated storage engine as its backend.  
+
+It's pretty easy to do, just replace the type `database_t` defined in file `storage_server.hpp` 
+under the `server` directory with the storage engine you prefered, the remaining modifications need to do will be 
+kept minimal.  
+
+The raft implementation has its own persistence and log compaction mechanisms, it's independent of the server.
+
+## License
+draft is licensed as [Boost Software License 1.0](LICENSE_1_0.txt).

build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+path=$(pwd -P)
+
+for d in raft client server; do
+    cd ${path}/${d}
+    ./build.sh
+done

clean.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+path=$(pwd -P)
+
+for d in client config server; do
+    cd ${path}/${d}
+    ./clean.sh
+done
+
+for d in a b c; do
+    cd ${path}/cluster/${d}
+
+    ./clean.sh
+    rm -f raft_server
+done
+
+cd ${path}
+rm -rf bin build

client/CMakeLists.txt

@@ -0,0 +1,28 @@
+#
+# Copyright (c) 2023-present DeepGrace (complex dot invoke at gmail dot com)
+#
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+#
+# Official repository: https://github.com/deepgrace/draft
+#
+
+set(CMAKE_VERBOSE_MAKEFILE ON)
+SET(CMAKE_CXX_FLAGS "-std=c++23 -Wall -O3 -Os -s")
+
+include(GNUInstallDirs)
+
+include_directories(${PROJECT_SOURCE_DIR}/client)
+
+function(add_file NAME)
+    add_executable("${NAME}" "${NAME}.cpp")
+    install(TARGETS ${NAME} DESTINATION ${PROJECT_SOURCE_DIR}/client)
+endfunction()
+
+file(GLOB SRCS "*.cpp")
+
+foreach(file-path ${SRCS})
+    string(REPLACE ".cpp" "" file-path-without-ext ${file-path})
+    get_filename_component(file-name ${file-path-without-ext} NAME)
+    add_file(${file-name})
+endforeach()

client/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+g++ -std=c++23 -Wall -O3 -Os -s -I . raft_client.cpp -o raft_client
+
+strip raft_client

client/clean.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+rm -f raft_client

client/connect.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if (( ${#} != 1 )); then
+      echo "Usage  : ${0} server"
+      echo "example: ${0} <a | b | c>"
+
+      exit 1
+fi
+
+source ../config/address.sh
+listen="${listens[${1}]}"
+
+if [[ "${listen}" == "" ]]; then
+      echo "${1} doesn't exist"
+      exit 1
+fi
+
+echo connected to ${listen}
+./raft_client ${listen/:/ }

client/raft_client.cpp

@@ -0,0 +1,40 @@
+//
+// Copyright (c) 2023-present DeepGrace (complex dot invoke at gmail dot com)
+//
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+// Official repository: https://github.com/deepgrace/draft
+//
+
+#include <raft_client.hpp>
+
+int main(int argc, char* argv[])
+{
+    if (argc != 3)
+    {
+        std::cerr << "Usage: " << argv[0] << " <host> <port>" << std::endl;
+
+        return 1;
+    }
+
+    std::string host = argv[1];
+    std::string port = argv[2];
+
+    std::string command;
+    net::io_context ioc;
+
+    raft_client c(ioc, host, port);
+    std::thread t([&ioc]{ ioc.run(); });
+
+    while (std::getline(std::cin, command))
+    {
+        if (!command.empty())
+            c.write(command.append("\r\n"));
+    }
+
+    c.close();
+    t.join();
+
+    return 0;
+}