From 4469423c41d8b1d9f59a338be44f06c86f6dca7e Mon Sep 17 00:00:00 2001 From: ivan770 Date: Tue, 7 Jul 2020 16:55:36 +0300 Subject: [PATCH 01/36] Add replication storage and event, use proxy DB --- spartan/src/node/mod.rs | 10 +- spartan/src/node/persistence.rs | 5 +- spartan/src/node/replication/event.rs | 12 +++ spartan/src/node/replication/mod.rs | 5 + spartan/src/node/replication/storage.rs | 127 ++++++++++++++++++++++++ 5 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 spartan/src/node/replication/event.rs create mode 100644 spartan/src/node/replication/mod.rs create mode 100644 spartan/src/node/replication/storage.rs diff --git a/spartan/src/node/mod.rs b/spartan/src/node/mod.rs index 82c0e33e..c3845350 100644 --- a/spartan/src/node/mod.rs +++ b/spartan/src/node/mod.rs @@ -10,6 +10,9 @@ pub mod manager; /// Persistence handler pub mod persistence; +/// Database replication +pub mod replication; + pub use exit::spawn_ctrlc_handler; pub use manager::Manager; pub use persistence::{load_from_fs, persist_manager, spawn_persistence}; @@ -18,8 +21,9 @@ use crate::config::Config; use futures_util::lock::{Mutex, MutexGuard}; use spartan_lib::core::{db::tree::TreeDatabase, message::Message}; use std::collections::{hash_map::RandomState, HashMap}; +use replication::storage::ReplicatedDatabase; -pub type DB = TreeDatabase; +pub type DB = ReplicatedDatabase>; type MutexDB = Mutex; /// Key-value node implementation @@ -36,7 +40,7 @@ impl<'a> Node<'a> { } /// Get locked queue instance - pub async fn get(&self, name: &str) -> Option>> { + pub async fn get(&self, name: &str) -> Option> { debug!("Obtaining queue \"{}\"", name); Some(self.queue(name)?.lock().await) } @@ -44,7 +48,7 @@ impl<'a> Node<'a> { /// Add queue entry to node pub fn add(&mut self, name: &'a str) { info!("Initializing queue \"{}\"", name); - self.db.insert(name, Mutex::new(TreeDatabase::default())); + self.db.insert(name, Mutex::new(DB::default())); } /// Load queues from config diff --git a/spartan/src/node/persistence.rs b/spartan/src/node/persistence.rs index 35ffb8f7..42900c65 100644 --- a/spartan/src/node/persistence.rs +++ b/spartan/src/node/persistence.rs @@ -1,9 +1,8 @@ -use super::Manager; +use super::{MutexDB, Manager}; use actix_rt::time::delay_for; use bincode::{deserialize, serialize, Error as BincodeError}; use futures_util::lock::Mutex; use futures_util::stream::{iter, StreamExt}; -use spartan_lib::core::{db::tree::TreeDatabase, message::Message}; use std::{io::Error, path::Path, time::Duration}; use thiserror::Error as ThisError; use tokio::fs::{read, write}; @@ -24,7 +23,7 @@ type PersistenceResult = Result; /// Persist database to provided path async fn persist_db( name: &str, - db: &Mutex>, + db: &MutexDB, path: &Path, ) -> Result<(), PersistenceError> { let db = db.lock().await; diff --git a/spartan/src/node/replication/event.rs b/spartan/src/node/replication/event.rs new file mode 100644 index 00000000..87e6b4a9 --- /dev/null +++ b/spartan/src/node/replication/event.rs @@ -0,0 +1,12 @@ +use serde::{Serialize, Deserialize}; +use spartan_lib::core::{payload::Identifiable, message::Message}; + +#[derive(Serialize, Deserialize)] +pub enum Event { + Push(Message), + Pop, + Requeue(::Id), + Delete(::Id), + Gc, + Clear, +} diff --git a/spartan/src/node/replication/mod.rs b/spartan/src/node/replication/mod.rs new file mode 100644 index 00000000..a7a1f23c --- /dev/null +++ b/spartan/src/node/replication/mod.rs @@ -0,0 +1,5 @@ +/// Replication storage (event log) and proxy database +pub mod storage; + +/// Replication event +pub mod event; \ No newline at end of file diff --git a/spartan/src/node/replication/storage.rs b/spartan/src/node/replication/storage.rs new file mode 100644 index 00000000..58c0a013 --- /dev/null +++ b/spartan/src/node/replication/storage.rs @@ -0,0 +1,127 @@ +use std::collections::BTreeMap; +use super::event::Event; +use spartan_lib::core::{message::Message, dispatcher::{StatusAwareDispatcher, SimpleDispatcher, simple::{PositionBasedDelete, Delete}}, payload::Identifiable}; +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize)] +pub struct ReplicationStorage { + next_index: u64, + gc_threshold: u64, + log: BTreeMap +} + +impl Default for ReplicationStorage { + fn default() -> Self { + ReplicationStorage { + next_index: 1, + gc_threshold: 0, + log: BTreeMap::new() + } + } +} + +impl ReplicationStorage { + pub fn push(&mut self, event: Event) { + self.log.insert(self.next_index, event); + self.next_index += 1; + } +} + +#[derive(Serialize, Deserialize)] +pub struct ReplicatedDatabase { + inner: DB, + storage: Option +} + +impl Default for ReplicatedDatabase +where + DB: Default +{ + fn default() -> Self { + ReplicatedDatabase { + inner: DB::default(), + storage: None + } + } +} + +impl ReplicatedDatabase { + pub fn push_event(&mut self, event: F) + where + F: FnOnce() -> Event + { + if let Some(storage) = &mut self.storage { + storage.push(event()); + } + } +} + +impl SimpleDispatcher for ReplicatedDatabase +where + DB: SimpleDispatcher, +{ + fn push(&mut self, message: Message) { + self.push_event(|| Event::Push(message.clone())); + + self.inner.push(message) + } + + fn peek(&self) -> Option<&Message> { + self.inner.peek() + } + + fn gc(&mut self) { + self.push_event(|| Event::Gc); + + self.inner.gc() + } + + fn size(&self) -> usize { + self.inner.size() + } + + fn clear(&mut self) { + self.push_event(|| Event::Clear); + + self.inner.clear() + } +} + +impl StatusAwareDispatcher for ReplicatedDatabase +where + DB: StatusAwareDispatcher +{ + fn pop(&mut self) -> Option<&Message> { + self.push_event(|| Event::Pop); + + self.inner.pop() + } + + fn requeue(&mut self, id: ::Id) -> Option<()> { + self.push_event(|| Event::Requeue(id)); + + self.inner.requeue(id) + } +} + +impl Delete for ReplicatedDatabase +where + DB: Delete +{ + fn delete(&mut self, id: ::Id) -> Option { + self.push_event(|| Event::Delete(id)); + + Delete::delete(&mut self.inner, id) + } +} + +impl PositionBasedDelete for ReplicatedDatabase +where + DB: PositionBasedDelete +{ + fn delete(&mut self, id: ::Id) -> Option { + self.push_event(|| Event::Delete(id)); + + PositionBasedDelete::delete(&mut self.inner, id) + } +} From 0ee012d9fb673056668aedd8ea055cf9154a1925 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Wed, 8 Jul 2020 13:04:28 +0300 Subject: [PATCH 02/36] Use separate storage implementations for primary and replica --- spartan/src/node/mod.rs | 2 +- spartan/src/node/persistence.rs | 8 +-- .../replication/{storage.rs => database.rs} | 50 +++++-------------- spartan/src/node/replication/event.rs | 4 +- spartan/src/node/replication/mod.rs | 5 +- spartan/src/node/replication/storage/mod.rs | 37 ++++++++++++++ .../src/node/replication/storage/primary.rs | 27 ++++++++++ .../src/node/replication/storage/replica.rs | 18 +++++++ 8 files changed, 104 insertions(+), 47 deletions(-) rename spartan/src/node/replication/{storage.rs => database.rs} (71%) create mode 100644 spartan/src/node/replication/storage/mod.rs create mode 100644 spartan/src/node/replication/storage/primary.rs create mode 100644 spartan/src/node/replication/storage/replica.rs diff --git a/spartan/src/node/mod.rs b/spartan/src/node/mod.rs index c3845350..e7b7a375 100644 --- a/spartan/src/node/mod.rs +++ b/spartan/src/node/mod.rs @@ -19,9 +19,9 @@ pub use persistence::{load_from_fs, persist_manager, spawn_persistence}; use crate::config::Config; use futures_util::lock::{Mutex, MutexGuard}; +use replication::database::ReplicatedDatabase; use spartan_lib::core::{db::tree::TreeDatabase, message::Message}; use std::collections::{hash_map::RandomState, HashMap}; -use replication::storage::ReplicatedDatabase; pub type DB = ReplicatedDatabase>; type MutexDB = Mutex; diff --git a/spartan/src/node/persistence.rs b/spartan/src/node/persistence.rs index 42900c65..e4f0c677 100644 --- a/spartan/src/node/persistence.rs +++ b/spartan/src/node/persistence.rs @@ -1,4 +1,4 @@ -use super::{MutexDB, Manager}; +use super::{Manager, MutexDB}; use actix_rt::time::delay_for; use bincode::{deserialize, serialize, Error as BincodeError}; use futures_util::lock::Mutex; @@ -21,11 +21,7 @@ pub enum PersistenceError { type PersistenceResult = Result; /// Persist database to provided path -async fn persist_db( - name: &str, - db: &MutexDB, - path: &Path, -) -> Result<(), PersistenceError> { +async fn persist_db(name: &str, db: &MutexDB, path: &Path) -> Result<(), PersistenceError> { let db = db.lock().await; info!("Saving database \"{}\"", name); diff --git a/spartan/src/node/replication/storage.rs b/spartan/src/node/replication/database.rs similarity index 71% rename from spartan/src/node/replication/storage.rs rename to spartan/src/node/replication/database.rs index 58c0a013..dc59ed8e 100644 --- a/spartan/src/node/replication/storage.rs +++ b/spartan/src/node/replication/database.rs @@ -1,46 +1,22 @@ -use std::collections::BTreeMap; -use super::event::Event; -use spartan_lib::core::{message::Message, dispatcher::{StatusAwareDispatcher, SimpleDispatcher, simple::{PositionBasedDelete, Delete}}, payload::Identifiable}; use serde::{Serialize, Deserialize}; - -#[derive(Serialize, Deserialize)] -pub struct ReplicationStorage { - next_index: u64, - gc_threshold: u64, - log: BTreeMap -} - -impl Default for ReplicationStorage { - fn default() -> Self { - ReplicationStorage { - next_index: 1, - gc_threshold: 0, - log: BTreeMap::new() - } - } -} - -impl ReplicationStorage { - pub fn push(&mut self, event: Event) { - self.log.insert(self.next_index, event); - self.next_index += 1; - } -} +use crate::node::replication::event::Event; +use spartan_lib::core::{message::Message, dispatcher::{StatusAwareDispatcher, SimpleDispatcher, simple::{PositionBasedDelete, Delete}}, payload::Identifiable}; +use super::storage::ReplicationStorage; #[derive(Serialize, Deserialize)] pub struct ReplicatedDatabase { inner: DB, - storage: Option + storage: Option, } impl Default for ReplicatedDatabase where - DB: Default + DB: Default, { fn default() -> Self { ReplicatedDatabase { inner: DB::default(), - storage: None + storage: None, } } } @@ -48,10 +24,10 @@ where impl ReplicatedDatabase { pub fn push_event(&mut self, event: F) where - F: FnOnce() -> Event + F: FnOnce() -> Event, { if let Some(storage) = &mut self.storage { - storage.push(event()); + storage.get_primary().push(event()); } } } @@ -89,7 +65,7 @@ where impl StatusAwareDispatcher for ReplicatedDatabase where - DB: StatusAwareDispatcher + DB: StatusAwareDispatcher, { fn pop(&mut self) -> Option<&Message> { self.push_event(|| Event::Pop); @@ -101,12 +77,12 @@ where self.push_event(|| Event::Requeue(id)); self.inner.requeue(id) - } + } } impl Delete for ReplicatedDatabase where - DB: Delete + DB: Delete, { fn delete(&mut self, id: ::Id) -> Option { self.push_event(|| Event::Delete(id)); @@ -117,11 +93,11 @@ where impl PositionBasedDelete for ReplicatedDatabase where - DB: PositionBasedDelete + DB: PositionBasedDelete, { fn delete(&mut self, id: ::Id) -> Option { self.push_event(|| Event::Delete(id)); PositionBasedDelete::delete(&mut self.inner, id) } -} +} \ No newline at end of file diff --git a/spartan/src/node/replication/event.rs b/spartan/src/node/replication/event.rs index 87e6b4a9..af551098 100644 --- a/spartan/src/node/replication/event.rs +++ b/spartan/src/node/replication/event.rs @@ -1,5 +1,5 @@ -use serde::{Serialize, Deserialize}; -use spartan_lib::core::{payload::Identifiable, message::Message}; +use serde::{Deserialize, Serialize}; +use spartan_lib::core::{message::Message, payload::Identifiable}; #[derive(Serialize, Deserialize)] pub enum Event { diff --git a/spartan/src/node/replication/mod.rs b/spartan/src/node/replication/mod.rs index a7a1f23c..92f3f458 100644 --- a/spartan/src/node/replication/mod.rs +++ b/spartan/src/node/replication/mod.rs @@ -2,4 +2,7 @@ pub mod storage; /// Replication event -pub mod event; \ No newline at end of file +pub mod event; + +/// Replication proxy database +pub mod database; diff --git a/spartan/src/node/replication/storage/mod.rs b/spartan/src/node/replication/storage/mod.rs new file mode 100644 index 00000000..1e34b872 --- /dev/null +++ b/spartan/src/node/replication/storage/mod.rs @@ -0,0 +1,37 @@ +/// Storage for replication primary host +pub mod primary; + +/// Storage for replica's +pub mod replica; + +use primary::PrimaryStorage; +use replica::ReplicaStorage; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub enum ReplicationStorage { + Primary(PrimaryStorage), + Replica(ReplicaStorage), +} + +impl Default for ReplicationStorage { + fn default() -> Self { + ReplicationStorage::Primary(PrimaryStorage::default()) + } +} + +impl ReplicationStorage { + pub fn get_primary(&mut self) -> &mut PrimaryStorage { + match self { + ReplicationStorage::Primary(storage) => storage, + _ => panic!("Replication storage is in replica mode.") + } + } + + pub fn get_replica(&mut self) -> &mut ReplicaStorage { + match self { + ReplicationStorage::Replica(storage) => storage, + _ => panic!("Replication storage is in primary mode.") + } + } +} diff --git a/spartan/src/node/replication/storage/primary.rs b/spartan/src/node/replication/storage/primary.rs new file mode 100644 index 00000000..0dd33514 --- /dev/null +++ b/spartan/src/node/replication/storage/primary.rs @@ -0,0 +1,27 @@ +use crate::node::replication::event::Event; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Serialize, Deserialize)] +pub struct PrimaryStorage { + next_index: u64, + gc_threshold: u64, + log: BTreeMap, +} + +impl Default for PrimaryStorage { + fn default() -> Self { + PrimaryStorage { + next_index: 1, + gc_threshold: 0, + log: BTreeMap::new() + } + } +} + +impl PrimaryStorage { + pub fn push(&mut self, event: Event) { + self.log.insert(self.next_index, event); + self.next_index += 1; + } +} \ No newline at end of file diff --git a/spartan/src/node/replication/storage/replica.rs b/spartan/src/node/replication/storage/replica.rs new file mode 100644 index 00000000..2062d553 --- /dev/null +++ b/spartan/src/node/replication/storage/replica.rs @@ -0,0 +1,18 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize)] +pub struct ReplicaStorage { + confirmed_index: u64, +} + +impl Default for ReplicaStorage { + fn default() -> Self { + ReplicaStorage { confirmed_index: 0 } + } +} + +impl ReplicaStorage { + pub fn confirm(&mut self) { + self.confirmed_index += 1; + } +} \ No newline at end of file From 1f59ad6db3ce6640cd8b8f9faf9e2b720253717a Mon Sep 17 00:00:00 2001 From: ivan770 Date: Wed, 8 Jul 2020 13:07:38 +0300 Subject: [PATCH 03/36] cargo fmt --- spartan/src/node/replication/database.rs | 15 +++++++++++---- spartan/src/node/replication/storage/mod.rs | 4 ++-- spartan/src/node/replication/storage/primary.rs | 4 ++-- spartan/src/node/replication/storage/replica.rs | 4 ++-- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/spartan/src/node/replication/database.rs b/spartan/src/node/replication/database.rs index dc59ed8e..f7402fff 100644 --- a/spartan/src/node/replication/database.rs +++ b/spartan/src/node/replication/database.rs @@ -1,7 +1,14 @@ -use serde::{Serialize, Deserialize}; -use crate::node::replication::event::Event; -use spartan_lib::core::{message::Message, dispatcher::{StatusAwareDispatcher, SimpleDispatcher, simple::{PositionBasedDelete, Delete}}, payload::Identifiable}; use super::storage::ReplicationStorage; +use crate::node::replication::event::Event; +use serde::{Deserialize, Serialize}; +use spartan_lib::core::{ + dispatcher::{ + simple::{Delete, PositionBasedDelete}, + SimpleDispatcher, StatusAwareDispatcher, + }, + message::Message, + payload::Identifiable, +}; #[derive(Serialize, Deserialize)] pub struct ReplicatedDatabase { @@ -100,4 +107,4 @@ where PositionBasedDelete::delete(&mut self.inner, id) } -} \ No newline at end of file +} diff --git a/spartan/src/node/replication/storage/mod.rs b/spartan/src/node/replication/storage/mod.rs index 1e34b872..6cd27516 100644 --- a/spartan/src/node/replication/storage/mod.rs +++ b/spartan/src/node/replication/storage/mod.rs @@ -24,14 +24,14 @@ impl ReplicationStorage { pub fn get_primary(&mut self) -> &mut PrimaryStorage { match self { ReplicationStorage::Primary(storage) => storage, - _ => panic!("Replication storage is in replica mode.") + _ => panic!("Replication storage is in replica mode."), } } pub fn get_replica(&mut self) -> &mut ReplicaStorage { match self { ReplicationStorage::Replica(storage) => storage, - _ => panic!("Replication storage is in primary mode.") + _ => panic!("Replication storage is in primary mode."), } } } diff --git a/spartan/src/node/replication/storage/primary.rs b/spartan/src/node/replication/storage/primary.rs index 0dd33514..a8a88044 100644 --- a/spartan/src/node/replication/storage/primary.rs +++ b/spartan/src/node/replication/storage/primary.rs @@ -14,7 +14,7 @@ impl Default for PrimaryStorage { PrimaryStorage { next_index: 1, gc_threshold: 0, - log: BTreeMap::new() + log: BTreeMap::new(), } } } @@ -24,4 +24,4 @@ impl PrimaryStorage { self.log.insert(self.next_index, event); self.next_index += 1; } -} \ No newline at end of file +} diff --git a/spartan/src/node/replication/storage/replica.rs b/spartan/src/node/replication/storage/replica.rs index 2062d553..e5e9ee43 100644 --- a/spartan/src/node/replication/storage/replica.rs +++ b/spartan/src/node/replication/storage/replica.rs @@ -1,4 +1,4 @@ -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] pub struct ReplicaStorage { @@ -15,4 +15,4 @@ impl ReplicaStorage { pub fn confirm(&mut self) { self.confirmed_index += 1; } -} \ No newline at end of file +} From 1b4e1a7fcd375d1ad77e7340b36504564ff2fd18 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Wed, 8 Jul 2020 13:22:52 +0300 Subject: [PATCH 04/36] Remove Default trait for ReplicationStorage --- spartan/src/node/replication/storage/mod.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/spartan/src/node/replication/storage/mod.rs b/spartan/src/node/replication/storage/mod.rs index 6cd27516..21291056 100644 --- a/spartan/src/node/replication/storage/mod.rs +++ b/spartan/src/node/replication/storage/mod.rs @@ -14,12 +14,6 @@ pub enum ReplicationStorage { Replica(ReplicaStorage), } -impl Default for ReplicationStorage { - fn default() -> Self { - ReplicationStorage::Primary(PrimaryStorage::default()) - } -} - impl ReplicationStorage { pub fn get_primary(&mut self) -> &mut PrimaryStorage { match self { From ee4cdf7fd66f89f43f5ea1ae075a939338b8c7bd Mon Sep 17 00:00:00 2001 From: ivan770 Date: Wed, 8 Jul 2020 18:48:45 +0300 Subject: [PATCH 05/36] Added GC to PrimaryStorage, nightly compiler required --- spartan/src/main.rs | 2 ++ spartan/src/node/replication/database.rs | 18 +++++++++--- .../src/node/replication/storage/primary.rs | 28 +++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/spartan/src/main.rs b/spartan/src/main.rs index f4dc3a95..d7532b0c 100644 --- a/spartan/src/main.rs +++ b/spartan/src/main.rs @@ -1,3 +1,5 @@ +#![feature(btree_drain_filter)] + #[macro_use] extern crate derive_new; diff --git a/spartan/src/node/replication/database.rs b/spartan/src/node/replication/database.rs index f7402fff..7478c802 100644 --- a/spartan/src/node/replication/database.rs +++ b/spartan/src/node/replication/database.rs @@ -29,13 +29,22 @@ where } impl ReplicatedDatabase { - pub fn push_event(&mut self, event: F) + fn call_storage(&mut self, f: F) + where + F: FnOnce(&mut ReplicationStorage), + { + self.storage.as_mut().map(f); + } + + fn push_event(&mut self, event: F) where F: FnOnce() -> Event, { - if let Some(storage) = &mut self.storage { - storage.get_primary().push(event()); - } + self.call_storage(|storage| storage.get_primary().push(event())); + } + + fn gc(&mut self) { + self.call_storage(|storage| storage.get_primary().gc()); } } @@ -56,6 +65,7 @@ where fn gc(&mut self) { self.push_event(|| Event::Gc); + ReplicatedDatabase::gc(self); self.inner.gc() } diff --git a/spartan/src/node/replication/storage/primary.rs b/spartan/src/node/replication/storage/primary.rs index a8a88044..f1ff1870 100644 --- a/spartan/src/node/replication/storage/primary.rs +++ b/spartan/src/node/replication/storage/primary.rs @@ -24,4 +24,32 @@ impl PrimaryStorage { self.log.insert(self.next_index, event); self.next_index += 1; } + + pub fn gc(&mut self) { + let gc_threshold = self.gc_threshold; + + self.log + .drain_filter(|index, _| *index <= gc_threshold) + .for_each(drop); + } +} + +#[cfg(test)] +mod tests { + use super::PrimaryStorage; + use crate::node::replication::event::Event; + + #[test] + fn test_gc() { + let mut storage = PrimaryStorage::default(); + + for _ in 0..6 { + storage.push(Event::Pop); + } + + storage.gc_threshold = 4; + storage.gc(); + + assert_eq!(storage.log.iter().map(|(k, _)| k).next(), Some(&5)); + } } From db2cba50a3ddf8f50774618061c5387ab0bcf1f7 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Thu, 9 Jul 2020 13:57:04 +0300 Subject: [PATCH 06/36] Persistence job, ensure correct storage is used in primary --- spartan/src/cli/commands/start.rs | 9 ++++-- spartan/src/config/mod.rs | 10 +++++++ spartan/src/config/replication.rs | 28 +++++++++++++++++++ spartan/src/node/replication/database.rs | 4 +++ spartan/src/node/replication/job.rs | 35 ++++++++++++++++++++++++ spartan/src/node/replication/mod.rs | 3 ++ 6 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 spartan/src/config/replication.rs create mode 100644 spartan/src/node/replication/job.rs diff --git a/spartan/src/cli/commands/start.rs b/spartan/src/cli/commands/start.rs index c992c52e..141ff489 100644 --- a/spartan/src/cli/commands/start.rs +++ b/spartan/src/cli/commands/start.rs @@ -2,8 +2,8 @@ use crate::{ cli::Server, http::server::{start_http_server, ServerError}, node::{ - gc::spawn_gc, load_from_fs, persistence::PersistenceError, spawn_ctrlc_handler, - spawn_persistence, Manager, + gc::spawn_gc, load_from_fs, persistence::PersistenceError, + replication::job::spawn_replication, spawn_ctrlc_handler, spawn_persistence, Manager, }, }; use actix_rt::System; @@ -74,6 +74,11 @@ impl StartCommand { let cloned_manager = manager.clone(); spawn(async move { spawn_persistence(&cloned_manager).await }); + debug!("Spawning replication job."); + + let cloned_manager = manager.clone(); + spawn(async move { spawn_replication(&cloned_manager).await }); + debug!("Spawning Ctrl-C handler"); let cloned_manager = manager.clone(); diff --git a/spartan/src/config/mod.rs b/spartan/src/config/mod.rs index bfa00b13..0de6bdaa 100644 --- a/spartan/src/config/mod.rs +++ b/spartan/src/config/mod.rs @@ -1,6 +1,11 @@ +/// Queue access key pub mod key; +/// Replication config +pub mod replication; + use key::Key; +use replication::Replication; use serde::{Deserialize, Serialize}; use std::{collections::HashSet, path::PathBuf}; @@ -42,6 +47,9 @@ pub struct Config { /// Queue access keys pub access_keys: Option>, + + /// Replication config + pub replication: Option, } #[cfg(not(test))] @@ -54,6 +62,7 @@ impl Default for Config { queues: Vec::new(), encryption_key: None, access_keys: None, + replication: None, } } } @@ -68,6 +77,7 @@ impl Default for Config { queues: vec![String::from("test")], encryption_key: None, access_keys: None, + replication: None, } } } diff --git a/spartan/src/config/replication.rs b/spartan/src/config/replication.rs new file mode 100644 index 00000000..544b7f61 --- /dev/null +++ b/spartan/src/config/replication.rs @@ -0,0 +1,28 @@ +use crate::node::replication::storage::ReplicationStorage; +use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; + +#[derive(Serialize, Deserialize)] +pub struct Primary { + destination: Vec, +} + +#[derive(Serialize, Deserialize)] +pub struct Replica { + host: SocketAddr, +} + +#[derive(Serialize, Deserialize)] +pub enum Replication { + Primary(Primary), + Replica(Replica), +} + +impl PartialEq for Replication { + fn eq(&self, other: &ReplicationStorage) -> bool { + match *self { + Replication::Primary(_) => matches!(*other, ReplicationStorage::Primary(_)), + Replication::Replica(_) => matches!(*other, ReplicationStorage::Replica(_)), + } + } +} diff --git a/spartan/src/node/replication/database.rs b/spartan/src/node/replication/database.rs index 7478c802..37bc9d0f 100644 --- a/spartan/src/node/replication/database.rs +++ b/spartan/src/node/replication/database.rs @@ -46,6 +46,10 @@ impl ReplicatedDatabase { fn gc(&mut self) { self.call_storage(|storage| storage.get_primary().gc()); } + + pub fn get_storage(&mut self) -> &mut Option { + &mut self.storage + } } impl SimpleDispatcher for ReplicatedDatabase diff --git a/spartan/src/node/replication/job.rs b/spartan/src/node/replication/job.rs new file mode 100644 index 00000000..4c520508 --- /dev/null +++ b/spartan/src/node/replication/job.rs @@ -0,0 +1,35 @@ +use super::storage::{primary::PrimaryStorage, ReplicationStorage}; +use crate::{config::replication::Replication, node::Manager}; + +async fn prepare_storage(manager: &Manager<'_>) { + for (_, db) in manager.node.db.iter() { + let mut db = db.lock().await; + + let storage = db + .get_storage() + .as_ref() + .filter(|storage| matches!(storage, ReplicationStorage::Primary(_))); + + if storage.is_none() { + db.get_storage() + .replace(ReplicationStorage::Primary(PrimaryStorage::default())); + } + } +} + +async fn replicate_manager(manager: &Manager<'_>) { + prepare_storage(manager).await; +} + +pub async fn spawn_replication(manager: &Manager<'_>) { + manager + .config + .replication + .as_ref() + .map(|config| async move { + match config { + Replication::Primary(_) => replicate_manager(manager).await, + Replication::Replica(_) => (), + }; + }); +} diff --git a/spartan/src/node/replication/mod.rs b/spartan/src/node/replication/mod.rs index 92f3f458..222e9161 100644 --- a/spartan/src/node/replication/mod.rs +++ b/spartan/src/node/replication/mod.rs @@ -6,3 +6,6 @@ pub mod event; /// Replication proxy database pub mod database; + +/// Replication job +pub mod job; From 7c480632c17a89721cf3be6d51d808ad306f23c0 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Thu, 9 Jul 2020 13:59:30 +0300 Subject: [PATCH 07/36] Remove unused PartialEq --- spartan/src/config/replication.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/spartan/src/config/replication.rs b/spartan/src/config/replication.rs index 544b7f61..50d6ce35 100644 --- a/spartan/src/config/replication.rs +++ b/spartan/src/config/replication.rs @@ -1,4 +1,3 @@ -use crate::node::replication::storage::ReplicationStorage; use serde::{Deserialize, Serialize}; use std::net::SocketAddr; @@ -17,12 +16,3 @@ pub enum Replication { Primary(Primary), Replica(Replica), } - -impl PartialEq for Replication { - fn eq(&self, other: &ReplicationStorage) -> bool { - match *self { - Replication::Primary(_) => matches!(*other, ReplicationStorage::Primary(_)), - Replication::Replica(_) => matches!(*other, ReplicationStorage::Replica(_)), - } - } -} From 53583c670ad264cc98ef91cb62e2ad803f9de2fb Mon Sep 17 00:00:00 2001 From: ivan770 Date: Thu, 9 Jul 2020 21:43:53 +0300 Subject: [PATCH 08/36] Try to fix Github Actions for test --- .github/workflows/test.yml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0c2543a4..1f9744c3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,12 +12,24 @@ jobs: linux: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Run tests - run: cargo test --verbose + - uses: actions/checkout@v2 + - name: Install latest nightly + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + - name: Run tests + uses: actions-rs/cargo@v1 + with: + command: test win: runs-on: windows-latest steps: - - uses: actions/checkout@v2 - - name: Run tests - run: cargo test --verbose \ No newline at end of file + - uses: actions/checkout@v2 + - name: Install latest nightly + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + - name: Run tests + uses: actions-rs/cargo@v1 + with: + command: test \ No newline at end of file From 7b91b005c25123848828f5f5b2458f66c7052ac9 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Thu, 9 Jul 2020 21:50:18 +0300 Subject: [PATCH 09/36] Override toolchain when testing --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f9744c3..2cfb6452 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,7 @@ jobs: uses: actions-rs/toolchain@v1 with: toolchain: nightly + override: true - name: Run tests uses: actions-rs/cargo@v1 with: @@ -29,6 +30,7 @@ jobs: uses: actions-rs/toolchain@v1 with: toolchain: nightly + override: true - name: Run tests uses: actions-rs/cargo@v1 with: From 66ee9ff76e5758fee55cee0a22b94973d7c4e9de Mon Sep 17 00:00:00 2001 From: ivan770 Date: Thu, 9 Jul 2020 22:03:42 +0300 Subject: [PATCH 10/36] Use nightly toolchain on release --- .github/workflows/docs.yml | 14 ++++++++--- .github/workflows/release.yml | 46 ++++++++++++++++++++++++----------- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a61bc2d7..ab6ef601 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,12 +10,20 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Build documentation - run: cargo doc --all --no-deps + - name: Install latest nightly + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + override: true + - name: Generate documentation + uses: actions-rs/cargo@v1 + with: + command: doc + args: --all --no-deps - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_branch: gh-pages - publish_dir: ./target/doc + publish_dir: target/doc force_orphan: true \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 25b91b53..e0976826 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,21 +10,39 @@ jobs: name: Build Linux runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Build - run: cargo build --release --verbose - - uses: actions/upload-artifact@v1 - with: - name: linux - path: target/release/spartan + - uses: actions/checkout@v2 + - name: Install latest nightly + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + override: true + - name: Build + uses: actions-rs/cargo@v1 + with: + command: build + args: --release + - name: Upload binary + uses: actions/upload-artifact@v1 + with: + name: linux + path: target/release/spartan win: name: Build Win runs-on: windows-latest steps: - - uses: actions/checkout@v2 - - name: Build - run: cargo build --release --verbose - - uses: actions/upload-artifact@v1 - with: - name: win - path: target/release/spartan.exe \ No newline at end of file + - uses: actions/checkout@v2 + - name: Install latest nightly + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + override: true + - name: Build + uses: actions-rs/cargo@v1 + with: + command: build + args: --release + - name: Upload binary + uses: actions/upload-artifact@v1 + with: + name: linux + path: target/release/spartan.exe \ No newline at end of file From 357dcd8871504be7cc8cee5d08b053d915b00a62 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Thu, 9 Jul 2020 22:04:41 +0300 Subject: [PATCH 11/36] Add line breaks to Actions configs --- .github/workflows/docs.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ab6ef601..6e77254a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,4 +26,4 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} publish_branch: gh-pages publish_dir: target/doc - force_orphan: true \ No newline at end of file + force_orphan: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e0976826..9e7cd613 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,4 +45,4 @@ jobs: uses: actions/upload-artifact@v1 with: name: linux - path: target/release/spartan.exe \ No newline at end of file + path: target/release/spartan.exe diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2cfb6452..4de9273c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,4 +34,4 @@ jobs: - name: Run tests uses: actions-rs/cargo@v1 with: - command: test \ No newline at end of file + command: test From 91dca0a3246bf446276a945b22e185f22a02b165 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Thu, 9 Jul 2020 22:06:37 +0300 Subject: [PATCH 12/36] Fix incorrect artifact name --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9e7cd613..8d29f56a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: name: linux path: target/release/spartan win: - name: Build Win + name: Build Windows runs-on: windows-latest steps: - uses: actions/checkout@v2 @@ -44,5 +44,5 @@ jobs: - name: Upload binary uses: actions/upload-artifact@v1 with: - name: linux + name: windows path: target/release/spartan.exe From 906403ad10c8a5269db921d77406f2e59ef4dbaf Mon Sep 17 00:00:00 2001 From: ivan770 Date: Sat, 11 Jul 2020 19:57:47 +0300 Subject: [PATCH 13/36] Added TCP stream wrapper, move replication job into submodule --- Cargo.lock | 104 ++++++++++----------- spartan/Cargo.toml | 2 +- spartan/src/config/replication.rs | 12 ++- spartan/src/node/replication/job.rs | 35 ------- spartan/src/node/replication/job/mod.rs | 71 ++++++++++++++ spartan/src/node/replication/job/stream.rs | 64 +++++++++++++ 6 files changed, 198 insertions(+), 90 deletions(-) delete mode 100644 spartan/src/node/replication/job.rs create mode 100644 spartan/src/node/replication/job/mod.rs create mode 100644 spartan/src/node/replication/job/stream.rs diff --git a/Cargo.lock b/Cargo.lock index e743d5c0..49dda726 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -261,18 +261,18 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.12.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602d785912f476e480434627e8732e6766b760c045bbf897d9dfaa9f4fbd399c" +checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" dependencies = [ "gimli", ] [[package]] -name = "adler32" -version = "1.1.0" +name = "adler" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" [[package]] name = "aho-corasick" @@ -289,7 +289,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" dependencies = [ - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -323,7 +323,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi", "libc", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -357,9 +357,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.49" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05100821de9e028f12ae3d189176b41ee198341eb8f369956407fea2f5cc666c" +checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" dependencies = [ "addr2line", "cfg-if", @@ -437,9 +437,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.54" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bbb73db36c1246e9034e307d0fba23f9a2e251faa47ade70c1bd252220c8311" +checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518" [[package]] name = "cfg-if" @@ -449,9 +449,9 @@ checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "chrono" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" +checksum = "c74d84029116787153e02106bf53e66828452a4b325cc8652b788b5967c0a0b6" dependencies = [ "num-integer", "num-traits", @@ -511,9 +511,9 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.8" +version = "0.99.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc655351f820d774679da6cdc23355a93de496867d8203496675162e17b1d671" +checksum = "298998b1cf6b5b2c8a7b023dfd45821825ce3ba8a8af55c921a0e734e4653f76" dependencies = [ "proc-macro2", "quote", @@ -590,9 +590,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" +checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e" dependencies = [ "cfg-if", "crc32fast", @@ -736,7 +736,7 @@ dependencies = [ "libc", "log", "rustc_version", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -752,9 +752,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc8e0c9bce37868955864dbecd2b1ab2bdf967e6f28066d65aaac620444b65c" +checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" [[package]] name = "h2" @@ -786,9 +786,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9586eedd4ce6b3c498bc3b4dd92fc9f11166aa908a914071953768066c67909" +checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" dependencies = [ "libc", ] @@ -801,7 +801,7 @@ checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" dependencies = [ "libc", "match_cfg", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -867,7 +867,7 @@ checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" dependencies = [ "socket2", "widestring", - "winapi 0.3.8", + "winapi 0.3.9", "winreg", ] @@ -901,9 +901,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.71" +version = "0.2.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" +checksum = "a9f8082297d534141b30c8d39e9b1773713ab50fdbe4ff30f750d063b3bfd701" [[package]] name = "linked-hash-map" @@ -975,11 +975,11 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" [[package]] name = "miniz_oxide" -version = "0.3.7" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" dependencies = [ - "adler32", + "adler", ] [[package]] @@ -1032,7 +1032,7 @@ checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" dependencies = [ "cfg-if", "libc", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1097,7 +1097,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1156,9 +1156,9 @@ dependencies = [ [[package]] name = "proc-macro-error" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678" +checksum = "fc175e9777c3116627248584e8f8b3e2987405cabe1c0adf7d1dd28f09dc7880" dependencies = [ "proc-macro-error-attr", "proc-macro2", @@ -1169,9 +1169,9 @@ dependencies = [ [[package]] name = "proc-macro-error-attr" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53" +checksum = "3cc9795ca17eb581285ec44936da7fc2335a3f34f2ddd13118b6f4d515435c50" dependencies = [ "proc-macro2", "quote", @@ -1259,9 +1259,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "regex" @@ -1287,7 +1287,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1370,9 +1370,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.55" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec2c5d7e739bc07a3e73381a39d61fdb5f671c60c1df26a130690665803d8226" +checksum = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" dependencies = [ "itoa", "ryu", @@ -1415,9 +1415,9 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "smallvec" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" +checksum = "3757cb9d89161a2f24e1cf78efa0c1fcff485d18e3f55e0aa3480824ddaa0f3f" [[package]] name = "socket2" @@ -1428,7 +1428,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1540,7 +1540,7 @@ dependencies = [ "rand", "redox_syscall", "remove_dir_all", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1606,7 +1606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" dependencies = [ "libc", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1634,7 +1634,7 @@ dependencies = [ "signal-hook-registry", "slab", "tokio-macros", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1750,9 +1750,9 @@ checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" [[package]] name = "unicode-width" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" @@ -1813,9 +1813,9 @@ checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" [[package]] name = "winapi" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", @@ -1839,7 +1839,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1854,7 +1854,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" dependencies = [ - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] diff --git a/spartan/Cargo.toml b/spartan/Cargo.toml index 6196787f..12163bd7 100644 --- a/spartan/Cargo.toml +++ b/spartan/Cargo.toml @@ -18,7 +18,7 @@ version = "1.0" [dependencies.tokio] version = "0.2" -features = ["macros", "rt-threaded", "fs"] +features = ["macros", "rt-threaded", "fs", "tcp"] [dependencies.structopt] version = "0.3" diff --git a/spartan/src/config/replication.rs b/spartan/src/config/replication.rs index 50d6ce35..6bad891d 100644 --- a/spartan/src/config/replication.rs +++ b/spartan/src/config/replication.rs @@ -1,14 +1,22 @@ use serde::{Deserialize, Serialize}; use std::net::SocketAddr; +/// Default amount of seconds between replication jobs +const fn default_replication_timer() -> u64 { + 180 +} + #[derive(Serialize, Deserialize)] pub struct Primary { - destination: Vec, + pub destination: Vec, + + #[serde(default = "default_replication_timer")] + pub replication_timer: u64, } #[derive(Serialize, Deserialize)] pub struct Replica { - host: SocketAddr, + pub host: SocketAddr, } #[derive(Serialize, Deserialize)] diff --git a/spartan/src/node/replication/job.rs b/spartan/src/node/replication/job.rs deleted file mode 100644 index 4c520508..00000000 --- a/spartan/src/node/replication/job.rs +++ /dev/null @@ -1,35 +0,0 @@ -use super::storage::{primary::PrimaryStorage, ReplicationStorage}; -use crate::{config::replication::Replication, node::Manager}; - -async fn prepare_storage(manager: &Manager<'_>) { - for (_, db) in manager.node.db.iter() { - let mut db = db.lock().await; - - let storage = db - .get_storage() - .as_ref() - .filter(|storage| matches!(storage, ReplicationStorage::Primary(_))); - - if storage.is_none() { - db.get_storage() - .replace(ReplicationStorage::Primary(PrimaryStorage::default())); - } - } -} - -async fn replicate_manager(manager: &Manager<'_>) { - prepare_storage(manager).await; -} - -pub async fn spawn_replication(manager: &Manager<'_>) { - manager - .config - .replication - .as_ref() - .map(|config| async move { - match config { - Replication::Primary(_) => replicate_manager(manager).await, - Replication::Replica(_) => (), - }; - }); -} diff --git a/spartan/src/node/replication/job/mod.rs b/spartan/src/node/replication/job/mod.rs new file mode 100644 index 00000000..d3026d13 --- /dev/null +++ b/spartan/src/node/replication/job/mod.rs @@ -0,0 +1,71 @@ +/// Tokio TCP stream abstraction +pub mod stream; + +use super::storage::{primary::PrimaryStorage, ReplicationStorage}; +use crate::{ + config::replication::{Primary, Replication}, + node::Manager, +}; +use actix_rt::time::delay_for; +use std::time::Duration; +use stream::{StreamPool, StreamResult}; +use tokio::io::Result as IoResult; + +async fn prepare_storage(manager: &Manager<'_>) { + for (_, db) in manager.node.db.iter() { + let mut db = db.lock().await; + + let storage = db + .get_storage() + .as_ref() + .filter(|storage| matches!(storage, ReplicationStorage::Primary(_))); + + if storage.is_none() { + db.get_storage() + .replace(ReplicationStorage::Primary(PrimaryStorage::default())); + } + } +} + +async fn replicate_manager(manager: &Manager<'_>, pool: &mut StreamPool) -> StreamResult<()> { + pool.ping().await?; + + Ok(()) +} + +async fn start_replication(manager: &Manager<'_>, pool: &mut StreamPool, config: &Primary) { + let timer = Duration::from_secs(config.replication_timer); + + loop { + delay_for(timer).await; + + match replicate_manager(manager, pool).await { + Err(e) => error!("Error happened during replication attempt: {}", e), + Ok(_) => info!("Database replicated successfully!") + } + } +} + +pub async fn spawn_replication(manager: &Manager<'_>) -> IoResult<()> { + manager + .config + .replication + .as_ref() + .map(|config| async move { + match config { + Replication::Primary(config) => { + prepare_storage(manager).await; + + match StreamPool::from_config(config).await { + Ok(mut pool) => start_replication(manager, &mut pool, config).await, + Err(e) => error!("Unable to open connection pool: {}", e), + } + } + Replication::Replica(_) => { + panic!("Starting replication job while in replica mode is not allowed") + } + }; + }); + + Ok(()) +} diff --git a/spartan/src/node/replication/job/stream.rs b/spartan/src/node/replication/job/stream.rs new file mode 100644 index 00000000..398f5ac9 --- /dev/null +++ b/spartan/src/node/replication/job/stream.rs @@ -0,0 +1,64 @@ +use crate::config::replication::Primary; +use bincode::{serialize, ErrorKind}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::{ + io::{AsyncWriteExt, Error as IoError}, + net::TcpStream, +}; + +#[derive(Error, Debug)] +pub enum StreamError { + #[error("Unable to serialize stream message: {0}")] + SerializationError(Box), + #[error("TCP connection error: {0}")] + SocketError(IoError), +} + +pub(super) type StreamResult = Result; + +#[derive(Serialize, Deserialize)] +pub enum StreamMessage { + Ping, +} + +pub(super) struct Stream(TcpStream); + +pub(super) struct StreamPool(Vec); + +impl Stream { + fn serialize(message: &StreamMessage) -> StreamResult> { + serialize(&message).map_err(|e| StreamError::SerializationError(e)) + } + + pub async fn ping(&mut self) -> Result<(), StreamError> { + self.0 + .write_all(&Self::serialize(&StreamMessage::Ping)?) + .await + .map_err(|e| StreamError::SocketError(e)) + } +} + +impl StreamPool { + pub async fn from_config(config: &Primary) -> StreamResult { + let mut pool = Vec::with_capacity(config.destination.len()); + + for host in &config.destination { + pool.push(Stream( + TcpStream::connect(host) + .await + .map_err(|e| StreamError::SocketError(e))?, + )); + } + + Ok(StreamPool(pool)) + } + + pub async fn ping(&mut self) -> StreamResult<()> { + for host in &mut self.0 { + host.ping().await?; + } + + Ok(()) + } +} From 90fc2b57e6822ad04aa463da162fe1230b32dc6d Mon Sep 17 00:00:00 2001 From: ivan770 Date: Sun, 12 Jul 2020 18:18:47 +0300 Subject: [PATCH 14/36] Added index asking --- spartan/src/node/replication/job/index.rs | 28 +++++++++++++ spartan/src/node/replication/job/mod.rs | 9 +++- spartan/src/node/replication/job/stream.rs | 48 +++++++++++++++++++--- 3 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 spartan/src/node/replication/job/index.rs diff --git a/spartan/src/node/replication/job/index.rs b/spartan/src/node/replication/job/index.rs new file mode 100644 index 00000000..b6992ed4 --- /dev/null +++ b/spartan/src/node/replication/job/index.rs @@ -0,0 +1,28 @@ +use super::stream::Stream; + +pub(super) struct RecvIndex<'a> { + stream: &'a mut Stream, + indexes: Vec<(Box, u64)>, +} + +impl<'a> RecvIndex<'a> { + pub fn new(stream: &'a mut Stream, indexes: Vec<(Box, u64)>) -> Self { + RecvIndex { stream, indexes } + } +} + +pub(super) struct BatchAskIndex<'a> { + batch: Vec>, +} + +impl<'a> BatchAskIndex<'a> { + pub fn with_capacity(capacity: usize) -> Self { + BatchAskIndex { + batch: Vec::with_capacity(capacity), + } + } + + pub fn push(&mut self, index: RecvIndex<'a>) { + self.batch.push(index); + } +} diff --git a/spartan/src/node/replication/job/mod.rs b/spartan/src/node/replication/job/mod.rs index d3026d13..fc8c8ed7 100644 --- a/spartan/src/node/replication/job/mod.rs +++ b/spartan/src/node/replication/job/mod.rs @@ -1,6 +1,9 @@ /// Tokio TCP stream abstraction pub mod stream; +/// Replication index exchange +pub mod index; + use super::storage::{primary::PrimaryStorage, ReplicationStorage}; use crate::{ config::replication::{Primary, Replication}, @@ -30,6 +33,10 @@ async fn prepare_storage(manager: &Manager<'_>) { async fn replicate_manager(manager: &Manager<'_>, pool: &mut StreamPool) -> StreamResult<()> { pool.ping().await?; + for host in pool.ask().await { + + } + Ok(()) } @@ -41,7 +48,7 @@ async fn start_replication(manager: &Manager<'_>, pool: &mut StreamPool, config: match replicate_manager(manager, pool).await { Err(e) => error!("Error happened during replication attempt: {}", e), - Ok(_) => info!("Database replicated successfully!") + Ok(_) => info!("Database replicated successfully!"), } } } diff --git a/spartan/src/node/replication/job/stream.rs b/spartan/src/node/replication/job/stream.rs index 398f5ac9..f47f601a 100644 --- a/spartan/src/node/replication/job/stream.rs +++ b/spartan/src/node/replication/job/stream.rs @@ -1,9 +1,10 @@ +use super::index::{BatchAskIndex, RecvIndex}; use crate::config::replication::Primary; -use bincode::{serialize, ErrorKind}; +use bincode::{deserialize, serialize, ErrorKind}; use serde::{Deserialize, Serialize}; use thiserror::Error; use tokio::{ - io::{AsyncWriteExt, Error as IoError}, + io::{AsyncReadExt, AsyncWriteExt, Error as IoError}, net::TcpStream, }; @@ -13,6 +14,8 @@ pub enum StreamError { SerializationError(Box), #[error("TCP connection error: {0}")] SocketError(IoError), + #[error("Protocol mismatch")] + ProtocolMismatch, } pub(super) type StreamResult = Result; @@ -20,26 +23,51 @@ pub(super) type StreamResult = Result; #[derive(Serialize, Deserialize)] pub enum StreamMessage { Ping, + AskIndex, + RecvIndex(Vec<(Box, u64)>), } pub(super) struct Stream(TcpStream); pub(super) struct StreamPool(Vec); -impl Stream { +impl<'a> Stream { fn serialize(message: &StreamMessage) -> StreamResult> { serialize(&message).map_err(|e| StreamError::SerializationError(e)) } - pub async fn ping(&mut self) -> Result<(), StreamError> { + pub async fn ping(&mut self) -> StreamResult<()> { self.0 .write_all(&Self::serialize(&StreamMessage::Ping)?) .await .map_err(|e| StreamError::SocketError(e)) } + + pub async fn ask(&'a mut self) -> StreamResult> { + let (mut receive, mut send) = self.0.split(); + + send.write_all(&Self::serialize(&StreamMessage::AskIndex)?) + .await + .map_err(|e| StreamError::SocketError(e))?; + + let mut buf = Vec::new(); + + receive + .read_to_end(&mut buf) + .await + .map_err(|e| StreamError::SocketError(e))?; + + let buf: StreamMessage = + deserialize(&buf).map_err(|e| StreamError::SerializationError(e))?; + + match buf { + StreamMessage::RecvIndex(recv) => Ok(RecvIndex::new(self, recv)), + _ => Err(StreamError::ProtocolMismatch), + } + } } -impl StreamPool { +impl<'a> StreamPool { pub async fn from_config(config: &Primary) -> StreamResult { let mut pool = Vec::with_capacity(config.destination.len()); @@ -61,4 +89,14 @@ impl StreamPool { Ok(()) } + + pub async fn ask(&'a mut self) -> StreamResult> { + let mut batch = BatchAskIndex::with_capacity(self.0.capacity()); + + for host in &mut self.0 { + batch.push(host.ask().await?); + } + + Ok(batch) + } } From aeb8d9bb4fdf8b1915beb93d69d2f1f4abfd048f Mon Sep 17 00:00:00 2001 From: ivan770 Date: Sun, 12 Jul 2020 18:28:40 +0300 Subject: [PATCH 15/36] Use boxed slices in read-only values --- spartan/src/node/replication/job/index.rs | 4 ++-- spartan/src/node/replication/job/stream.rs | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/spartan/src/node/replication/job/index.rs b/spartan/src/node/replication/job/index.rs index b6992ed4..a55bdec4 100644 --- a/spartan/src/node/replication/job/index.rs +++ b/spartan/src/node/replication/job/index.rs @@ -2,11 +2,11 @@ use super::stream::Stream; pub(super) struct RecvIndex<'a> { stream: &'a mut Stream, - indexes: Vec<(Box, u64)>, + indexes: Box<[(Box, u64)]>, } impl<'a> RecvIndex<'a> { - pub fn new(stream: &'a mut Stream, indexes: Vec<(Box, u64)>) -> Self { + pub fn new(stream: &'a mut Stream, indexes: Box<[(Box, u64)]>) -> Self { RecvIndex { stream, indexes } } } diff --git a/spartan/src/node/replication/job/stream.rs b/spartan/src/node/replication/job/stream.rs index f47f601a..c218bbe2 100644 --- a/spartan/src/node/replication/job/stream.rs +++ b/spartan/src/node/replication/job/stream.rs @@ -29,7 +29,7 @@ pub enum StreamMessage { pub(super) struct Stream(TcpStream); -pub(super) struct StreamPool(Vec); +pub(super) struct StreamPool(Box<[Stream]>); impl<'a> Stream { fn serialize(message: &StreamMessage) -> StreamResult> { @@ -50,6 +50,7 @@ impl<'a> Stream { .await .map_err(|e| StreamError::SocketError(e))?; + // TODO: Optimize by using pre-allocated buffer let mut buf = Vec::new(); receive @@ -61,7 +62,7 @@ impl<'a> Stream { deserialize(&buf).map_err(|e| StreamError::SerializationError(e))?; match buf { - StreamMessage::RecvIndex(recv) => Ok(RecvIndex::new(self, recv)), + StreamMessage::RecvIndex(recv) => Ok(RecvIndex::new(self, recv.into_boxed_slice())), _ => Err(StreamError::ProtocolMismatch), } } @@ -79,11 +80,11 @@ impl<'a> StreamPool { )); } - Ok(StreamPool(pool)) + Ok(StreamPool(pool.into_boxed_slice())) } pub async fn ping(&mut self) -> StreamResult<()> { - for host in &mut self.0 { + for host in &mut *self.0 { host.ping().await?; } @@ -91,9 +92,9 @@ impl<'a> StreamPool { } pub async fn ask(&'a mut self) -> StreamResult> { - let mut batch = BatchAskIndex::with_capacity(self.0.capacity()); + let mut batch = BatchAskIndex::with_capacity(self.0.len()); - for host in &mut self.0 { + for host in &mut *self.0 { batch.push(host.ask().await?); } From a1e187573e054a3d0f00bf03d569ca1151b77ebc Mon Sep 17 00:00:00 2001 From: ivan770 Date: Sun, 12 Jul 2020 19:22:33 +0300 Subject: [PATCH 16/36] Log slice exchange --- spartan/src/node/replication/job/error.rs | 17 ++++ spartan/src/node/replication/job/index.rs | 34 ++++++- spartan/src/node/replication/job/mod.rs | 12 +-- spartan/src/node/replication/job/stream.rs | 90 ++++++++++--------- .../src/node/replication/storage/primary.rs | 4 + 5 files changed, 110 insertions(+), 47 deletions(-) create mode 100644 spartan/src/node/replication/job/error.rs diff --git a/spartan/src/node/replication/job/error.rs b/spartan/src/node/replication/job/error.rs new file mode 100644 index 00000000..9d0664d4 --- /dev/null +++ b/spartan/src/node/replication/job/error.rs @@ -0,0 +1,17 @@ +use bincode::ErrorKind; +use thiserror::Error; +use tokio::io::Error as IoError; + +#[derive(Error, Debug)] +pub(super) enum ReplicationError { + #[error("Unable to serialize stream message: {0}")] + SerializationError(Box), + #[error("TCP connection error: {0}")] + SocketError(IoError), + #[error("Protocol mismatch")] + ProtocolMismatch, + #[error("Queue configuration mismatch")] + QueueConfigMismatch, +} + +pub(super) type ReplicationResult = Result; diff --git a/spartan/src/node/replication/job/index.rs b/spartan/src/node/replication/job/index.rs index a55bdec4..d3cf63e9 100644 --- a/spartan/src/node/replication/job/index.rs +++ b/spartan/src/node/replication/job/index.rs @@ -1,4 +1,8 @@ -use super::stream::Stream; +use super::{ + error::{ReplicationError, ReplicationResult}, + stream::Stream, +}; +use crate::node::Manager; pub(super) struct RecvIndex<'a> { stream: &'a mut Stream, @@ -9,6 +13,26 @@ impl<'a> RecvIndex<'a> { pub fn new(stream: &'a mut Stream, indexes: Box<[(Box, u64)]>) -> Self { RecvIndex { stream, indexes } } + + pub async fn sync(&mut self, manager: &Manager<'_>) -> ReplicationResult<()> { + for (name, start) in self.indexes.iter() { + self.stream + .send_range( + manager + .queue(name) + .await + .map_err(|_| ReplicationError::QueueConfigMismatch)? + .get_storage() + .as_mut() + .expect("Replication storage is uninitialized") + .get_primary() + .slice(*start), + ) + .await?; + } + + Ok(()) + } } pub(super) struct BatchAskIndex<'a> { @@ -25,4 +49,12 @@ impl<'a> BatchAskIndex<'a> { pub fn push(&mut self, index: RecvIndex<'a>) { self.batch.push(index); } + + pub async fn sync(&mut self, manager: &Manager<'_>) -> ReplicationResult<()> { + for host in &mut self.batch { + host.sync(manager).await?; + } + + Ok(()) + } } diff --git a/spartan/src/node/replication/job/mod.rs b/spartan/src/node/replication/job/mod.rs index fc8c8ed7..0b752672 100644 --- a/spartan/src/node/replication/job/mod.rs +++ b/spartan/src/node/replication/job/mod.rs @@ -4,14 +4,18 @@ pub mod stream; /// Replication index exchange pub mod index; +/// Replication error +pub mod error; + use super::storage::{primary::PrimaryStorage, ReplicationStorage}; use crate::{ config::replication::{Primary, Replication}, node::Manager, }; use actix_rt::time::delay_for; +use error::ReplicationResult; use std::time::Duration; -use stream::{StreamPool, StreamResult}; +use stream::StreamPool; use tokio::io::Result as IoResult; async fn prepare_storage(manager: &Manager<'_>) { @@ -30,12 +34,10 @@ async fn prepare_storage(manager: &Manager<'_>) { } } -async fn replicate_manager(manager: &Manager<'_>, pool: &mut StreamPool) -> StreamResult<()> { +async fn replicate_manager(manager: &Manager<'_>, pool: &mut StreamPool) -> ReplicationResult<()> { pool.ping().await?; - for host in pool.ask().await { - - } + pool.ask().await?.sync(manager).await?; Ok(()) } diff --git a/spartan/src/node/replication/job/stream.rs b/spartan/src/node/replication/job/stream.rs index c218bbe2..02da9560 100644 --- a/spartan/src/node/replication/job/stream.rs +++ b/spartan/src/node/replication/job/stream.rs @@ -1,30 +1,27 @@ -use super::index::{BatchAskIndex, RecvIndex}; -use crate::config::replication::Primary; -use bincode::{deserialize, serialize, ErrorKind}; +use super::{ + error::{ReplicationError, ReplicationResult}, + index::{BatchAskIndex, RecvIndex}, +}; +use crate::{config::replication::Primary, node::replication::event::Event}; +use bincode::{deserialize, serialize}; use serde::{Deserialize, Serialize}; -use thiserror::Error; use tokio::{ - io::{AsyncReadExt, AsyncWriteExt, Error as IoError}, + io::{AsyncReadExt, AsyncWriteExt}, net::TcpStream, }; -#[derive(Error, Debug)] -pub enum StreamError { - #[error("Unable to serialize stream message: {0}")] - SerializationError(Box), - #[error("TCP connection error: {0}")] - SocketError(IoError), - #[error("Protocol mismatch")] - ProtocolMismatch, -} - -pub(super) type StreamResult = Result; - -#[derive(Serialize, Deserialize)] -pub enum StreamMessage { +#[derive(Serialize)] +pub enum StreamRequest<'a> { Ping, AskIndex, + SendRange(Box<[(&'a u64, &'a Event)]>), +} + +#[derive(Deserialize)] +pub enum StreamResponse { + Pong, RecvIndex(Vec<(Box, u64)>), + RecvRange, } pub(super) struct Stream(TcpStream); @@ -32,23 +29,19 @@ pub(super) struct Stream(TcpStream); pub(super) struct StreamPool(Box<[Stream]>); impl<'a> Stream { - fn serialize(message: &StreamMessage) -> StreamResult> { - serialize(&message).map_err(|e| StreamError::SerializationError(e)) - } - - pub async fn ping(&mut self) -> StreamResult<()> { - self.0 - .write_all(&Self::serialize(&StreamMessage::Ping)?) - .await - .map_err(|e| StreamError::SocketError(e)) + fn serialize(message: &StreamRequest) -> ReplicationResult> { + serialize(&message).map_err(|e| ReplicationError::SerializationError(e)) } - pub async fn ask(&'a mut self) -> StreamResult> { + pub async fn exchange( + &mut self, + message: &StreamRequest<'_>, + ) -> ReplicationResult { let (mut receive, mut send) = self.0.split(); - send.write_all(&Self::serialize(&StreamMessage::AskIndex)?) + send.write_all(&Self::serialize(message)?) .await - .map_err(|e| StreamError::SocketError(e))?; + .map_err(|e| ReplicationError::SocketError(e))?; // TODO: Optimize by using pre-allocated buffer let mut buf = Vec::new(); @@ -56,34 +49,49 @@ impl<'a> Stream { receive .read_to_end(&mut buf) .await - .map_err(|e| StreamError::SocketError(e))?; + .map_err(|e| ReplicationError::SocketError(e))?; + + Ok(deserialize(&buf).map_err(|e| ReplicationError::SerializationError(e))?) + } + + pub async fn ping(&mut self) -> ReplicationResult<()> { + match self.exchange(&StreamRequest::Ping).await? { + StreamResponse::Pong => Ok(()), + _ => Err(ReplicationError::ProtocolMismatch), + } + } - let buf: StreamMessage = - deserialize(&buf).map_err(|e| StreamError::SerializationError(e))?; + pub async fn ask(&'a mut self) -> ReplicationResult> { + match self.exchange(&StreamRequest::AskIndex).await? { + StreamResponse::RecvIndex(recv) => Ok(RecvIndex::new(self, recv.into_boxed_slice())), + _ => Err(ReplicationError::ProtocolMismatch), + } + } - match buf { - StreamMessage::RecvIndex(recv) => Ok(RecvIndex::new(self, recv.into_boxed_slice())), - _ => Err(StreamError::ProtocolMismatch), + pub async fn send_range(&mut self, range: Box<[(&u64, &Event)]>) -> ReplicationResult<()> { + match self.exchange(&StreamRequest::SendRange(range)).await? { + StreamResponse::RecvRange => Ok(()), + _ => Err(ReplicationError::ProtocolMismatch), } } } impl<'a> StreamPool { - pub async fn from_config(config: &Primary) -> StreamResult { + pub async fn from_config(config: &Primary) -> ReplicationResult { let mut pool = Vec::with_capacity(config.destination.len()); for host in &config.destination { pool.push(Stream( TcpStream::connect(host) .await - .map_err(|e| StreamError::SocketError(e))?, + .map_err(|e| ReplicationError::SocketError(e))?, )); } Ok(StreamPool(pool.into_boxed_slice())) } - pub async fn ping(&mut self) -> StreamResult<()> { + pub async fn ping(&mut self) -> ReplicationResult<()> { for host in &mut *self.0 { host.ping().await?; } @@ -91,7 +99,7 @@ impl<'a> StreamPool { Ok(()) } - pub async fn ask(&'a mut self) -> StreamResult> { + pub async fn ask(&'a mut self) -> ReplicationResult> { let mut batch = BatchAskIndex::with_capacity(self.0.len()); for host in &mut *self.0 { diff --git a/spartan/src/node/replication/storage/primary.rs b/spartan/src/node/replication/storage/primary.rs index f1ff1870..9daa6eb5 100644 --- a/spartan/src/node/replication/storage/primary.rs +++ b/spartan/src/node/replication/storage/primary.rs @@ -32,6 +32,10 @@ impl PrimaryStorage { .drain_filter(|index, _| *index <= gc_threshold) .for_each(drop); } + + pub fn slice(&self, start: u64) -> Box<[(&u64, &Event)]> { + self.log.range(start..).collect() + } } #[cfg(test)] From bfc498f19cbc4c6580a7d90c17052ff6c653e2de Mon Sep 17 00:00:00 2001 From: ivan770 Date: Sun, 12 Jul 2020 21:07:09 +0300 Subject: [PATCH 17/36] Move TCP messages into submodule --- spartan/src/node/replication/job/mod.rs | 2 +- spartan/src/node/replication/job/stream.rs | 42 +++++++------------ spartan/src/node/replication/messages/mod.rs | 5 +++ .../src/node/replication/messages/primary.rs | 16 +++++++ .../src/node/replication/messages/replica.rs | 1 + spartan/src/node/replication/mod.rs | 3 ++ 6 files changed, 41 insertions(+), 28 deletions(-) create mode 100644 spartan/src/node/replication/messages/mod.rs create mode 100644 spartan/src/node/replication/messages/primary.rs create mode 100644 spartan/src/node/replication/messages/replica.rs diff --git a/spartan/src/node/replication/job/mod.rs b/spartan/src/node/replication/job/mod.rs index 0b752672..5b09bd59 100644 --- a/spartan/src/node/replication/job/mod.rs +++ b/spartan/src/node/replication/job/mod.rs @@ -49,8 +49,8 @@ async fn start_replication(manager: &Manager<'_>, pool: &mut StreamPool, config: delay_for(timer).await; match replicate_manager(manager, pool).await { - Err(e) => error!("Error happened during replication attempt: {}", e), Ok(_) => info!("Database replicated successfully!"), + Err(e) => error!("Error happened during replication attempt: {}", e), } } } diff --git a/spartan/src/node/replication/job/stream.rs b/spartan/src/node/replication/job/stream.rs index 02da9560..7f8bac2d 100644 --- a/spartan/src/node/replication/job/stream.rs +++ b/spartan/src/node/replication/job/stream.rs @@ -2,41 +2,29 @@ use super::{ error::{ReplicationError, ReplicationResult}, index::{BatchAskIndex, RecvIndex}, }; -use crate::{config::replication::Primary, node::replication::event::Event}; +use crate::{ + config::replication::Primary, + node::replication::{ + event::Event, + messages::primary::{Request, Response}, + }, +}; use bincode::{deserialize, serialize}; -use serde::{Deserialize, Serialize}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::TcpStream, }; -#[derive(Serialize)] -pub enum StreamRequest<'a> { - Ping, - AskIndex, - SendRange(Box<[(&'a u64, &'a Event)]>), -} - -#[derive(Deserialize)] -pub enum StreamResponse { - Pong, - RecvIndex(Vec<(Box, u64)>), - RecvRange, -} - pub(super) struct Stream(TcpStream); pub(super) struct StreamPool(Box<[Stream]>); impl<'a> Stream { - fn serialize(message: &StreamRequest) -> ReplicationResult> { + fn serialize(message: &Request) -> ReplicationResult> { serialize(&message).map_err(|e| ReplicationError::SerializationError(e)) } - pub async fn exchange( - &mut self, - message: &StreamRequest<'_>, - ) -> ReplicationResult { + pub async fn exchange(&mut self, message: &Request<'_>) -> ReplicationResult { let (mut receive, mut send) = self.0.split(); send.write_all(&Self::serialize(message)?) @@ -55,22 +43,22 @@ impl<'a> Stream { } pub async fn ping(&mut self) -> ReplicationResult<()> { - match self.exchange(&StreamRequest::Ping).await? { - StreamResponse::Pong => Ok(()), + match self.exchange(&Request::Ping).await? { + Response::Pong => Ok(()), _ => Err(ReplicationError::ProtocolMismatch), } } pub async fn ask(&'a mut self) -> ReplicationResult> { - match self.exchange(&StreamRequest::AskIndex).await? { - StreamResponse::RecvIndex(recv) => Ok(RecvIndex::new(self, recv.into_boxed_slice())), + match self.exchange(&Request::AskIndex).await? { + Response::RecvIndex(recv) => Ok(RecvIndex::new(self, recv.into_boxed_slice())), _ => Err(ReplicationError::ProtocolMismatch), } } pub async fn send_range(&mut self, range: Box<[(&u64, &Event)]>) -> ReplicationResult<()> { - match self.exchange(&StreamRequest::SendRange(range)).await? { - StreamResponse::RecvRange => Ok(()), + match self.exchange(&Request::SendRange(range)).await? { + Response::RecvRange => Ok(()), _ => Err(ReplicationError::ProtocolMismatch), } } diff --git a/spartan/src/node/replication/messages/mod.rs b/spartan/src/node/replication/messages/mod.rs new file mode 100644 index 00000000..117ec2a6 --- /dev/null +++ b/spartan/src/node/replication/messages/mod.rs @@ -0,0 +1,5 @@ +/// Primary node messages +pub mod primary; + +/// Replica node messages +pub mod replica; diff --git a/spartan/src/node/replication/messages/primary.rs b/spartan/src/node/replication/messages/primary.rs new file mode 100644 index 00000000..f8f9ac2f --- /dev/null +++ b/spartan/src/node/replication/messages/primary.rs @@ -0,0 +1,16 @@ +use crate::node::replication::event::Event; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize)] +pub enum Request<'a> { + Ping, + AskIndex, + SendRange(Box<[(&'a u64, &'a Event)]>), +} + +#[derive(Deserialize)] +pub enum Response { + Pong, + RecvIndex(Vec<(Box, u64)>), + RecvRange, +} diff --git a/spartan/src/node/replication/messages/replica.rs b/spartan/src/node/replication/messages/replica.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/spartan/src/node/replication/messages/replica.rs @@ -0,0 +1 @@ + diff --git a/spartan/src/node/replication/mod.rs b/spartan/src/node/replication/mod.rs index 222e9161..150eb502 100644 --- a/spartan/src/node/replication/mod.rs +++ b/spartan/src/node/replication/mod.rs @@ -9,3 +9,6 @@ pub mod database; /// Replication job pub mod job; + +/// Replication TCP messages +pub mod messages; From c94cb96df29fb55148d4815d3dd6002ffdb2876c Mon Sep 17 00:00:00 2001 From: ivan770 Date: Mon, 13 Jul 2020 11:04:55 +0300 Subject: [PATCH 18/36] Serializable and deserializable message for primary and replica --- Cargo.lock | 10 +++++++ spartan/Cargo.toml | 4 +++ spartan/src/node/replication/job/stream.rs | 27 ++++++++++++------- spartan/src/node/replication/message.rs | 17 ++++++++++++ spartan/src/node/replication/messages/mod.rs | 5 ---- .../src/node/replication/messages/primary.rs | 16 ----------- .../src/node/replication/messages/replica.rs | 1 - spartan/src/node/replication/mod.rs | 2 +- .../src/node/replication/storage/primary.rs | 8 ++++-- 9 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 spartan/src/node/replication/message.rs delete mode 100644 spartan/src/node/replication/messages/mod.rs delete mode 100644 spartan/src/node/replication/messages/primary.rs delete mode 100644 spartan/src/node/replication/messages/replica.rs diff --git a/Cargo.lock b/Cargo.lock index 49dda726..d3b9d756 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,6 +961,15 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" +dependencies = [ + "serde", +] + [[package]] name = "memchr" version = "2.3.3" @@ -1443,6 +1452,7 @@ dependencies = [ "derive-new", "futures-util", "log", + "maybe-owned", "once_cell", "pretty_env_logger", "serde", diff --git a/spartan/Cargo.toml b/spartan/Cargo.toml index 12163bd7..2855083f 100644 --- a/spartan/Cargo.toml +++ b/spartan/Cargo.toml @@ -57,5 +57,9 @@ version = "0.3" [dependencies.once_cell] version = "1.4" +[dependencies.maybe-owned] +version = "0.3" +features = ["serde"] + [dev-dependencies.tempfile] version = "3.1" \ No newline at end of file diff --git a/spartan/src/node/replication/job/stream.rs b/spartan/src/node/replication/job/stream.rs index 7f8bac2d..b0b74bdd 100644 --- a/spartan/src/node/replication/job/stream.rs +++ b/spartan/src/node/replication/job/stream.rs @@ -6,10 +6,11 @@ use crate::{ config::replication::Primary, node::replication::{ event::Event, - messages::primary::{Request, Response}, + message::{PrimaryRequest, ReplicaRequest}, }, }; use bincode::{deserialize, serialize}; +use maybe_owned::MaybeOwned; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::TcpStream, @@ -20,11 +21,14 @@ pub(super) struct Stream(TcpStream); pub(super) struct StreamPool(Box<[Stream]>); impl<'a> Stream { - fn serialize(message: &Request) -> ReplicationResult> { + fn serialize(message: &PrimaryRequest) -> ReplicationResult> { serialize(&message).map_err(|e| ReplicationError::SerializationError(e)) } - pub async fn exchange(&mut self, message: &Request<'_>) -> ReplicationResult { + pub async fn exchange( + &mut self, + message: &PrimaryRequest<'_>, + ) -> ReplicationResult { let (mut receive, mut send) = self.0.split(); send.write_all(&Self::serialize(message)?) @@ -43,22 +47,25 @@ impl<'a> Stream { } pub async fn ping(&mut self) -> ReplicationResult<()> { - match self.exchange(&Request::Ping).await? { - Response::Pong => Ok(()), + match self.exchange(&PrimaryRequest::Ping).await? { + ReplicaRequest::Pong => Ok(()), _ => Err(ReplicationError::ProtocolMismatch), } } pub async fn ask(&'a mut self) -> ReplicationResult> { - match self.exchange(&Request::AskIndex).await? { - Response::RecvIndex(recv) => Ok(RecvIndex::new(self, recv.into_boxed_slice())), + match self.exchange(&PrimaryRequest::AskIndex).await? { + ReplicaRequest::RecvIndex(recv) => Ok(RecvIndex::new(self, recv)), _ => Err(ReplicationError::ProtocolMismatch), } } - pub async fn send_range(&mut self, range: Box<[(&u64, &Event)]>) -> ReplicationResult<()> { - match self.exchange(&Request::SendRange(range)).await? { - Response::RecvRange => Ok(()), + pub async fn send_range( + &mut self, + range: Box<[(MaybeOwned<'a, u64>, MaybeOwned<'a, Event>)]>, + ) -> ReplicationResult<()> { + match self.exchange(&PrimaryRequest::SendRange(range)).await? { + ReplicaRequest::RecvRange => Ok(()), _ => Err(ReplicationError::ProtocolMismatch), } } diff --git a/spartan/src/node/replication/message.rs b/spartan/src/node/replication/message.rs new file mode 100644 index 00000000..6b951997 --- /dev/null +++ b/spartan/src/node/replication/message.rs @@ -0,0 +1,17 @@ +use crate::node::replication::event::Event; +use maybe_owned::MaybeOwned; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub enum PrimaryRequest<'a> { + Ping, + AskIndex, + SendRange(Box<[(MaybeOwned<'a, u64>, MaybeOwned<'a, Event>)]>), +} + +#[derive(Serialize, Deserialize)] +pub enum ReplicaRequest { + Pong, + RecvIndex(Box<[(Box, u64)]>), + RecvRange, +} diff --git a/spartan/src/node/replication/messages/mod.rs b/spartan/src/node/replication/messages/mod.rs deleted file mode 100644 index 117ec2a6..00000000 --- a/spartan/src/node/replication/messages/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -/// Primary node messages -pub mod primary; - -/// Replica node messages -pub mod replica; diff --git a/spartan/src/node/replication/messages/primary.rs b/spartan/src/node/replication/messages/primary.rs deleted file mode 100644 index f8f9ac2f..00000000 --- a/spartan/src/node/replication/messages/primary.rs +++ /dev/null @@ -1,16 +0,0 @@ -use crate::node::replication::event::Event; -use serde::{Deserialize, Serialize}; - -#[derive(Serialize)] -pub enum Request<'a> { - Ping, - AskIndex, - SendRange(Box<[(&'a u64, &'a Event)]>), -} - -#[derive(Deserialize)] -pub enum Response { - Pong, - RecvIndex(Vec<(Box, u64)>), - RecvRange, -} diff --git a/spartan/src/node/replication/messages/replica.rs b/spartan/src/node/replication/messages/replica.rs deleted file mode 100644 index 8b137891..00000000 --- a/spartan/src/node/replication/messages/replica.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/spartan/src/node/replication/mod.rs b/spartan/src/node/replication/mod.rs index 150eb502..124b3840 100644 --- a/spartan/src/node/replication/mod.rs +++ b/spartan/src/node/replication/mod.rs @@ -11,4 +11,4 @@ pub mod database; pub mod job; /// Replication TCP messages -pub mod messages; +pub mod message; diff --git a/spartan/src/node/replication/storage/primary.rs b/spartan/src/node/replication/storage/primary.rs index 9daa6eb5..8b5824da 100644 --- a/spartan/src/node/replication/storage/primary.rs +++ b/spartan/src/node/replication/storage/primary.rs @@ -1,4 +1,5 @@ use crate::node::replication::event::Event; +use maybe_owned::MaybeOwned; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -33,8 +34,11 @@ impl PrimaryStorage { .for_each(drop); } - pub fn slice(&self, start: u64) -> Box<[(&u64, &Event)]> { - self.log.range(start..).collect() + pub fn slice(&self, start: u64) -> Box<[(MaybeOwned<'_, u64>, MaybeOwned<'_, Event>)]> { + self.log + .range(start..) + .map(|(k, v)| (MaybeOwned::Borrowed(k), MaybeOwned::Borrowed(v))) + .collect() } } From 942d0e6f2053ab24842d39efec359b979dcc93f5 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Mon, 13 Jul 2020 14:19:30 +0300 Subject: [PATCH 19/36] Replica command WIP --- spartan/src/cli/commands/mod.rs | 3 + spartan/src/cli/commands/replica.rs | 119 ++++++++++++++++++++++++ spartan/src/cli/mod.rs | 4 +- spartan/src/main.rs | 1 + spartan/src/node/replication/job/mod.rs | 13 +-- 5 files changed, 131 insertions(+), 9 deletions(-) create mode 100644 spartan/src/cli/commands/replica.rs diff --git a/spartan/src/cli/commands/mod.rs b/spartan/src/cli/commands/mod.rs index ca091e63..2f988350 100644 --- a/spartan/src/cli/commands/mod.rs +++ b/spartan/src/cli/commands/mod.rs @@ -3,3 +3,6 @@ pub mod start; /// `init` command pub mod init; + +/// `replica` command +pub mod replica; diff --git a/spartan/src/cli/commands/replica.rs b/spartan/src/cli/commands/replica.rs new file mode 100644 index 00000000..b72024ba --- /dev/null +++ b/spartan/src/cli/commands/replica.rs @@ -0,0 +1,119 @@ +use crate::{ + cli::Server, + config::replication::Replication, + node::{ + load_from_fs, + persistence::PersistenceError, + replication::message::{PrimaryRequest, ReplicaRequest}, + Manager, + }, +}; +use bincode::{deserialize, serialize, ErrorKind}; +use std::future::Future; +use structopt::StructOpt; +use thiserror::Error; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt, Error as IoError}, + net::{TcpListener, TcpStream}, +}; + +#[derive(Error, Debug)] +pub enum ReplicaCommandError { + #[error("Manager persistence error: {0}")] + PersistenceError(PersistenceError), + #[error("Unable to find replica node config")] + ReplicaConfigNotFound, + #[error("TCP socket error: {0}")] + SocketError(IoError), + #[error("Packet serialization error: {0}")] + SerializationError(Box), +} + +#[derive(StructOpt)] +pub struct ReplicaCommand {} + +impl ReplicaCommand { + pub async fn dispatch(&self, server: &Server) -> Result<(), ReplicaCommandError> { + let config = server.config().expect("Config not loaded"); + let mut manager = Manager::new(config); + + load_from_fs(&mut manager) + .await + .map_err(|e| ReplicaCommandError::PersistenceError(e))?; + + let mut socket = match config + .replication + .as_ref() + .ok_or_else(|| ReplicaCommandError::ReplicaConfigNotFound)? + { + Replication::Replica(config) => TcpListener::bind(config.host) + .await + .map_err(|e| ReplicaCommandError::SocketError(e)), + _ => Err(ReplicaCommandError::ReplicaConfigNotFound), + }?; + + match socket.accept().await { + Ok((mut socket, _)) => { + ReplicaSocket::new(&manager, &mut socket) + .exchange(accept_connection) + .await? + } + Err(e) => Err(ReplicaCommandError::SocketError(e))?, + } + + Ok(()) + } +} + +struct ReplicaSocket<'a> { + manager: &'a Manager<'a>, + socket: &'a mut TcpStream, +} + +impl<'a> ReplicaSocket<'a> { + fn new(manager: &'a Manager<'a>, socket: &'a mut TcpStream) -> Self { + ReplicaSocket { manager, socket } + } + + async fn exchange(&mut self, f: F) -> Result<(), ReplicaCommandError> + where + F: FnOnce(PrimaryRequest<'static>, &'a Manager<'a>) -> Fut, + Fut: Future, + { + let (mut receive, mut send) = self.socket.split(); + + let mut buf = Vec::new(); + + receive + .read_to_end(&mut buf) + .await + .map_err(|e| ReplicaCommandError::SocketError(e))?; + + let request = f( + deserialize(&buf).map_err(|e| ReplicaCommandError::SerializationError(e))?, + self.manager, + ) + .await; + + send.write_all( + &serialize(&request).map_err(|e| ReplicaCommandError::SerializationError(e))?, + ) + .await + .map_err(|e| ReplicaCommandError::SocketError(e))?; + + Ok(()) + } +} + +async fn accept_connection( + request: PrimaryRequest<'static>, + manager: &Manager<'_>, +) -> ReplicaRequest { + match request { + PrimaryRequest::Ping => { + info!("Pong!"); + ReplicaRequest::Pong + }, + _ => unimplemented!(), + } +} diff --git a/spartan/src/cli/mod.rs b/spartan/src/cli/mod.rs index 89125c5e..c39118aa 100644 --- a/spartan/src/cli/mod.rs +++ b/spartan/src/cli/mod.rs @@ -2,7 +2,7 @@ mod commands; use crate::config::Config; -use commands::{init::InitCommand, start::StartCommand}; +use commands::{init::InitCommand, replica::ReplicaCommand, start::StartCommand}; use std::{ io::Error, path::{Path, PathBuf}, @@ -17,6 +17,8 @@ pub enum Command { Start(StartCommand), #[structopt(about = "Initialize configuration file")] Init(InitCommand), + #[structopt(about = "Start replication server")] + Replica(ReplicaCommand), } #[derive(StructOpt)] diff --git a/spartan/src/main.rs b/spartan/src/main.rs index d7532b0c..e781b7e8 100644 --- a/spartan/src/main.rs +++ b/spartan/src/main.rs @@ -48,6 +48,7 @@ async fn main() -> Result<(), Error> { match server.command() { Start(command) => command.dispatch(server).await?, Init(command) => command.dispatch(server).await?, + Replica(command) => command.dispatch(server).await?, }; Ok(()) diff --git a/spartan/src/node/replication/job/mod.rs b/spartan/src/node/replication/job/mod.rs index 5b09bd59..3f0ea091 100644 --- a/spartan/src/node/replication/job/mod.rs +++ b/spartan/src/node/replication/job/mod.rs @@ -56,11 +56,8 @@ async fn start_replication(manager: &Manager<'_>, pool: &mut StreamPool, config: } pub async fn spawn_replication(manager: &Manager<'_>) -> IoResult<()> { - manager - .config - .replication - .as_ref() - .map(|config| async move { + Ok(match manager.config.replication.as_ref() { + Some(config) => { match config { Replication::Primary(config) => { prepare_storage(manager).await; @@ -74,7 +71,7 @@ pub async fn spawn_replication(manager: &Manager<'_>) -> IoResult<()> { panic!("Starting replication job while in replica mode is not allowed") } }; - }); - - Ok(()) + }, + None => () + }) } From a54a4dd255f1e4a0cdec08e6459f37d271333538 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Mon, 13 Jul 2020 18:12:58 +0300 Subject: [PATCH 20/36] Add TCP framing --- Cargo.lock | 1 + spartan/Cargo.toml | 3 ++ spartan/src/cli/commands/replica.rs | 63 +++++++++++++--------- spartan/src/node/replication/job/error.rs | 2 + spartan/src/node/replication/job/mod.rs | 4 +- spartan/src/node/replication/job/stream.rs | 34 ++++++------ 6 files changed, 63 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d3b9d756..dcf92027 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1461,6 +1461,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tokio-util 0.3.1", "toml", "uuid", ] diff --git a/spartan/Cargo.toml b/spartan/Cargo.toml index 2855083f..321ceebb 100644 --- a/spartan/Cargo.toml +++ b/spartan/Cargo.toml @@ -61,5 +61,8 @@ version = "1.4" version = "0.3" features = ["serde"] +[dependencies.tokio-util] +version = "0.3" + [dev-dependencies.tempfile] version = "3.1" \ No newline at end of file diff --git a/spartan/src/cli/commands/replica.rs b/spartan/src/cli/commands/replica.rs index b72024ba..458a3c16 100644 --- a/spartan/src/cli/commands/replica.rs +++ b/spartan/src/cli/commands/replica.rs @@ -9,13 +9,15 @@ use crate::{ }, }; use bincode::{deserialize, serialize, ErrorKind}; +use futures_util::{SinkExt, StreamExt}; use std::future::Future; use structopt::StructOpt; use thiserror::Error; use tokio::{ - io::{AsyncReadExt, AsyncWriteExt, Error as IoError}, + io::Error as IoError, net::{TcpListener, TcpStream}, }; +use tokio_util::codec::{BytesCodec, Decoder, Framed}; #[derive(Error, Debug)] pub enum ReplicaCommandError { @@ -25,6 +27,8 @@ pub enum ReplicaCommandError { ReplicaConfigNotFound, #[error("TCP socket error: {0}")] SocketError(IoError), + #[error("Empty TCP socket")] + EmptySocket, #[error("Packet serialization error: {0}")] SerializationError(Box), } @@ -52,27 +56,30 @@ impl ReplicaCommand { _ => Err(ReplicaCommandError::ReplicaConfigNotFound), }?; - match socket.accept().await { - Ok((mut socket, _)) => { - ReplicaSocket::new(&manager, &mut socket) - .exchange(accept_connection) - .await? + loop { + match socket.accept().await { + Ok((socket, _)) => { + ReplicaSocket::new(&manager, socket) + .exchange(accept_connection) + .await? + } + Err(e) => Err(ReplicaCommandError::SocketError(e))?, } - Err(e) => Err(ReplicaCommandError::SocketError(e))?, } - - Ok(()) } } struct ReplicaSocket<'a> { manager: &'a Manager<'a>, - socket: &'a mut TcpStream, + socket: Framed, } impl<'a> ReplicaSocket<'a> { - fn new(manager: &'a Manager<'a>, socket: &'a mut TcpStream) -> Self { - ReplicaSocket { manager, socket } + fn new(manager: &'a Manager<'a>, socket: TcpStream) -> Self { + ReplicaSocket { + manager, + socket: BytesCodec::new().framed(socket), + } } async fn exchange(&mut self, f: F) -> Result<(), ReplicaCommandError> @@ -80,14 +87,10 @@ impl<'a> ReplicaSocket<'a> { F: FnOnce(PrimaryRequest<'static>, &'a Manager<'a>) -> Fut, Fut: Future, { - let (mut receive, mut send) = self.socket.split(); - - let mut buf = Vec::new(); - - receive - .read_to_end(&mut buf) - .await - .map_err(|e| ReplicaCommandError::SocketError(e))?; + let buf = match self.socket.next().await { + Some(r) => r.map_err(|e| ReplicaCommandError::SocketError(e))?, + None => Err(ReplicaCommandError::EmptySocket)?, + }; let request = f( deserialize(&buf).map_err(|e| ReplicaCommandError::SerializationError(e))?, @@ -95,11 +98,19 @@ impl<'a> ReplicaSocket<'a> { ) .await; - send.write_all( - &serialize(&request).map_err(|e| ReplicaCommandError::SerializationError(e))?, - ) - .await - .map_err(|e| ReplicaCommandError::SocketError(e))?; + self.socket + .send( + serialize(&request) + .map_err(|e| ReplicaCommandError::SerializationError(e))? + .into(), + ) + .await + .map_err(|e| ReplicaCommandError::SocketError(e))?; + + self.socket + .flush() + .await + .map_err(|e| ReplicaCommandError::SocketError(e))?; Ok(()) } @@ -113,7 +124,7 @@ async fn accept_connection( PrimaryRequest::Ping => { info!("Pong!"); ReplicaRequest::Pong - }, + } _ => unimplemented!(), } } diff --git a/spartan/src/node/replication/job/error.rs b/spartan/src/node/replication/job/error.rs index 9d0664d4..96521bba 100644 --- a/spartan/src/node/replication/job/error.rs +++ b/spartan/src/node/replication/job/error.rs @@ -8,6 +8,8 @@ pub(super) enum ReplicationError { SerializationError(Box), #[error("TCP connection error: {0}")] SocketError(IoError), + #[error("TCP socket is empty")] + EmptySocket, #[error("Protocol mismatch")] ProtocolMismatch, #[error("Queue configuration mismatch")] diff --git a/spartan/src/node/replication/job/mod.rs b/spartan/src/node/replication/job/mod.rs index 3f0ea091..40fac470 100644 --- a/spartan/src/node/replication/job/mod.rs +++ b/spartan/src/node/replication/job/mod.rs @@ -71,7 +71,7 @@ pub async fn spawn_replication(manager: &Manager<'_>) -> IoResult<()> { panic!("Starting replication job while in replica mode is not allowed") } }; - }, - None => () + } + None => (), }) } diff --git a/spartan/src/node/replication/job/stream.rs b/spartan/src/node/replication/job/stream.rs index b0b74bdd..e0738677 100644 --- a/spartan/src/node/replication/job/stream.rs +++ b/spartan/src/node/replication/job/stream.rs @@ -10,13 +10,12 @@ use crate::{ }, }; use bincode::{deserialize, serialize}; +use futures_util::{SinkExt, StreamExt}; use maybe_owned::MaybeOwned; -use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpStream, -}; +use tokio::net::TcpStream; +use tokio_util::codec::{BytesCodec, Decoder, Framed}; -pub(super) struct Stream(TcpStream); +pub(super) struct Stream(Framed); pub(super) struct StreamPool(Box<[Stream]>); @@ -29,20 +28,21 @@ impl<'a> Stream { &mut self, message: &PrimaryRequest<'_>, ) -> ReplicationResult { - let (mut receive, mut send) = self.0.split(); - - send.write_all(&Self::serialize(message)?) + self.0 + .send(Self::serialize(message)?.into()) .await .map_err(|e| ReplicationError::SocketError(e))?; - // TODO: Optimize by using pre-allocated buffer - let mut buf = Vec::new(); - - receive - .read_to_end(&mut buf) + self.0 + .flush() .await .map_err(|e| ReplicationError::SocketError(e))?; + let buf = match self.0.next().await { + Some(r) => r.map_err(|e| ReplicationError::SocketError(e))?, + None => Err(ReplicationError::EmptySocket)?, + }; + Ok(deserialize(&buf).map_err(|e| ReplicationError::SerializationError(e))?) } @@ -77,9 +77,11 @@ impl<'a> StreamPool { for host in &config.destination { pool.push(Stream( - TcpStream::connect(host) - .await - .map_err(|e| ReplicationError::SocketError(e))?, + BytesCodec::new().framed( + TcpStream::connect(host) + .await + .map_err(|e| ReplicationError::SocketError(e))?, + ), )); } From e76f04d94f852fcfcb96f286011b4bf6c3349a8b Mon Sep 17 00:00:00 2001 From: ivan770 Date: Mon, 13 Jul 2020 18:26:41 +0300 Subject: [PATCH 21/36] Use boxed slice instead of Vec in replication config --- spartan/src/config/replication.rs | 2 +- spartan/src/node/replication/job/stream.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spartan/src/config/replication.rs b/spartan/src/config/replication.rs index 6bad891d..a5ac0a85 100644 --- a/spartan/src/config/replication.rs +++ b/spartan/src/config/replication.rs @@ -8,7 +8,7 @@ const fn default_replication_timer() -> u64 { #[derive(Serialize, Deserialize)] pub struct Primary { - pub destination: Vec, + pub destination: Box<[SocketAddr]>, #[serde(default = "default_replication_timer")] pub replication_timer: u64, diff --git a/spartan/src/node/replication/job/stream.rs b/spartan/src/node/replication/job/stream.rs index e0738677..d101c5a3 100644 --- a/spartan/src/node/replication/job/stream.rs +++ b/spartan/src/node/replication/job/stream.rs @@ -75,7 +75,7 @@ impl<'a> StreamPool { pub async fn from_config(config: &Primary) -> ReplicationResult { let mut pool = Vec::with_capacity(config.destination.len()); - for host in &config.destination { + for host in &*config.destination { pool.push(Stream( BytesCodec::new().framed( TcpStream::connect(host) From f2bc8040620e9d41a00a09fca131763a31c4b2e5 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Tue, 14 Jul 2020 20:08:13 +0300 Subject: [PATCH 22/36] Ensure correct storage is used in replica --- spartan/src/cli/commands/replica.rs | 81 ++++++++++++------- spartan/src/node/mod.rs | 4 + spartan/src/node/replication/job/index.rs | 1 + spartan/src/node/replication/job/mod.rs | 23 ++---- spartan/src/node/replication/job/stream.rs | 7 +- spartan/src/node/replication/message.rs | 6 +- spartan/src/node/replication/storage/mod.rs | 17 ++++ .../src/node/replication/storage/replica.rs | 4 + 8 files changed, 96 insertions(+), 47 deletions(-) diff --git a/spartan/src/cli/commands/replica.rs b/spartan/src/cli/commands/replica.rs index 458a3c16..f4585b52 100644 --- a/spartan/src/cli/commands/replica.rs +++ b/spartan/src/cli/commands/replica.rs @@ -4,7 +4,10 @@ use crate::{ node::{ load_from_fs, persistence::PersistenceError, - replication::message::{PrimaryRequest, ReplicaRequest}, + replication::{ + message::{PrimaryRequest, ReplicaRequest}, + storage::{replica::ReplicaStorage, ReplicationStorage}, + }, Manager, }, }; @@ -45,6 +48,13 @@ impl ReplicaCommand { .await .map_err(|e| ReplicaCommandError::PersistenceError(e))?; + ReplicationStorage::prepare( + &manager, + |storage| matches!(storage, ReplicationStorage::Replica(_)), + || ReplicationStorage::Replica(ReplicaStorage::default()), + ) + .await; + let mut socket = match config .replication .as_ref() @@ -84,35 +94,35 @@ impl<'a> ReplicaSocket<'a> { async fn exchange(&mut self, f: F) -> Result<(), ReplicaCommandError> where - F: FnOnce(PrimaryRequest<'static>, &'a Manager<'a>) -> Fut, + F: Fn(PrimaryRequest<'static>, &'a Manager<'a>) -> Fut, Fut: Future, { - let buf = match self.socket.next().await { - Some(r) => r.map_err(|e| ReplicaCommandError::SocketError(e))?, - None => Err(ReplicaCommandError::EmptySocket)?, - }; - - let request = f( - deserialize(&buf).map_err(|e| ReplicaCommandError::SerializationError(e))?, - self.manager, - ) - .await; + loop { + let buf = match self.socket.next().await { + Some(r) => r.map_err(|e| ReplicaCommandError::SocketError(e))?, + None => Err(ReplicaCommandError::EmptySocket)?, + }; - self.socket - .send( - serialize(&request) - .map_err(|e| ReplicaCommandError::SerializationError(e))? - .into(), + let request = f( + deserialize(&buf).map_err(|e| ReplicaCommandError::SerializationError(e))?, + self.manager, ) - .await - .map_err(|e| ReplicaCommandError::SocketError(e))?; + .await; - self.socket - .flush() - .await - .map_err(|e| ReplicaCommandError::SocketError(e))?; + self.socket + .send( + serialize(&request) + .map_err(|e| ReplicaCommandError::SerializationError(e))? + .into(), + ) + .await + .map_err(|e| ReplicaCommandError::SocketError(e))?; - Ok(()) + self.socket + .flush() + .await + .map_err(|e| ReplicaCommandError::SocketError(e))?; + } } } @@ -121,10 +131,25 @@ async fn accept_connection( manager: &Manager<'_>, ) -> ReplicaRequest { match request { - PrimaryRequest::Ping => { - info!("Pong!"); - ReplicaRequest::Pong + PrimaryRequest::Ping => ReplicaRequest::Pong, + PrimaryRequest::AskIndex => { + let mut indexes = Vec::with_capacity(manager.config.queues.len()); + + for (name, db) in manager.node.iter() { + let index = db + .lock() + .await + .get_storage() + .as_mut() + .expect("No database present") + .get_replica() + .get_index(); + + indexes.push((name.to_string().into_boxed_str(), index)); + } + + ReplicaRequest::RecvIndex(indexes.into_boxed_slice()) } - _ => unimplemented!(), + PrimaryRequest::SendRange(queue, range) => ReplicaRequest::RecvRange, } } diff --git a/spartan/src/node/mod.rs b/spartan/src/node/mod.rs index e7b7a375..0452b4f0 100644 --- a/spartan/src/node/mod.rs +++ b/spartan/src/node/mod.rs @@ -51,6 +51,10 @@ impl<'a> Node<'a> { self.db.insert(name, Mutex::new(DB::default())); } + pub fn iter(&'a self) -> impl Iterator { + self.db.iter() + } + /// Load queues from config pub fn load_from_config(&mut self, config: &'a Config) { config.queues.iter().for_each(|queue| self.add(queue)); diff --git a/spartan/src/node/replication/job/index.rs b/spartan/src/node/replication/job/index.rs index d3cf63e9..6e67d148 100644 --- a/spartan/src/node/replication/job/index.rs +++ b/spartan/src/node/replication/job/index.rs @@ -18,6 +18,7 @@ impl<'a> RecvIndex<'a> { for (name, start) in self.indexes.iter() { self.stream .send_range( + name, manager .queue(name) .await diff --git a/spartan/src/node/replication/job/mod.rs b/spartan/src/node/replication/job/mod.rs index 40fac470..13f8f395 100644 --- a/spartan/src/node/replication/job/mod.rs +++ b/spartan/src/node/replication/job/mod.rs @@ -18,22 +18,6 @@ use std::time::Duration; use stream::StreamPool; use tokio::io::Result as IoResult; -async fn prepare_storage(manager: &Manager<'_>) { - for (_, db) in manager.node.db.iter() { - let mut db = db.lock().await; - - let storage = db - .get_storage() - .as_ref() - .filter(|storage| matches!(storage, ReplicationStorage::Primary(_))); - - if storage.is_none() { - db.get_storage() - .replace(ReplicationStorage::Primary(PrimaryStorage::default())); - } - } -} - async fn replicate_manager(manager: &Manager<'_>, pool: &mut StreamPool) -> ReplicationResult<()> { pool.ping().await?; @@ -60,7 +44,12 @@ pub async fn spawn_replication(manager: &Manager<'_>) -> IoResult<()> { Some(config) => { match config { Replication::Primary(config) => { - prepare_storage(manager).await; + ReplicationStorage::prepare( + manager, + |storage| matches!(storage, ReplicationStorage::Primary(_)), + || ReplicationStorage::Primary(PrimaryStorage::default()), + ) + .await; match StreamPool::from_config(config).await { Ok(mut pool) => start_replication(manager, &mut pool, config).await, diff --git a/spartan/src/node/replication/job/stream.rs b/spartan/src/node/replication/job/stream.rs index d101c5a3..eed4388b 100644 --- a/spartan/src/node/replication/job/stream.rs +++ b/spartan/src/node/replication/job/stream.rs @@ -12,6 +12,7 @@ use crate::{ use bincode::{deserialize, serialize}; use futures_util::{SinkExt, StreamExt}; use maybe_owned::MaybeOwned; +use std::borrow::Cow; use tokio::net::TcpStream; use tokio_util::codec::{BytesCodec, Decoder, Framed}; @@ -62,9 +63,13 @@ impl<'a> Stream { pub async fn send_range( &mut self, + queue: &str, range: Box<[(MaybeOwned<'a, u64>, MaybeOwned<'a, Event>)]>, ) -> ReplicationResult<()> { - match self.exchange(&PrimaryRequest::SendRange(range)).await? { + match self + .exchange(&PrimaryRequest::SendRange(Cow::Borrowed(queue), range)) + .await? + { ReplicaRequest::RecvRange => Ok(()), _ => Err(ReplicationError::ProtocolMismatch), } diff --git a/spartan/src/node/replication/message.rs b/spartan/src/node/replication/message.rs index 6b951997..c193ef51 100644 --- a/spartan/src/node/replication/message.rs +++ b/spartan/src/node/replication/message.rs @@ -1,12 +1,16 @@ use crate::node::replication::event::Event; use maybe_owned::MaybeOwned; use serde::{Deserialize, Serialize}; +use std::borrow::Cow; #[derive(Serialize, Deserialize)] pub enum PrimaryRequest<'a> { Ping, AskIndex, - SendRange(Box<[(MaybeOwned<'a, u64>, MaybeOwned<'a, Event>)]>), + SendRange( + Cow<'a, str>, + Box<[(MaybeOwned<'a, u64>, MaybeOwned<'a, Event>)]>, + ), } #[derive(Serialize, Deserialize)] diff --git a/spartan/src/node/replication/storage/mod.rs b/spartan/src/node/replication/storage/mod.rs index 21291056..f6bd6c72 100644 --- a/spartan/src/node/replication/storage/mod.rs +++ b/spartan/src/node/replication/storage/mod.rs @@ -4,6 +4,7 @@ pub mod primary; /// Storage for replica's pub mod replica; +use crate::node::Manager; use primary::PrimaryStorage; use replica::ReplicaStorage; use serde::{Deserialize, Serialize}; @@ -28,4 +29,20 @@ impl ReplicationStorage { _ => panic!("Replication storage is in primary mode."), } } + + pub async fn prepare(manager: &Manager<'_>, filter: F, replace: R) + where + F: Fn(&&ReplicationStorage) -> bool + Copy, + R: Fn() -> ReplicationStorage, + { + for (_, db) in manager.node.db.iter() { + let mut db = db.lock().await; + + let storage = db.get_storage().as_ref().filter(filter); + + if storage.is_none() { + db.get_storage().replace(replace()); + } + } + } } diff --git a/spartan/src/node/replication/storage/replica.rs b/spartan/src/node/replication/storage/replica.rs index e5e9ee43..663968ea 100644 --- a/spartan/src/node/replication/storage/replica.rs +++ b/spartan/src/node/replication/storage/replica.rs @@ -12,6 +12,10 @@ impl Default for ReplicaStorage { } impl ReplicaStorage { + pub fn get_index(&self) -> u64 { + self.confirmed_index + } + pub fn confirm(&mut self) { self.confirmed_index += 1; } From 256b0e9be93286e68700bd304fc97ef47f062d8c Mon Sep 17 00:00:00 2001 From: ivan770 Date: Wed, 15 Jul 2020 12:22:55 +0300 Subject: [PATCH 23/36] Added event slice applying --- spartan/src/cli/commands/replica.rs | 19 ++++-- spartan/src/node/replication/database.rs | 15 +++++ spartan/src/node/replication/event.rs | 58 ++++++++++++++++++- spartan/src/node/replication/job/stream.rs | 3 +- spartan/src/node/replication/message.rs | 3 +- .../src/node/replication/storage/replica.rs | 4 +- 6 files changed, 92 insertions(+), 10 deletions(-) diff --git a/spartan/src/cli/commands/replica.rs b/spartan/src/cli/commands/replica.rs index f4585b52..1b7778e8 100644 --- a/spartan/src/cli/commands/replica.rs +++ b/spartan/src/cli/commands/replica.rs @@ -5,6 +5,7 @@ use crate::{ load_from_fs, persistence::PersistenceError, replication::{ + event::Event, message::{PrimaryRequest, ReplicaRequest}, storage::{replica::ReplicaStorage, ReplicationStorage}, }, @@ -95,7 +96,7 @@ impl<'a> ReplicaSocket<'a> { async fn exchange(&mut self, f: F) -> Result<(), ReplicaCommandError> where F: Fn(PrimaryRequest<'static>, &'a Manager<'a>) -> Fut, - Fut: Future, + Fut: Future>, { loop { let buf = match self.socket.next().await { @@ -126,10 +127,10 @@ impl<'a> ReplicaSocket<'a> { } } -async fn accept_connection( +async fn accept_connection<'a>( request: PrimaryRequest<'static>, - manager: &Manager<'_>, -) -> ReplicaRequest { + manager: &Manager<'a>, +) -> ReplicaRequest<'a> { match request { PrimaryRequest::Ping => ReplicaRequest::Pong, PrimaryRequest::AskIndex => { @@ -150,6 +151,14 @@ async fn accept_connection( ReplicaRequest::RecvIndex(indexes.into_boxed_slice()) } - PrimaryRequest::SendRange(queue, range) => ReplicaRequest::RecvRange, + PrimaryRequest::SendRange(queue, range) => { + match manager.queue(&queue).await { + Ok(mut db) => { + Event::apply_events(&mut *db, range); + ReplicaRequest::RecvRange + } + Err(_) => ReplicaRequest::QueueNotFound(queue), + } + } } } diff --git a/spartan/src/node/replication/database.rs b/spartan/src/node/replication/database.rs index 37bc9d0f..d53dedc3 100644 --- a/spartan/src/node/replication/database.rs +++ b/spartan/src/node/replication/database.rs @@ -9,6 +9,7 @@ use spartan_lib::core::{ message::Message, payload::Identifiable, }; +use std::ops::{Deref, DerefMut}; #[derive(Serialize, Deserialize)] pub struct ReplicatedDatabase { @@ -28,6 +29,20 @@ where } } +impl Deref for ReplicatedDatabase { + type Target = DB; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for ReplicatedDatabase { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + impl ReplicatedDatabase { fn call_storage(&mut self, f: F) where diff --git a/spartan/src/node/replication/event.rs b/spartan/src/node/replication/event.rs index af551098..b22a0efe 100644 --- a/spartan/src/node/replication/event.rs +++ b/spartan/src/node/replication/event.rs @@ -1,5 +1,11 @@ +use crate::node::DB; +use maybe_owned::MaybeOwned; use serde::{Deserialize, Serialize}; -use spartan_lib::core::{message::Message, payload::Identifiable}; +use spartan_lib::core::{ + dispatcher::{simple::PositionBasedDelete, SimpleDispatcher, StatusAwareDispatcher}, + message::Message, + payload::Identifiable, +}; #[derive(Serialize, Deserialize)] pub enum Event { @@ -10,3 +16,53 @@ pub enum Event { Gc, Clear, } + +impl Event { + fn apply_event(self, queue: &mut DB) { + let queue = &mut **queue; + + match self { + Event::Push(message) => queue.push(message), + Event::Pop => { + queue.pop(); + } + Event::Requeue(id) => { + queue.requeue(id); + } + Event::Delete(id) => { + queue.delete(id); + } + Event::Gc => { + queue.gc(); + } + Event::Clear => { + queue.clear(); + } + } + } + + pub fn apply_events( + queue: &mut DB, + events: Box<[(MaybeOwned<'_, u64>, MaybeOwned<'_, Event>)]>, + ) -> Option<()> { + let index = *events.last()?.0; + + // into_vec allows to use owned event + events + .into_vec() + .into_iter() + .for_each(|(_, event)| match event { + MaybeOwned::Owned(event) => event.apply_event(queue), + MaybeOwned::Borrowed(_) => unreachable!(), + }); + + queue + .get_storage() + .as_mut() + .expect("No storage provided") + .get_replica() + .confirm(index); + + Some(()) + } +} diff --git a/spartan/src/node/replication/job/stream.rs b/spartan/src/node/replication/job/stream.rs index eed4388b..b8e02e6f 100644 --- a/spartan/src/node/replication/job/stream.rs +++ b/spartan/src/node/replication/job/stream.rs @@ -28,7 +28,7 @@ impl<'a> Stream { pub async fn exchange( &mut self, message: &PrimaryRequest<'_>, - ) -> ReplicationResult { + ) -> ReplicationResult> { self.0 .send(Self::serialize(message)?.into()) .await @@ -71,6 +71,7 @@ impl<'a> Stream { .await? { ReplicaRequest::RecvRange => Ok(()), + ReplicaRequest::QueueNotFound(queue) => Ok(warn!("Queue {} not found on replica", queue)), _ => Err(ReplicationError::ProtocolMismatch), } } diff --git a/spartan/src/node/replication/message.rs b/spartan/src/node/replication/message.rs index c193ef51..eb12cea7 100644 --- a/spartan/src/node/replication/message.rs +++ b/spartan/src/node/replication/message.rs @@ -14,8 +14,9 @@ pub enum PrimaryRequest<'a> { } #[derive(Serialize, Deserialize)] -pub enum ReplicaRequest { +pub enum ReplicaRequest<'a> { Pong, RecvIndex(Box<[(Box, u64)]>), RecvRange, + QueueNotFound(Cow<'a, str>), } diff --git a/spartan/src/node/replication/storage/replica.rs b/spartan/src/node/replication/storage/replica.rs index 663968ea..6c2b5e65 100644 --- a/spartan/src/node/replication/storage/replica.rs +++ b/spartan/src/node/replication/storage/replica.rs @@ -16,7 +16,7 @@ impl ReplicaStorage { self.confirmed_index } - pub fn confirm(&mut self) { - self.confirmed_index += 1; + pub fn confirm(&mut self, index: u64) { + self.confirmed_index = index; } } From 0133be496d55fdd4a9f14bcd8d9c32734d5fd05c Mon Sep 17 00:00:00 2001 From: ivan770 Date: Wed, 15 Jul 2020 15:08:43 +0300 Subject: [PATCH 24/36] Move jobs into separate submodule --- spartan/src/cli/commands/replica.rs | 17 +++++++---------- spartan/src/cli/commands/start.rs | 9 ++++++--- spartan/src/{node => jobs}/exit.rs | 3 ++- spartan/src/{node => jobs}/gc.rs | 4 ++-- spartan/src/jobs/mod.rs | 11 +++++++++++ spartan/src/{node => jobs}/persistence.rs | 7 +++---- .../job => jobs/replication}/error.rs | 0 .../job => jobs/replication}/index.rs | 0 .../job => jobs/replication}/mod.rs | 6 ++++-- .../job => jobs/replication}/stream.rs | 4 +++- spartan/src/main.rs | 3 +++ spartan/src/node/mod.rs | 19 ++++++------------- spartan/src/node/replication/mod.rs | 3 --- 13 files changed, 47 insertions(+), 39 deletions(-) rename spartan/src/{node => jobs}/exit.rs (83%) rename spartan/src/{node => jobs}/gc.rs (96%) create mode 100644 spartan/src/jobs/mod.rs rename spartan/src/{node => jobs}/persistence.rs (95%) rename spartan/src/{node/replication/job => jobs/replication}/error.rs (100%) rename spartan/src/{node/replication/job => jobs/replication}/index.rs (100%) rename spartan/src/{node/replication/job => jobs/replication}/mod.rs (94%) rename spartan/src/{node/replication/job => jobs/replication}/stream.rs (96%) diff --git a/spartan/src/cli/commands/replica.rs b/spartan/src/cli/commands/replica.rs index 1b7778e8..07c854f1 100644 --- a/spartan/src/cli/commands/replica.rs +++ b/spartan/src/cli/commands/replica.rs @@ -1,9 +1,8 @@ use crate::{ cli::Server, config::replication::Replication, + jobs::persistence::{load_from_fs, PersistenceError}, node::{ - load_from_fs, - persistence::PersistenceError, replication::{ event::Event, message::{PrimaryRequest, ReplicaRequest}, @@ -151,14 +150,12 @@ async fn accept_connection<'a>( ReplicaRequest::RecvIndex(indexes.into_boxed_slice()) } - PrimaryRequest::SendRange(queue, range) => { - match manager.queue(&queue).await { - Ok(mut db) => { - Event::apply_events(&mut *db, range); - ReplicaRequest::RecvRange - } - Err(_) => ReplicaRequest::QueueNotFound(queue), + PrimaryRequest::SendRange(queue, range) => match manager.queue(&queue).await { + Ok(mut db) => { + Event::apply_events(&mut *db, range); + ReplicaRequest::RecvRange } - } + Err(_) => ReplicaRequest::QueueNotFound(queue), + }, } } diff --git a/spartan/src/cli/commands/start.rs b/spartan/src/cli/commands/start.rs index 141ff489..a834c159 100644 --- a/spartan/src/cli/commands/start.rs +++ b/spartan/src/cli/commands/start.rs @@ -1,10 +1,13 @@ use crate::{ cli::Server, http::server::{start_http_server, ServerError}, - node::{ - gc::spawn_gc, load_from_fs, persistence::PersistenceError, - replication::job::spawn_replication, spawn_ctrlc_handler, spawn_persistence, Manager, + jobs::{ + exit::spawn_ctrlc_handler, + gc::spawn_gc, + persistence::{load_from_fs, spawn_persistence, PersistenceError}, + replication::spawn_replication, }, + node::Manager, }; use actix_rt::System; use actix_web::web::Data; diff --git a/spartan/src/node/exit.rs b/spartan/src/jobs/exit.rs similarity index 83% rename from spartan/src/node/exit.rs rename to spartan/src/jobs/exit.rs index 246e197a..add3467c 100644 --- a/spartan/src/node/exit.rs +++ b/spartan/src/jobs/exit.rs @@ -1,4 +1,5 @@ -use super::{persist_manager, Manager}; +use super::persistence::persist_manager; +use crate::node::Manager; use actix_rt::signal::ctrl_c; use std::process::exit; diff --git a/spartan/src/node/gc.rs b/spartan/src/jobs/gc.rs similarity index 96% rename from spartan/src/node/gc.rs rename to spartan/src/jobs/gc.rs index ca2e460d..323aeff3 100644 --- a/spartan/src/node/gc.rs +++ b/spartan/src/jobs/gc.rs @@ -1,4 +1,4 @@ -use super::Manager; +use crate::node::Manager; use actix_rt::time::delay_for; use futures_util::stream::{iter, StreamExt}; use spartan_lib::core::dispatcher::SimpleDispatcher; @@ -6,7 +6,7 @@ use std::time::Duration; /// Concurrently iterates over all databases in node, and executes GC on them. async fn execute_gc(manager: &Manager<'_>) { - iter(manager.node.db.iter()) + iter(manager.node.iter()) .for_each_concurrent(None, |(name, db)| async move { let mut db = db.lock().await; diff --git a/spartan/src/jobs/mod.rs b/spartan/src/jobs/mod.rs new file mode 100644 index 00000000..be4f2124 --- /dev/null +++ b/spartan/src/jobs/mod.rs @@ -0,0 +1,11 @@ +/// Ctrl-C handler +pub mod exit; + +/// GC handler +pub mod gc; + +/// Persistence handler +pub mod persistence; + +/// Replication job +pub mod replication; diff --git a/spartan/src/node/persistence.rs b/spartan/src/jobs/persistence.rs similarity index 95% rename from spartan/src/node/persistence.rs rename to spartan/src/jobs/persistence.rs index e4f0c677..dea0ede8 100644 --- a/spartan/src/node/persistence.rs +++ b/spartan/src/jobs/persistence.rs @@ -1,7 +1,6 @@ -use super::{Manager, MutexDB}; +use crate::node::{Manager, MutexDB}; use actix_rt::time::delay_for; use bincode::{deserialize, serialize, Error as BincodeError}; -use futures_util::lock::Mutex; use futures_util::stream::{iter, StreamExt}; use std::{io::Error, path::Path, time::Duration}; use thiserror::Error as ThisError; @@ -40,7 +39,7 @@ async fn persist_db(name: &str, db: &MutexDB, path: &Path) -> Result<(), Persist /// Persist all databases from manager pub async fn persist_manager(manager: &Manager<'_>) { - iter(manager.node.db.iter()) + iter(manager.node.iter()) .for_each_concurrent(None, |(name, db)| async move { match persist_db(name, db, &manager.config.path).await { Err(PersistenceError::SerializationError(e)) => { @@ -72,7 +71,7 @@ pub async fn load_from_fs(manager: &mut Manager<'_>) -> PersistenceResult<()> { match read(manager.config.path.join(queue)).await { Ok(file_buf) => { let db = deserialize(&file_buf).map_err(PersistenceError::InvalidFileFormat)?; - manager.node.db.insert(queue, Mutex::new(db)); + manager.node.add_db(queue, db); } Err(e) => warn!("Unable to load database {}: {}", queue, e), } diff --git a/spartan/src/node/replication/job/error.rs b/spartan/src/jobs/replication/error.rs similarity index 100% rename from spartan/src/node/replication/job/error.rs rename to spartan/src/jobs/replication/error.rs diff --git a/spartan/src/node/replication/job/index.rs b/spartan/src/jobs/replication/index.rs similarity index 100% rename from spartan/src/node/replication/job/index.rs rename to spartan/src/jobs/replication/index.rs diff --git a/spartan/src/node/replication/job/mod.rs b/spartan/src/jobs/replication/mod.rs similarity index 94% rename from spartan/src/node/replication/job/mod.rs rename to spartan/src/jobs/replication/mod.rs index 13f8f395..03269fe7 100644 --- a/spartan/src/node/replication/job/mod.rs +++ b/spartan/src/jobs/replication/mod.rs @@ -7,10 +7,12 @@ pub mod index; /// Replication error pub mod error; -use super::storage::{primary::PrimaryStorage, ReplicationStorage}; use crate::{ config::replication::{Primary, Replication}, - node::Manager, + node::{ + replication::storage::{primary::PrimaryStorage, ReplicationStorage}, + Manager, + }, }; use actix_rt::time::delay_for; use error::ReplicationResult; diff --git a/spartan/src/node/replication/job/stream.rs b/spartan/src/jobs/replication/stream.rs similarity index 96% rename from spartan/src/node/replication/job/stream.rs rename to spartan/src/jobs/replication/stream.rs index b8e02e6f..12d39c5a 100644 --- a/spartan/src/node/replication/job/stream.rs +++ b/spartan/src/jobs/replication/stream.rs @@ -71,7 +71,9 @@ impl<'a> Stream { .await? { ReplicaRequest::RecvRange => Ok(()), - ReplicaRequest::QueueNotFound(queue) => Ok(warn!("Queue {} not found on replica", queue)), + ReplicaRequest::QueueNotFound(queue) => { + Ok(warn!("Queue {} not found on replica", queue)) + } _ => Err(ReplicationError::ProtocolMismatch), } } diff --git a/spartan/src/main.rs b/spartan/src/main.rs index e781b7e8..55f062ed 100644 --- a/spartan/src/main.rs +++ b/spartan/src/main.rs @@ -24,6 +24,9 @@ mod routing; /// Configuration mod config; +/// Background jobs +mod jobs; + /// Utilities for easier development pub mod utils; diff --git a/spartan/src/node/mod.rs b/spartan/src/node/mod.rs index 0452b4f0..4fa5ff95 100644 --- a/spartan/src/node/mod.rs +++ b/spartan/src/node/mod.rs @@ -1,21 +1,10 @@ -/// Ctrl-C handler -pub mod exit; - -/// GC handler -pub mod gc; - /// Node manager pub mod manager; -/// Persistence handler -pub mod persistence; - /// Database replication pub mod replication; -pub use exit::spawn_ctrlc_handler; pub use manager::Manager; -pub use persistence::{load_from_fs, persist_manager, spawn_persistence}; use crate::config::Config; use futures_util::lock::{Mutex, MutexGuard}; @@ -24,7 +13,7 @@ use spartan_lib::core::{db::tree::TreeDatabase, message::Message}; use std::collections::{hash_map::RandomState, HashMap}; pub type DB = ReplicatedDatabase>; -type MutexDB = Mutex; +pub type MutexDB = Mutex; /// Key-value node implementation #[derive(Default)] @@ -47,8 +36,12 @@ impl<'a> Node<'a> { /// Add queue entry to node pub fn add(&mut self, name: &'a str) { + self.add_db(name, DB::default()) + } + + pub fn add_db(&mut self, name: &'a str, db: DB) { info!("Initializing queue \"{}\"", name); - self.db.insert(name, Mutex::new(DB::default())); + self.db.insert(name, Mutex::new(db)); } pub fn iter(&'a self) -> impl Iterator { diff --git a/spartan/src/node/replication/mod.rs b/spartan/src/node/replication/mod.rs index 124b3840..3acc6ce6 100644 --- a/spartan/src/node/replication/mod.rs +++ b/spartan/src/node/replication/mod.rs @@ -7,8 +7,5 @@ pub mod event; /// Replication proxy database pub mod database; -/// Replication job -pub mod job; - /// Replication TCP messages pub mod message; From ec8ba243165b8b32565f0db9726010676750f698 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Wed, 15 Jul 2020 20:15:06 +0300 Subject: [PATCH 25/36] Concurrency, clippy lints --- spartan/src/cli/commands/replica.rs | 18 +++++------ spartan/src/jobs/replication/index.rs | 8 +++-- spartan/src/jobs/replication/mod.rs | 41 +++++++++++++------------- spartan/src/jobs/replication/stream.rs | 26 ++++++++-------- 4 files changed, 48 insertions(+), 45 deletions(-) diff --git a/spartan/src/cli/commands/replica.rs b/spartan/src/cli/commands/replica.rs index 07c854f1..8ecf8391 100644 --- a/spartan/src/cli/commands/replica.rs +++ b/spartan/src/cli/commands/replica.rs @@ -46,7 +46,7 @@ impl ReplicaCommand { load_from_fs(&mut manager) .await - .map_err(|e| ReplicaCommandError::PersistenceError(e))?; + .map_err(ReplicaCommandError::PersistenceError)?; ReplicationStorage::prepare( &manager, @@ -62,7 +62,7 @@ impl ReplicaCommand { { Replication::Replica(config) => TcpListener::bind(config.host) .await - .map_err(|e| ReplicaCommandError::SocketError(e)), + .map_err(ReplicaCommandError::SocketError), _ => Err(ReplicaCommandError::ReplicaConfigNotFound), }?; @@ -73,7 +73,7 @@ impl ReplicaCommand { .exchange(accept_connection) .await? } - Err(e) => Err(ReplicaCommandError::SocketError(e))?, + Err(e) => return Err(ReplicaCommandError::SocketError(e)), } } } @@ -99,12 +99,12 @@ impl<'a> ReplicaSocket<'a> { { loop { let buf = match self.socket.next().await { - Some(r) => r.map_err(|e| ReplicaCommandError::SocketError(e))?, - None => Err(ReplicaCommandError::EmptySocket)?, + Some(r) => r.map_err(ReplicaCommandError::SocketError)?, + None => return Err(ReplicaCommandError::EmptySocket), }; let request = f( - deserialize(&buf).map_err(|e| ReplicaCommandError::SerializationError(e))?, + deserialize(&buf).map_err(ReplicaCommandError::SerializationError)?, self.manager, ) .await; @@ -112,16 +112,16 @@ impl<'a> ReplicaSocket<'a> { self.socket .send( serialize(&request) - .map_err(|e| ReplicaCommandError::SerializationError(e))? + .map_err(ReplicaCommandError::SerializationError)? .into(), ) .await - .map_err(|e| ReplicaCommandError::SocketError(e))?; + .map_err(ReplicaCommandError::SocketError)?; self.socket .flush() .await - .map_err(|e| ReplicaCommandError::SocketError(e))?; + .map_err(ReplicaCommandError::SocketError)?; } } } diff --git a/spartan/src/jobs/replication/index.rs b/spartan/src/jobs/replication/index.rs index 6e67d148..43eeb359 100644 --- a/spartan/src/jobs/replication/index.rs +++ b/spartan/src/jobs/replication/index.rs @@ -3,6 +3,7 @@ use super::{ stream::Stream, }; use crate::node::Manager; +use futures_util::{stream::iter, StreamExt, TryStreamExt}; pub(super) struct RecvIndex<'a> { stream: &'a mut Stream, @@ -52,9 +53,10 @@ impl<'a> BatchAskIndex<'a> { } pub async fn sync(&mut self, manager: &Manager<'_>) -> ReplicationResult<()> { - for host in &mut self.batch { - host.sync(manager).await?; - } + iter(self.batch.iter_mut()) + .map(Ok) + .try_for_each_concurrent(None, |host| async move { host.sync(manager).await }) + .await?; Ok(()) } diff --git a/spartan/src/jobs/replication/mod.rs b/spartan/src/jobs/replication/mod.rs index 03269fe7..afa19d9d 100644 --- a/spartan/src/jobs/replication/mod.rs +++ b/spartan/src/jobs/replication/mod.rs @@ -42,27 +42,26 @@ async fn start_replication(manager: &Manager<'_>, pool: &mut StreamPool, config: } pub async fn spawn_replication(manager: &Manager<'_>) -> IoResult<()> { - Ok(match manager.config.replication.as_ref() { - Some(config) => { - match config { - Replication::Primary(config) => { - ReplicationStorage::prepare( - manager, - |storage| matches!(storage, ReplicationStorage::Primary(_)), - || ReplicationStorage::Primary(PrimaryStorage::default()), - ) - .await; + if let Some(config) = manager.config.replication.as_ref() { + match config { + Replication::Primary(config) => { + ReplicationStorage::prepare( + manager, + |storage| matches!(storage, ReplicationStorage::Primary(_)), + || ReplicationStorage::Primary(PrimaryStorage::default()), + ) + .await; - match StreamPool::from_config(config).await { - Ok(mut pool) => start_replication(manager, &mut pool, config).await, - Err(e) => error!("Unable to open connection pool: {}", e), - } + match StreamPool::from_config(config).await { + Ok(mut pool) => start_replication(manager, &mut pool, config).await, + Err(e) => error!("Unable to open connection pool: {}", e), } - Replication::Replica(_) => { - panic!("Starting replication job while in replica mode is not allowed") - } - }; - } - None => (), - }) + } + Replication::Replica(_) => { + panic!("Starting replication job while in replica mode is not allowed") + } + }; + } + + Ok(()) } diff --git a/spartan/src/jobs/replication/stream.rs b/spartan/src/jobs/replication/stream.rs index 12d39c5a..685dbb37 100644 --- a/spartan/src/jobs/replication/stream.rs +++ b/spartan/src/jobs/replication/stream.rs @@ -10,7 +10,7 @@ use crate::{ }, }; use bincode::{deserialize, serialize}; -use futures_util::{SinkExt, StreamExt}; +use futures_util::{stream::iter, SinkExt, StreamExt, TryStreamExt}; use maybe_owned::MaybeOwned; use std::borrow::Cow; use tokio::net::TcpStream; @@ -22,7 +22,7 @@ pub(super) struct StreamPool(Box<[Stream]>); impl<'a> Stream { fn serialize(message: &PrimaryRequest) -> ReplicationResult> { - serialize(&message).map_err(|e| ReplicationError::SerializationError(e)) + serialize(&message).map_err(ReplicationError::SerializationError) } pub async fn exchange( @@ -32,19 +32,19 @@ impl<'a> Stream { self.0 .send(Self::serialize(message)?.into()) .await - .map_err(|e| ReplicationError::SocketError(e))?; + .map_err(ReplicationError::SocketError)?; self.0 .flush() .await - .map_err(|e| ReplicationError::SocketError(e))?; + .map_err(ReplicationError::SocketError)?; let buf = match self.0.next().await { - Some(r) => r.map_err(|e| ReplicationError::SocketError(e))?, - None => Err(ReplicationError::EmptySocket)?, + Some(r) => r.map_err(ReplicationError::SocketError)?, + None => return Err(ReplicationError::EmptySocket), }; - Ok(deserialize(&buf).map_err(|e| ReplicationError::SerializationError(e))?) + Ok(deserialize(&buf).map_err(ReplicationError::SerializationError)?) } pub async fn ping(&mut self) -> ReplicationResult<()> { @@ -72,7 +72,8 @@ impl<'a> Stream { { ReplicaRequest::RecvRange => Ok(()), ReplicaRequest::QueueNotFound(queue) => { - Ok(warn!("Queue {} not found on replica", queue)) + warn!("Queue {} not found on replica", queue); + Ok(()) } _ => Err(ReplicationError::ProtocolMismatch), } @@ -88,7 +89,7 @@ impl<'a> StreamPool { BytesCodec::new().framed( TcpStream::connect(host) .await - .map_err(|e| ReplicationError::SocketError(e))?, + .map_err(ReplicationError::SocketError)?, ), )); } @@ -97,9 +98,10 @@ impl<'a> StreamPool { } pub async fn ping(&mut self) -> ReplicationResult<()> { - for host in &mut *self.0 { - host.ping().await?; - } + iter(self.0.iter_mut()) + .map(Ok) + .try_for_each_concurrent(None, |stream| async move { stream.ping().await }) + .await?; Ok(()) } From 2841eccb0dd1f13a7a2b0182ba384c786e1144aa Mon Sep 17 00:00:00 2001 From: ivan770 Date: Wed, 15 Jul 2020 20:26:15 +0300 Subject: [PATCH 26/36] Move event applying into separate trait --- spartan/src/cli/commands/replica.rs | 4 ++-- spartan/src/node/replication/event.rs | 28 ++++++++++++++++++--------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/spartan/src/cli/commands/replica.rs b/spartan/src/cli/commands/replica.rs index 8ecf8391..8bd0198e 100644 --- a/spartan/src/cli/commands/replica.rs +++ b/spartan/src/cli/commands/replica.rs @@ -4,7 +4,7 @@ use crate::{ jobs::persistence::{load_from_fs, PersistenceError}, node::{ replication::{ - event::Event, + event::ApplyEvent, message::{PrimaryRequest, ReplicaRequest}, storage::{replica::ReplicaStorage, ReplicationStorage}, }, @@ -152,7 +152,7 @@ async fn accept_connection<'a>( } PrimaryRequest::SendRange(queue, range) => match manager.queue(&queue).await { Ok(mut db) => { - Event::apply_events(&mut *db, range); + db.apply_events(range); ReplicaRequest::RecvRange } Err(_) => ReplicaRequest::QueueNotFound(queue), diff --git a/spartan/src/node/replication/event.rs b/spartan/src/node/replication/event.rs index b22a0efe..52cb3bb5 100644 --- a/spartan/src/node/replication/event.rs +++ b/spartan/src/node/replication/event.rs @@ -17,11 +17,22 @@ pub enum Event { Clear, } -impl Event { - fn apply_event(self, queue: &mut DB) { - let queue = &mut **queue; +pub trait ApplyEvent { + /// Apply single event to database + fn apply_event(&mut self, event: Event); - match self { + /// Apply slice of events to database + fn apply_events( + &mut self, + events: Box<[(MaybeOwned<'_, u64>, MaybeOwned<'_, Event>)]>, + ) -> Option<()>; +} + +impl ApplyEvent for DB { + fn apply_event(&mut self, event: Event) { + let queue = &mut **self; + + match event { Event::Push(message) => queue.push(message), Event::Pop => { queue.pop(); @@ -41,8 +52,8 @@ impl Event { } } - pub fn apply_events( - queue: &mut DB, + fn apply_events( + &mut self, events: Box<[(MaybeOwned<'_, u64>, MaybeOwned<'_, Event>)]>, ) -> Option<()> { let index = *events.last()?.0; @@ -52,12 +63,11 @@ impl Event { .into_vec() .into_iter() .for_each(|(_, event)| match event { - MaybeOwned::Owned(event) => event.apply_event(queue), + MaybeOwned::Owned(event) => self.apply_event(event), MaybeOwned::Borrowed(_) => unreachable!(), }); - queue - .get_storage() + self.get_storage() .as_mut() .expect("No storage provided") .get_replica() From 19dd5190ef68e27f1ddf6ccd0014deb9585338e6 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Wed, 15 Jul 2020 20:38:18 +0300 Subject: [PATCH 27/36] Ctrl-C and persistence in replica command --- spartan/src/cli/commands/replica.rs | 16 ++++++++++++---- spartan/src/cli/commands/start.rs | 8 -------- spartan/src/jobs/exit.rs | 2 ++ spartan/src/jobs/gc.rs | 2 ++ spartan/src/jobs/persistence.rs | 2 ++ spartan/src/jobs/replication/mod.rs | 2 ++ 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/spartan/src/cli/commands/replica.rs b/spartan/src/cli/commands/replica.rs index 8bd0198e..0d3345b7 100644 --- a/spartan/src/cli/commands/replica.rs +++ b/spartan/src/cli/commands/replica.rs @@ -1,7 +1,7 @@ use crate::{ cli::Server, config::replication::Replication, - jobs::persistence::{load_from_fs, PersistenceError}, + jobs::{persistence::{load_from_fs, PersistenceError, spawn_persistence}, exit::spawn_ctrlc_handler}, node::{ replication::{ event::ApplyEvent, @@ -13,12 +13,12 @@ use crate::{ }; use bincode::{deserialize, serialize, ErrorKind}; use futures_util::{SinkExt, StreamExt}; -use std::future::Future; +use std::{sync::Arc, future::Future}; use structopt::StructOpt; use thiserror::Error; use tokio::{ io::Error as IoError, - net::{TcpListener, TcpStream}, + net::{TcpListener, TcpStream}, spawn, }; use tokio_util::codec::{BytesCodec, Decoder, Framed}; @@ -40,7 +40,7 @@ pub enum ReplicaCommandError { pub struct ReplicaCommand {} impl ReplicaCommand { - pub async fn dispatch(&self, server: &Server) -> Result<(), ReplicaCommandError> { + pub async fn dispatch(&self, server: &'static Server) -> Result<(), ReplicaCommandError> { let config = server.config().expect("Config not loaded"); let mut manager = Manager::new(config); @@ -48,6 +48,14 @@ impl ReplicaCommand { .await .map_err(ReplicaCommandError::PersistenceError)?; + let manager = Arc::new(manager); + + let cloned_manager = manager.clone(); + spawn(async move { spawn_ctrlc_handler(&cloned_manager).await }); + + let cloned_manager = manager.clone(); + spawn(async move { spawn_persistence(&cloned_manager).await }); + ReplicationStorage::prepare( &manager, |storage| matches!(storage, ReplicationStorage::Replica(_)), diff --git a/spartan/src/cli/commands/start.rs b/spartan/src/cli/commands/start.rs index a834c159..fc9383b5 100644 --- a/spartan/src/cli/commands/start.rs +++ b/spartan/src/cli/commands/start.rs @@ -67,23 +67,15 @@ impl StartCommand { let manager = Data::new(manager); - debug!("Spawning GC handler."); - let cloned_manager = manager.clone(); spawn(async move { spawn_gc(&cloned_manager).await }); - debug!("Spawning persistence job."); - let cloned_manager = manager.clone(); spawn(async move { spawn_persistence(&cloned_manager).await }); - debug!("Spawning replication job."); - let cloned_manager = manager.clone(); spawn(async move { spawn_replication(&cloned_manager).await }); - debug!("Spawning Ctrl-C handler"); - let cloned_manager = manager.clone(); spawn(async move { spawn_ctrlc_handler(&cloned_manager).await }); diff --git a/spartan/src/jobs/exit.rs b/spartan/src/jobs/exit.rs index add3467c..de962689 100644 --- a/spartan/src/jobs/exit.rs +++ b/spartan/src/jobs/exit.rs @@ -7,6 +7,8 @@ use std::process::exit; /// /// Listens to Ctrl-C signal, and after receiving one starts persisting database. pub async fn spawn_ctrlc_handler(manager: &Manager<'_>) { + debug!("Spawning Ctrl-C handler"); + ctrl_c().await.expect("Unable to listen to Ctrl-C signal."); persist_manager(manager).await; diff --git a/spartan/src/jobs/gc.rs b/spartan/src/jobs/gc.rs index 323aeff3..f3e65879 100644 --- a/spartan/src/jobs/gc.rs +++ b/spartan/src/jobs/gc.rs @@ -23,6 +23,8 @@ async fn execute_gc(manager: &Manager<'_>) { /// /// Periodically iterates over all databases in node, and executes GC on them. pub async fn spawn_gc(manager: &Manager<'_>) { + debug!("Spawning GC handler."); + let timer = Duration::from_secs(manager.config.gc_timer); loop { diff --git a/spartan/src/jobs/persistence.rs b/spartan/src/jobs/persistence.rs index 17c6379e..ba146b3c 100644 --- a/spartan/src/jobs/persistence.rs +++ b/spartan/src/jobs/persistence.rs @@ -56,6 +56,8 @@ pub async fn persist_manager(manager: &Manager<'_>) { /// Persistence job handler, that persists all databases from manager pub async fn spawn_persistence(manager: &Manager<'_>) { + debug!("Spawning persistence job."); + let timer = Duration::from_secs(manager.config.persistence_timer); loop { diff --git a/spartan/src/jobs/replication/mod.rs b/spartan/src/jobs/replication/mod.rs index afa19d9d..6ca792ad 100644 --- a/spartan/src/jobs/replication/mod.rs +++ b/spartan/src/jobs/replication/mod.rs @@ -42,6 +42,8 @@ async fn start_replication(manager: &Manager<'_>, pool: &mut StreamPool, config: } pub async fn spawn_replication(manager: &Manager<'_>) -> IoResult<()> { + debug!("Spawning replication job."); + if let Some(config) = manager.config.replication.as_ref() { match config { Replication::Primary(config) => { From c7afdb0c4bf22fb9f4b3bfa1287fe4c9fd8507de Mon Sep 17 00:00:00 2001 From: ivan770 Date: Wed, 15 Jul 2020 20:55:50 +0300 Subject: [PATCH 28/36] Move primary and replica stuff into their own submodules --- spartan/src/cli/commands/replica.rs | 130 +++--------------- .../{replication/mod.rs => replication.rs} | 18 +-- spartan/src/node/replication/mod.rs | 6 + .../replication/primary}/error.rs | 4 +- .../replication/primary}/index.rs | 12 +- spartan/src/node/replication/primary/mod.rs | 8 ++ .../replication/primary}/stream.rs | 47 +++---- spartan/src/node/replication/replica/error.rs | 20 +++ spartan/src/node/replication/replica/mod.rs | 97 +++++++++++++ 9 files changed, 183 insertions(+), 159 deletions(-) rename spartan/src/jobs/{replication/mod.rs => replication.rs} (85%) rename spartan/src/{jobs/replication => node/replication/primary}/error.rs (81%) rename spartan/src/{jobs/replication => node/replication/primary}/index.rs (78%) create mode 100644 spartan/src/node/replication/primary/mod.rs rename spartan/src/{jobs/replication => node/replication/primary}/stream.rs (61%) create mode 100644 spartan/src/node/replication/replica/error.rs create mode 100644 spartan/src/node/replication/replica/mod.rs diff --git a/spartan/src/cli/commands/replica.rs b/spartan/src/cli/commands/replica.rs index 0d3345b7..755b42ee 100644 --- a/spartan/src/cli/commands/replica.rs +++ b/spartan/src/cli/commands/replica.rs @@ -1,52 +1,37 @@ use crate::{ cli::Server, config::replication::Replication, - jobs::{persistence::{load_from_fs, PersistenceError, spawn_persistence}, exit::spawn_ctrlc_handler}, + jobs::{ + exit::spawn_ctrlc_handler, + persistence::{load_from_fs, spawn_persistence}, + }, node::{ replication::{ - event::ApplyEvent, - message::{PrimaryRequest, ReplicaRequest}, + replica::{ + accept_connection, + error::{ReplicaError, ReplicaResult}, + ReplicaSocket, + }, storage::{replica::ReplicaStorage, ReplicationStorage}, }, Manager, }, }; -use bincode::{deserialize, serialize, ErrorKind}; -use futures_util::{SinkExt, StreamExt}; -use std::{sync::Arc, future::Future}; +use std::sync::Arc; use structopt::StructOpt; -use thiserror::Error; -use tokio::{ - io::Error as IoError, - net::{TcpListener, TcpStream}, spawn, -}; -use tokio_util::codec::{BytesCodec, Decoder, Framed}; - -#[derive(Error, Debug)] -pub enum ReplicaCommandError { - #[error("Manager persistence error: {0}")] - PersistenceError(PersistenceError), - #[error("Unable to find replica node config")] - ReplicaConfigNotFound, - #[error("TCP socket error: {0}")] - SocketError(IoError), - #[error("Empty TCP socket")] - EmptySocket, - #[error("Packet serialization error: {0}")] - SerializationError(Box), -} +use tokio::{net::TcpListener, spawn}; #[derive(StructOpt)] pub struct ReplicaCommand {} impl ReplicaCommand { - pub async fn dispatch(&self, server: &'static Server) -> Result<(), ReplicaCommandError> { + pub async fn dispatch(&self, server: &'static Server) -> ReplicaResult<()> { let config = server.config().expect("Config not loaded"); let mut manager = Manager::new(config); load_from_fs(&mut manager) .await - .map_err(ReplicaCommandError::PersistenceError)?; + .map_err(ReplicaError::PersistenceError)?; let manager = Arc::new(manager); @@ -66,12 +51,12 @@ impl ReplicaCommand { let mut socket = match config .replication .as_ref() - .ok_or_else(|| ReplicaCommandError::ReplicaConfigNotFound)? + .ok_or_else(|| ReplicaError::ReplicaConfigNotFound)? { Replication::Replica(config) => TcpListener::bind(config.host) .await - .map_err(ReplicaCommandError::SocketError), - _ => Err(ReplicaCommandError::ReplicaConfigNotFound), + .map_err(ReplicaError::SocketError), + _ => Err(ReplicaError::ReplicaConfigNotFound), }?; loop { @@ -81,89 +66,8 @@ impl ReplicaCommand { .exchange(accept_connection) .await? } - Err(e) => return Err(ReplicaCommandError::SocketError(e)), + Err(e) => return Err(ReplicaError::SocketError(e)), } } } } - -struct ReplicaSocket<'a> { - manager: &'a Manager<'a>, - socket: Framed, -} - -impl<'a> ReplicaSocket<'a> { - fn new(manager: &'a Manager<'a>, socket: TcpStream) -> Self { - ReplicaSocket { - manager, - socket: BytesCodec::new().framed(socket), - } - } - - async fn exchange(&mut self, f: F) -> Result<(), ReplicaCommandError> - where - F: Fn(PrimaryRequest<'static>, &'a Manager<'a>) -> Fut, - Fut: Future>, - { - loop { - let buf = match self.socket.next().await { - Some(r) => r.map_err(ReplicaCommandError::SocketError)?, - None => return Err(ReplicaCommandError::EmptySocket), - }; - - let request = f( - deserialize(&buf).map_err(ReplicaCommandError::SerializationError)?, - self.manager, - ) - .await; - - self.socket - .send( - serialize(&request) - .map_err(ReplicaCommandError::SerializationError)? - .into(), - ) - .await - .map_err(ReplicaCommandError::SocketError)?; - - self.socket - .flush() - .await - .map_err(ReplicaCommandError::SocketError)?; - } - } -} - -async fn accept_connection<'a>( - request: PrimaryRequest<'static>, - manager: &Manager<'a>, -) -> ReplicaRequest<'a> { - match request { - PrimaryRequest::Ping => ReplicaRequest::Pong, - PrimaryRequest::AskIndex => { - let mut indexes = Vec::with_capacity(manager.config.queues.len()); - - for (name, db) in manager.node.iter() { - let index = db - .lock() - .await - .get_storage() - .as_mut() - .expect("No database present") - .get_replica() - .get_index(); - - indexes.push((name.to_string().into_boxed_str(), index)); - } - - ReplicaRequest::RecvIndex(indexes.into_boxed_slice()) - } - PrimaryRequest::SendRange(queue, range) => match manager.queue(&queue).await { - Ok(mut db) => { - db.apply_events(range); - ReplicaRequest::RecvRange - } - Err(_) => ReplicaRequest::QueueNotFound(queue), - }, - } -} diff --git a/spartan/src/jobs/replication/mod.rs b/spartan/src/jobs/replication.rs similarity index 85% rename from spartan/src/jobs/replication/mod.rs rename to spartan/src/jobs/replication.rs index 6ca792ad..12ed73b1 100644 --- a/spartan/src/jobs/replication/mod.rs +++ b/spartan/src/jobs/replication.rs @@ -1,26 +1,18 @@ -/// Tokio TCP stream abstraction -pub mod stream; - -/// Replication index exchange -pub mod index; - -/// Replication error -pub mod error; - use crate::{ config::replication::{Primary, Replication}, node::{ - replication::storage::{primary::PrimaryStorage, ReplicationStorage}, + replication::{ + primary::{error::PrimaryResult, stream::StreamPool}, + storage::{primary::PrimaryStorage, ReplicationStorage}, + }, Manager, }, }; use actix_rt::time::delay_for; -use error::ReplicationResult; use std::time::Duration; -use stream::StreamPool; use tokio::io::Result as IoResult; -async fn replicate_manager(manager: &Manager<'_>, pool: &mut StreamPool) -> ReplicationResult<()> { +async fn replicate_manager(manager: &Manager<'_>, pool: &mut StreamPool) -> PrimaryResult<()> { pool.ping().await?; pool.ask().await?.sync(manager).await?; diff --git a/spartan/src/node/replication/mod.rs b/spartan/src/node/replication/mod.rs index 3acc6ce6..b9f97d49 100644 --- a/spartan/src/node/replication/mod.rs +++ b/spartan/src/node/replication/mod.rs @@ -9,3 +9,9 @@ pub mod database; /// Replication TCP messages pub mod message; + +/// Primary replication node +pub mod primary; + +/// Replica replication node +pub mod replica; diff --git a/spartan/src/jobs/replication/error.rs b/spartan/src/node/replication/primary/error.rs similarity index 81% rename from spartan/src/jobs/replication/error.rs rename to spartan/src/node/replication/primary/error.rs index 96521bba..29e5a48f 100644 --- a/spartan/src/jobs/replication/error.rs +++ b/spartan/src/node/replication/primary/error.rs @@ -3,7 +3,7 @@ use thiserror::Error; use tokio::io::Error as IoError; #[derive(Error, Debug)] -pub(super) enum ReplicationError { +pub enum PrimaryError { #[error("Unable to serialize stream message: {0}")] SerializationError(Box), #[error("TCP connection error: {0}")] @@ -16,4 +16,4 @@ pub(super) enum ReplicationError { QueueConfigMismatch, } -pub(super) type ReplicationResult = Result; +pub type PrimaryResult = Result; diff --git a/spartan/src/jobs/replication/index.rs b/spartan/src/node/replication/primary/index.rs similarity index 78% rename from spartan/src/jobs/replication/index.rs rename to spartan/src/node/replication/primary/index.rs index 43eeb359..b93da9bb 100644 --- a/spartan/src/jobs/replication/index.rs +++ b/spartan/src/node/replication/primary/index.rs @@ -1,11 +1,11 @@ use super::{ - error::{ReplicationError, ReplicationResult}, + error::{PrimaryError, PrimaryResult}, stream::Stream, }; use crate::node::Manager; use futures_util::{stream::iter, StreamExt, TryStreamExt}; -pub(super) struct RecvIndex<'a> { +pub struct RecvIndex<'a> { stream: &'a mut Stream, indexes: Box<[(Box, u64)]>, } @@ -15,7 +15,7 @@ impl<'a> RecvIndex<'a> { RecvIndex { stream, indexes } } - pub async fn sync(&mut self, manager: &Manager<'_>) -> ReplicationResult<()> { + pub async fn sync(&mut self, manager: &Manager<'_>) -> PrimaryResult<()> { for (name, start) in self.indexes.iter() { self.stream .send_range( @@ -23,7 +23,7 @@ impl<'a> RecvIndex<'a> { manager .queue(name) .await - .map_err(|_| ReplicationError::QueueConfigMismatch)? + .map_err(|_| PrimaryError::QueueConfigMismatch)? .get_storage() .as_mut() .expect("Replication storage is uninitialized") @@ -37,7 +37,7 @@ impl<'a> RecvIndex<'a> { } } -pub(super) struct BatchAskIndex<'a> { +pub struct BatchAskIndex<'a> { batch: Vec>, } @@ -52,7 +52,7 @@ impl<'a> BatchAskIndex<'a> { self.batch.push(index); } - pub async fn sync(&mut self, manager: &Manager<'_>) -> ReplicationResult<()> { + pub async fn sync(&mut self, manager: &Manager<'_>) -> PrimaryResult<()> { iter(self.batch.iter_mut()) .map(Ok) .try_for_each_concurrent(None, |host| async move { host.sync(manager).await }) diff --git a/spartan/src/node/replication/primary/mod.rs b/spartan/src/node/replication/primary/mod.rs new file mode 100644 index 00000000..32f1d37b --- /dev/null +++ b/spartan/src/node/replication/primary/mod.rs @@ -0,0 +1,8 @@ +/// Primary error +pub mod error; + +/// Entry index exchange +pub mod index; + +/// Network stream +pub mod stream; diff --git a/spartan/src/jobs/replication/stream.rs b/spartan/src/node/replication/primary/stream.rs similarity index 61% rename from spartan/src/jobs/replication/stream.rs rename to spartan/src/node/replication/primary/stream.rs index 685dbb37..b691388b 100644 --- a/spartan/src/jobs/replication/stream.rs +++ b/spartan/src/node/replication/primary/stream.rs @@ -1,5 +1,5 @@ use super::{ - error::{ReplicationError, ReplicationResult}, + error::{PrimaryError, PrimaryResult}, index::{BatchAskIndex, RecvIndex}, }; use crate::{ @@ -16,56 +16,53 @@ use std::borrow::Cow; use tokio::net::TcpStream; use tokio_util::codec::{BytesCodec, Decoder, Framed}; -pub(super) struct Stream(Framed); +pub struct Stream(Framed); -pub(super) struct StreamPool(Box<[Stream]>); +pub struct StreamPool(Box<[Stream]>); impl<'a> Stream { - fn serialize(message: &PrimaryRequest) -> ReplicationResult> { - serialize(&message).map_err(ReplicationError::SerializationError) + fn serialize(message: &PrimaryRequest) -> PrimaryResult> { + serialize(&message).map_err(PrimaryError::SerializationError) } pub async fn exchange( &mut self, message: &PrimaryRequest<'_>, - ) -> ReplicationResult> { + ) -> PrimaryResult> { self.0 .send(Self::serialize(message)?.into()) .await - .map_err(ReplicationError::SocketError)?; + .map_err(PrimaryError::SocketError)?; - self.0 - .flush() - .await - .map_err(ReplicationError::SocketError)?; + self.0.flush().await.map_err(PrimaryError::SocketError)?; let buf = match self.0.next().await { - Some(r) => r.map_err(ReplicationError::SocketError)?, - None => return Err(ReplicationError::EmptySocket), + Some(r) => r.map_err(PrimaryError::SocketError)?, + None => return Err(PrimaryError::EmptySocket), }; - Ok(deserialize(&buf).map_err(ReplicationError::SerializationError)?) + Ok(deserialize(&buf).map_err(PrimaryError::SerializationError)?) } - pub async fn ping(&mut self) -> ReplicationResult<()> { + async fn ping(&mut self) -> PrimaryResult<()> { match self.exchange(&PrimaryRequest::Ping).await? { ReplicaRequest::Pong => Ok(()), - _ => Err(ReplicationError::ProtocolMismatch), + _ => Err(PrimaryError::ProtocolMismatch), } } - pub async fn ask(&'a mut self) -> ReplicationResult> { + async fn ask(&'a mut self) -> PrimaryResult> { match self.exchange(&PrimaryRequest::AskIndex).await? { ReplicaRequest::RecvIndex(recv) => Ok(RecvIndex::new(self, recv)), - _ => Err(ReplicationError::ProtocolMismatch), + _ => Err(PrimaryError::ProtocolMismatch), } } - pub async fn send_range( + pub(super) async fn send_range( &mut self, queue: &str, range: Box<[(MaybeOwned<'a, u64>, MaybeOwned<'a, Event>)]>, - ) -> ReplicationResult<()> { + ) -> PrimaryResult<()> { match self .exchange(&PrimaryRequest::SendRange(Cow::Borrowed(queue), range)) .await? @@ -75,13 +72,13 @@ impl<'a> Stream { warn!("Queue {} not found on replica", queue); Ok(()) } - _ => Err(ReplicationError::ProtocolMismatch), + _ => Err(PrimaryError::ProtocolMismatch), } } } impl<'a> StreamPool { - pub async fn from_config(config: &Primary) -> ReplicationResult { + pub async fn from_config(config: &Primary) -> PrimaryResult { let mut pool = Vec::with_capacity(config.destination.len()); for host in &*config.destination { @@ -89,7 +86,7 @@ impl<'a> StreamPool { BytesCodec::new().framed( TcpStream::connect(host) .await - .map_err(ReplicationError::SocketError)?, + .map_err(PrimaryError::SocketError)?, ), )); } @@ -97,7 +94,7 @@ impl<'a> StreamPool { Ok(StreamPool(pool.into_boxed_slice())) } - pub async fn ping(&mut self) -> ReplicationResult<()> { + pub async fn ping(&mut self) -> PrimaryResult<()> { iter(self.0.iter_mut()) .map(Ok) .try_for_each_concurrent(None, |stream| async move { stream.ping().await }) @@ -106,7 +103,7 @@ impl<'a> StreamPool { Ok(()) } - pub async fn ask(&'a mut self) -> ReplicationResult> { + pub async fn ask(&'a mut self) -> PrimaryResult> { let mut batch = BatchAskIndex::with_capacity(self.0.len()); for host in &mut *self.0 { diff --git a/spartan/src/node/replication/replica/error.rs b/spartan/src/node/replication/replica/error.rs new file mode 100644 index 00000000..573dac0c --- /dev/null +++ b/spartan/src/node/replication/replica/error.rs @@ -0,0 +1,20 @@ +use crate::jobs::persistence::PersistenceError; +use bincode::ErrorKind; +use thiserror::Error; +use tokio::io::Error as IoError; + +#[derive(Error, Debug)] +pub enum ReplicaError { + #[error("Manager persistence error: {0}")] + PersistenceError(PersistenceError), + #[error("Unable to find replica node config")] + ReplicaConfigNotFound, + #[error("TCP socket error: {0}")] + SocketError(IoError), + #[error("Empty TCP socket")] + EmptySocket, + #[error("Packet serialization error: {0}")] + SerializationError(Box), +} + +pub type ReplicaResult = Result; diff --git a/spartan/src/node/replication/replica/mod.rs b/spartan/src/node/replication/replica/mod.rs new file mode 100644 index 00000000..216aad08 --- /dev/null +++ b/spartan/src/node/replication/replica/mod.rs @@ -0,0 +1,97 @@ +/// Replica error +pub mod error; + +use crate::node::{ + replication::{ + event::ApplyEvent, + message::{PrimaryRequest, ReplicaRequest}, + }, + Manager, +}; +use bincode::{deserialize, serialize}; +use error::{ReplicaError, ReplicaResult}; +use futures_util::{SinkExt, StreamExt}; +use std::future::Future; +use tokio::net::TcpStream; +use tokio_util::codec::{BytesCodec, Decoder, Framed}; + +pub struct ReplicaSocket<'a> { + manager: &'a Manager<'a>, + socket: Framed, +} + +impl<'a> ReplicaSocket<'a> { + pub fn new(manager: &'a Manager<'a>, socket: TcpStream) -> Self { + ReplicaSocket { + manager, + socket: BytesCodec::new().framed(socket), + } + } + + pub async fn exchange(&mut self, f: F) -> ReplicaResult<()> + where + F: Fn(PrimaryRequest<'static>, &'a Manager<'a>) -> Fut, + Fut: Future>, + { + loop { + let buf = match self.socket.next().await { + Some(r) => r.map_err(ReplicaError::SocketError)?, + None => return Err(ReplicaError::EmptySocket), + }; + + let request = f( + deserialize(&buf).map_err(ReplicaError::SerializationError)?, + self.manager, + ) + .await; + + self.socket + .send( + serialize(&request) + .map_err(ReplicaError::SerializationError)? + .into(), + ) + .await + .map_err(ReplicaError::SocketError)?; + + self.socket + .flush() + .await + .map_err(ReplicaError::SocketError)?; + } + } +} + +pub async fn accept_connection<'a>( + request: PrimaryRequest<'static>, + manager: &Manager<'a>, +) -> ReplicaRequest<'a> { + match request { + PrimaryRequest::Ping => ReplicaRequest::Pong, + PrimaryRequest::AskIndex => { + let mut indexes = Vec::with_capacity(manager.config.queues.len()); + + for (name, db) in manager.node.iter() { + let index = db + .lock() + .await + .get_storage() + .as_mut() + .expect("No database present") + .get_replica() + .get_index(); + + indexes.push((name.to_string().into_boxed_str(), index)); + } + + ReplicaRequest::RecvIndex(indexes.into_boxed_slice()) + } + PrimaryRequest::SendRange(queue, range) => match manager.queue(&queue).await { + Ok(mut db) => { + db.apply_events(range); + ReplicaRequest::RecvRange + } + Err(_) => ReplicaRequest::QueueNotFound(queue), + }, + } +} From 2e831c5c4bc8481af33ee1eee9fe995c844aa33b Mon Sep 17 00:00:00 2001 From: ivan770 Date: Wed, 15 Jul 2020 21:02:08 +0300 Subject: [PATCH 29/36] Move replication storage preparation to node --- spartan/src/cli/commands/replica.rs | 13 +++++++------ spartan/src/jobs/replication.rs | 13 +++++++------ spartan/src/node/mod.rs | 18 +++++++++++++++++- spartan/src/node/replication/storage/mod.rs | 17 ----------------- 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/spartan/src/cli/commands/replica.rs b/spartan/src/cli/commands/replica.rs index 755b42ee..c2745182 100644 --- a/spartan/src/cli/commands/replica.rs +++ b/spartan/src/cli/commands/replica.rs @@ -41,12 +41,13 @@ impl ReplicaCommand { let cloned_manager = manager.clone(); spawn(async move { spawn_persistence(&cloned_manager).await }); - ReplicationStorage::prepare( - &manager, - |storage| matches!(storage, ReplicationStorage::Replica(_)), - || ReplicationStorage::Replica(ReplicaStorage::default()), - ) - .await; + manager + .node + .prepare_replication( + |storage| matches!(storage, ReplicationStorage::Replica(_)), + || ReplicationStorage::Replica(ReplicaStorage::default()), + ) + .await; let mut socket = match config .replication diff --git a/spartan/src/jobs/replication.rs b/spartan/src/jobs/replication.rs index 12ed73b1..51383db5 100644 --- a/spartan/src/jobs/replication.rs +++ b/spartan/src/jobs/replication.rs @@ -39,12 +39,13 @@ pub async fn spawn_replication(manager: &Manager<'_>) -> IoResult<()> { if let Some(config) = manager.config.replication.as_ref() { match config { Replication::Primary(config) => { - ReplicationStorage::prepare( - manager, - |storage| matches!(storage, ReplicationStorage::Primary(_)), - || ReplicationStorage::Primary(PrimaryStorage::default()), - ) - .await; + manager + .node + .prepare_replication( + |storage| matches!(storage, ReplicationStorage::Primary(_)), + || ReplicationStorage::Primary(PrimaryStorage::default()), + ) + .await; match StreamPool::from_config(config).await { Ok(mut pool) => start_replication(manager, &mut pool, config).await, diff --git a/spartan/src/node/mod.rs b/spartan/src/node/mod.rs index 4fa5ff95..0293ee5d 100644 --- a/spartan/src/node/mod.rs +++ b/spartan/src/node/mod.rs @@ -8,7 +8,7 @@ pub use manager::Manager; use crate::config::Config; use futures_util::lock::{Mutex, MutexGuard}; -use replication::database::ReplicatedDatabase; +use replication::{database::ReplicatedDatabase, storage::ReplicationStorage}; use spartan_lib::core::{db::tree::TreeDatabase, message::Message}; use std::collections::{hash_map::RandomState, HashMap}; @@ -52,4 +52,20 @@ impl<'a> Node<'a> { pub fn load_from_config(&mut self, config: &'a Config) { config.queues.iter().for_each(|queue| self.add(queue)); } + + pub async fn prepare_replication(&self, filter: F, replace: R) + where + F: Fn(&&ReplicationStorage) -> bool + Copy, + R: Fn() -> ReplicationStorage, + { + for (_, db) in self.iter() { + let mut db = db.lock().await; + + let storage = db.get_storage().as_ref().filter(filter); + + if storage.is_none() { + db.get_storage().replace(replace()); + } + } + } } diff --git a/spartan/src/node/replication/storage/mod.rs b/spartan/src/node/replication/storage/mod.rs index f6bd6c72..21291056 100644 --- a/spartan/src/node/replication/storage/mod.rs +++ b/spartan/src/node/replication/storage/mod.rs @@ -4,7 +4,6 @@ pub mod primary; /// Storage for replica's pub mod replica; -use crate::node::Manager; use primary::PrimaryStorage; use replica::ReplicaStorage; use serde::{Deserialize, Serialize}; @@ -29,20 +28,4 @@ impl ReplicationStorage { _ => panic!("Replication storage is in primary mode."), } } - - pub async fn prepare(manager: &Manager<'_>, filter: F, replace: R) - where - F: Fn(&&ReplicationStorage) -> bool + Copy, - R: Fn() -> ReplicationStorage, - { - for (_, db) in manager.node.db.iter() { - let mut db = db.lock().await; - - let storage = db.get_storage().as_ref().filter(filter); - - if storage.is_none() { - db.get_storage().replace(replace()); - } - } - } } From 4b5cda78fba5657d30a4d577d8dc771fe3850182 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Wed, 15 Jul 2020 21:33:34 +0300 Subject: [PATCH 30/36] Drop stream if socket is empty in replica, add try timer --- spartan/src/cli/commands/replica.rs | 16 ++-- spartan/src/config/replication.rs | 8 ++ spartan/src/node/replication/replica/mod.rs | 84 +++++++++++++-------- 3 files changed, 71 insertions(+), 37 deletions(-) diff --git a/spartan/src/cli/commands/replica.rs b/spartan/src/cli/commands/replica.rs index c2745182..112604d4 100644 --- a/spartan/src/cli/commands/replica.rs +++ b/spartan/src/cli/commands/replica.rs @@ -49,25 +49,27 @@ impl ReplicaCommand { ) .await; - let mut socket = match config + let config = match config .replication .as_ref() .ok_or_else(|| ReplicaError::ReplicaConfigNotFound)? { - Replication::Replica(config) => TcpListener::bind(config.host) - .await - .map_err(ReplicaError::SocketError), + Replication::Replica(config) => Ok(config), _ => Err(ReplicaError::ReplicaConfigNotFound), }?; + let mut socket = TcpListener::bind(config.host) + .await + .map_err(ReplicaError::SocketError)?; + loop { match socket.accept().await { Ok((socket, _)) => { - ReplicaSocket::new(&manager, socket) + ReplicaSocket::new(&manager, config, socket) .exchange(accept_connection) - .await? + .await } - Err(e) => return Err(ReplicaError::SocketError(e)), + Err(e) => error!("Unable to accept TCP connection: {}", e), } } } diff --git a/spartan/src/config/replication.rs b/spartan/src/config/replication.rs index a5ac0a85..29b0cb52 100644 --- a/spartan/src/config/replication.rs +++ b/spartan/src/config/replication.rs @@ -6,6 +6,11 @@ const fn default_replication_timer() -> u64 { 180 } +/// Default amount of seconds between replica command restart tries +const fn default_replica_try_timer() -> u64 { + 5 +} + #[derive(Serialize, Deserialize)] pub struct Primary { pub destination: Box<[SocketAddr]>, @@ -17,6 +22,9 @@ pub struct Primary { #[derive(Serialize, Deserialize)] pub struct Replica { pub host: SocketAddr, + + #[serde(default = "default_replica_try_timer")] + pub try_timer: u64, } #[derive(Serialize, Deserialize)] diff --git a/spartan/src/node/replication/replica/mod.rs b/spartan/src/node/replication/replica/mod.rs index 216aad08..757ab8c5 100644 --- a/spartan/src/node/replication/replica/mod.rs +++ b/spartan/src/node/replication/replica/mod.rs @@ -1,64 +1,88 @@ /// Replica error pub mod error; -use crate::node::{ - replication::{ - event::ApplyEvent, - message::{PrimaryRequest, ReplicaRequest}, +use crate::{ + config::replication::Replica, + node::{ + replication::{ + event::ApplyEvent, + message::{PrimaryRequest, ReplicaRequest}, + }, + Manager, }, - Manager, }; +use actix_rt::time::delay_for; use bincode::{deserialize, serialize}; use error::{ReplicaError, ReplicaResult}; use futures_util::{SinkExt, StreamExt}; -use std::future::Future; +use std::{future::Future, time::Duration}; use tokio::net::TcpStream; use tokio_util::codec::{BytesCodec, Decoder, Framed}; pub struct ReplicaSocket<'a> { manager: &'a Manager<'a>, + config: &'a Replica, socket: Framed, } impl<'a> ReplicaSocket<'a> { - pub fn new(manager: &'a Manager<'a>, socket: TcpStream) -> Self { + pub fn new(manager: &'a Manager<'a>, config: &'a Replica, socket: TcpStream) -> Self { ReplicaSocket { manager, + config, socket: BytesCodec::new().framed(socket), } } - pub async fn exchange(&mut self, f: F) -> ReplicaResult<()> + pub async fn exchange(&mut self, f: F) where - F: Fn(PrimaryRequest<'static>, &'a Manager<'a>) -> Fut, + F: Fn(PrimaryRequest<'static>, &'a Manager<'a>) -> Fut + Copy, Fut: Future>, { + let timer = Duration::from_secs(self.config.try_timer); + loop { - let buf = match self.socket.next().await { - Some(r) => r.map_err(ReplicaError::SocketError)?, - None => return Err(ReplicaError::EmptySocket), - }; + delay_for(timer).await; + + match self.process(f).await { + Err(ReplicaError::EmptySocket) => return, + Err(e) => error!("Error occured during replication process: {}", e), + _ => (), + } + } + } - let request = f( - deserialize(&buf).map_err(ReplicaError::SerializationError)?, - self.manager, + async fn process(&mut self, f: F) -> ReplicaResult<()> + where + F: Fn(PrimaryRequest<'static>, &'a Manager<'a>) -> Fut, + Fut: Future>, + { + let buf = match self.socket.next().await { + Some(r) => r.map_err(ReplicaError::SocketError)?, + None => return Err(ReplicaError::EmptySocket), + }; + + let request = f( + deserialize(&buf).map_err(ReplicaError::SerializationError)?, + self.manager, + ) + .await; + + self.socket + .send( + serialize(&request) + .map_err(ReplicaError::SerializationError)? + .into(), ) - .await; + .await + .map_err(ReplicaError::SocketError)?; - self.socket - .send( - serialize(&request) - .map_err(ReplicaError::SerializationError)? - .into(), - ) - .await - .map_err(ReplicaError::SocketError)?; + self.socket + .flush() + .await + .map_err(ReplicaError::SocketError)?; - self.socket - .flush() - .await - .map_err(ReplicaError::SocketError)?; - } + Ok(()) } } From 133fc3e7ea0998f3a9c935bcc67d6b9c502eb95f Mon Sep 17 00:00:00 2001 From: ivan770 Date: Thu, 16 Jul 2020 19:08:06 +0300 Subject: [PATCH 31/36] Added GC threshold changing --- Cargo.lock | 10 ++++++ spartan/Cargo.toml | 3 ++ spartan/src/jobs/replication.rs | 12 ++++++- spartan/src/node/replication/primary/index.rs | 32 +++++++++++++++++++ spartan/src/node/replication/replica/mod.rs | 2 ++ .../src/node/replication/storage/primary.rs | 4 +++ 6 files changed, 62 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index dcf92027..7141f625 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -871,6 +871,15 @@ dependencies = [ "winreg", ] +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.6" @@ -1451,6 +1460,7 @@ dependencies = [ "bincode", "derive-new", "futures-util", + "itertools", "log", "maybe-owned", "once_cell", diff --git a/spartan/Cargo.toml b/spartan/Cargo.toml index 321ceebb..b0435db8 100644 --- a/spartan/Cargo.toml +++ b/spartan/Cargo.toml @@ -64,5 +64,8 @@ features = ["serde"] [dependencies.tokio-util] version = "0.3" +[dependencies.itertools] +version = "0.9" + [dev-dependencies.tempfile] version = "3.1" \ No newline at end of file diff --git a/spartan/src/jobs/replication.rs b/spartan/src/jobs/replication.rs index 51383db5..eb3c44d7 100644 --- a/spartan/src/jobs/replication.rs +++ b/spartan/src/jobs/replication.rs @@ -13,9 +13,17 @@ use std::time::Duration; use tokio::io::Result as IoResult; async fn replicate_manager(manager: &Manager<'_>, pool: &mut StreamPool) -> PrimaryResult<()> { + debug!("Pinging stream pool."); pool.ping().await?; - pool.ask().await?.sync(manager).await?; + debug!("Asking stream pool for indexes."); + let mut batch = pool.ask().await?; + + debug!("Starting event slice sync."); + batch.sync(manager).await?; + + debug!("Setting GC threshold."); + batch.set_gc(manager).await; Ok(()) } @@ -26,6 +34,8 @@ async fn start_replication(manager: &Manager<'_>, pool: &mut StreamPool, config: loop { delay_for(timer).await; + info!("Starting database replication."); + match replicate_manager(manager, pool).await { Ok(_) => info!("Database replicated successfully!"), Err(e) => error!("Error happened during replication attempt: {}", e), diff --git a/spartan/src/node/replication/primary/index.rs b/spartan/src/node/replication/primary/index.rs index b93da9bb..59dce820 100644 --- a/spartan/src/node/replication/primary/index.rs +++ b/spartan/src/node/replication/primary/index.rs @@ -4,6 +4,11 @@ use super::{ }; use crate::node::Manager; use futures_util::{stream::iter, StreamExt, TryStreamExt}; +use itertools::Itertools; +use std::{ + collections::hash_map::DefaultHasher, + hash::{Hash, Hasher}, +}; pub struct RecvIndex<'a> { stream: &'a mut Stream, @@ -60,4 +65,31 @@ impl<'a> BatchAskIndex<'a> { Ok(()) } + + pub async fn set_gc(&self, manager: &Manager<'_>) { + let iter = self + .batch + .iter() + .map(|index| index.indexes.iter()) + .flatten() + .sorted_by(|a, b| Ord::cmp(&a, &b)) + .unique_by(|(name, _)| { + let mut hasher = DefaultHasher::new(); + name.hash(&mut hasher); + hasher.finish() + }); + + for (queue, index) in iter { + let mut m = manager + .queue(&queue) + .await + .expect("set_gc called without sync before"); + + m.get_storage() + .as_mut() + .unwrap() + .get_primary() + .set_gc_threshold(*index); + } + } } diff --git a/spartan/src/node/replication/replica/mod.rs b/spartan/src/node/replication/replica/mod.rs index 757ab8c5..36e6db42 100644 --- a/spartan/src/node/replication/replica/mod.rs +++ b/spartan/src/node/replication/replica/mod.rs @@ -93,6 +93,7 @@ pub async fn accept_connection<'a>( match request { PrimaryRequest::Ping => ReplicaRequest::Pong, PrimaryRequest::AskIndex => { + debug!("Preparing indexes for primary node."); let mut indexes = Vec::with_capacity(manager.config.queues.len()); for (name, db) in manager.node.iter() { @@ -112,6 +113,7 @@ pub async fn accept_connection<'a>( } PrimaryRequest::SendRange(queue, range) => match manager.queue(&queue).await { Ok(mut db) => { + debug!("Applying event slice."); db.apply_events(range); ReplicaRequest::RecvRange } diff --git a/spartan/src/node/replication/storage/primary.rs b/spartan/src/node/replication/storage/primary.rs index 8b5824da..fc6f5d64 100644 --- a/spartan/src/node/replication/storage/primary.rs +++ b/spartan/src/node/replication/storage/primary.rs @@ -40,6 +40,10 @@ impl PrimaryStorage { .map(|(k, v)| (MaybeOwned::Borrowed(k), MaybeOwned::Borrowed(v))) .collect() } + + pub fn set_gc_threshold(&mut self, threshold: u64) { + self.gc_threshold = threshold; + } } #[cfg(test)] From 6b0497cfef87b24e74e485e518f2b27381663291 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Thu, 16 Jul 2020 19:22:19 +0300 Subject: [PATCH 32/36] Reset TCP socket on error in replication job --- spartan/src/config/replication.rs | 8 ++++++++ spartan/src/jobs/replication.rs | 22 +++++++++++++++++---- spartan/src/node/replication/replica/mod.rs | 5 ++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/spartan/src/config/replication.rs b/spartan/src/config/replication.rs index 29b0cb52..81e6d4e9 100644 --- a/spartan/src/config/replication.rs +++ b/spartan/src/config/replication.rs @@ -6,6 +6,11 @@ const fn default_replication_timer() -> u64 { 180 } +/// Default amount of seconds between replication job restart tries +const fn default_primary_try_timer() -> u64 { + 10 +} + /// Default amount of seconds between replica command restart tries const fn default_replica_try_timer() -> u64 { 5 @@ -17,6 +22,9 @@ pub struct Primary { #[serde(default = "default_replication_timer")] pub replication_timer: u64, + + #[serde(default = "default_primary_try_timer")] + pub try_timer: u64, } #[derive(Serialize, Deserialize)] diff --git a/spartan/src/jobs/replication.rs b/spartan/src/jobs/replication.rs index eb3c44d7..c8e7f6f8 100644 --- a/spartan/src/jobs/replication.rs +++ b/spartan/src/jobs/replication.rs @@ -2,7 +2,7 @@ use crate::{ config::replication::{Primary, Replication}, node::{ replication::{ - primary::{error::PrimaryResult, stream::StreamPool}, + primary::{error::{PrimaryError, PrimaryResult}, stream::StreamPool}, storage::{primary::PrimaryStorage, ReplicationStorage}, }, Manager, @@ -38,6 +38,14 @@ async fn start_replication(manager: &Manager<'_>, pool: &mut StreamPool, config: match replicate_manager(manager, pool).await { Ok(_) => info!("Database replicated successfully!"), + Err(PrimaryError::EmptySocket) => { + error!("Empty TCP socket"); + return; + }, + Err(PrimaryError::SocketError(e)) => { + error!("TCP socket error: {}", e); + return; + } Err(e) => error!("Error happened during replication attempt: {}", e), } } @@ -57,9 +65,15 @@ pub async fn spawn_replication(manager: &Manager<'_>) -> IoResult<()> { ) .await; - match StreamPool::from_config(config).await { - Ok(mut pool) => start_replication(manager, &mut pool, config).await, - Err(e) => error!("Unable to open connection pool: {}", e), + let timer = Duration::from_secs(config.try_timer); + + loop { + delay_for(timer).await; + + match StreamPool::from_config(config).await { + Ok(mut pool) => start_replication(manager, &mut pool, config).await, + Err(e) => error!("Unable to open connection pool: {}", e), + } } } Replication::Replica(_) => { diff --git a/spartan/src/node/replication/replica/mod.rs b/spartan/src/node/replication/replica/mod.rs index 36e6db42..3dfd3d1b 100644 --- a/spartan/src/node/replication/replica/mod.rs +++ b/spartan/src/node/replication/replica/mod.rs @@ -45,7 +45,10 @@ impl<'a> ReplicaSocket<'a> { delay_for(timer).await; match self.process(f).await { - Err(ReplicaError::EmptySocket) => return, + Err(ReplicaError::EmptySocket) => { + error!("Empty TCP socket"); + return + }, Err(e) => error!("Error occured during replication process: {}", e), _ => (), } From ae257eca483074b5e1e4fd11033d31e8a77198f9 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Fri, 17 Jul 2020 14:33:49 +0300 Subject: [PATCH 33/36] Remove Option return in apply_events --- spartan/src/jobs/replication.rs | 9 ++++--- spartan/src/node/replication/event.rs | 26 +++++++++------------ spartan/src/node/replication/replica/mod.rs | 4 ++-- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/spartan/src/jobs/replication.rs b/spartan/src/jobs/replication.rs index c8e7f6f8..8f013375 100644 --- a/spartan/src/jobs/replication.rs +++ b/spartan/src/jobs/replication.rs @@ -2,7 +2,10 @@ use crate::{ config::replication::{Primary, Replication}, node::{ replication::{ - primary::{error::{PrimaryError, PrimaryResult}, stream::StreamPool}, + primary::{ + error::{PrimaryError, PrimaryResult}, + stream::StreamPool, + }, storage::{primary::PrimaryStorage, ReplicationStorage}, }, Manager, @@ -41,7 +44,7 @@ async fn start_replication(manager: &Manager<'_>, pool: &mut StreamPool, config: Err(PrimaryError::EmptySocket) => { error!("Empty TCP socket"); return; - }, + } Err(PrimaryError::SocketError(e)) => { error!("TCP socket error: {}", e); return; @@ -66,7 +69,7 @@ pub async fn spawn_replication(manager: &Manager<'_>) -> IoResult<()> { .await; let timer = Duration::from_secs(config.try_timer); - + loop { delay_for(timer).await; diff --git a/spartan/src/node/replication/event.rs b/spartan/src/node/replication/event.rs index 52cb3bb5..09ad41ce 100644 --- a/spartan/src/node/replication/event.rs +++ b/spartan/src/node/replication/event.rs @@ -22,10 +22,7 @@ pub trait ApplyEvent { fn apply_event(&mut self, event: Event); /// Apply slice of events to database - fn apply_events( - &mut self, - events: Box<[(MaybeOwned<'_, u64>, MaybeOwned<'_, Event>)]>, - ) -> Option<()>; + fn apply_events(&mut self, events: Box<[(MaybeOwned<'_, u64>, MaybeOwned<'_, Event>)]>); } impl ApplyEvent for DB { @@ -52,11 +49,8 @@ impl ApplyEvent for DB { } } - fn apply_events( - &mut self, - events: Box<[(MaybeOwned<'_, u64>, MaybeOwned<'_, Event>)]>, - ) -> Option<()> { - let index = *events.last()?.0; + fn apply_events(&mut self, events: Box<[(MaybeOwned<'_, u64>, MaybeOwned<'_, Event>)]>) { + let index = events.last().map(|(index, _)| **index); // into_vec allows to use owned event events @@ -67,12 +61,14 @@ impl ApplyEvent for DB { MaybeOwned::Borrowed(_) => unreachable!(), }); - self.get_storage() - .as_mut() - .expect("No storage provided") - .get_replica() - .confirm(index); + index.and_then(|index| { + self.get_storage() + .as_mut() + .expect("No storage provided") + .get_replica() + .confirm(index); - Some(()) + Some(()) + }); } } diff --git a/spartan/src/node/replication/replica/mod.rs b/spartan/src/node/replication/replica/mod.rs index 3dfd3d1b..c877c491 100644 --- a/spartan/src/node/replication/replica/mod.rs +++ b/spartan/src/node/replication/replica/mod.rs @@ -47,8 +47,8 @@ impl<'a> ReplicaSocket<'a> { match self.process(f).await { Err(ReplicaError::EmptySocket) => { error!("Empty TCP socket"); - return - }, + return; + } Err(e) => error!("Error occured during replication process: {}", e), _ => (), } From 56d870bca2975148d4a49ec29027b69c30b4f5e1 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Fri, 17 Jul 2020 14:47:46 +0300 Subject: [PATCH 34/36] Removed unnecessary references --- spartan/src/node/replication/primary/index.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spartan/src/node/replication/primary/index.rs b/spartan/src/node/replication/primary/index.rs index 59dce820..db38dda4 100644 --- a/spartan/src/node/replication/primary/index.rs +++ b/spartan/src/node/replication/primary/index.rs @@ -72,7 +72,7 @@ impl<'a> BatchAskIndex<'a> { .iter() .map(|index| index.indexes.iter()) .flatten() - .sorted_by(|a, b| Ord::cmp(&a, &b)) + .sorted_by(|a, b| Ord::cmp(a, b)) .unique_by(|(name, _)| { let mut hasher = DefaultHasher::new(); name.hash(&mut hasher); From 8c67f2e310ba36e7ea2e9140767fea5618c6b171 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Mon, 20 Jul 2020 11:03:19 +0300 Subject: [PATCH 35/36] Documentation, fix index response in AskIndex, support start with replica config --- spartan/src/cli/mod.rs | 2 + spartan/src/jobs/replication.rs | 3 +- spartan/src/node/mod.rs | 3 ++ spartan/src/node/replication/database.rs | 8 +++- spartan/src/node/replication/event.rs | 2 + spartan/src/node/replication/message.rs | 2 + spartan/src/node/replication/mod.rs | 48 +++++++++++++++++-- spartan/src/node/replication/primary/index.rs | 10 ++++ spartan/src/node/replication/primary/mod.rs | 2 +- spartan/src/node/replication/storage/mod.rs | 10 ++++ .../src/node/replication/storage/replica.rs | 2 +- 11 files changed, 83 insertions(+), 9 deletions(-) diff --git a/spartan/src/cli/mod.rs b/spartan/src/cli/mod.rs index c39118aa..4304a933 100644 --- a/spartan/src/cli/mod.rs +++ b/spartan/src/cli/mod.rs @@ -11,6 +11,7 @@ use structopt::StructOpt; use tokio::fs::read; use toml::from_slice; +/// Enum of all available CLI commands #[derive(StructOpt)] pub enum Command { #[structopt(about = "Start Spartan MQ server")] @@ -21,6 +22,7 @@ pub enum Command { Replica(ReplicaCommand), } +/// Server with config and selected command #[derive(StructOpt)] pub struct Server { /// Server configuration path diff --git a/spartan/src/jobs/replication.rs b/spartan/src/jobs/replication.rs index 8f013375..30023a4d 100644 --- a/spartan/src/jobs/replication.rs +++ b/spartan/src/jobs/replication.rs @@ -80,7 +80,8 @@ pub async fn spawn_replication(manager: &Manager<'_>) -> IoResult<()> { } } Replication::Replica(_) => { - panic!("Starting replication job while in replica mode is not allowed") + warn!("Primary node started with replica configuration!"); + warn!("Event log will be disabled for this session."); } }; } diff --git a/spartan/src/node/mod.rs b/spartan/src/node/mod.rs index 0293ee5d..5c81c614 100644 --- a/spartan/src/node/mod.rs +++ b/spartan/src/node/mod.rs @@ -12,7 +12,10 @@ use replication::{database::ReplicatedDatabase, storage::ReplicationStorage}; use spartan_lib::core::{db::tree::TreeDatabase, message::Message}; use std::collections::{hash_map::RandomState, HashMap}; +/// Conjucted type of replicated database based on tree database pub type DB = ReplicatedDatabase>; + +/// Mutexed database pub type MutexDB = Mutex; /// Key-value node implementation diff --git a/spartan/src/node/replication/database.rs b/spartan/src/node/replication/database.rs index d53dedc3..3591d732 100644 --- a/spartan/src/node/replication/database.rs +++ b/spartan/src/node/replication/database.rs @@ -13,7 +13,11 @@ use std::ops::{Deref, DerefMut}; #[derive(Serialize, Deserialize)] pub struct ReplicatedDatabase { + /// Proxied database inner: DB, + + /// Replication storage + /// None if replication is not enabled storage: Option, } @@ -55,11 +59,11 @@ impl ReplicatedDatabase { where F: FnOnce() -> Event, { - self.call_storage(|storage| storage.get_primary().push(event())); + self.call_storage(|storage| storage.map_primary(|storage| storage.push(event()))); } fn gc(&mut self) { - self.call_storage(|storage| storage.get_primary().gc()); + self.call_storage(|storage| storage.map_primary(|storage| storage.gc())); } pub fn get_storage(&mut self) -> &mut Option { diff --git a/spartan/src/node/replication/event.rs b/spartan/src/node/replication/event.rs index 09ad41ce..501f5588 100644 --- a/spartan/src/node/replication/event.rs +++ b/spartan/src/node/replication/event.rs @@ -7,6 +7,8 @@ use spartan_lib::core::{ payload::Identifiable, }; +/// Database event +/// Only events that mutate database are present here #[derive(Serialize, Deserialize)] pub enum Event { Push(Message), diff --git a/spartan/src/node/replication/message.rs b/spartan/src/node/replication/message.rs index eb12cea7..d3e31946 100644 --- a/spartan/src/node/replication/message.rs +++ b/spartan/src/node/replication/message.rs @@ -1,3 +1,5 @@ +//! See replication module documentation for messages description + use crate::node::replication::event::Event; use maybe_owned::MaybeOwned; use serde::{Deserialize, Serialize}; diff --git a/spartan/src/node/replication/mod.rs b/spartan/src/node/replication/mod.rs index b9f97d49..0813ef31 100644 --- a/spartan/src/node/replication/mod.rs +++ b/spartan/src/node/replication/mod.rs @@ -1,7 +1,47 @@ -/// Replication storage (event log) and proxy database +//! # General schema +//! +//! ``` +//! +-----------+ +---------------+ +//! | | | | +//! | Ping +----><----+ Pong | +//! | | | | +//! | AskIndex +----><----+ RecvIndex | +//! | | | | +//! | SendRange +----><--+-+ RecvRange | +//! | | ^ | | +//! +-----------+ +-+ QueueNotFound | +//! | | +//! +---------------+ +//! ``` +//! +//! This schema describes order, in which messages are sent between primary and replica nodes. +//! +//! Primary is on the left and always sends messages first, while replica node is on the right, and only responds to them. +//! +//! # Messages +//! +//! ## Ping and Pong +//! +//! These messages are used to check, if all replicas are still online. +//! +//! While just the TCP ping can be used, it's better to check if replica has the same replication protocol as primary. +//! +//! ## AskIndex and RecvIndex +//! +//! After we check replica health, we need to ask about last received index of each queue. +//! +//! Replica sends back RecvIndex message, that contains data about available queues and their indexes. +//! +//! ## SendRange and RecvRange (and probably QueueNotFound) +//! +//! With `queue = index` array of each replica available, primary node takes new events from event log of each queue, and sends it to replica. +//! +//! Replica responds with either RecvRange in case if everything is OK, or with QueueNotFound in case if primary node sent queue, that doesn't exist. + +/// Replication storage (event log) pub mod storage; -/// Replication event +/// Database event pub mod event; /// Replication proxy database @@ -10,8 +50,8 @@ pub mod database; /// Replication TCP messages pub mod message; -/// Primary replication node +/// Primary node pub mod primary; -/// Replica replication node +/// Replica node pub mod replica; diff --git a/spartan/src/node/replication/primary/index.rs b/spartan/src/node/replication/primary/index.rs index db38dda4..bcb2cdaf 100644 --- a/spartan/src/node/replication/primary/index.rs +++ b/spartan/src/node/replication/primary/index.rs @@ -66,6 +66,16 @@ impl<'a> BatchAskIndex<'a> { Ok(()) } + /// Set GC threshold of each queue to minimal index of all replica's + /// + /// Example: + /// + /// ```no_run + /// First replica: [("TestQueue", 2), ("NextQueue", 3)] + /// Second replica: [("TestQueue", 1), ("AnotherQueue", 4)] + /// + /// Result: [("AnotherQueue", 4), ("NextQueue", 3), ("TestQueue", 1)] + /// ``` pub async fn set_gc(&self, manager: &Manager<'_>) { let iter = self .batch diff --git a/spartan/src/node/replication/primary/mod.rs b/spartan/src/node/replication/primary/mod.rs index 32f1d37b..b579ea77 100644 --- a/spartan/src/node/replication/primary/mod.rs +++ b/spartan/src/node/replication/primary/mod.rs @@ -1,7 +1,7 @@ /// Primary error pub mod error; -/// Entry index exchange +/// Queue index exchange pub mod index; /// Network stream diff --git a/spartan/src/node/replication/storage/mod.rs b/spartan/src/node/replication/storage/mod.rs index 21291056..bb96f07e 100644 --- a/spartan/src/node/replication/storage/mod.rs +++ b/spartan/src/node/replication/storage/mod.rs @@ -28,4 +28,14 @@ impl ReplicationStorage { _ => panic!("Replication storage is in primary mode."), } } + + pub fn map_primary(&mut self, f: F) + where + F: FnOnce(&mut PrimaryStorage), + { + match self { + ReplicationStorage::Primary(storage) => f(storage), + ReplicationStorage::Replica(_) => (), + } + } } diff --git a/spartan/src/node/replication/storage/replica.rs b/spartan/src/node/replication/storage/replica.rs index 6c2b5e65..6098603f 100644 --- a/spartan/src/node/replication/storage/replica.rs +++ b/spartan/src/node/replication/storage/replica.rs @@ -13,7 +13,7 @@ impl Default for ReplicaStorage { impl ReplicaStorage { pub fn get_index(&self) -> u64 { - self.confirmed_index + self.confirmed_index + 1 } pub fn confirm(&mut self, index: u64) { From a8de970337e66efc22fe2b0379a42079ae82935d Mon Sep 17 00:00:00 2001 From: ivan770 Date: Mon, 20 Jul 2020 11:22:54 +0300 Subject: [PATCH 36/36] Add replication documentation to README --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index 93ae98af..2579f686 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ cargo build --release * `persistence_timer` - Amount of seconds between each database write to disk (default: `900`). * `gc_timer` - Amount of seconds between each GC job wake (GC cycle times vary, default: `300`). * `access_keys` - Table of queue access keys. Anonymous access to queues will not be permitted if this key has any value. +* `replication` - Replication configuration for both primary and replica nodes. #### `access_keys` Spartan has authentication and authorization mechanism using access keys. @@ -90,3 +91,31 @@ Example of valid HTTP Authorization header: ``` Authorization: Bearer IHaveAccessToAllQueues ``` + +#### `replication` +Spartan also has support for queue replication. + +Replication process will be restarted in case of any minor error (protocol or queue config mismatch). + +If there is any problem with TCP socket, then connection will be dropped and re-opened with each replica. + +##### Primary + +The following config will start primary node that communicates with one replica every 180 seconds (default value): +```toml +replication = { Primary = { destination = ["127.0.0.1:12345"] } } +``` + +You may also use `replication_timer` key to change amount of seconds between each replication: +```toml +replication = { Primary = { destination = ["127.0.0.1:12345"], replication_timer = 30 } } +``` + +##### Replica + +Change your replication config to following example: +```toml +replication = { Replica = { host = "127.0.0.1:12345" } } +``` + +Then, start replica node with `startan replica` command.