forked from madara-alliance/madara-orchestrator
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feat : setup scripts (madara-alliance#181)
* feat : added basic scripts for setup and functions * changelog * chore : refactor * chore : refactor implementation * chore : refactored code according to comments * feat : refactor * feat : refactor cron provider
Showing
17 changed files
with
436 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,111 @@ | ||
use std::time::Duration; | ||
|
||
use async_trait::async_trait; | ||
use aws_sdk_eventbridge::types::{InputTransformer, RuleState, Target}; | ||
use aws_sdk_sqs::types::QueueAttributeName; | ||
|
||
use crate::cron::Cron; | ||
use crate::setup::SetupConfig; | ||
|
||
pub struct AWSEventBridge {} | ||
|
||
#[async_trait] | ||
#[allow(unreachable_patterns)] | ||
impl Cron for AWSEventBridge { | ||
async fn create_cron( | ||
&self, | ||
config: &SetupConfig, | ||
cron_time: Duration, | ||
trigger_rule_name: String, | ||
) -> color_eyre::Result<()> { | ||
let config = match config { | ||
SetupConfig::AWS(config) => config, | ||
_ => panic!("Unsupported Event Bridge configuration"), | ||
}; | ||
let event_bridge_client = aws_sdk_eventbridge::Client::new(config); | ||
event_bridge_client | ||
.put_rule() | ||
.name(&trigger_rule_name) | ||
.schedule_expression(duration_to_rate_string(cron_time)) | ||
.state(RuleState::Enabled) | ||
.send() | ||
.await?; | ||
|
||
Ok(()) | ||
} | ||
async fn add_cron_target_queue( | ||
&self, | ||
config: &SetupConfig, | ||
target_queue_name: String, | ||
message: String, | ||
trigger_rule_name: String, | ||
) -> color_eyre::Result<()> { | ||
let config = match config { | ||
SetupConfig::AWS(config) => config, | ||
_ => panic!("Unsupported Event Bridge configuration"), | ||
}; | ||
let event_bridge_client = aws_sdk_eventbridge::Client::new(config); | ||
let sqs_client = aws_sdk_sqs::Client::new(config); | ||
let queue_url = sqs_client.get_queue_url().queue_name(target_queue_name).send().await?; | ||
|
||
let queue_attributes = sqs_client | ||
.get_queue_attributes() | ||
.queue_url(queue_url.queue_url.unwrap()) | ||
.attribute_names(QueueAttributeName::QueueArn) | ||
.send() | ||
.await?; | ||
let queue_arn = queue_attributes.attributes().unwrap().get(&QueueAttributeName::QueueArn).unwrap(); | ||
|
||
// Create the EventBridge target with the input transformer | ||
let input_transformer = | ||
InputTransformer::builder().input_paths_map("$.time", "time").input_template(message).build()?; | ||
|
||
event_bridge_client | ||
.put_targets() | ||
.rule(trigger_rule_name) | ||
.targets( | ||
Target::builder() | ||
.id(uuid::Uuid::new_v4().to_string()) | ||
.arn(queue_arn) | ||
.input_transformer(input_transformer) | ||
.build()?, | ||
) | ||
.send() | ||
.await?; | ||
|
||
Ok(()) | ||
} | ||
} | ||
|
||
fn duration_to_rate_string(duration: Duration) -> String { | ||
let total_secs = duration.as_secs(); | ||
let total_mins = duration.as_secs() / 60; | ||
let total_hours = duration.as_secs() / 3600; | ||
let total_days = duration.as_secs() / 86400; | ||
|
||
if total_days > 0 { | ||
format!("rate({} day{})", total_days, if total_days == 1 { "" } else { "s" }) | ||
} else if total_hours > 0 { | ||
format!("rate({} hour{})", total_hours, if total_hours == 1 { "" } else { "s" }) | ||
} else if total_mins > 0 { | ||
format!("rate({} minute{})", total_mins, if total_mins == 1 { "" } else { "s" }) | ||
} else { | ||
format!("rate({} second{})", total_secs, if total_secs == 1 { "" } else { "s" }) | ||
} | ||
} | ||
|
||
#[cfg(test)] | ||
mod event_bridge_utils_test { | ||
use rstest::rstest; | ||
|
||
use super::*; | ||
|
||
#[rstest] | ||
fn test_duration_to_rate_string() { | ||
assert_eq!(duration_to_rate_string(Duration::from_secs(60)), "rate(1 minute)"); | ||
assert_eq!(duration_to_rate_string(Duration::from_secs(120)), "rate(2 minutes)"); | ||
assert_eq!(duration_to_rate_string(Duration::from_secs(30)), "rate(30 seconds)"); | ||
assert_eq!(duration_to_rate_string(Duration::from_secs(3600)), "rate(1 hour)"); | ||
assert_eq!(duration_to_rate_string(Duration::from_secs(86400)), "rate(1 day)"); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
use std::time::Duration; | ||
|
||
use async_trait::async_trait; | ||
use lazy_static::lazy_static; | ||
|
||
use crate::queue::job_queue::{WorkerTriggerMessage, WorkerTriggerType}; | ||
use crate::setup::SetupConfig; | ||
|
||
pub mod event_bridge; | ||
|
||
lazy_static! { | ||
pub static ref CRON_DURATION: Duration = Duration::from_mins(1); | ||
// TODO : we can take this from clap. | ||
pub static ref TARGET_QUEUE_NAME: String = String::from("madara_orchestrator_worker_trigger_queue"); | ||
pub static ref WORKER_TRIGGERS: Vec<WorkerTriggerType> = vec![ | ||
WorkerTriggerType::Snos, | ||
WorkerTriggerType::Proving, | ||
WorkerTriggerType::DataSubmission, | ||
WorkerTriggerType::UpdateState | ||
]; | ||
pub static ref WORKER_TRIGGER_RULE_NAME: String = String::from("worker_trigger_scheduled"); | ||
} | ||
|
||
#[async_trait] | ||
pub trait Cron { | ||
async fn create_cron( | ||
&self, | ||
config: &SetupConfig, | ||
cron_time: Duration, | ||
trigger_rule_name: String, | ||
) -> color_eyre::Result<()>; | ||
async fn add_cron_target_queue( | ||
&self, | ||
config: &SetupConfig, | ||
target_queue_name: String, | ||
message: String, | ||
trigger_rule_name: String, | ||
) -> color_eyre::Result<()>; | ||
async fn setup(&self, config: SetupConfig) -> color_eyre::Result<()> { | ||
self.create_cron(&config, *CRON_DURATION, WORKER_TRIGGER_RULE_NAME.clone()).await?; | ||
for triggers in WORKER_TRIGGERS.iter() { | ||
self.add_cron_target_queue( | ||
&config, | ||
TARGET_QUEUE_NAME.clone(), | ||
get_worker_trigger_message(triggers.clone())?, | ||
WORKER_TRIGGER_RULE_NAME.clone(), | ||
) | ||
.await?; | ||
} | ||
Ok(()) | ||
} | ||
} | ||
|
||
fn get_worker_trigger_message(worker_trigger_type: WorkerTriggerType) -> color_eyre::Result<String> { | ||
let message = WorkerTriggerMessage { worker: worker_trigger_type }; | ||
Ok(serde_json::to_string(&message)?) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,97 @@ | ||
use std::process::Command; | ||
use std::sync::Arc; | ||
|
||
use aws_config::environment::EnvironmentVariableCredentialsProvider; | ||
use aws_config::{from_env, Region, SdkConfig}; | ||
use aws_credential_types::provider::ProvideCredentials; | ||
use utils::env_utils::get_env_var_or_panic; | ||
use utils::settings::env::EnvSettingsProvider; | ||
|
||
use crate::alerts::aws_sns::AWSSNS; | ||
use crate::alerts::Alerts; | ||
use crate::config::{get_aws_config, ProviderConfig}; | ||
use crate::cron::Cron; | ||
use crate::data_storage::aws_s3::AWSS3; | ||
use crate::data_storage::DataStorage; | ||
use crate::queue::QueueProvider; | ||
|
||
#[derive(Clone)] | ||
pub enum SetupConfig { | ||
AWS(SdkConfig), | ||
} | ||
|
||
pub enum ConfigType { | ||
AWS, | ||
} | ||
|
||
async fn setup_config(client_type: ConfigType) -> SetupConfig { | ||
match client_type { | ||
ConfigType::AWS => { | ||
let region_provider = Region::new(get_env_var_or_panic("AWS_REGION")); | ||
let creds = EnvironmentVariableCredentialsProvider::new().provide_credentials().await.unwrap(); | ||
SetupConfig::AWS(from_env().region(region_provider).credentials_provider(creds).load().await) | ||
} | ||
} | ||
} | ||
|
||
// TODO : move this to main.rs after moving to clap. | ||
pub async fn setup_cloud() -> color_eyre::Result<()> { | ||
log::info!("Setting up cloud."); | ||
let settings_provider = EnvSettingsProvider {}; | ||
let provider_config = Arc::new(ProviderConfig::AWS(Box::new(get_aws_config(&settings_provider).await))); | ||
|
||
log::info!("Setting up data storage."); | ||
match get_env_var_or_panic("DATA_STORAGE").as_str() { | ||
"s3" => { | ||
let s3 = Box::new(AWSS3::new_with_settings(&settings_provider, provider_config.clone()).await); | ||
s3.setup(Box::new(settings_provider.clone())).await? | ||
} | ||
_ => panic!("Unsupported Storage Client"), | ||
} | ||
log::info!("Data storage setup completed ✅"); | ||
|
||
log::info!("Setting up queues"); | ||
match get_env_var_or_panic("QUEUE_PROVIDER").as_str() { | ||
"sqs" => { | ||
let config = setup_config(ConfigType::AWS).await; | ||
let sqs = Box::new(crate::queue::sqs::SqsQueue {}); | ||
sqs.setup(config).await? | ||
} | ||
_ => panic!("Unsupported Queue Client"), | ||
} | ||
log::info!("Queues setup completed ✅"); | ||
|
||
log::info!("Setting up cron"); | ||
match get_env_var_or_panic("CRON_PROVIDER").as_str() { | ||
"event_bridge" => { | ||
let config = setup_config(ConfigType::AWS).await; | ||
let event_bridge = Box::new(crate::cron::event_bridge::AWSEventBridge {}); | ||
event_bridge.setup(config).await? | ||
} | ||
_ => panic!("Unsupported Event Bridge Client"), | ||
} | ||
log::info!("Cron setup completed ✅"); | ||
|
||
log::info!("Setting up alerts."); | ||
match get_env_var_or_panic("ALERTS").as_str() { | ||
"sns" => { | ||
let sns = Box::new(AWSSNS::new_with_settings(&settings_provider, provider_config).await); | ||
sns.setup(Box::new(settings_provider)).await? | ||
} | ||
_ => panic!("Unsupported Alert Client"), | ||
} | ||
log::info!("Alerts setup completed ✅"); | ||
|
||
Ok(()) | ||
} | ||
|
||
pub async fn setup_db() -> color_eyre::Result<()> { | ||
// We run the js script in the folder root: | ||
log::info!("Setting up database."); | ||
|
||
Command::new("node").arg("migrate-mongo-config.js").output()?; | ||
|
||
log::info!("Database setup completed ✅"); | ||
|
||
Ok(()) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Submodule madara-bootstrapper
updated
29 files
+7 −33 | .github/workflows/test-bridge.yml | |
+1 −3 | .gitignore | |
+0 −83 | Cargo.lock | |
+0 −3 | Cargo.toml | |
+ − | src/.DS_Store | |
+0 −22 | src/configs/devnet.json | |
+8 −23 | src/contract_clients/config.rs | |
+16 −15 | src/contract_clients/core_contract.rs | |
+6 −6 | src/contract_clients/eth_bridge.rs | |
+0 −2 | src/contract_clients/legacy_class.rs | |
+1 −2 | src/contract_clients/mod.rs | |
+18 −39 | src/contract_clients/starknet_sovereign.rs | |
+18 −39 | src/contract_clients/starknet_validity.rs | |
+13 −13 | src/contract_clients/token_bridge.rs | |
+19 −15 | src/contract_clients/utils.rs | |
+126 −161 | src/main.rs | |
+6 −6 | src/setup_scripts/account_setup.rs | |
+1 −1 | src/setup_scripts/argent.rs | |
+10 −10 | src/setup_scripts/braavos.rs | |
+5 −7 | src/setup_scripts/core_contract.rs | |
+9 −12 | src/setup_scripts/erc20_bridge.rs | |
+17 −20 | src/setup_scripts/eth_bridge.rs | |
+9 −14 | src/setup_scripts/udc.rs | |
+1 −2 | src/tests/constants.rs | |
+47 −44 | src/tests/erc20_bridge.rs | |
+42 −40 | src/tests/eth_bridge.rs | |
+0 −172 | src/tests/madara.rs | |
+58 −93 | src/tests/mod.rs | |
+1 −1 | src/utils/mod.rs |