massalabs · bilboquet · May 28, 2024 · May 20, 2024
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -3,29 +3,57 @@ name = "massa-slot-replayer"
 version = "0.1.0"
 edition = "2021"
 
+[features]
+db_storage_backend = ["execution-trace"]
+file_storage_backend = ["execution-trace"]
+execution-trace = []
+
 [dependencies]
+cfg-if = "1.0.0"
 clap = { version = "4.4", features = ["derive", "cargo"] }
-parking_lot = { version = "0.12", features = ["deadlock_detection"] }
+copy_dir = "0.1.3"
+glob = "0.3.1"
 num = "=0.4"
+parking_lot = { version = "0.12", features = ["deadlock_detection"] }
+prost = { version = "=0.12" }
+reqwest = { version = "0.12.4", features = ["blocking", "json"] }
 rocksdb = "0.21"
-glob = "0.3.1"
-tokio = "1.23"
+serde = { version = "1.0.202", features = ["derive"] }
+serde_json = "1.0.117"
 tempfile = "3.10"
-copy_dir = "0.1.3"
+tokio = "1.23"
+tracing = { version = "0.1", "features" = [
+    "max_level_debug",
+    "release_max_level_debug",
+] }
+tracing-subscriber = { version = "0.3" }
+
+massa_time = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_time" }
+massa_models = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_models" }
+massa_db_exports = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_db_exports" }
+massa_db_worker = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_db_worker" }
+massa_ledger_exports = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_ledger_exports" }
+massa_ledger_worker = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_ledger_worker" }
+massa_async_pool = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_async_pool" }
+massa_pos_exports = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_pos_exports" }
+massa_pos_worker = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_pos_worker" }
+massa_executed_ops = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_executed_ops" }
+massa_final_state = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_final_state" }
+massa_hash = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_hash" }
+massa_execution_exports = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_execution_exports", features = [
+    "execution-info",
+    "execution-trace",
 
-massa_time = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_time" }
-massa_models = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_models" }
-massa_db_exports = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_db_exports" }
-massa_db_worker = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_db_worker"}
-massa_ledger_exports = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_ledger_exports" }
-massa_ledger_worker = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_ledger_worker" }
-massa_async_pool = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_async_pool" }
-massa_pos_exports = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_pos_exports" }
-massa_pos_worker = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_pos_worker" }
-massa_executed_ops = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_executed_ops" }
-massa_final_state = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_final_state" }
-massa_execution_exports = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_execution_exports" }
-massa_execution_worker = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_execution_worker" }
-massa_versioning = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_versioning" }
-massa_metrics = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_metrics" }
-massa_wallet = { git = "https://github.com/massalabs/massa", branch = "main", package = "massa_wallet" }
+] }
+massa_execution_worker = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_execution_worker", features = [
+    "execution-info",
+    "file_storage_backend",
+    "slot-replayer",
+] }
+massa_versioning = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_versioning" }
+massa_metrics = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_metrics" }
+massa-proto-rs = { git = "https://github.com/massalabs/massa-proto-rs", "rev" = "38950875a7aa406fedc4f0b8336864e5ff290f2c" }
+massa_serialization = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_serialization" }
+massa_signature = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_signature" }
+massa_storage = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_storage" }
+massa_wallet = { git = "https://github.com/massalabs/massa", rev = "a804a2cdb688b99bf10296a5f3329fb31b613c0d", package = "massa_wallet" }
diff --git a/Readme.md b/Readme.md
@@ -11,8 +11,12 @@ Block replay tool for Massa blockchain (for debugging, testing & performance che
 
 ## List snapshots
 
-* cargo run -- --path /tmp/massa_8_g4j_3n/massa-node/storage/ledger/rocks_db/ --initial_roll_path /tmp/compile_massa_5tm8z0am/massa-node/base_config/initial_rolls.json list-snapshot
+* cargo run --features file_storage_backend -- --path /tmp/massa_8_g4j_3n/massa-node/storage/ledger/rocks_db/ --initial_roll_path /tmp/compile_massa_5tm8z0am/massa-node/base_config/initial_rolls.json list-snapshot
 
 ## Replay blocks
 
-* cargo run -- --path /tmp/massa_8_g4j_3n/massa-node/storage/ledger/rocks_db/ --initial_roll_path /tmp/compile_massa_5tm8z0am/massa-node/base_config/initial_rolls.json replay -b /tmp/massa_8_g4j_3n/massa-node/dump/blocks/ --backup /tmp/massa_8_g4j_3n/massa-node/storage/ledger/rocks_db/backup_40_0/
+* cargo run --features file_storage_backend -- --path /tmp/massa_8_g4j_3n/massa-node/storage/ledger/rocks_db/ --initial_roll_path /tmp/compile_massa_5tm8z0am/massa-node/base_config/initial_rolls.json replay -b /tmp/massa_8_g4j_3n/massa-node/dump/blocks/ --backup /tmp/massa_8_g4j_3n/massa-node/storage/ledger/rocks_db/backup_35_0/ --until_slot 36,13
+
+## Parsing logs
+
+* Logs can be easily parsed with [lnav](https://lnav.org/). Use Shift-P to pretty print the json data :)
diff --git a/src/args.rs b/src/args.rs
@@ -1,3 +1,4 @@
+use std::error::Error;
 use std::path::PathBuf;
 
 use clap::{Args, Parser, Subcommand};
@@ -6,9 +7,17 @@ use clap::{Args, Parser, Subcommand};
 #[command(name = "slot_replayer_try_1")]
 #[command(about = "Slot replay", long_about = None)]
 pub struct Cli {
-    #[arg(short = 'p', long = "path", help = "Path of an existing db (e.g: /tmp/massa_8_g4j_3n/massa-node/storage/ledger/rocks_db")]
+    #[arg(
+        short = 'p',
+        long = "path",
+        help = "Path of an existing db (e.g: /tmp/massa_8_g4j_3n/massa-node/storage/ledger/rocks_db"
+    )]
     pub(crate) db_path: PathBuf,
-    #[arg(short = 'r', long = "initial_roll_path", help = "Filepath to initial_rolls.json")]
+    #[arg(
+        short = 'r',
+        long = "initial_roll_path",
+        help = "Filepath to initial_rolls.json"
+    )]
     pub(crate) initial_rolls_path: PathBuf,
     #[command(subcommand)]
     pub(crate) command: Commands,
@@ -30,13 +39,21 @@ pub struct ReplayArgs {
         help = "Folder where to find block dumped as .bin file"
     )]
     pub(crate) dump_block_path: PathBuf,
+    #[arg(long = "backup", help = "Folder where to find db backup")]
+    pub(crate) db_backup_path: PathBuf,
     #[arg(
-        long = "backup",
-        help = "Folder where to find db backup"
+        long = "until_slot",
+        help = "Replay from last slot defined into backup to given slot period, if not specified will replay until blocks are available. ex: `--slot 40,2`",
+        value_parser = parse_slot,
     )]
-    pub(crate) db_backup_path: PathBuf,
+    pub(crate) until_slot: Option<(u64, u8)>,
 }
 
-
-
-
+fn parse_slot(s: &str) -> Result<(u64, u8), Box<dyn Error + Send + Sync + 'static>> {
+
+    let (period_, thread_) = s.split_once(',')
+        .ok_or("Slot must be specified as PERIOD,THREAD, ex: `--slt 40,2`")?;
+    let period = period_.parse::<u64>()?;
+    let thread = thread_.parse::<u8>()?;
+    Ok((period, thread))
+}
diff --git a/src/block_storer.rs b/src/block_storer.rs
@@ -0,0 +1,59 @@
+/* use serde::Deserialize;
+
+const KEY: &str = "X-Api-Key";
+
+#[derive(Deserialize, Debug)]
+struct CursorIn {
+    b: String,
+    bool: String,
+    bs: String,
+    l: String,
+    m: String,
+    n: String,
+    ns: String,
+    null: String,
+    s: String,
+    ss: String,
+}
+
+#[derive(Deserialize, Debug)]
+struct Cursor {
+    hash: CursorIn,
+    point_in_time: CursorIn,
+}
+
+#[derive(Deserialize, Debug)]
+struct BlockDescriptor {
+    id: String,
+    timestamp: u64,
+    #[serde(rename(deserialize = "totalFee"))]
+    total_fee: u64,
+    #[serde(rename(deserialize = "operationCount"))]
+    operation_count: u64,
+    #[serde(rename(deserialize = "creatorAddress"))]
+    creator_address: String,
+}
+
+#[derive(Deserialize, Debug)]
+struct Blocks {
+    records: Vec<BlockDescriptor>,
+    #[serde(rename(deserialize = "nextCursor"))]
+    next_cursor: Cursor,
+}
+
+pub(crate) fn fetch_block_from_node_storer() -> Result<(), reqwest::Error> {
+    let url = "https://ikfuju74kk.execute-api.eu-west-3.amazonaws.com/Prod/".to_string()
+    // + "/operation/O1a86XqERaR5MSuLx4HebYt4f2U5jJfDW7p4JMobL9714xGMsZp";
+    + "/block";
+    let x_api_key = "3epwEGrAlb2MnQwAK3hFfonYJAXXwUx5Z7dHFmp3";
+
+    let client = reqwest::blocking::Client::new();
+    let res = client.get(&url).header(KEY, x_api_key).send().unwrap();
+    let value: Blocks = res.json().unwrap();
+    println!("################################################################################");
+    println!("body = {:?}", value);
+    println!("################################################################################");
+
+    Ok(())
+}
+ */
diff --git a/src/config.rs b/src/config.rs
@@ -1,5 +1,5 @@
 // std lib
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 
 use massa_async_pool::AsyncPoolConfig;
 use massa_db_exports::MassaDBConfig;
@@ -8,7 +8,20 @@ use massa_execution_exports::{ExecutionConfig, GasCosts, StorageCostsConstants};
 use massa_final_state::FinalStateConfig;
 use massa_ledger_exports::LedgerConfig;
 use massa_models::address::Address;
-use massa_models::config::{DENUNCIATION_EXPIRE_PERIODS, ENDORSEMENT_COUNT, GENESIS_TIMESTAMP, INITIAL_DRAW_SEED, MAX_ASYNC_POOL_LENGTH, MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE, MAX_BOOTSTRAP_VERSIONING_ELEMENTS_SIZE, MAX_DATASTORE_KEY_LENGTH, MAX_DATASTORE_VALUE_LENGTH, MAX_DEFERRED_CREDITS_LENGTH, MAX_DENUNCIATIONS_PER_BLOCK_HEADER, MAX_DENUNCIATION_CHANGES_LENGTH, MAX_FUNCTION_NAME_LENGTH, MAX_PARAMETERS_SIZE, MAX_PRODUCTION_STATS_LENGTH, MAX_ROLLS_COUNT_LENGTH, MIP_STORE_STATS_BLOCK_CONSIDERED, PERIODS_PER_CYCLE, POS_SAVED_CYCLES, T0, THREAD_COUNT, SELECTOR_DRAW_CACHE_SIZE, CHANNEL_SIZE, GENESIS_KEY, MAX_ASYNC_GAS, ASYNC_MSG_CST_GAS_COST, MAX_GAS_PER_BLOCK, ROLL_PRICE, BLOCK_REWARD, OPERATION_VALIDITY_PERIODS, POS_MISS_RATE_DEACTIVATION_THRESHOLD, MAX_BYTECODE_LENGTH, BASE_OPERATION_GAS_COST, ROLL_COUNT_TO_SLASH_ON_DENUNCIATION, MAX_EVENT_DATA_SIZE, CHAINID, LEDGER_COST_PER_BYTE, LEDGER_ENTRY_BASE_COST, LEDGER_ENTRY_DATASTORE_BASE_SIZE};
+use massa_models::config::{
+    ASYNC_MSG_CST_GAS_COST, BASE_OPERATION_GAS_COST, BLOCK_REWARD, CHAINID, CHANNEL_SIZE,
+    DENUNCIATION_EXPIRE_PERIODS, ENDORSEMENT_COUNT, GENESIS_KEY, GENESIS_TIMESTAMP,
+    INITIAL_DRAW_SEED, LEDGER_COST_PER_BYTE, LEDGER_ENTRY_BASE_COST,
+    LEDGER_ENTRY_DATASTORE_BASE_SIZE, MAX_ASYNC_GAS, MAX_ASYNC_POOL_LENGTH,
+    MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE, MAX_BOOTSTRAP_VERSIONING_ELEMENTS_SIZE,
+    MAX_BYTECODE_LENGTH, MAX_DATASTORE_KEY_LENGTH, MAX_DATASTORE_VALUE_LENGTH,
+    MAX_DEFERRED_CREDITS_LENGTH, MAX_DENUNCIATIONS_PER_BLOCK_HEADER,
+    MAX_DENUNCIATION_CHANGES_LENGTH, MAX_EVENT_DATA_SIZE, MAX_FUNCTION_NAME_LENGTH,
+    MAX_GAS_PER_BLOCK, MAX_PARAMETERS_SIZE, MAX_PRODUCTION_STATS_LENGTH, MAX_ROLLS_COUNT_LENGTH,
+    MIP_STORE_STATS_BLOCK_CONSIDERED, OPERATION_VALIDITY_PERIODS, PERIODS_PER_CYCLE,
+    POS_MISS_RATE_DEACTIVATION_THRESHOLD, POS_SAVED_CYCLES, ROLL_COUNT_TO_SLASH_ON_DENUNCIATION,
+    ROLL_PRICE, SELECTOR_DRAW_CACHE_SIZE, T0, THREAD_COUNT,
+};
 use massa_pos_exports::{PoSConfig, SelectorConfig};
 use massa_time::MassaTime;
 use massa_versioning::versioning::MipStatsConfig;
@@ -88,19 +101,6 @@ pub fn get_final_state_config(
     let executed_ops_config = get_executed_ops_config();
     let executed_denunciations_config = get_executed_denunciations_config();
 
-    // let initial_rolls_path = match initial_rolls_path {
-    //     Some(p) => p,
-    //     None => path
-    //         .parent()
-    //         .unwrap()
-    //         .parent()
-    //         .unwrap()
-    //         .parent()
-    //         .unwrap()
-    //         .join("base_config")
-    //         .join("initial_rolls.json"),
-    // };
-
     println!("initial_rolls_path: {:?}", initial_rolls_path);
     FinalStateConfig {
         ledger_config,
@@ -133,7 +133,6 @@ pub fn get_selector_config() -> SelectorConfig {
 }
 
 pub fn get_execution_config(last_start_period: u64, gas_costs_folder: &PathBuf) -> ExecutionConfig {
-
     // Storage costs constants
     let storage_costs_constants = StorageCostsConstants {
         ledger_cost_per_byte: LEDGER_COST_PER_BYTE,
@@ -146,11 +145,12 @@ pub fn get_execution_config(last_start_period: u64, gas_costs_folder: &PathBuf)
     // gas costs
     let gas_costs = GasCosts::new(
         gas_costs_folder.join("abi_gas_costs.json"), // SETTINGS.execution.abi_gas_costs_file.clone(),
-        gas_costs_folder.join("wasm_gas_costs.json") // SETTINGS.execution.wasm_gas_costs_file.clone(),
-    ).expect("Failed to load gas costs");
+        gas_costs_folder.join("wasm_gas_costs.json"), // SETTINGS.execution.wasm_gas_costs_file.clone(),
+    )
+    .expect("Failed to load gas costs");
 
     ExecutionConfig {
-        max_final_events: 10000, // SETTINGS.execution.max_final_events,
+        max_final_events: 10000,   // SETTINGS.execution.max_final_events,
         readonly_queue_length: 10, // SETTINGS.execution.readonly_queue_length,
         cursor_delay: MassaTime::from_millis(2000), // SETTINGS.execution.cursor_delay,
         max_async_gas: MAX_ASYNC_GAS,
@@ -175,15 +175,15 @@ pub fn get_execution_config(last_start_period: u64, gas_costs_folder: &PathBuf)
         base_operation_gas_cost: BASE_OPERATION_GAS_COST,
         last_start_period, // final_state.read().get_last_start_period(),
         hd_cache_path: PathBuf::from("storage/cache/rocks_db"), // SETTINGS.execution.hd_cache_path.clone(),
-        lru_cache_size: 200,// SETTINGS.execution.lru_cache_size,
-        hd_cache_size: 2000,// SETTINGS.execution.hd_cache_size,
-        snip_amount: 10,// SETTINGS.execution.snip_amount,
+        lru_cache_size: 200, // SETTINGS.execution.lru_cache_size,
+        hd_cache_size: 2000, // SETTINGS.execution.hd_cache_size,
+        snip_amount: 10,     // SETTINGS.execution.snip_amount,
         roll_count_to_slash_on_denunciation: ROLL_COUNT_TO_SLASH_ON_DENUNCIATION,
         denunciation_expire_periods: DENUNCIATION_EXPIRE_PERIODS,
         broadcast_enabled: false, // SETTINGS.api.enable_broadcast,
         broadcast_slot_execution_output_channel_capacity: 5000, // SETTINGS
-            // .execution
-            // .broadcast_slot_execution_output_channel_capacity,
+        // .execution
+        // .broadcast_slot_execution_output_channel_capacity,
         max_event_size: MAX_EVENT_DATA_SIZE,
         max_function_length: MAX_FUNCTION_NAME_LENGTH,
         max_parameter_length: MAX_PARAMETERS_SIZE,
@@ -193,9 +193,9 @@ pub fn get_execution_config(last_start_period: u64, gas_costs_folder: &PathBuf)
         #[cfg(not(feature = "execution-trace"))]
         broadcast_traces_enabled: false,
         broadcast_slot_execution_traces_channel_capacity: 5000, // SETTINGS
-            // .execution
-            // .broadcast_slot_execution_traces_channel_capacity,
+        // .execution
+        // .broadcast_slot_execution_traces_channel_capacity,
         max_execution_traces_slot_limit: 320, // SETTINGS.execution.execution_traces_limit,
         block_dump_folder_path: PathBuf::from(""),
     }
-}
+}