wire format validation PoC

gossip/Cargo.toml

@@ -76,12 +76,14 @@ static_assertions = { workspace = true }
 thiserror = { workspace = true }
 
 [dev-dependencies]
+anyhow = { workspace = true }
 bs58 = { workspace = true }
 criterion = { workspace = true }
 num_cpus = { workspace = true }
 rand0-7 = { workspace = true }
 rand_chacha0-2 = { workspace = true }
 serial_test = { workspace = true }
+solana-net-utils = { workspace = true, features = ["dev-context-only-utils"] }
 solana-perf = { workspace = true, features = ["dev-context-only-utils"] }
 solana-runtime = { workspace = true, features = ["dev-context-only-utils"] }
 solana-sdk = { workspace = true }

gossip/src/lib.rs

@@ -46,3 +46,5 @@ extern crate solana_frozen_abi_macro;
 
 #[macro_use]
 extern crate solana_metrics;
+
+mod wire_format_tests;

gossip/src/wire_format_tests.rs

@@ -0,0 +1,65 @@
+#![allow(clippy::arithmetic_side_effects)]
+
+#[cfg(test)]
+mod tests {
+
+    use {
+        crate::protocol::Protocol,
+        serde::Serialize,
+        solana_net_utils::tooling_for_tests::{hexdump, validate_packet_format},
+        solana_sanitize::Sanitize,
+        std::path::PathBuf,
+    };
+
+    fn parse_gossip(bytes: &[u8]) -> anyhow::Result<Protocol> {
+        let pkt: Protocol = solana_perf::packet::deserialize_from_with_limit(bytes)?;
+        pkt.sanitize()?;
+        Ok(pkt)
+    }
+
+    fn serialize<T: Serialize>(pkt: T) -> Vec<u8> {
+        bincode::serialize(&pkt).unwrap()
+    }
+
+    fn find_differences(a: &[u8], b: &[u8]) -> Option<usize> {
+        if a.len() != b.len() {
+            return Some(a.len().min(b.len()));
+        }
+        for (idx, (e1, e2)) in a.iter().zip(b).enumerate() {
+            if e1 != e2 {
+                return Some(idx);
+            }
+        }
+        None
+    }
+
+    /// Test the ability of gossip parsers to understand and re-serialize a corpus of
+    /// packets captured from mainnet.
+    ///
+    /// This test requires external files and is not run by default.
+    /// Export the "GOSSIP_WIRE_FORMAT_PACKETS" variable to run this test
+    #[test]
+    fn test_gossip_wire_format() {
+        solana_logger::setup();
+        let path_base = match std::env::var_os("GOSSIP_WIRE_FORMAT_PACKETS") {
+            Some(p) => PathBuf::from(p),
+            None => {
+                eprintln!("Test requires GOSSIP_WIRE_FORMAT_PACKETS env variable, skipping!");
+                return;
+            }
+        };
+        for entry in
+            std::fs::read_dir(path_base).expect("Expecting env var to point to a directory")
+        {
+            let entry = entry.expect("Expecting a readable file");
+            validate_packet_format(
+                &entry.path(),
+                parse_gossip,
+                serialize,
+                hexdump,
+                find_differences,
+            )
+            .unwrap();
+        }
+    }
+}

ledger/Cargo.toml

@@ -10,6 +10,7 @@ license = { workspace = true }
 edition = { workspace = true }
 
 [dependencies]
+anyhow = { workspace = true }
 assert_matches = { workspace = true }
 bincode = { workspace = true }
 bitflags = { workspace = true, features = ["serde"] }
@@ -55,6 +56,7 @@ solana-frozen-abi-macro = { workspace = true, optional = true, features = [
 ] }
 solana-measure = { workspace = true }
 solana-metrics = { workspace = true }
+solana-net-utils = { workspace = true }
 solana-perf = { workspace = true }
 solana-program-runtime = { workspace = true, features = ["metrics"] }
 solana-pubkey = { workspace = true }
@@ -95,6 +97,7 @@ bs58 = { workspace = true }
 criterion = { workspace = true }
 solana-account-decoder = { workspace = true }
 solana-logger = { workspace = true }
+solana-net-utils = { workspace = true, features = ["dev-context-only-utils"] }
 solana-runtime = { workspace = true, features = ["dev-context-only-utils"] }
 spl-pod = { workspace = true }
 test-case = { workspace = true }

ledger/src/lib.rs

@@ -51,3 +51,5 @@ extern crate solana_frozen_abi_macro;
 pub mod macro_reexports {
     pub use solana_accounts_db::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE;
 }
+
+mod wire_format_tests;

ledger/src/wire_format_tests.rs

@@ -0,0 +1,93 @@
+#![allow(clippy::arithmetic_side_effects)]
+
+#[cfg(test)]
+mod tests {
+    use {
+        crate::shred::Shred,
+        solana_net_utils::tooling_for_tests::{hexdump, validate_packet_format},
+        std::path::PathBuf,
+    };
+
+    fn parse_turbine(bytes: &[u8]) -> anyhow::Result<Shred> {
+        let shred = Shred::new_from_serialized_shred(bytes.to_owned())
+            .map_err(|_e| anyhow::anyhow!("Can not deserialize"))?;
+        shred
+            .sanitize()
+            .map_err(|_e| anyhow::anyhow!("Failed sanitize"))?;
+        Ok(shred)
+    }
+
+    fn serialize(pkt: Shred) -> Vec<u8> {
+        pkt.payload().to_vec()
+    }
+
+    fn find_differences(a: &[u8], b: &[u8]) -> Option<usize> {
+        if a.len() != b.len() {
+            return Some(a.len());
+        }
+        for (idx, (e1, e2)) in a.iter().zip(b).enumerate() {
+            if e1 != e2 {
+                return Some(idx);
+            }
+        }
+        None
+    }
+
+    fn show_packet(bytes: &[u8]) -> anyhow::Result<()> {
+        let shred = parse_turbine(bytes)?;
+        let merkle_root = shred.merkle_root();
+        let chained_merkle_root = shred.chained_merkle_root();
+        let rtx_sign = shred.retransmitter_signature();
+
+        println!("=== {} bytes ===", bytes.len());
+        println!(
+            "Shred ID={ID:?} ErasureSetID={ESI:?}",
+            ID = shred.id(),
+            ESI = shred.erasure_set()
+        );
+        println!(
+            "Shred merkle root {:X?}, chained root {:X?}, rtx_sign {:X?}",
+            merkle_root.map(|v| v.as_ref().to_vec()),
+            chained_merkle_root.map(|v| v.as_ref().to_vec()),
+            rtx_sign.map(|v| v.as_ref().to_vec())
+        );
+        println!(
+            "Data shreds: {:?}, Coding shreds: {:?}",
+            shred.num_data_shreds(),
+            shred.num_coding_shreds()
+        );
+        hexdump(bytes)?;
+        println!("===");
+        Ok(())
+    }
+
+    /// Test the ability of turbine parser to understand and re-serialize a corpus of
+    /// packets captured from mainnet.
+    ///
+    /// This test requires external files and is not run by default.
+    /// Export the "TURBINE_WIRE_FORMAT_PACKETS" env variable to run this test.
+    #[test]
+    fn test_turbine_wire_format() {
+        solana_logger::setup();
+        let path_base = match std::env::var_os("TURBINE_WIRE_FORMAT_PACKETS") {
+            Some(p) => PathBuf::from(p),
+            None => {
+                eprintln!("Test requires TURBINE_WIRE_FORMAT_PACKETS env variable, skipping!");
+                return;
+            }
+        };
+        for entry in
+            std::fs::read_dir(path_base).expect("Expecting env var to point to a directory")
+        {
+            let entry = entry.expect("Expecting a readable file");
+            validate_packet_format(
+                &entry.path(),
+                parse_turbine,
+                serialize,
+                show_packet,
+                find_differences,
+            )
+            .unwrap();
+        }
+    }
+}

net-utils/Cargo.toml

@@ -15,9 +15,11 @@ anyhow = { workspace = true }
 bincode = { workspace = true }
 bytes = { workspace = true }
 clap = { version = "3.1.5", features = ["cargo"], optional = true }
+hxdmp = { version = "0.2.1", optional = true }
 itertools = { workspace = true }
 log = { workspace = true }
 nix = { workspace = true, features = ["socket"] }
+pcap-file = { version = "2.0.0", optional = true }
 rand = { workspace = true }
 serde = { workspace = true }
 serde_derive = { workspace = true }
@@ -34,7 +36,7 @@ solana-logger = { workspace = true }
 [features]
 default = []
 clap = ["dep:clap", "dep:solana-logger", "dep:solana-version"]
-dev-context-only-utils = []
+dev-context-only-utils = ["dep:pcap-file", "dep:hxdmp"]
 
 [lib]
 name = "solana_net_utils"

net-utils/src/lib.rs

@@ -2,6 +2,9 @@
 mod ip_echo_client;
 mod ip_echo_server;
 
+#[cfg(feature = "dev-context-only-utils")]
+pub mod tooling_for_tests;
+
 pub use ip_echo_server::{
     ip_echo_server, IpEchoServer, DEFAULT_IP_ECHO_SERVER_THREADS, MAX_PORT_COUNT_PER_MESSAGE,
     MINIMUM_IP_ECHO_SERVER_THREADS,