diff --git a/Cargo.lock b/Cargo.lock index 320fc8957ba..db9f25c9fb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,9 +43,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.4" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" +checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" dependencies = [ "getrandom 0.2.11", "once_cell", @@ -54,13 +54,14 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if 1.0.0", "once_cell", "version_check 0.9.4", + "zerocopy", ] [[package]] @@ -635,7 +636,7 @@ dependencies = [ "regex", "rustc-hash", "shlex 1.1.0", - "syn 2.0.18", + "syn 2.0.43", ] [[package]] @@ -691,9 +692,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.2.1" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ae9db68ad7fac5fe51304d20f016c911539251075a214f8e663babefa35187" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytemuck" @@ -1213,11 +1214,10 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" dependencies = [ - "cfg-if 1.0.0", "crossbeam-utils", ] @@ -1270,13 +1270,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.8" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" -dependencies = [ - "cfg-if 1.0.0", - "lazy_static", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crypto" @@ -1447,7 +1443,7 @@ dependencies = [ [[package]] name = "encoding_rs" version = "0.8.29" -source = "git+https://github.com/xiongjiwei/encoding_rs.git?rev=68e0bc5a72a37a78228d80cd98047326559cf43c#68e0bc5a72a37a78228d80cd98047326559cf43c" +source = "git+https://github.com/tikv/encoding_rs.git?rev=68e0bc5a72a37a78228d80cd98047326559cf43c#68e0bc5a72a37a78228d80cd98047326559cf43c" dependencies = [ "cfg-if 1.0.0", ] @@ -1835,7 +1831,6 @@ dependencies = [ "fs2", "lazy_static", "libc 0.2.151", - "maligned", "online_config", "openssl", "parking_lot 0.12.1", @@ -1954,7 +1949,7 @@ dependencies = [ [[package]] name = "fs2" version = "0.4.3" -source = "git+https://github.com/tabokie/fs2-rs?branch=tikv#cd503764a19a99d74c1ab424dd13d6bcd093fcae" +source = "git+https://github.com/tikv/fs2-rs?branch=tikv#cd503764a19a99d74c1ab424dd13d6bcd093fcae" dependencies = [ "libc 0.2.151", "winapi 0.3.9", @@ -2382,7 +2377,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.7", "allocator-api2", ] @@ -2639,7 +2634,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16d4bde3a7105e59c66a4104cfe9606453af1c7a0eac78cb7d5bc263eb762a70" dependencies = [ - "ahash 0.7.4", + "ahash 0.7.7", "atty", "indexmap 1.6.2", "itoa 1.0.1", @@ -2876,7 +2871,7 @@ dependencies = [ [[package]] name = "librocksdb_sys" version = "0.1.0" -source = "git+https://github.com/tikv/rust-rocksdb.git#73ba736143699fa623486c335527dd2a284bd0df" +source = "git+https://github.com/tikv/rust-rocksdb.git#c247909c279e89b3eabb4f200d580ee50f27fda6" dependencies = [ "bindgen 0.65.1", "bzip2-sys", @@ -2895,7 +2890,7 @@ dependencies = [ [[package]] name = "libtitan_sys" version = "0.0.1" -source = "git+https://github.com/tikv/rust-rocksdb.git#73ba736143699fa623486c335527dd2a284bd0df" +source = "git+https://github.com/tikv/rust-rocksdb.git#c247909c279e89b3eabb4f200d580ee50f27fda6" dependencies = [ "bzip2-sys", "cc", @@ -2982,12 +2977,6 @@ dependencies = [ "libc 0.2.151", ] -[[package]] -name = "maligned" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e88c3cbe8288f77f293e48a28b3232e3defd203a6d839fa7f68ea4329e83464" - [[package]] name = "match-template" version = "0.0.1" @@ -3372,7 +3361,7 @@ checksum = "9e6a0fd4f737c707bd9086cc16c925f294943eb62eb71499e9fd4cf71f8b9f4e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.43", ] [[package]] @@ -3475,9 +3464,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.16.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "online_config" @@ -3908,7 +3897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b69d39aab54d069e7f2fe8cb970493e7834601ca2d8c65fd7bbd183578080d1" dependencies = [ "proc-macro2", - "syn 2.0.18", + "syn 2.0.43", ] [[package]] @@ -3949,9 +3938,9 @@ checksum = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -4714,7 +4703,7 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f06953bb8b9e4307cb7ccc0d9d018e2ddd25a30d32831f631ce4fe8f17671f7" dependencies = [ - "ahash 0.7.4", + "ahash 0.7.7", "bitflags 1.3.2", "instant", "num-traits", @@ -4737,7 +4726,7 @@ dependencies = [ [[package]] name = "rocksdb" version = "0.3.0" -source = "git+https://github.com/tikv/rust-rocksdb.git#73ba736143699fa623486c335527dd2a284bd0df" +source = "git+https://github.com/tikv/rust-rocksdb.git#c247909c279e89b3eabb4f200d580ee50f27fda6" dependencies = [ "libc 0.2.151", "librocksdb_sys", @@ -5356,7 +5345,7 @@ dependencies = [ [[package]] name = "slog-global" version = "0.1.0" -source = "git+https://github.com/breeswish/slog-global.git?rev=d592f88e4dbba5eb439998463054f1a44fbf17b9#d592f88e4dbba5eb439998463054f1a44fbf17b9" +source = "git+https://github.com/tikv/slog-global.git?rev=d592f88e4dbba5eb439998463054f1a44fbf17b9#d592f88e4dbba5eb439998463054f1a44fbf17b9" dependencies = [ "arc-swap", "lazy_static", @@ -5455,7 +5444,7 @@ dependencies = [ [[package]] name = "snappy-sys" version = "0.1.0" -source = "git+https://github.com/busyjay/rust-snappy.git?branch=static-link#8c12738bad811397600455d6982aff754ea2ac44" +source = "git+https://github.com/tikv/rust-snappy.git?branch=static-link#8c12738bad811397600455d6982aff754ea2ac44" dependencies = [ "cmake", "libc 0.2.151", @@ -5632,7 +5621,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.18", + "syn 2.0.43", ] [[package]] @@ -5670,9 +5659,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" dependencies = [ "proc-macro2", "quote", @@ -6242,7 +6231,7 @@ dependencies = [ "codec", "collections", "crc32fast", - "encoding_rs 0.8.29 (git+https://github.com/xiongjiwei/encoding_rs.git?rev=68e0bc5a72a37a78228d80cd98047326559cf43c)", + "encoding_rs 0.8.29 (git+https://github.com/tikv/encoding_rs.git?rev=68e0bc5a72a37a78228d80cd98047326559cf43c)", "error_code", "hex 0.4.2", "kvproto", @@ -6918,7 +6907,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.43", ] [[package]] @@ -6978,7 +6967,7 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "static_assertions", ] @@ -7601,6 +7590,26 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.43", +] + [[package]] name = "zeroize" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 249ee380b37..757f154b165 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -202,10 +202,10 @@ rusoto_sts = { git = "https://github.com/tikv/rusoto", branch = "gh1482-s3-addr- # NOTICE: use openssl for signature to support fips 140 tame-oauth = { git = "https://github.com/tikv/tame-oauth", branch = "fips-0.9" } -snappy-sys = { git = "https://github.com/busyjay/rust-snappy.git", branch = "static-link" } +snappy-sys = { git = "https://github.com/tikv/rust-snappy.git", branch = "static-link" } # remove this when https://github.com/danburkert/fs2-rs/pull/42 is merged. -fs2 = { git = "https://github.com/tabokie/fs2-rs", branch = "tikv" } +fs2 = { git = "https://github.com/tikv/fs2-rs", branch = "tikv" } # Remove this when a new version is release. We need to solve rust-lang/cmake-rs#143. cmake = { git = "https://github.com/rust-lang/cmake-rs" } @@ -315,9 +315,12 @@ batch-system = { path = "components/batch-system" } case_macros = { path = "components/case_macros" } causal_ts = { path = "components/causal_ts" } cdc = { path = "components/cdc", default-features = false } -# Do not enable default features that implicitly enables oldtime which is +# Do not enable default-features because it implicitly enables oldtime which is # vulnerable to RUSTSEC-2020-0071, see more in deny.toml. chrono = { version = "0.4", default-features = false } +# Do not enable default-features because it implicitly enables the unsound +# "atty" crate, see more about RUSTSEC-2021-0145 in deny.toml. +clap = { version = "2.32", default-features = false, features = ["suggestions", "vec_map"] } cloud = { path = "components/cloud" } codec = { path = "components/codec" } collections = { path = "components/collections" } @@ -384,6 +387,8 @@ tracker = { path = "components/tracker" } txn_types = { path = "components/txn_types" } # External libs raft = { version = "0.7.0", default-features = false, features = ["protobuf-codec"] } +raft-engine = { git = "https://github.com/tikv/raft-engine.git", features = ["swap"] } +raft-engine-ctl = { git = "https://github.com/tikv/raft-engine.git" } grpcio = { version = "0.10.4", default-features = false, features = ["openssl", "protobuf-codec", "nightly"] } grpcio-health = { version = "0.10.4", default-features = false, features = ["protobuf-codec"] } tipb = { git = "https://github.com/pingcap/tipb.git" } @@ -392,7 +397,7 @@ yatp = { git = "https://github.com/tikv/yatp.git", branch = "master" } tokio-timer = { git = "https://github.com/tikv/tokio", branch = "tokio-timer-hotfix" } tokio-executor = { git = "https://github.com/tikv/tokio", branch = "tokio-timer-hotfix" } slog = { version = "2.3", features = ["max_level_trace", "release_max_level_debug"] } -slog-global = { version = "0.1", git = "https://github.com/breeswish/slog-global.git", rev = "d592f88e4dbba5eb439998463054f1a44fbf17b9" } +slog-global = { version = "0.1", git = "https://github.com/tikv/slog-global.git", rev = "d592f88e4dbba5eb439998463054f1a44fbf17b9" } tracing-active-tree = { git = "https://github.com/tikv/tracing-active-tree.git", features = ["coarsetime"], rev = "a71f8f8148f88ab759deb6d3e1d62d07ab218347" } # This `tracing` is only used for `tracing-active-tree`, enable its attributes only. tracing = { version = "0.1.39", default-features = false, features = [ "attributes", "std" ] } diff --git a/clippy.toml b/clippy.toml index c1a11598a5a..15e0f1f549c 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,16 +1,55 @@ -disallowed-methods = [ - { path = "std::thread::Builder::spawn", reason = "Wrapper function `::spawn_wrapper` should be used instead, refer to https://github.com/tikv/tikv/pull/12442 for more details." }, +[[disallowed-methods]] +path = "std::thread::Builder::spawn" +reason = """ +Wrapper function `::spawn_wrapper` +should be used instead, refer to https://github.com/tikv/tikv/pull/12442 for more details. +""" +[[disallowed-methods]] +path = "tokio::runtime::builder::Builder::on_thread_start" +reason = """ +Adding hooks directly will omit system hooks, please use +::with_sys_and_custom_hooks +refer to https://github.com/tikv/tikv/pull/12442 and https://github.com/tikv/tikv/pull/15017 for more details. +""" +[[disallowed-methods]] +path = "tokio::runtime::builder::Builder::on_thread_stop" +reason = """ +Adding hooks directly will omit system hooks, please use +::with_sys_and_custom_hooks +refer to https://github.com/tikv/tikv/pull/12442 and https://github.com/tikv/tikv/pull/15017 for more details. +""" +[[disallowed-methods]] +path = "futures_executor::thread_pool::ThreadPoolBuilder::after_start" +reason = """ +Adding hooks directly will omit system hooks, please use +::with_sys_and_custom_hooks +refer to https://github.com/tikv/tikv/pull/12442 and https://github.com/tikv/tikv/pull/15017 for more details. +""" +[[disallowed-methods]] +path = "futures_executor::thread_pool::ThreadPoolBuilder::before_stop" +reason = """ +Adding hooks directly will omit system hooks, please use +::with_sys_and_custom_hooks +refer to https://github.com/tikv/tikv/pull/12442 and https://github.com/tikv/tikv/pull/15017 for more details. +""" - { path = "tokio::runtime::builder::Builder::on_thread_start", reason = "Adding hooks directly will omit system hooks, please use ::with_sys_and_custom_hooks refer to https://github.com/tikv/tikv/pull/12442 and https://github.com/tikv/tikv/pull/15017 for more details." }, - { path = "tokio::runtime::builder::Builder::on_thread_stop", reason = "Adding hooks directly will omit system hooks, please use ::with_sys_and_custom_hooks refer to https://github.com/tikv/tikv/pull/12442 and https://github.com/tikv/tikv/pull/15017 for more details." }, +# See more about RUSTSEC-2020-0071 in deny.toml. +[[disallowed-methods]] +path = "time::now" +reason = "time::now is unsound, see RUSTSEC-2020-0071" +[[disallowed-methods]] +path = "time::at" +reason = "time::at is unsound, see RUSTSEC-2020-0071" +[[disallowed-methods]] +path = "time::at_utc" +reason = "time::at_utc is unsound, see RUSTSEC-2020-0071" - { path = "futures_executor::thread_pool::ThreadPoolBuilder::after_start", reason = "Adding hooks directly will omit system hooks, please use ::with_sys_and_custom_hooks refer to https://github.com/tikv/tikv/pull/12442 and https://github.com/tikv/tikv/pull/15017 for more details." }, - { path = "futures_executor::thread_pool::ThreadPoolBuilder::before_stop", reason = "Adding hooks directly will omit system hooks, please use ::with_sys_and_custom_hooks refer to https://github.com/tikv/tikv/pull/12442 and https://github.com/tikv/tikv/pull/15017 for more details." }, +# See more about RUSTSEC-2023-0072 in deny.toml. +[[disallowed-methods]] +path = "openssl::x509::store::X509StoreRef::objects" +reason = """ +X509StoreRef::objects is unsound, see RUSTSEC-2020-0071 +""" - # See more about RUSTSEC-2020-0071 in deny.toml. - { path = "time::now", reason = "time::now is unsound, see RUSTSEC-2020-0071" }, - { path = "time::at", reason = "time::at is unsound, see RUSTSEC-2020-0071" }, - { path = "time::at_utc", reason = "time::at_utc is unsound, see RUSTSEC-2020-0071" }, -] avoid-breaking-exported-api = false upper-case-acronyms-aggressive = true diff --git a/cmd/tikv-ctl/Cargo.toml b/cmd/tikv-ctl/Cargo.toml index 056bc602224..e55ef234e8d 100644 --- a/cmd/tikv-ctl/Cargo.toml +++ b/cmd/tikv-ctl/Cargo.toml @@ -44,7 +44,7 @@ nortcheck = ["engine_rocks/nortcheck"] api_version = { workspace = true } backup = { workspace = true } cdc = { workspace = true } -clap = "2.32" +clap = { workspace = true } collections = { workspace = true } concurrency_manager = { workspace = true } crossbeam = "0.8" @@ -67,8 +67,8 @@ pd_client = { workspace = true } prometheus = { version = "0.13", features = ["nightly"] } protobuf = { version = "2.8", features = ["bytes"] } raft = { workspace = true } -raft-engine = { git = "https://github.com/tikv/raft-engine.git" } -raft-engine-ctl = { git = "https://github.com/tikv/raft-engine.git" } +raft-engine = { workspace = true } +raft-engine-ctl = { workspace = true } raft_log_engine = { workspace = true } raftstore = { workspace = true } regex = "1" diff --git a/cmd/tikv-server/Cargo.toml b/cmd/tikv-server/Cargo.toml index be02fe4915b..fdc42f35c3a 100644 --- a/cmd/tikv-server/Cargo.toml +++ b/cmd/tikv-server/Cargo.toml @@ -35,13 +35,13 @@ nortcheck = ["server/nortcheck"] pprof-fp = ["tikv/pprof-fp"] [dependencies] -clap = "2.32" +clap = { workspace = true } crypto = { workspace = true } encryption_export = { workspace = true } engine_traits = { workspace = true } keys = { workspace = true } kvproto = { workspace = true } -raft-engine = { git = "https://github.com/tikv/raft-engine.git" } +raft-engine = { workspace = true } regex = "1" serde_json = { version = "1.0", features = ["preserve_order"] } server = { workspace = true } diff --git a/components/file_system/Cargo.toml b/components/file_system/Cargo.toml index ef7df46936d..5c778f87454 100644 --- a/components/file_system/Cargo.toml +++ b/components/file_system/Cargo.toml @@ -29,7 +29,6 @@ tikv_util = { workspace = true } tokio = { version = "1.5", features = ["time"] } [dev-dependencies] -maligned = "0.2.1" tempfile = "3.0" [target.'cfg(target_os = "linux")'.dependencies] diff --git a/components/file_system/src/io_stats/biosnoop.rs b/components/file_system/src/io_stats/biosnoop.rs index 6b804bfed87..2267193a3ec 100644 --- a/components/file_system/src/io_stats/biosnoop.rs +++ b/components/file_system/src/io_stats/biosnoop.rs @@ -150,10 +150,11 @@ pub fn get_io_type() -> IoType { } pub fn fetch_io_bytes() -> [IoBytes; IoType::COUNT] { - let mut bytes = Default::default(); + let mut bytes: [IoBytes; IoType::COUNT] = Default::default(); unsafe { if let Some(ctx) = BPF_CONTEXT.as_mut() { for io_type in IoType::iter() { + let mut io_type = io_type; let io_type_buf_ptr = &mut io_type as *mut IoType as *mut u8; let mut io_type_buf = std::slice::from_raw_parts_mut(io_type_buf_ptr, std::mem::size_of::()); @@ -269,15 +270,19 @@ pub fn flush_io_latency_metrics() { } } +pub fn get_thread_io_bytes_total() -> Result { + Err("unimplemented".into()) +} + #[cfg(test)] mod tests { use std::{ io::{Read, Seek, SeekFrom, Write}, + os::unix::fs::OpenOptionsExt, sync::{Arc, Condvar, Mutex}, }; use libc::O_DIRECT; - use maligned::{AsBytes, AsBytesMut, A512}; use rand::Rng; use tempfile::TempDir; use test::Bencher; @@ -286,7 +291,7 @@ mod tests { fetch_io_bytes, flush_io_latency_metrics, get_io_type, init, set_io_type, BPF_CONTEXT, MAX_THREAD_IDX, }; - use crate::{metrics::*, IoType, OpenOptions}; + use crate::{io_stats::A512, metrics::*, IoType, OpenOptions}; #[test] fn test_biosnoop() { @@ -311,10 +316,10 @@ mod tests { .custom_flags(O_DIRECT) .open(&file_path) .unwrap(); - let mut w = vec![A512::default(); 2]; - w.as_bytes_mut()[512] = 42; + let mut w = Box::new(A512([0u8; 512 * 2])); + w.0[512] = 42; let mut compaction_bytes_before = fetch_io_bytes()[IoType::Compaction as usize]; - f.write(w.as_bytes()).unwrap(); + f.write(&w.0).unwrap(); f.sync_all().unwrap(); let compaction_bytes = fetch_io_bytes()[IoType::Compaction as usize]; assert_ne!((compaction_bytes - compaction_bytes_before).write, 0); @@ -330,8 +335,8 @@ mod tests { .custom_flags(O_DIRECT) .open(&file_path) .unwrap(); - let mut r = vec![A512::default(); 2]; - assert_ne!(f.read(&mut r.as_bytes_mut()).unwrap(), 0); + let mut r = Box::new(A512([0u8; 512 * 2])); + assert_ne!(f.read(&mut r.0).unwrap(), 0); drop(f); }) .join() @@ -450,10 +455,10 @@ mod tests { .open(&file_path) .unwrap(); - let mut w = vec![A512::default(); 1]; - w.as_bytes_mut()[64] = 42; + let mut w = Box::new(A512([0u8; 512 * 1])); + w.0[64] = 42; for _ in 1..=100 { - f.write(w.as_bytes()).unwrap(); + f.write(&w.0).unwrap(); } f.sync_all().unwrap(); @@ -472,12 +477,12 @@ mod tests { .open(&file_path) .unwrap(); - let mut w = vec![A512::default(); 1]; - w.as_bytes_mut()[64] = 42; + let mut w = Box::new(A512([0u8; 512 * 1])); + w.0[64] = 42; b.iter(|| { set_io_type(IoType::ForegroundWrite); - f.write(w.as_bytes()).unwrap(); + f.write(&w.0).unwrap(); f.sync_all().unwrap(); }); } @@ -493,10 +498,10 @@ mod tests { .open(&file_path) .unwrap(); - let mut w = vec![A512::default(); 2]; - w.as_bytes_mut()[64] = 42; + let mut w = Box::new(A512([0u8; 512 * 2])); + w.0[64] = 42; for _ in 0..100 { - f.write(w.as_bytes()).unwrap(); + f.write(&w.0).unwrap(); } f.sync_all().unwrap(); drop(f); @@ -507,12 +512,12 @@ mod tests { .custom_flags(O_DIRECT) .open(&file_path) .unwrap(); - let mut r = vec![A512::default(); 2]; + let mut r = Box::new(A512([0u8; 512 * 2])); b.iter(|| { set_io_type(IoType::ForegroundRead); f.seek(SeekFrom::Start(rng.gen_range(0..100) * 512)) .unwrap(); - assert_ne!(f.read(&mut r.as_bytes_mut()).unwrap(), 0); + assert_ne!(f.read(&mut r.0).unwrap(), 0); }); } } diff --git a/components/file_system/src/io_stats/mod.rs b/components/file_system/src/io_stats/mod.rs index 216c6ad7897..f65d7c57884 100644 --- a/components/file_system/src/io_stats/mod.rs +++ b/components/file_system/src/io_stats/mod.rs @@ -47,6 +47,14 @@ mod proc; #[cfg(all(target_os = "linux", not(feature = "bcc-iosnoop")))] pub use proc::*; +// A struct assists testing IO stats. +// +// O_DIRECT requires I/O to be 512-byte aligned. +// See https://man7.org/linux/man-pages/man2/open.2.html#NOTES +#[cfg(test)] +#[repr(align(512))] +pub(crate) struct A512(pub [u8; SZ]); + #[cfg(test)] mod tests { use tikv_util::sys::thread::StdThreadBuildWrapper; diff --git a/components/file_system/src/io_stats/proc.rs b/components/file_system/src/io_stats/proc.rs index d66a04aa4e8..8da6876d70f 100644 --- a/components/file_system/src/io_stats/proc.rs +++ b/components/file_system/src/io_stats/proc.rs @@ -196,12 +196,11 @@ mod tests { }; use libc::O_DIRECT; - use maligned::{AsBytes, AsBytesMut, A512}; use tempfile::{tempdir, tempdir_in}; use tikv_util::sys::thread::StdThreadBuildWrapper; use super::*; - use crate::{OpenOptions, WithIoType}; + use crate::{io_stats::A512, OpenOptions, WithIoType}; #[test] fn test_read_bytes() { @@ -216,8 +215,8 @@ mod tests { .custom_flags(O_DIRECT) .open(&file_path) .unwrap(); - let w = vec![A512::default(); 10]; - f.write_all(w.as_bytes()).unwrap(); + let w = Box::new(A512([0u8; 512 * 10])); + f.write_all(&w.0).unwrap(); f.sync_all().unwrap(); } let mut f = OpenOptions::new() @@ -225,10 +224,10 @@ mod tests { .custom_flags(O_DIRECT) .open(&file_path) .unwrap(); - let mut w = vec![A512::default(); 1]; + let mut w = A512([0u8; 512]); let base_local_bytes = id.fetch_io_bytes().unwrap(); for i in 1..=10 { - f.read_exact(w.as_bytes_mut()).unwrap(); + f.read_exact(&mut w.0).unwrap(); let local_bytes = id.fetch_io_bytes().unwrap(); assert_eq!(i * 512 + base_local_bytes.read, local_bytes.read); @@ -247,10 +246,10 @@ mod tests { .custom_flags(O_DIRECT) .open(file_path) .unwrap(); - let w = vec![A512::default(); 8]; + let w = Box::new(A512([0u8; 512 * 8])); let base_local_bytes = id.fetch_io_bytes().unwrap(); for i in 1..=10 { - f.write_all(w.as_bytes()).unwrap(); + f.write_all(&w.0).unwrap(); f.sync_all().unwrap(); let local_bytes = id.fetch_io_bytes().unwrap(); @@ -275,8 +274,8 @@ mod tests { .custom_flags(O_DIRECT) .open(file_path) .unwrap(); - let w = vec![A512::default(); 8]; - f.write_all(w.as_bytes()).unwrap(); + let w = Box::new(A512([0u8; 512 * 8])); + f.write_all(&w.0).unwrap(); f.sync_all().unwrap(); tx1.send(()).unwrap(); tx1.send(()).unwrap(); @@ -293,8 +292,8 @@ mod tests { .custom_flags(O_DIRECT) .open(file_path) .unwrap(); - let w = vec![A512::default(); 8]; - f.write_all(w.as_bytes()).unwrap(); + let w = Box::new(A512([0u8; 512 * 8])); + f.write_all(&w.0).unwrap(); f.sync_all().unwrap(); tx2.send(()).unwrap(); tx2.send(()).unwrap(); diff --git a/components/raft_log_engine/Cargo.toml b/components/raft_log_engine/Cargo.toml index 913a0d18ae1..dc98bae0d5e 100644 --- a/components/raft_log_engine/Cargo.toml +++ b/components/raft_log_engine/Cargo.toml @@ -18,7 +18,7 @@ num_cpus = "1" online_config = { workspace = true } protobuf = "2" raft = { workspace = true } -raft-engine = { git = "https://github.com/tikv/raft-engine.git", features = ["swap"] } +raft-engine = { workspace = true } serde = "1.0" serde_derive = "1.0" slog = { workspace = true } diff --git a/components/server/Cargo.toml b/components/server/Cargo.toml index cf2d653bf52..c03c42e8f71 100644 --- a/components/server/Cargo.toml +++ b/components/server/Cargo.toml @@ -40,7 +40,7 @@ backup-stream = { workspace = true } causal_ts = { workspace = true } cdc = { workspace = true } chrono = { workspace = true } -clap = "2.32" +clap = { workspace = true } collections = { workspace = true } concurrency_manager = { workspace = true } crossbeam = "0.8" diff --git a/components/tidb_query_datatype/Cargo.toml b/components/tidb_query_datatype/Cargo.toml index 5f52580d454..33aff4c7b5e 100644 --- a/components/tidb_query_datatype/Cargo.toml +++ b/components/tidb_query_datatype/Cargo.toml @@ -17,7 +17,7 @@ chrono-tz = "0.5.1" codec = { workspace = true } collections = { workspace = true } crc32fast = "1.2" -encoding_rs = { git = "https://github.com/xiongjiwei/encoding_rs.git", rev = "68e0bc5a72a37a78228d80cd98047326559cf43c" } +encoding_rs = { git = "https://github.com/tikv/encoding_rs.git", rev = "68e0bc5a72a37a78228d80cd98047326559cf43c" } error_code = { workspace = true } hex = "0.4" kvproto = { workspace = true } diff --git a/deny.toml b/deny.toml index 1cc62067c7c..8bd0ea33df7 100644 --- a/deny.toml +++ b/deny.toml @@ -43,18 +43,31 @@ multiple-versions = "allow" [advisories] vulnerability = "deny" unmaintained = "allow" # FIXME: Deny it. -unsound = "allow" # FIXME: Deny it. -yanked = "allow" # FIXME: Deny it. +unsound = "deny" +yanked = "deny" notice = "warn" ignore = [ # Ignore time 0.1 RUSTSEC-2020-0071 as 1) we have taken measures (see # clippy.toml) to mitigate the issue and 2) time 0.1 has no fix availble. # - # Note: Upgrading to time 0.3 do fix the issue but it's an imcompatible + # NB: Upgrading to time 0.3 do fix the issue but it's an imcompatible # versoin which removes some necessary APIs (`time::precise_time_ns`) that # requiresd by TiKV. # See https://github.com/time-rs/time/blob/8067540c/CHANGELOG.md#L703 "RUSTSEC-2020-0071", + # Ignore RUSTSEC-2023-0072 as we bans the unsound `X509StoreRef::objects`. + # + # NB: Upgrading rust-openssl the latest version do fix the issue but it + # also upgrade the OpenSSL to v3.x which causes performance degradation. + # See https://github.com/openssl/openssl/issues/17064 + "RUSTSEC-2023-0072", + # Ignore RUSTSEC-2023-0072 (unsound issue of "atty" crate) as it only + # affects Windows plaform which is not supported offically by TiKV, and 2) + # we have disabled the clap feature "color" so that the "atty" crate is not + # included in production code. + # + # TODO: Upgrade clap to v4.x. + "RUSTSEC-2021-0145", ] [licenses] @@ -82,6 +95,6 @@ exceptions = [ ] [sources] -unknown-git = "allow" # FIXME: Deny it. +unknown-git = "deny" unknown-registry = "deny" -allow-org = { github = ["tikv"] } +allow-org = { github = ["tikv", "pingcap", "rust-lang"] }