Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update metriken to replace heatmaps #52

Merged
merged 33 commits into from
Oct 17, 2023
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
b95ff47
update metriken to replace heatmaps
brayniac Sep 27, 2023
458664e
update
brayniac Sep 27, 2023
1a1a945
correct percentiles to be windowed instead of cumulative
brayniac Sep 28, 2023
09c8c69
fix admin endpoint metrics
brayniac Oct 11, 2023
1d81453
dataspec: add histogram merge operation
mihirn Oct 9, 2023
0ad47a7
clippy nonsense
mihirn Oct 11, 2023
2b50e24
merge main
brayniac Oct 11, 2023
0ddaf76
Merge branch 'main' into metriken
brayniac Oct 11, 2023
6abce4b
use warp again
brayniac Oct 11, 2023
c18a3a2
fixup
brayniac Oct 11, 2023
0b2bbe2
make that lazy
brayniac Oct 11, 2023
9973df7
rustfmt
brayniac Oct 11, 2023
82cecf8
address clippy lints
brayniac Oct 11, 2023
32e3d91
use published dependencies
brayniac Oct 12, 2023
1600c2a
fix admin percentiles
brayniac Oct 12, 2023
ce8f82e
manifest cleanup
brayniac Oct 12, 2023
f3dd272
cleanup
brayniac Oct 12, 2023
487534b
update macro name
brayniac Oct 12, 2023
38aa60f
remove snapshot buffer and maintain previous and deltas only
brayniac Oct 12, 2023
3130e11
naming and structure
brayniac Oct 12, 2023
0cabee4
reduce duplication
brayniac Oct 12, 2023
9455b70
consolidate metrics snapshot types
brayniac Oct 13, 2023
30895dc
consolidate logic for human and json stats
brayniac Oct 13, 2023
d71ea20
rustfmt
brayniac Oct 13, 2023
658a1ea
need to update pelikan before merge
brayniac Oct 14, 2023
7c83892
update pelikan again
brayniac Oct 14, 2023
c7808b7
latencies should be in microseconds
brayniac Oct 15, 2023
756d6ea
these were already rates
brayniac Oct 15, 2023
7c87742
copy paste error
brayniac Oct 15, 2023
1d82755
fix cli output
brayniac Oct 17, 2023
253f959
switch repo and address feedback
brayniac Oct 17, 2023
e375db5
regenerate lockfile
brayniac Oct 17, 2023
901f771
rustfmt
brayniac Oct 17, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
466 changes: 252 additions & 214 deletions Cargo.lock

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ repository = { workspace = true }
license = { workspace = true }

[workspace.dependencies]
histogram = "0.7.4"
serde = "1.0.185"
histogram = { git = "https://github.com/brayniac/rustcommon", branch = "metriken-histograms" }
serde = { version = "1.0.185", features = ["derive"] }

[dependencies]
ahash = "0.8.3"
Expand All @@ -35,7 +35,7 @@ hyper = { version = "1.0.0-rc.4", features = ["http1", "http2", "client"]}
histogram = { workspace = true }
humantime = "2.1.0"
momento = "0.31.0"
metriken = "0.2.3"
metriken = { git = "https://github.com/brayniac/rustcommon", branch = "metriken-histograms" }
mio = "0.8.8"
net = { git = "https://github.com/pelikan-io/pelikan" }
paste = "1.0.14"
Expand All @@ -56,8 +56,8 @@ slab = "0.4.9"
tokio = { version = "1.32.0", features = ["full"] }
tokio-boring = "2.1.5"
toml = "0.7.6"
zipf = "7.0.1"
warp = "0.3.6"
zipf = "7.0.1"

[workspace]
members = [
Expand Down
46 changes: 20 additions & 26 deletions lib/dataspec/src/histogram.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use serde::{Deserialize, Serialize};
use thiserror::Error;

use histogram::Histogram as _Histogram;
use histogram::Snapshot;

/// This histogram is a sparse, columnar representation of the regular
/// Histogram. It is significantly smaller than a regular Histogram
Expand All @@ -14,13 +14,12 @@ use histogram::Histogram as _Histogram;
pub struct Histogram {
/// parameters representing the resolution and the range of
/// the histogram tracking request latencies
pub m: u32,
pub r: u32,
pub n: u32,
pub grouping_power: u8,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I might end up changing this to include the entire Config when we move this to rustcommon, but I think this is fine for now.

pub max_value_power: u8,
/// indices for the non-zero buckets in the histogram
pub index: Vec<usize>,
/// histogram bucket counts corresponding to the indices
pub count: Vec<u32>,
pub count: Vec<u64>,
}

/// Errors returned for operations on histograms.
Expand All @@ -32,7 +31,7 @@ pub enum Error {
}

impl Histogram {
fn add_bucket(&mut self, idx: usize, n: u32) {
fn add_bucket(&mut self, idx: usize, n: u64) {
self.index.push(idx);
self.count.push(n);
}
Expand All @@ -43,14 +42,13 @@ impl Histogram {
/// Buckets which have values in both histograms are allowed to wrap.
#[allow(clippy::comparison_chain)]
pub fn merge(&self, h: &Histogram) -> Result<Histogram, Error> {
if self.m != h.m || self.r != h.r || self.n != h.n {
if self.grouping_power != h.grouping_power || self.max_value_power != h.max_value_power {
return Err(Error::MismatchedParams);
}

let mut histogram = Histogram {
m: self.m,
r: self.r,
n: self.n,
grouping_power: self.grouping_power,
max_value_power: self.max_value_power,
index: Vec::new(),
count: Vec::new(),
};
Expand Down Expand Up @@ -89,12 +87,12 @@ impl Histogram {
}
}

impl From<&_Histogram> for Histogram {
fn from(histogram: &_Histogram) -> Self {
impl From<&Snapshot> for Histogram {
fn from(snapshot: &Snapshot) -> Self {
let mut index = Vec::new();
let mut count = Vec::new();

for (i, bucket) in histogram
for (i, bucket) in snapshot
.into_iter()
.enumerate()
.filter(|(_i, bucket)| bucket.count() != 0)
Expand All @@ -103,11 +101,10 @@ impl From<&_Histogram> for Histogram {
count.push(bucket.count());
}

let p = histogram.parameters();
let config = snapshot.config();
Self {
m: p.0,
r: p.1,
n: p.2,
grouping_power: config.grouping_power(),
max_value_power: config.max_value_power(),
index,
count,
}
Expand All @@ -121,25 +118,22 @@ mod tests {
#[test]
fn merge() {
let h1 = Histogram {
m: 0,
r: 7,
n: 32,
grouping_power: 8,
max_value_power: 32,
index: vec![1, 3, 5],
count: vec![6, 12, 7],
};

let h2 = Histogram {
m: 0,
r: 7,
n: 32,
grouping_power: 8,
max_value_power: 32,
index: Vec::new(),
count: Vec::new(),
};

let h3 = Histogram {
m: 0,
r: 7,
n: 32,
grouping_power: 8,
max_value_power: 32,
index: vec![2, 3, 4, 11],
count: vec![5, 7, 3, 15],
};
Expand Down
3 changes: 2 additions & 1 deletion lib/dataspec/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
//! by any consumer of the produced data to parse the files.

pub mod histogram;
// use ::histogram::Snapshot;
brayniac marked this conversation as resolved.
Show resolved Hide resolved
pub use crate::histogram::Histogram;

use serde::{Deserialize, Serialize};
Expand Down Expand Up @@ -49,7 +50,7 @@ pub struct ClientStats {
pub connections: Connections,
pub requests: Requests,
pub responses: Responses,
pub request_latency: Histogram,
pub response_latency: Histogram,
}

#[derive(Serialize, Deserialize)]
Expand Down
Loading
Loading