Skip to content

Commit f857aa3

Browse files
ktsaouktsaou
authored
optimized ses and added des (netdata#4470)
* optimized ses and added des * added coefficient of variation * fix bug identified by @vlvkobal: use all available points when resampling is required and the timeframe is not enough for a single point
1 parent 0a78758 commit f857aa3

15 files changed

+428
-52
lines changed

‎CMakeLists.txt

+2
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,8 @@ set(API_PLUGIN_FILES
393393
web/api/queries/stddev/stddev.h
394394
web/api/queries/ses/ses.c
395395
web/api/queries/ses/ses.h
396+
web/api/queries/des/des.c
397+
web/api/queries/des/des.h
396398
)
397399

398400
set(STREAMING_PLUGIN_FILES

‎Makefile.am

+2
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,8 @@ API_PLUGIN_FILES = \
291291
web/api/exporters/shell/allmetrics_shell.h \
292292
web/api/queries/average/average.c \
293293
web/api/queries/average/average.h \
294+
web/api/queries/des/des.c \
295+
web/api/queries/des/des.h \
294296
web/api/queries/incremental_sum/incremental_sum.c \
295297
web/api/queries/incremental_sum/incremental_sum.h \
296298
web/api/queries/max/max.c \

‎configure.ac

+1
Original file line numberDiff line numberDiff line change
@@ -609,6 +609,7 @@ AC_CONFIG_FILES([
609609
web/api/exporters/prometheus/Makefile
610610
web/api/queries/Makefile
611611
web/api/queries/average/Makefile
612+
web/api/queries/des/Makefile
612613
web/api/queries/incremental_sum/Makefile
613614
web/api/queries/max/Makefile
614615
web/api/queries/median/Makefile

‎cppcheck.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,14 @@ processors=$(grep -c ^processor /proc/cpuinfo)
1212
base="$(dirname "${0}")"
1313
[ "${base}" = "." ] && base="${PWD}"
1414

15-
cd "${base}/src" || exit 1
15+
cd "${base}" || exit 1
1616

1717
[ ! -d "cppcheck-build" ] && mkdir "cppcheck-build"
1818

1919
file="${1}"
2020
shift
2121
# shellcheck disable=SC2235
22-
([ "${file}" = "${base}" ] || [ -z "${file}" ]) && file="${base}/src"
22+
([ "${file}" = "${base}" ] || [ -z "${file}" ]) && file="${base}"
2323

2424
"${cppcheck}" \
2525
-j ${processors} \

‎web/api/queries/des/Makefile.am

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# SPDX-License-Identifier: GPL-3.0-or-later
2+
3+
AUTOMAKE_OPTIONS = subdir-objects
4+
MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5+
6+
dist_noinst_DATA = \
7+
README.md \
8+
$(NULL)

‎web/api/queries/des/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# double exponential smoothing

‎web/api/queries/des/des.c

+112
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
// SPDX-License-Identifier: GPL-3.0-or-later
2+
3+
#include "des.h"
4+
5+
6+
// ----------------------------------------------------------------------------
7+
// single exponential smoothing
8+
9+
struct grouping_des {
10+
calculated_number alpha;
11+
calculated_number alpha_other;
12+
calculated_number beta;
13+
calculated_number beta_other;
14+
15+
calculated_number level;
16+
calculated_number trend;
17+
18+
size_t count;
19+
};
20+
21+
#define MAX_WINDOW_SIZE 10
22+
23+
static inline void set_alpha(RRDR *r, struct grouping_des *g) {
24+
// https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
25+
// A commonly used value for alpha is 2 / (N + 1)
26+
calculated_number window = (r->group > MAX_WINDOW_SIZE) ? MAX_WINDOW_SIZE : r->group;
27+
28+
g->alpha = 2.0 / ((calculated_number)window + 1.0);
29+
g->alpha_other = 1.0 - g->alpha;
30+
31+
//info("alpha for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->alpha);
32+
}
33+
34+
static inline void set_beta(RRDR *r, struct grouping_des *g) {
35+
// https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
36+
// A commonly used value for alpha is 2 / (N + 1)
37+
calculated_number window = (r->group > MAX_WINDOW_SIZE) ? MAX_WINDOW_SIZE : r->group;
38+
39+
g->beta = 2.0 / ((calculated_number)window + 1.0);
40+
g->beta_other = 1.0 - g->beta;
41+
42+
//info("beta for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->beta);
43+
}
44+
45+
void *grouping_init_des(RRDR *r) {
46+
struct grouping_des *g = (struct grouping_des *)malloc(sizeof(struct grouping_des));
47+
set_alpha(r, g);
48+
set_beta(r, g);
49+
g->level = 0.0;
50+
g->trend = 0.0;
51+
g->count = 0;
52+
return g;
53+
}
54+
55+
// resets when switches dimensions
56+
// so, clear everything to restart
57+
void grouping_reset_des(RRDR *r) {
58+
struct grouping_des *g = (struct grouping_des *)r->grouping_data;
59+
g->level = 0.0;
60+
g->trend = 0.0;
61+
g->count = 0;
62+
63+
// fprintf(stderr, "\nDES: ");
64+
65+
}
66+
67+
void grouping_free_des(RRDR *r) {
68+
freez(r->grouping_data);
69+
r->grouping_data = NULL;
70+
}
71+
72+
void grouping_add_des(RRDR *r, calculated_number value) {
73+
struct grouping_des *g = (struct grouping_des *)r->grouping_data;
74+
75+
if(isnormal(value)) {
76+
if(likely(g->count > 0)) {
77+
// we have at least a number so far
78+
79+
if(unlikely(g->count == 1)) {
80+
// the second value we got
81+
g->trend = value - g->trend;
82+
g->level = value;
83+
}
84+
85+
// for the values, except the first
86+
calculated_number last_level = g->level;
87+
g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend));
88+
g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend);
89+
}
90+
else {
91+
// the first value we got
92+
g->level = g->trend = value;
93+
}
94+
95+
g->count++;
96+
}
97+
98+
//fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend);
99+
}
100+
101+
calculated_number grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
102+
struct grouping_des *g = (struct grouping_des *)r->grouping_data;
103+
104+
if(unlikely(!g->count || !isnormal(g->level))) {
105+
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
106+
return 0.0;
107+
}
108+
109+
//fprintf(stderr, " RESULT for %zu values = " CALCULATED_NUMBER_FORMAT " \n", g->count, g->level);
110+
111+
return g->level;
112+
}

‎web/api/queries/des/des.h

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// SPDX-License-Identifier: GPL-3.0-or-later
2+
3+
#ifndef NETDATA_API_QUERIES_DES_H
4+
#define NETDATA_API_QUERIES_DES_H
5+
6+
#include "../query.h"
7+
#include "../rrdr.h"
8+
9+
extern void *grouping_init_des(RRDR *r);
10+
extern void grouping_reset_des(RRDR *r);
11+
extern void grouping_free_des(RRDR *r);
12+
extern void grouping_add_des(RRDR *r, calculated_number value);
13+
extern calculated_number grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
14+
15+
#endif //NETDATA_API_QUERIES_DES_H

‎web/api/queries/query.c

+18-3
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "sum/sum.h"
1313
#include "stddev/stddev.h"
1414
#include "ses/ses.h"
15+
#include "des/des.h"
1516

1617
// ----------------------------------------------------------------------------
1718

@@ -31,9 +32,23 @@ static struct {
3132
, { "median" , 0, RRDR_GROUPING_MEDIAN , grouping_init_median , grouping_reset_median , grouping_free_median , grouping_add_median , grouping_flush_median }
3233
, { "min" , 0, RRDR_GROUPING_MIN , grouping_init_min , grouping_reset_min , grouping_free_min , grouping_add_min , grouping_flush_min }
3334
, { "max" , 0, RRDR_GROUPING_MAX , grouping_init_max , grouping_reset_max , grouping_free_max , grouping_add_max , grouping_flush_max }
34-
, { "ses" , 0, RRDR_GROUPING_SES , grouping_init_ses , grouping_reset_ses , grouping_free_ses , grouping_add_ses , grouping_flush_ses }
35-
, { "stddev" , 0, RRDR_GROUPING_STDDEV , grouping_init_stddev , grouping_reset_stddev , grouping_free_stddev , grouping_add_stddev , grouping_flush_stddev }
3635
, { "sum" , 0, RRDR_GROUPING_SUM , grouping_init_sum , grouping_reset_sum , grouping_free_sum , grouping_add_sum , grouping_flush_sum }
36+
37+
// stddev module provides mean, variance and coefficient of variation
38+
, { "stddev" , 0, RRDR_GROUPING_STDDEV , grouping_init_stddev , grouping_reset_stddev , grouping_free_stddev , grouping_add_stddev , grouping_flush_stddev }
39+
, { "cv" , 0, RRDR_GROUPING_CV , grouping_init_stddev , grouping_reset_stddev , grouping_free_stddev , grouping_add_stddev , grouping_flush_coefficient_of_variation }
40+
//, { "mean" , 0, RRDR_GROUPING_MEAN , grouping_init_stddev , grouping_reset_stddev , grouping_free_stddev , grouping_add_stddev , grouping_flush_mean }
41+
//, { "variance" , 0, RRDR_GROUPING_VARIANCE , grouping_init_stddev , grouping_reset_stddev , grouping_free_stddev , grouping_add_stddev , grouping_flush_variance }
42+
43+
// single exponential smoothing or exponential weighted moving average
44+
, { "ses" , 0, RRDR_GROUPING_SES , grouping_init_ses , grouping_reset_ses , grouping_free_ses , grouping_add_ses , grouping_flush_ses }
45+
, { "ema" , 0, RRDR_GROUPING_SES , grouping_init_ses , grouping_reset_ses , grouping_free_ses , grouping_add_ses , grouping_flush_ses }
46+
, { "ewma" , 0, RRDR_GROUPING_SES , grouping_init_ses , grouping_reset_ses , grouping_free_ses , grouping_add_ses , grouping_flush_ses }
47+
48+
// double exponential smoothing
49+
, { "des" , 0, RRDR_GROUPING_DES , grouping_init_des , grouping_reset_des , grouping_free_des , grouping_add_des , grouping_flush_des }
50+
51+
// terminator
3752
, { NULL , 0, RRDR_GROUPING_UNDEFINED , grouping_init_average , grouping_reset_average , grouping_free_average , grouping_add_average , grouping_flush_average }
3853
};
3954

@@ -431,7 +446,7 @@ RRDR *rrd2rrdr(
431446
info("INTERNAL CHECK: %s: requested gtime %ld secs, is greater than the desired duration %ld secs", st->id, group_time_requested, duration);
432447
#endif
433448

434-
group = points_requested; // use all the points
449+
group = available_points; // use all the points
435450
}
436451
else {
437452
// the points we should group to satisfy gtime

‎web/api/queries/query.h

+11-9
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,17 @@
44
#define NETDATA_API_DATA_QUERY_H
55

66
typedef enum rrdr_grouping {
7-
RRDR_GROUPING_UNDEFINED = 0,
8-
RRDR_GROUPING_AVERAGE = 1,
9-
RRDR_GROUPING_MIN = 2,
10-
RRDR_GROUPING_MAX = 3,
11-
RRDR_GROUPING_SUM = 4,
12-
RRDR_GROUPING_INCREMENTAL_SUM = 5,
13-
RRDR_GROUPING_MEDIAN = 6,
14-
RRDR_GROUPING_STDDEV = 7,
15-
RRDR_GROUPING_SES = 8,
7+
RRDR_GROUPING_UNDEFINED = 0,
8+
RRDR_GROUPING_AVERAGE,
9+
RRDR_GROUPING_MIN,
10+
RRDR_GROUPING_MAX,
11+
RRDR_GROUPING_SUM,
12+
RRDR_GROUPING_INCREMENTAL_SUM,
13+
RRDR_GROUPING_MEDIAN,
14+
RRDR_GROUPING_STDDEV,
15+
RRDR_GROUPING_CV,
16+
RRDR_GROUPING_SES,
17+
RRDR_GROUPING_DES,
1618
} RRDR_GROUPING;
1719

1820
extern const char *group_method2string(RRDR_GROUPING group);

‎web/api/queries/ses/README.md

+45-1
Original file line numberDiff line numberDiff line change
@@ -1 +1,45 @@
1-
# single exponential smoothing
1+
# Single (or Simple) Exponential Smoothing (`ses`)
2+
3+
> This query is also available as `ema` and `ewma`.
4+
5+
An exponential moving average (`ema`), also known as an exponentially weighted moving average (`ewma`)
6+
is a first-order infinite impulse response filter that applies weighting factors which decrease
7+
exponentially. The weighting for each older datum decreases exponentially, never reaching zero.
8+
9+
In simple terms, this is like an average value, but more recent values are given more weight.
10+
11+
Netdata automatically adjusts the weight based on the number of values processed, using the formula:
12+
13+
```
14+
alpha = 2 / (number_of_values + 1)
15+
```
16+
17+
## how to use
18+
19+
Use it in alarms like this:
20+
21+
```
22+
alarm: my_alarm
23+
on: my_chart
24+
lookup: ses -1m unaligned of my_dimension
25+
warn: $this > 1000
26+
```
27+
28+
`ses` does not change the units. For example, if the chart units is `requests/sec`, the exponential
29+
moving average will be again expressed in the same units.
30+
31+
It can also be used in APIs and badges as `&group=ses` in the URL.
32+
33+
## Examples
34+
35+
Examining last 1 minute `successful` web server responses:
36+
37+
- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&dimensions=success&group=min&after=-60&label=min)
38+
- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&dimensions=success&group=average&after=-60&label=average&value_color=yellow)
39+
- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&dimensions=success&group=ses&after=-60&label=single+exponential+smoothing&value_color=orange)
40+
- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&dimensions=success&group=max&after=-60&label=max)
41+
42+
## References
43+
44+
- [https://en.wikipedia.org/wiki/Moving_average#exponential-moving-average](https://en.wikipedia.org/wiki/Moving_average#exponential-moving-average)
45+
- [https://en.wikipedia.org/wiki/Exponential_smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing).

‎web/api/queries/ses/ses.c

+9-19
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,16 @@
88

99
struct grouping_ses {
1010
calculated_number alpha;
11-
calculated_number alpha_older;
11+
calculated_number alpha_other;
1212
calculated_number level;
1313
size_t count;
14-
size_t has_data;
1514
};
1615

1716
static inline void set_alpha(RRDR *r, struct grouping_ses *g) {
18-
g->alpha = 1.0 / r->group;
19-
g->alpha_older = 1 - g->alpha;
17+
// https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
18+
// A commonly used value for alpha is 2 / (N + 1)
19+
g->alpha = 2.0 / ((calculated_number)r->group + 1.0);
20+
g->alpha_other = 1 - g->alpha;
2021
}
2122

2223
void *grouping_init_ses(RRDR *r) {
@@ -32,7 +33,6 @@ void grouping_reset_ses(RRDR *r) {
3233
struct grouping_ses *g = (struct grouping_ses *)r->grouping_data;
3334
g->level = 0.0;
3435
g->count = 0;
35-
g->has_data = 0;
3636
}
3737

3838
void grouping_free_ses(RRDR *r) {
@@ -44,31 +44,21 @@ void grouping_add_ses(RRDR *r, calculated_number value) {
4444
struct grouping_ses *g = (struct grouping_ses *)r->grouping_data;
4545

4646
if(isnormal(value)) {
47-
if(unlikely(!g->has_data)) {
47+
if(unlikely(!g->count))
4848
g->level = value;
49-
g->has_data = 1;
50-
}
51-
52-
g->level = g->alpha * value + g->alpha_older * g->level;
5349

50+
g->level = g->alpha * value + g->alpha_other * g->level;
5451
g->count++;
5552
}
5653
}
5754

5855
calculated_number grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
5956
struct grouping_ses *g = (struct grouping_ses *)r->grouping_data;
6057

61-
calculated_number value;
62-
6358
if(unlikely(!g->count || !isnormal(g->level))) {
64-
value = 0.0;
6559
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
60+
return 0.0;
6661
}
67-
else {
68-
value = g->level;
69-
}
70-
71-
g->count = 0;
7262

73-
return value;
63+
return g->level;
7464
}

0 commit comments

Comments
 (0)