Merge pull request #3 from Japannext/master

v1.0.15

.github/workflows/release.yml

@@ -246,6 +246,7 @@ jobs:
         with:
           retry_on: error
           max_attempts: 3
+          timeout_minutes: 10
           command: >-
             docker build
             --build-arg VCS_REF=${{ github.sha }}

.github/workflows/tests.yml

@@ -7,7 +7,6 @@ on:
 jobs:
   tests:
     runs-on: ubuntu-latest
-
     steps:
       - uses: actions/checkout@v2
       - name: Install packages
@@ -16,10 +15,10 @@ jobs:
           sudo apt-get install -y \
             build-essential \
             python3-dev
-      - name: Set up Python 3.x
+      - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:
-          python-version: '3.x'
+          python-version: '3.8'
           architecture: 'x64'
       - name: Install dependencies
         id: install-deps

CHANGELOG.md

@@ -1,3 +1,15 @@
+## v1.0.15
+
+### New features
+* Storing metrics locally and displaying a dashboard
+* Can configure a default landing page in preferences
+* Keeping track of Last login for all users
+* InfluxDB 2.0 webhook added
+### Bug fixes
+* Do no crash whenever a plugin fails to load
+* Widgets pretty print was not working properly
+* Failed webhook actions did not register as failed properly
+
 ## v1.0.14
 
 ### New features
@@ -7,7 +19,6 @@
 * Resized Condition box to get more input space
 * Snooze filters can discard alerts
 * Retro apply Snooze filters to all alerts
-
 ### Bug fixes
 * Going back to wsgiref. It was working fine. Waitress is just having issues with TLS
 
@@ -25,7 +36,7 @@
 * Moving Unix socket management out of the falcon API
 * Using Waitress for Unix socket and TCP socket
 * Secrets are now bootstrapped using random numbers and are stored in the backend database
-* Dedicated middleware for logging 
+* Dedicated middleware for logging
 ### Bug fixes
 * When changing tabs or refreshing, webUI row tables are not flickering anymore
 * Throttled alerts generated duplicate entries

Pipfile

@@ -7,7 +7,7 @@ falcon = "*"
 falcon-auth = "*"
 PyJWT = "==1.7.1"
 Jinja2 = "*"
-PyYAML = "*"
+PyYAML = "5.4.1"
 requests_unixsocket = "*"
 urllib3 = "*"
 tinydb = "*"

VERSION

@@ -1 +1 @@
-1.0.14
+1.0.15

setup.py

@@ -40,7 +40,7 @@
     install_requires = [
         'Jinja2',
         'PyJWT==1.7.1',
-        'PyYAML',
+        'PyYAML==5.4.1',
         'click',
         'falcon',
         'falcon-auth',

snooze/api/base.py

@@ -108,10 +108,14 @@ def load_plugin_routes(self):
             try:
                 spec.loader.exec_module(plugin_module)
                 log.debug("Found custom routes for `{}`".format(plugin.name))
-            except:
+            except FileNotFoundError:
                 # Loading default
                 log.debug("Loading default route for `{}`".format(plugin.name))
                 plugin_module = import_module("snooze.plugins.core.basic.{}.route".format(self.api_type))
+            except Exception as e:
+                log.exception(e)
+                log.debug("Skip loading plugin `{}` routes".format(plugin.name))
+                continue
             primary = plugin.metadata.get('primary') or None
             duplicate_policy = plugin.metadata.get('duplicate_policy') or 'update'
             authorization_policy = plugin.metadata.get('authorization_policy')

snooze/api/falcon.py

@@ -23,6 +23,7 @@
 from snooze.api.base import Api, BasicRoute
 from snooze.api.static import StaticRoute
 from snooze.utils import config, write_config
+from snooze.utils.functions import ensure_kv
 
 from logging import getLogger
 log = getLogger('snooze.api')
@@ -160,6 +161,8 @@ def on_post(self, req, resp):
                     if not isinstance(alerts, list):
                         alerts = [alerts]
                     for alert in alerts:
+                        for key, val in req.params.items():
+                            alert = ensure_kv(alert, val, *key.split('.'))
                         rec = self.core.process_record(alert)
                         rec_list.append(rec)
             except Exception as e:
@@ -392,8 +395,9 @@ def on_post(self, req, resp):
         if self.enabled:
             self.authenticate(req, resp)
             user = self.parse_user(req.context['user'])
+            preferences = None
             if self.userplugin:
-                self.userplugin.manage_db(user)
+                _, preferences = self.userplugin.manage_db(user)
             self.inject_permissions(user)
             log.debug("Context user: {}".format(user))
             token = self.api.jwt_auth.get_auth_token(user)
@@ -403,6 +407,8 @@ def on_post(self, req, resp):
             resp.media = {
                 'token': token,
             }
+            if preferences:
+                resp.media['default_page'] = preferences.get('default_page')
         else:
             resp.content_type = falcon.MEDIA_JSON
             resp.status = falcon.HTTP_409

snooze/core.py

@@ -38,7 +38,10 @@ def __init__(self, conf):
         self.plugins = []
         self.process_plugins = []
         self.stats = Stats(self)
-        self.stats.init('process_record_duration', 'summary', 'snooze_record_process_duration', 'Average time spend processing a record', ['source'])
+        self.stats.init('process_alert_duration', 'summary', 'snooze_process_alert_duration', 'Average time spend processing a alert', ['source', 'environment', 'severity'])
+        self.stats.init('alert_hit', 'counter', 'snooze_alert_hit', 'Counter of received alerts', ['source', 'environment', 'severity'])
+        self.stats.init('alert_snoozed', 'counter', 'snooze_alert_snoozed', 'Counter of snoozed alerts', ['name'])
+        self.stats.init('alert_throttled', 'counter', 'snooze_alert_throttled', 'Counter of throttled alerts', ['name'])
         self.stats.init('notification_sent', 'counter', 'snooze_notification_sent', 'Counter of notification sent', ['name', 'action'])
         self.stats.init('notification_error', 'counter', 'snooze_notification_error', 'Counter of notification that failed', ['name', 'action'])
         self.bootstrap_db()
@@ -92,6 +95,8 @@ def load_plugins(self):
     def process_record(self, record):
         data = {}
         source = record.get('source', 'unknown')
+        environment = record.get('environment', 'unknown')
+        severity = record.get('severity', 'unknown')
         record['ttl'] = self.housekeeper.conf.get('record_ttl', 86400)
         record['state'] = ''
         record['plugins'] = []
@@ -100,7 +105,7 @@ def process_record(self, record):
         except Exception as e:
             log.warning(e)
             record['timestamp'] = datetime.now().astimezone().strftime("%Y-%m-%dT%H:%M:%S%z")
-        with self.stats.time('process_record_duration', {'source': source}):
+        with self.stats.time('process_alert_duration', {'source': source, 'environment': environment, 'severity': severity}):
             for plugin in self.process_plugins:
                 try:
                     log.debug("Executing plugin {} on {}".format(plugin.name, record))
@@ -126,6 +131,9 @@ def process_record(self, record):
             else:
                 log.debug("Writing record {}".format(record))
                 data = self.db.write('record', record)
+        environment = record.get('environment', 'unknown')
+        severity = record.get('severity', 'unknown')
+        self.stats.inc('alert_hit', {'source': source, 'environment': environment, 'severity': severity})
         return data
 
     def get_core_plugin(self, plugin_name):

snooze/db/file/database.py

@@ -58,7 +58,7 @@ def init_db(self, conf):
 
     def cleanup_timeout(self, collection):
         mutex.acquire()
-        log.debug("Cleanup collection {}".format(collection))
+        #log.debug("Cleanup collection {}".format(collection))
         now = datetime.datetime.now().timestamp()
         aggregate_results = self.db.table(collection).search(Query().ttl >= 0)
         aggregate_results = list(map(lambda doc: {'_id': doc.doc_id, 'timeout': doc['ttl'] + doc['date_epoch']}, aggregate_results))
@@ -69,7 +69,7 @@ def cleanup_timeout(self, collection):
 
     def cleanup_orphans(self, collection, key, col_ref, key_ref):
         mutex.acquire()
-        log.debug("Cleanup collection {} by finding {} in collection {} matching {}".format(collection, key, col_ref, key_ref))
+        #log.debug("Cleanup collection {} by finding {} in collection {} matching {}".format(collection, key, col_ref, key_ref))
         results = list(map(lambda doc: doc[key_ref], self.db.table(col_ref).all()))
         aggregate_results = self.db.table(collection).search(~ (Query()[key].one_of(results)))
         aggregate_results = list(map(lambda doc: {'_id': doc.doc_id}, aggregate_results))
@@ -79,9 +79,11 @@ def cleanup_orphans(self, collection, key, col_ref, key_ref):
 
     def delete_aggregates(self, collection, aggregate_results):
         ids = list(map(lambda doc: doc['_id'], aggregate_results))
-        deleted_results = self.db.table(collection).remove(doc_ids=ids)
-        deleted_count = len(deleted_results)
-        log.debug('Removed {} documents in {}'.format(deleted_count, collection))
+        deleted_count = 0
+        if ids:
+            deleted_results = self.db.table(collection).remove(doc_ids=ids)
+            deleted_count = len(deleted_results)
+            log.debug('Removed {} documents in {}'.format(deleted_count, collection))
         return deleted_count
 
     def write(self, collection, obj, primary = None, duplicate_policy='update', update_time=True, constant=None):
@@ -127,12 +129,12 @@ def write(self, collection, obj, primary = None, duplicate_policy='update', upda
                         rejected.append(o)
                     elif duplicate_policy == 'replace':
                         log.debug('Replacing with: {}'.format(o))
-                        self.db.table(collection).remove(doc_ids=[doc_id])
-                        self.db.table(collection).insert(o)
+                        table.remove(doc_ids=[doc_id])
+                        table.insert(o)
                         replaced.append(o)
                     else:
                         log.debug('Updating with: {}'.format(o))
-                        self.db.table(collection).update(o, doc_ids=[doc_id])
+                        table.update(o, doc_ids=[doc_id])
                         updated.append(doc)
                 else:
                     log.error("UID {} not found. Skipping...".format(o['uid']))
@@ -152,12 +154,12 @@ def write(self, collection, obj, primary = None, duplicate_policy='update', upda
                             rejected.append(o)
                         elif duplicate_policy == 'replace':
                             log.debug('Replace with: {}'.format(o))
-                            self.db.table(collection).remove(doc_ids=[doc_id])
-                            self.db.table(collection).insert(o)
+                            table.remove(doc_ids=[doc_id])
+                            table.insert(o)
                             replaced.append(o)
                         else:
                             log.debug('Update with: {}'.format(o))
-                            self.db.table(collection).update(o, doc_ids=[doc_id])
+                            table.update(o, doc_ids=[doc_id])
                             updated.append(o)
                 else:
                     log.debug("Could not find document with primary {}. Inserting instead".format(primary))
@@ -175,6 +177,36 @@ def write(self, collection, obj, primary = None, duplicate_policy='update', upda
         mutex.release()
         return {'data': {'added': deepcopy(added), 'updated': deepcopy(updated), 'replaced': deepcopy(replaced),'rejected': deepcopy(rejected)}}
 
+    def inc(self, collection, field, labels={}):
+        now = int((datetime.datetime.now().timestamp() // 3600) * 3600)
+        table = self.db.table(collection)
+        query = Query()
+        mutex.acquire()
+        keys = []
+        added = []
+        updated = []
+        if labels:
+            for k,v in labels.items():
+                keys.append(field+'__'+k+'__'+v)
+        else:
+            keys.append(field)
+        for key in keys:
+            result = table.search((query.date == now) & (query.key == key))
+            if result:
+                result = result[0]
+                result['value'] = result.get('value', 0) + 1
+                table.update(result, doc_ids=[result.doc_id])
+                log.debug('Updated in {} metric {}'.format(collection, result))
+                updated.append(deepcopy(result))
+            else:
+                result = {'date': now, 'type': 'counter', 'key': key}
+                result['value'] = 1
+                table.insert(result)
+                log.debug('Inserted in {} metric {}'.format(collection, result))
+                added.append(deepcopy(result))
+        mutex.release()
+        return {'data': {'added': added, 'updated': updated}}
+
     def update_fields(self, collection, fields, condition=[]):
         log.debug("Update collection '{}' with fields '{}' based on the following search".format(collection, fields))
         total = 0
@@ -189,6 +221,52 @@ def update_fields(self, collection, fields, condition=[]):
         log.debug("Updated {} fields".format(total))
         return total
 
+    def compute_stats(self, collection, date_from, date_until, groupby='hour'):
+        log.debug("Compute metrics on `{}` from {} until {} grouped by {}".format(collection, date_from, date_until, groupby))
+        date_from = date_from.replace(minute=0, second=0, microsecond=0)
+        if collection not in self.db.tables():
+            log.debug("Compute stats: collection {} does not exist".format(collection))
+            return {'data': [], 'count': 0}
+        if groupby == 'hour':
+            date_format = '%Y-%m-%dT%H:00%z'
+        elif groupby == 'day':
+            date_format = '%Y-%m-%dT00:00%z'
+        elif groupby == 'month':
+            date_format = '%Y-%m-01T00:00%z'
+        elif groupby == 'year':
+            date_format = '%Y-01-01T00:00%z'
+        elif groupby == 'week':
+            date_format = '%Y-%VT00:00%z'
+        elif groupby == 'weekday':
+            date_format = '%u'
+        else:
+            date_format = '%Y-%m-%dT%H:00%z'
+        date_from = date_from.timestamp()
+        date_until = date_until.timestamp()
+        table = self.db.table(collection)
+        results = table.search((Query().date >= date_from) & (Query().date <= date_until))
+        if len(results) == 0:
+            log.debug("Compute stats: No data found within time interval")
+            return {'data': [], 'count': 0}
+        groups = {}
+        res = []
+        for doc in results:
+            date_range = datetime.date.fromtimestamp(doc['date']).strftime(date_format)
+            if date_range not in groups:
+                groups[date_range] = {doc['key']: {'value': 0}}
+            elif doc['key'] not in groups[date_range]:
+                groups[date_range][doc['key']] = {'value': 0}
+            groups[date_range][doc['key']]['value'] += doc['value']
+        for date, v in groups.items():
+            entry = {'_id': date, 'data': []}
+            for key, doc in v.items():
+                entry['data'].append({'key': key, 'value': doc['value']})
+            res.append(entry)
+        results_agg = sorted(res, key=lambda d: d['_id'])
+        count = len(results_agg)
+        log.debug("Compute stats: Got {} results".format(count))
+        return {'data': results_agg, 'count': count}
+
     def search(self, collection, condition=[], nb_per_page=0, page_number=1, orderby="", asc=True):
         mutex.acquire()
         tinydb_search = self.convert(condition)