Add common parsing functionality for collector messages.

index.js

@@ -14,6 +14,7 @@ module.exports = {
     IngestC : require('./al_servicec').IngestC,
     AzcollectC : require('./azcollectc').AzcollectC,
     EndpointsC : require('./al_servicec').EndpointsC,
-    AlLog : require('./al_log')
+    AlLog : require('./al_log'),
+    Parse: require('./parse')
 };
 

package.json

@@ -24,7 +24,8 @@
     "protobufjs": "^6.8.8",
     "request": "^2.88.0",
     "request-promise-native": "^1.0.5",
-    "retry": "^0.12.0"
+    "retry": "^0.12.0",
+    "rewire": "^4.0.1"
   },
   "author": "Alert Logic Inc."
 }

parse.js

@@ -0,0 +1,95 @@
+/* -----------------------------------------------------------------------------
+ * @copyright (C) 2019, Alert Logic, Inc
+ * @doc
+ *
+ * Message parse utilities common for collector functions.
+ * Message timestamp and type property paths are based on Azure event schema definitions
+ * https://docs.microsoft.com/en-us/azure/azure-monitor/platform/activity-log-schema#mapping-to-diagnostic-logs-schema
+ * https://docs.microsoft.com/en-us/azure/azure-monitor/platform/tutorial-dashboards
+ *
+ * @end
+ * -----------------------------------------------------------------------------
+ */
+
+/*
+ * For the ISO8601 timestamp, '2018-12-19T08:18:21.1834546Z'
+ */
+const ISO8601_MICROSEC_OFFSET = 20;
+
+var getProp = function(path, obj, defaultVal = null) {
+    var reduceFun = function(xs, x) {
+        return (xs && xs[x]) ? xs[x] : defaultVal;
+    };
+    return path.reduce(reduceFun, obj);
+};
+
+var defaultTs = function() {
+    return {
+        sec: Math.floor(Date.now() / 1000),
+        usec: null
+    };
+};
+
+var parseTs = function(ts) {
+    var milli = Date.parse(ts);
+    if (isNaN(milli)) {
+        return defaultTs();
+    } else {
+        var micro = parseTsUsec(ts);
+        return {
+            sec: Math.floor(milli / 1000),
+            usec: micro
+        };
+    }
+};
+
+var parseTsUsec = function(ts) {
+    var micro = null;
+    try {
+        // extracts microseconds from ISO8601 timestamp, like '2018-12-19T08:18:21.1834546Z'
+        if (ts.length > ISO8601_MICROSEC_OFFSET) {
+            var microStr = ts.slice(ISO8601_MICROSEC_OFFSET, ISO8601_MICROSEC_OFFSET + 6).replace(/Z|\+.*$/g, '');
+            while (microStr && microStr.length > 0 && microStr.length < 6) {
+                microStr += '0';
+            }
+            micro = Number.parseInt(microStr);
+            micro = Number.isInteger(micro) ? micro : null;
+        }
+        return micro;
+    } catch (err) {
+        // Unable to get microseconds from a timestamp. Do nothing.
+        return null;
+    }
+};
+
+var iteratePropPaths = function(paths, msg) {
+    return paths.reduce(function(acc, v) {
+        if (acc) {
+            return acc;
+        } else {
+            const propVal = getProp(v.path, msg);
+            if (v.override) {
+                return propVal ? v.override : propVal;
+            } else {
+                return propVal;
+            }
+        }
+    }, null);
+};
+
+var getMsgTs = function(msg, tsPaths) {
+    var msgTs = iteratePropPaths(tsPaths, msg);
+    return msgTs ? parseTs(msgTs) : defaultTs();
+};
+
+var getMsgTypeId = function(msg, typeIdPaths, defaultVal = null) {
+    var msgType = iteratePropPaths(typeIdPaths, msg);
+    return msgType ? msgType : defaultVal;
+};
+
+module.exports = {
+    iteratePropPaths: iteratePropPaths,
+    getMsgTs: getMsgTs,
+    getMsgTypeId: getMsgTypeId
+};
+

test/parse_test.js

@@ -0,0 +1,205 @@
+/* -----------------------------------------------------------------------------
+ * @copyright (C) 2018, Alert Logic, Inc
+ * @doc
+ * 
+ * Unit tests for EHub functions
+ * 
+ * @end
+ * -----------------------------------------------------------------------------
+ */
+
+const assert = require('assert');
+const rewire = require('rewire');
+const sinon = require('sinon');
+
+var parseWire = rewire('../parse');
+
+const parse = require('../parse');
+
+describe('Common parse functions unit tests.', function() {
+    var clock;
+
+    before(function() {
+        clock = sinon.useFakeTimers({now: 1234567890});
+    });
+    after(function() {
+        clock.restore();
+    });
+
+    it('Simple OK test', function(done) {
+        var privGetProp = parseWire.__get__('getProp');
+        var obj = {
+            a: {
+                aa: 1,
+                ab: 2
+            },
+            b: 2,
+            c: 3
+        };
+        assert.equal(privGetProp(['a', 'aa'], obj), 1);
+        assert.equal(privGetProp(['b'], obj), 2);
+        assert.deepEqual(privGetProp(['a'], obj), {aa: 1, ab: 2});
+
+        done();
+    });
+
+    it('Empty path', function(done) {
+        var privGetProp = parseWire.__get__('getProp');
+        var obj = {
+            a: {
+                aa: 1,
+                ab: 2
+            },
+            b: 2,
+            c: 3
+        };
+        assert.deepEqual(privGetProp([], obj), obj);
+
+        done();
+    });
+
+    it('Empty obj', function(done) {
+        var privGetProp = parseWire.__get__('getProp');
+        assert.deepEqual(privGetProp(['a'], {}), null);
+
+        done();
+    });
+
+    it('Wrong path', function(done) {
+        var privGetProp = parseWire.__get__('getProp');
+        var obj = {
+            a: {
+                aa: 1,
+                ab: 2
+            },
+            b: 2,
+            c: 3
+        };
+        assert.deepEqual(privGetProp(['d'], obj), null);
+        assert.deepEqual(privGetProp(['d', 'da'], obj), null);
+
+        done();
+    });
+
+    it('Ok test with default', function(done) {
+        var privGetProp = parseWire.__get__('getProp');
+        var obj = {
+            a: {
+                aa: 1,
+                ab: 2
+            },
+            b: 2,
+            c: 3
+        };
+        assert.deepEqual(privGetProp(['d'], obj, 'default'), 'default');
+        assert.deepEqual(privGetProp(['d', 'da'], obj, 'default'), 'default');
+
+        done();
+    });
+
+    it('Parse timestamp ISO-8601', function(done) {
+        var privParseTs = parseWire.__get__('parseTs');
+        assert.deepEqual(privParseTs('2018-12-19T08:18:21.13Z'), {sec: 1545207501, usec: 130000});
+        assert.deepEqual(privParseTs('2018-12-19T08:18:21Z'), {sec: 1545207501, usec: null});
+        assert.deepEqual(privParseTs('2018-12-19T08:18:21.1357685Z'), {sec: 1545207501, usec: 135768});
+        assert.deepEqual(privParseTs('2018-12-19T08:18:21.1351Z'), {sec: 1545207501, usec: 135100});
+        assert.deepEqual(privParseTs('2018-12-10T00:03:46.61618+00:00'), {sec: 1544400226, usec: 616180});
+        assert.deepEqual(privParseTs('2018-12-10T00:03:46.1+00:00'), {sec: 1544400226, usec: 100000});
+        assert.deepEqual(privParseTs('2018-12-10T00:03:46.12+00:00'), {sec: 1544400226, usec: 120000});
+        assert.deepEqual(privParseTs('2018-12-10T00:03:46.123+00:00'), {sec: 1544400226, usec: 123000});
+        assert.deepEqual(privParseTs('2018-12-10T00:03:46.1234+00:00'), {sec: 1544400226, usec: 123400});
+        assert.deepEqual(privParseTs('2018-12-10T00:03:46.12345+00:00'), {sec: 1544400226, usec: 123450});
+        assert.deepEqual(privParseTs('2018-12-10T00:03:46.6161822+00:00'), {sec: 1544400226, usec: 616182});
+
+        done();
+    });
+
+    it('Wrong timestamp input', function(done) {
+        var privParseTs = parseWire.__get__('parseTs');
+        assert.deepEqual(privParseTs('foo'), {sec: 1234567, usec: null});
+
+        done();
+    });
+
+    it('Test override values', function(done) {
+        const testPaths = [
+            { path: ['eventTimestamp'] },
+            { path: ['time'], override: 'foo' },
+            { path: ['CreationTime'] }
+        ];
+        const testObj = {
+            some: 'value',
+            time: 'some-time'
+        };
+        assert.deepEqual(parse.iteratePropPaths(testPaths, testObj), 'foo');
+
+        done();
+    });
+
+    it('Gets the message TS successfully', function(done){
+        const testPaths = [
+            { path: ['time'] }
+        ];
+        const testObj = {
+            time: '2019-03-08T15:03:16+00:00'
+        };
+
+        const parsedTs = parse.getMsgTs(testObj, testPaths);
+        const assertion = {
+            sec: 1552057396,
+            usec: 0
+        };
+
+        assert.deepEqual(parsedTs, assertion);
+
+        done();
+    });
+
+    it('Gets the default TS if the key is wrong', function(done){
+        const testPaths = [
+            { path: ['time'] }
+        ];
+        const testObj = {
+            someOtherTimeKey: '2019-03-08T15:03:16+00:00'
+        };
+
+        const parsedTs = parse.getMsgTs(testObj, testPaths);
+
+        assert.equal(parsedTs.usec, null);
+        assert(typeof parsedTs.sec === "number");
+
+        done();
+    });
+
+    it('Gets the type ID successfully', function(done){
+        const testPaths = [
+            { path: ['typeId'] }
+        ];
+        const testObj = {
+            typeId: 'foobar'
+        };
+
+        const parsedType = parse.getMsgTypeId(testObj, testPaths);
+
+        assert.deepEqual(parsedType, 'foobar');
+
+        done();
+    });
+
+    it('Uses the type ID default value successfully', function(done){
+        const testPaths = [
+            { path: ['typeId'] }
+        ];
+        const testObj = {
+            typeIdOtherKey: 'foobar'
+        };
+
+        const parsedType = parse.getMsgTypeId(testObj, testPaths, 'someother');
+
+        assert.deepEqual(parsedType, 'someother');
+
+        done();
+    });
+
+});
+