From d9313f52e365f91b6c3efc6998911b983c8aea1b Mon Sep 17 00:00:00 2001 From: rakhimundhada15 <60738948+rakhimundhada15@users.noreply.github.com> Date: Mon, 21 Oct 2024 12:23:04 +0530 Subject: [PATCH] [GCP ]Filter the logs base on logNames and timestamps (#379) * Filter the logs base on logNames and timestamps --- .../cfn/googlestackdriver-collector.template | 7 ++ collectors/googlestackdriver/collector.js | 53 +++++++-- collectors/googlestackdriver/package.json | 34 +++--- collectors/googlestackdriver/test/test.js | 108 +++++++++++++++++- ps_spec.yml | 2 +- 5 files changed, 172 insertions(+), 32 deletions(-) diff --git a/collectors/googlestackdriver/cfn/googlestackdriver-collector.template b/collectors/googlestackdriver/cfn/googlestackdriver-collector.template index aafa8d1f..537cd93e 100644 --- a/collectors/googlestackdriver/cfn/googlestackdriver-collector.template +++ b/collectors/googlestackdriver/cfn/googlestackdriver-collector.template @@ -57,6 +57,10 @@ "Description": "JSON list of google resources to poll logs from. In the format /", "Type": "String" }, + "LogNameFilters": { + "Description": "A JSON list of Google Cloud log names used to filter logs. The format should be 'service.googleapis.com%2Flog_type', for example: 'cloudaudit.googleapis.com%2Factivity'.", + "Type": "String" + }, "CollectionStartTs": { "Description": "Timestamp when log collection starts. For example, 2020-01-13T16:00:00Z", "Type": "String", @@ -113,6 +117,9 @@ "CollectorStreams": { "Ref": "GoogleResourceIds" }, + "CollectorParamString2": { + "Ref": "LogNameFilters" + }, "CollectionStartTs": { "Ref": "CollectionStartTs" } diff --git a/collectors/googlestackdriver/collector.js b/collectors/googlestackdriver/collector.js index 606d4bce..2b5fbc97 100644 --- a/collectors/googlestackdriver/collector.js +++ b/collectors/googlestackdriver/collector.js @@ -61,13 +61,15 @@ class GooglestackdriverCollector extends PawsCollector { endTs = moment(startTs).add(this.pollInterval, 'seconds').toISOString(); } const resourceNames = JSON.parse(process.env.collector_streams); - const initialStates = resourceNames.map(stream => ({ - stream, - nextPage:null, - since: startTs, - until: endTs, - poll_interval_sec: 1 - })); + const initialStates = resourceNames + .filter(stream => stream && stream.trim() !== "") // Filter out empty or invalid values + .map(stream => ({ + stream, + nextPage: null, + since: startTs, + until: endTs, + poll_interval_sec: 1 + })); return callback(null, initialStates, 1); } @@ -97,9 +99,7 @@ class GooglestackdriverCollector extends PawsCollector { AlLogger.info(`GSTA000001 Collecting data from ${state.since} till ${state.until} for ${state.stream}`); - // TODO: figure out a better way to format this. I'm pretty sure that it needs the newlines in it. - const filter = `timestamp >= "${state.since}" -timestamp < "${state.until}"`; + const filter = collector.generateFilter(state); let pagesRetireved = 0; @@ -182,6 +182,39 @@ timestamp < "${state.until}"`; }); } + generateFilter(state) { + const logNames = process.env.paws_collector_param_string_2 ? JSON.parse(process.env.paws_collector_param_string_2) : []; + const filterConditions = []; + let logFilterCondition; + + if (logNames.length > 0) { + logNames.forEach(logName => { + const trimmedlogName = logName?.trim(); + + if (trimmedlogName) { + const encodelogName = this.isUriEncoded(logName) ? logName : encodeURIComponent(logName); + filterConditions.push(`logName:"${encodelogName}"`); + } else { + AlLogger.warn("Skipping empty log ID."); + } + }); + if (filterConditions.length > 0) { + logFilterCondition = filterConditions.join(" OR "); + } + } + // Construct the basic timestamp filter + let filterQuery = `timestamp >= "${state.since}" AND timestamp < "${state.until}"`; + if (logFilterCondition) { + // Combine the LogName and timesamp filter + filterQuery = `${filterQuery} AND (${logFilterCondition})`; + } + return filterQuery; + } + + isUriEncoded(logName) { + return decodeURIComponent(logName) !== logName; + } + _getNextCollectionState(curState, nextPage) { // Reset the page size for the next collection if it's less than the maximum diff --git a/collectors/googlestackdriver/package.json b/collectors/googlestackdriver/package.json index 0631da7a..a83cba70 100644 --- a/collectors/googlestackdriver/package.json +++ b/collectors/googlestackdriver/package.json @@ -1,6 +1,6 @@ { "name": "googlestackdriver-collector", - "version": "1.2.14", + "version": "1.2.15", "description": "Alert Logic AWS based Googlestackdriver Log Collector", "repository": {}, "private": true, @@ -9,28 +9,28 @@ "test": "JUNIT_REPORT_PATH=./test/report.xml nyc --reporter=text mocha --colors" }, "devDependencies": { - "@aws-sdk/client-cloudformation": "^3.632.0", - "@aws-sdk/client-cloudwatch": "^3.632.0", - "@aws-sdk/client-dynamodb": "^3.632.0", - "@aws-sdk/client-kms": "^3.632.0", - "@aws-sdk/client-lambda": "^3.632.0", - "@aws-sdk/client-s3": "^3.633.0", - "@aws-sdk/client-sqs": "^3.632.0", - "@aws-sdk/client-ssm": "^3.632.0", + "@aws-sdk/client-cloudformation": "^3.666.0", + "@aws-sdk/client-cloudwatch": "^3.666.0", + "@aws-sdk/client-dynamodb": "^3.666.0", + "@aws-sdk/client-kms": "^3.666.0", + "@aws-sdk/client-lambda": "^3.666.0", + "@aws-sdk/client-s3": "^3.666.0", + "@aws-sdk/client-sqs": "^3.666.0", + "@aws-sdk/client-ssm": "^3.666.0", "jshint": "^2.13.6", "mocha": "^10.7.3", "mocha-jenkins-reporter": "^0.4.8", - "nyc": "^17.0.0", + "nyc": "^17.1.0", "rewire": "^7.0.0", - "sinon": "^18.0.0" + "sinon": "^19.0.2" }, "dependencies": { - "@alertlogic/al-collector-js": "3.0.12", - "@alertlogic/paws-collector": "2.2.5", - "async": "^3.2.5", - "debug": "^4.3.6", - "google-auth-library": "^9.13.0", - "googleapis": "^140.0.1", + "@alertlogic/al-collector-js": "3.0.14", + "@alertlogic/paws-collector": "2.2.6", + "async": "^3.2.6", + "debug": "^4.3.7", + "google-auth-library": "^9.14.1", + "googleapis": "^144.0.0", "moment": "2.30.1" }, "author": "Alert Logic Inc." diff --git a/collectors/googlestackdriver/test/test.js b/collectors/googlestackdriver/test/test.js index eab9f07f..be82bc80 100644 --- a/collectors/googlestackdriver/test/test.js +++ b/collectors/googlestackdriver/test/test.js @@ -293,10 +293,8 @@ describe('Unit Tests', function() { const startDate = moment().subtract(3, 'days'); let since = startDate.toISOString(); let until = startDate.add(2, 'days').toISOString(); - const filter = `timestamp >= "${since}" -timestamp < "${until}"`; + const filter = `timestamp >= "${since}" AND timestamp < "${until}"`; let nextPage = { pageToken: 'http://somenextpage.com', "pageSize": 1000, "resourceNames": ["projects/a-fake-project"], filter }; - logginClientStub.callsFake(() => { return new Promise((res, rej) => { res({ @@ -539,4 +537,106 @@ timestamp < "${until}"`; }); }); }); - }); \ No newline at end of file + describe('log filter Tests', function() { + it('should generate correct filter excluding empty logType values', function (done) { + let ctx = { + invokedFunctionArn: googlestackdriverMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + done(); + }, + succeed: function () { + done(); + } + }; + + GooglestackdriverCollector.load().then(function (creds) { + var collector = new GooglestackdriverCollector(ctx, creds, 'googlestackdriver'); + const startDate = moment().subtract(20, 'minutes'); + let since = startDate.toISOString(); + let until = startDate.add(collector.pollInterval, 'seconds').toISOString(); + process.env.paws_collector_param_string_2 = "[\"cloudaudit.googleapis.com%2Factivity\",\"\",\"cloudfunctions.googleapis.com/cloud-functions\"]"; + const curState = { + since: since, + until: until, + poll_interval_sec: 1, + stream: 'projects/imran-49253', + }; + // Expected filter string + const expectedFilter = `timestamp >= "${since}" AND timestamp < "${until}" AND (logName:"cloudaudit.googleapis.com%2Factivity" OR logName:"cloudfunctions.googleapis.com%2Fcloud-functions")`; + + // Call the function to generate the filter + const filter = collector.generateFilter(curState); + assert.equal(expectedFilter, filter); + done(); + }); + }); + + it('should generate filter without logNameFilter when all logTypes are empty', function (done) { + let ctx = { + invokedFunctionArn: googlestackdriverMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + done(); + }, + succeed: function () { + done(); + } + }; + + GooglestackdriverCollector.load().then(function (creds) { + var collector = new GooglestackdriverCollector(ctx, creds, 'googlestackdriver'); + const startDate = moment().subtract(20, 'minutes'); + let since = startDate.toISOString(); + let until = startDate.add(collector.pollInterval, 'seconds').toISOString(); + process.env.paws_collector_param_string_2 = "[\"\",\"\"]"; + const curState = { + since: since, + until: until, + poll_interval_sec: 1, + stream: 'projects/imran-49253', + }; + // Expected filter string + const expectedFilter = `timestamp >= "${since}" AND timestamp < "${until}"`; + + // Call the function to generate the filter + const filter = collector.generateFilter(curState); + assert.equal(expectedFilter, filter); + done(); + }); + }); + it('should handle case when logTypes is undefined or null', function (done) { + let ctx = { + invokedFunctionArn: googlestackdriverMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + done(); + }, + succeed: function () { + done(); + } + }; + + GooglestackdriverCollector.load().then(function (creds) { + var collector = new GooglestackdriverCollector(ctx, creds, 'googlestackdriver'); + const startDate = moment().subtract(20, 'minutes'); + let since = startDate.toISOString(); + let until = startDate.add(collector.pollInterval, 'seconds').toISOString(); + const curState = { + since: since, + until: until, + poll_interval_sec: 1, + stream: 'projects/imran-49253', + }; + // Expected filter string + const expectedFilter = `timestamp >= "${since}" AND timestamp < "${until}"`; + + // Call the function to generate the filter + const filter = collector.generateFilter(curState); + assert.equal(expectedFilter, filter); + done(); + }); + }); + }); +}); + \ No newline at end of file diff --git a/ps_spec.yml b/ps_spec.yml index 8eccd4bd..46346428 100644 --- a/ps_spec.yml +++ b/ps_spec.yml @@ -177,7 +177,7 @@ stages: - ./build_collector.sh googlestackdriver env: ALPS_SERVICE_NAME: "paws-googlestackdriver-collector" - ALPS_SERVICE_VERSION: "1.2.14" #set the value from collector package json + ALPS_SERVICE_VERSION: "1.2.15" #set the value from collector package json outputs: file: ./googlestackdriver-collector* packagers: