Skip to content

Commit

Permalink
[ENG-52244]Upgrade collectors aws-sdk version from v2 to v3 and rewri…
Browse files Browse the repository at this point in the history 
…te functionality and test cases (#354)

* Upgrade aws-sdk v2 to v3 with callback and fixed the functionality
  • Loading branch information
@rakhimundhada15
rakhimundhada15 authored Nov 6, 2023
1 parent 4a10d4d commit 54b903c
Show file tree
Hide file tree
Showing 4 changed files with 208 additions and 128 deletions.
14 changes: 10 additions & 4 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@alertlogic/paws-collector",
"version": "2.1.21",
"version": "2.2.0",
"license": "MIT",
"description": "Alert Logic AWS based API Poll Log Collector Library",
"repository": {
Expand All @@ -21,7 +21,14 @@
"rel": "npm publish --access=public"
},
"devDependencies": {
"aws-sdk-mock": "*",
"@aws-sdk/client-cloudformation": "^3.423.0",
"@aws-sdk/client-cloudwatch": "^3.423.0",
"@aws-sdk/client-dynamodb": "^3.423.0",
"@aws-sdk/client-kms": "^3.423.0",
"@aws-sdk/client-lambda": "^3.423.0",
"@aws-sdk/client-s3": "^3.423.0",
"@aws-sdk/client-sqs": "^3.423.0",
"@aws-sdk/client-ssm": "^3.423.0",
"clone": "*",
"jshint": "^2.13.6",
"mocha": "^10.2.0",
Expand All @@ -31,8 +38,7 @@
"yargs": "^17.6.2"
},
"dependencies": {
"@alertlogic/al-aws-collector-js": "4.1.22",
"aws-sdk": "*",
"@alertlogic/al-aws-collector-js": "4.1.24",
"async": "^3.2.4",
"datadog-lambda-js": "^6.85.0",
"debug": "^4.3.4",
Expand Down
62 changes: 38 additions & 24 deletions paws_collector.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,21 @@

const async = require('async');
const debug = require('debug')('index');
const AWS = require('aws-sdk');
const {
CloudWatch
} = require("@aws-sdk/client-cloudwatch"),
{
DynamoDB
} = require("@aws-sdk/client-dynamodb"),
{
KMS
} = require("@aws-sdk/client-kms"),
{
SQS
} = require("@aws-sdk/client-sqs"),
{
SSM
} = require("@aws-sdk/client-ssm");
const fs = require('fs');
const moment = require('moment');
const ddLambda = require('datadog-lambda-js');
Expand Down Expand Up @@ -55,7 +69,7 @@ function getPawsParamStoreParam(){
if (process.env.ssm_direct) return resolve(fs.readFileSync(CREDS_FILE_PATH, 'utf-8'));
else return resolve(fs.readFileSync(CREDS_FILE_PATH));
}
var ssm = new AWS.SSM();
var ssm = new SSM();
var params = {
Name: process.env.paws_secret_param_name
};
Expand Down Expand Up @@ -93,7 +107,7 @@ function getDecryptedPawsCredentials(credsBuffer) {
};
return resolve(PAWS_DECRYPTED_CREDS);
} else {
const kms = new AWS.KMS();
const kms = new KMS();
kms.decrypt(
{CiphertextBlob: credsBuffer},
(err, data) => {
Expand All @@ -103,7 +117,7 @@ function getDecryptedPawsCredentials(credsBuffer) {
PAWS_DECRYPTED_CREDS = {
auth_type: process.env.paws_api_auth_type,
client_id: process.env.paws_api_client_id,
secret: data.Plaintext.toString('ascii')
secret: new TextDecoder("utf-8").decode(data.Plaintext)
};

return resolve(PAWS_DECRYPTED_CREDS);
Expand Down Expand Up @@ -244,23 +258,23 @@ class PawsCollector extends AlAwsCollector {

setPawsSecret(secretValue){
const encryptPromise = new Promise((resolve, reject) => {
const kms = new AWS.KMS();
const kms = new KMS();
const params = {
KeyId: process.env.paws_kms_key_arn,
Plaintext: secretValue
Plaintext: new TextEncoder().encode(secretValue)
};
kms.encrypt(params, function(err, data) {
if (err) {
return reject(err, err.stack);
}
const base64 = new Buffer(data.CiphertextBlob).toString('base64');
const base64 = Buffer.from(data.CiphertextBlob).toString('base64');
return resolve(base64);
});
});

return encryptPromise.then((base64) => {
return new Promise((resolve, reject) => {
var ssm = new AWS.SSM();
var ssm = new SSM();
var params = {
Name: process.env.paws_secret_param_name,
Type: 'String',
Expand All @@ -276,7 +290,7 @@ class PawsCollector extends AlAwsCollector {
AlLogger.error('PAWS000300 Error setting new secret', err);
return err;
});
})
});
}

registerPawsCollector(event, callback) {
Expand Down Expand Up @@ -343,7 +357,7 @@ class PawsCollector extends AlAwsCollector {
// there is a lot of logic here. not sur eif there is a better way of deduping. trying for an MVP
checkStateSqsMessage(stateSqsMsg, asyncCallback) {
const collector = this;
const DDB = new AWS.DynamoDB(DDB_OPTIONS);
const DDB = new DynamoDB(DDB_OPTIONS);

const params = {
Key: {
Expand All @@ -354,7 +368,7 @@ class PawsCollector extends AlAwsCollector {
ConsistentRead: true
}

const getItemPromise = DDB.getItem(params).promise();
const getItemPromise = DDB.getItem(params);

getItemPromise.then(data => {
// if the item is alread there, try and see if it is a duplicate
Expand Down Expand Up @@ -402,7 +416,7 @@ class PawsCollector extends AlAwsCollector {

updateStateDBEntry(stateSqsMsg, Status, asyncCallback) {
const collector = this;
const DDB = new AWS.DynamoDB(DDB_OPTIONS);
const DDB = new DynamoDB(DDB_OPTIONS);

const updateParams = {
Key: {
Expand Down Expand Up @@ -608,7 +622,7 @@ class PawsCollector extends AlAwsCollector {
}
reportApiThrottling(callback) {
// TODO: report collector status via Ingest/agentstatus
var cloudwatch = new AWS.CloudWatch({apiVersion: '2010-08-01'});
var cloudwatch = new CloudWatch({apiVersion: '2010-08-01'});
const params = {
MetricData: [
{
Expand Down Expand Up @@ -639,7 +653,7 @@ class PawsCollector extends AlAwsCollector {
* @param callback
*/
reportErrorToIngestApi(error, callback) {
var cloudwatch = new AWS.CloudWatch({apiVersion: '2010-08-01'});
var cloudwatch = new CloudWatch({apiVersion: '2010-08-01'});
const params = {
MetricData: [
{
Expand Down Expand Up @@ -677,7 +691,7 @@ class PawsCollector extends AlAwsCollector {
* @param error
*/
reportClientError(error, callback) {
var cloudwatch = new AWS.CloudWatch({ apiVersion: '2010-08-01' });
var cloudwatch = new CloudWatch({ apiVersion: '2010-08-01' });
const params = {
MetricData: [
{
Expand Down Expand Up @@ -711,7 +725,7 @@ class PawsCollector extends AlAwsCollector {
const delayDuration = moment.duration(nowMoment.diff(lastCollectedMoment));
const collectionDelaySec = Math.floor(delayDuration.asSeconds());

var cloudwatch = new AWS.CloudWatch({apiVersion: '2010-08-01'});
var cloudwatch = new CloudWatch({apiVersion: '2010-08-01'});
const params = {
MetricData: [
{
Expand All @@ -738,7 +752,7 @@ class PawsCollector extends AlAwsCollector {
};

reportDuplicateLogCount(duplicateCount, callback) {
var cloudwatch = new AWS.CloudWatch({ apiVersion: '2010-08-01' });
var cloudwatch = new CloudWatch({ apiVersion: '2010-08-01' });
const params = {
MetricData: [
{
Expand Down Expand Up @@ -770,7 +784,7 @@ class PawsCollector extends AlAwsCollector {
* @returns
*/
reportCollectorStatus(status, callback) {
var cloudwatch = new AWS.CloudWatch({apiVersion: '2010-08-01'});
var cloudwatch = new CloudWatch({apiVersion: '2010-08-01'});
const params = {
MetricData: [
{
Expand Down Expand Up @@ -807,7 +821,7 @@ class PawsCollector extends AlAwsCollector {

_storeCollectionStateArray(pawsState, privCollectorStates, invocationTimeout, callback) {
let collector = this;
var sqs = new AWS.SQS({apiVersion: '2012-11-05'});
var sqs = new SQS({apiVersion: '2012-11-05'});
const nextInvocationTimeout = invocationTimeout ? invocationTimeout : collector.pollInterval;
let SQSMsgs = privCollectorStates.map(function(privState, index) {
let pState = pawsState;
Expand Down Expand Up @@ -846,7 +860,7 @@ class PawsCollector extends AlAwsCollector {

_storeCollectionStateSingle(pawsState, privCollectorState, invocationTimeout, callback) {
let collector = this;
var sqs = new AWS.SQS({apiVersion: '2012-11-05'});
var sqs = new SQS({apiVersion: '2012-11-05'});
const nextInvocationTimeout = invocationTimeout ? invocationTimeout : collector.pollInterval;
pawsState.priv_collector_state = privCollectorState;

Expand Down Expand Up @@ -902,7 +916,7 @@ class PawsCollector extends AlAwsCollector {
*/
removeDuplicatedItem(logs, paramName, asyncCallback) {
let collector = this;
const ddb = new AWS.DynamoDB();
const ddb = new DynamoDB();
let uniqueLogs = [];
var promises = [];
let duplicateCount = 0;
Expand All @@ -921,7 +935,7 @@ class PawsCollector extends AlAwsCollector {
let promise = new Promise((resolve, reject) => {
ddb.putItem(params, (err, res) => {
if (err) {
if (err.code === 'ConditionalCheckFailedException') {
if (err.name === 'ConditionalCheckFailedException') {
duplicateCount++;
return resolve(null);
} else {
Expand Down Expand Up @@ -1024,9 +1038,9 @@ class PawsCollector extends AlAwsCollector {
* @returns
*/
dDBBatchWriteItem(tableName, batch) {
const ddb = new AWS.DynamoDB(DDB_OPTIONS);
const ddb = new DynamoDB(DDB_OPTIONS);
const batchParams = { RequestItems: { [tableName]: batch } };
return ddb.batchWriteItem(batchParams).promise();
return ddb.batchWriteItem(batchParams);
}

/**
Expand Down
14 changes: 14 additions & 0 deletions test/paws_stub.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
const sinon = require('sinon');
const sandbox = sinon.createSandbox();
function mock(service, methodName, callback) {
return sandbox.stub(service.prototype, methodName).callsFake(callback);
}

function restore(serviceName, methodName) {
serviceName.prototype[methodName].restore();
}

module.exports = {
mock: mock,
restore: restore
};
Loading

0 comments on commit 54b903c

Please sign in to comment.