Skip to content

Commit

Permalink
Merge pull request #377 from vseryakov/master
Browse files Browse the repository at this point in the history 
Changed search-filter to regexp and new filter command to work with saved raw data
  • Loading branch information
@iann0036
iann0036 authored Jul 22, 2024
2 parents cdfb0c0 + feee093 commit 0cd878b
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 80 deletions.
21 changes: 21 additions & 0 deletions cli/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ Options:
--output-logical-id-mapping <filename> filename for logical to physical id mapping
--cfn-deletion-policy <Delete|Retain> add DeletionPolicy in CloudFormation output
--search-filter <value> search filter for discovered resources ('or search' can be comma separated, 'and search' can be '&' separated.)
--regex-filter <value> regexp filter for discovered resources to include in the output
--services <value> list of services to include (can be comma separated (default: ALL))
--exclude-services <value> list of services to exclude (can be comma separated)
--sort-output sort resources by their ID before outputting
Expand Down Expand Up @@ -224,6 +225,26 @@ Filtering by whether the JSON responses of the AWS SDK calls contain a specified
former2 generate --output-terraform "tf.hcl" --search-filter "myapp"
```

Generate CloudFormation output for EC2 excluding instances with volumes/ENIs

```
former2 generate --output-cloudformation "cfn.yaml" --services EC2 --regex-filter '"f2type":(?!"(ec2.instance|ec2.volume|ec2.networkinterface))'
```

## filter

The `filter` command will use saved raw data output from previous `generate` run to produce the outputs instead of queryng the cloud every time you need to change the filter.

The use case which inspired this command was to produce EC2 CFN file without instances and volumes because autoscaling groups take care of launching instances.

```
former2 filter \
--output-cloudformation "cloudformation.yml" \
--input-file "debug.json" \
--regex-filter '"f2type":(?!"(ec2.instance|elbv2.loadbalancerlistenercertificate|ec2.volume|ec2.networkinterface))' \
--sort-output
```

## Security

Calls to the AWS service API endpoints are made directly with the JavaScript SDK. Recording data is kept entirely in memory or on local disk and is never sent over the internet or anywhere else. You should take care to remove any sensitive data (passwords etc.) when sharing your generated code/templates with others.
180 changes: 100 additions & 80 deletions cli/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ var resource_tag_cache = {};
const iaclangselect = "typescript";

function $(selector) { return new $obj(selector) }
const isAllIncludes = (arr, target) => arr.every(el => target.includes(el));
$obj = function (selector) { };
$obj.prototype.bootstrapTable = function (action, data) {
if (action == "append") {
Expand Down Expand Up @@ -135,7 +134,59 @@ for (var i=0; i<items.length; i++) {
f2log = function(msg){};
f2trace = function(err){};

async function main(opts) {
function saveOutput(opts) {
if (opts.sortOutput) {
cli_resources = cli_resources.sort((a, b) => (a.f2id > b.f2id) ? 1 : -1);
}

if (opts.outputCloudformation || opts.outputTerraform) {
var output_objects = [], jsonres;

for (var i=0; i<cli_resources.length; i++) {
jsonres = null;
if (opts.searchFilter) {
jsonres = JSON.stringify(cli_resources[i]);
if (opts.searchFilter.includes(",")) {
if (!opts.searchFilter.split(",").some((el) => jsonres.includes(el))) continue;
} else
if (opts.searchFilter.includes("&")) {
if (!opts.searchFilter.split("&").every((el) => jsonres.includes(el))) continue;
} else {
if (!jsonres.includes(opts.searchFilter)) continue;
}
}
if (opts.regexFilter) {
if (!jsonres) jsonres = JSON.stringify(cli_resources[i]);
const ok = opts.regexFilter.test(jsonres);
f2log(`${ok?"":"NOT-"}MATCHED: ${jsonres}`);
if (!ok) continue;
}
output_objects.push({
'id': cli_resources[i].f2id,
'type': cli_resources[i].f2type,
'data': cli_resources[i].f2data,
'region': cli_resources[i].f2region
});
}

var tracked_resources = performF2Mappings(output_objects);
var mapped_outputs = compileOutputs(tracked_resources, opts.cfnDeletionPolicy);

if (opts.outputLogicalIdMapping) {
fs.writeFileSync(opts.outputLogicalIdMapping, JSON.stringify(getLogicalToPhysicalIdMap()))
}

if (opts.outputCloudformation) {
fs.writeFileSync(opts.outputCloudformation, mapped_outputs['cfn']);
}

if (opts.outputTerraform) {
fs.writeFileSync(opts.outputTerraform, mapped_outputs['tf']);
}
}
}

function parseOpts(opts) {
if (!opts.outputRawData && !opts.outputCloudformation && !opts.outputTerraform) {
throw new Error('You must specify an output type');
}
Expand All @@ -146,12 +197,36 @@ async function main(opts) {
f2debug = function(msg){ console.log(Date.now().toString() + ": " + msg); };
}

if (opts.regexFilter) {
opts.regexFilter = new RegExp(opts.regexFilter);
}

if (opts.cfnDeletionPolicy && opts.cfnDeletionPolicy != "Delete" && opts.cfnDeletionPolicy != "Retain") {
throw new Error('You must specify --cfn-deletion-policy value in [Delete, Retain]');
}

outputMapCdk = function(){};
outputMapCdkv2 = function(){};
outputMapTroposphere = function(){};
outputMapPulumi = function(){};
outputMapCdktf = function(){};
if (!opts.outputCloudformation) { outputMapCfn = function(){}; }
if (!opts.outputTerraform) { outputMapTf = function(){}; }

if (opts.includeDefaultResources) {
include_default_resources = true;
}

}

async function main(opts) {

if (opts.profile) {
AWS.config.credentials = new AWS.SharedIniFileCredentials({profile: opts.profile});
if (!opts.region) {
var profiles = AWS.util.getProfilesFromSharedConfig(AWS.util.iniLoader, "");
if (profiles[opts.profile]) opts.region = profiles[opts.profile].region;
}
}

if (AWS.config.region) {
Expand Down Expand Up @@ -187,18 +262,6 @@ async function main(opts) {
sections = sections.filter(val => val); // reindex
}

if (opts.cfnDeletionPolicy && opts.cfnDeletionPolicy != "Delete" && opts.cfnDeletionPolicy != "Retain") {
throw new Error('You must specify --cfn-deletion-policy value in [Delete, Retain]');
}

outputMapCdk = function(){};
outputMapCdkv2 = function(){};
outputMapTroposphere = function(){};
outputMapPulumi = function(){};
outputMapCdktf = function(){};
if (!opts.outputCloudformation) { outputMapCfn = function(){}; }
if (!opts.outputTerraform) { outputMapTf = function(){}; }

const b1 = new cliprogress.SingleBar({
format: _colors.cyan('{bar}') + ' {percentage}% ({value}/{total} services completed)',
barCompleteChar: '\u2588',
Expand Down Expand Up @@ -231,77 +294,12 @@ async function main(opts) {

b1.stop();

if (opts.sortOutput) {
cli_resources = cli_resources.sort((a, b) => (a.f2id > b.f2id) ? 1 : -1);
}

if (opts.outputRawData) {
fs.writeFileSync(opts.outputRawData, JSON.stringify(cli_resources, null, 4));
}

if (opts.outputCloudformation || opts.outputTerraform) {
var output_objects = [];
saveOutput(opts);

for (var i=0; i<cli_resources.length; i++) {
if (opts.searchFilter) {
var jsonres = JSON.stringify(cli_resources[i]);
if (opts.searchFilter.includes(",")) {
for (var searchterm of opts.searchFilter.split(",")) {
if (jsonres.includes(searchterm)) {
output_objects.push({
'id': cli_resources[i].f2id,
'type': cli_resources[i].f2type,
'data': cli_resources[i].f2data,
'region': cli_resources[i].f2region
});
break;
}
}
} else if (opts.searchFilter.includes("&")) {
const searchWords = opts.searchFilter.split("&")
if (isAllIncludes(searchWords, jsonres)) {
output_objects.push({
'id': cli_resources[i].f2id,
'type': cli_resources[i].f2type,
'data': cli_resources[i].f2data,
'region': cli_resources[i].f2region
});
}
} else {
if (jsonres.includes(opts.searchFilter)) {
output_objects.push({
'id': cli_resources[i].f2id,
'type': cli_resources[i].f2type,
'data': cli_resources[i].f2data,
'region': cli_resources[i].f2region
});
}
}
} else {
output_objects.push({
'id': cli_resources[i].f2id,
'type': cli_resources[i].f2type,
'data': cli_resources[i].f2data,
'region': cli_resources[i].f2region
});
}
}

var tracked_resources = performF2Mappings(output_objects);
var mapped_outputs = compileOutputs(tracked_resources, opts.cfnDeletionPolicy);

if(opts.outputLogicalIdMapping) {
fs.writeFileSync(opts.outputLogicalIdMapping, JSON.stringify(getLogicalToPhysicalIdMap()))
}

if (opts.outputCloudformation) {
fs.writeFileSync(opts.outputCloudformation, mapped_outputs['cfn']);
}

if (opts.outputTerraform) {
fs.writeFileSync(opts.outputTerraform, mapped_outputs['tf']);
}
}
}

let validation = false;
Expand All @@ -315,6 +313,7 @@ cliargs
.option('--output-logical-id-mapping <filename>', 'filename for logical to physical id mapping')
.option('--cfn-deletion-policy <Delete|Retain>', 'add DeletionPolicy in CloudFormation output')
.option('--search-filter <value>', 'search filter for discovered resources (can be comma separated)')
.option('--regex-filter <regex>', 'search filter as a RegExp for discovered resources')
.option('--services <value>', 'list of services to include (can be comma separated (default: ALL))')
.option('--exclude-services <value>', 'list of services to exclude (can be comma separated)')
.option('--sort-output', 'sort resources by their ID before outputting')
Expand All @@ -324,6 +323,7 @@ cliargs
.option('--proxy <protocol://host:port>', 'use proxy')
.option('--debug', 'log debugging messages')
.action(async (opts) => {
parseOpts(opts);
// The followings are here to silence Node runtime complaining about event emitter listeners
// due to the number of TLS requests that suddenly go out to AWS APIs. This is harmless here
require('events').EventEmitter.defaultMaxListeners = 1000;
Expand All @@ -337,6 +337,26 @@ cliargs
}
});

cliargs
.command('filter')
.description('load resources from file and writes them to the specified file')
.requiredOption('--input-file <filename>', 'filename with raw data from the generate command')
.option('--output-cloudformation <filename>', 'filename for CloudFormation output')
.option('--output-terraform <filename>', 'filename for Terraform output')
.option('--output-logical-id-mapping <filename>', 'filename for logical to physical id mapping')
.option('--cfn-deletion-policy <Delete|Retain>', 'add DeletionPolicy in CloudFormation output')
.option('--search-filter <value>', 'search filter for discovered resources (can be comma separated)')
.option('--regex-filter <regex>', 'search filter as a RegExp for discovered resources')
.option('--sort-output', 'sort resources by their ID before outputting')
.option('--include-default-resources', 'include default resources such as default VPCs and their subnets')
.option('--debug', 'log debugging messages')
.action((opts) => {
parseOpts(opts);
validation = true;
cli_resources = JSON.parse(fs.readFileSync(opts.inputFile).toString());
saveOutput(opts);
});

cliargs.parse(process.argv);
if (!validation) {
cliargs.help();
Expand Down

0 comments on commit 0cd878b

Please sign in to comment.