batch logged warnings, fix behavior with regressions

ee/vulnerability-dashboard/scripts/update-reports.js

@@ -17,7 +17,7 @@ module.exports = {
     // [?] Wondering where all the performance notes and commented-out methods of processing API data went?
     // You can find them on this commit -» https://github.com/fleetdm/fleet-vulnerability-dashboard/blob/1c58578c149d97307ae288000c80257b29bb3126/scripts/update-reports.js
     let assert = require('assert');
-
+    let loggedWarningsFromThisScriptRun = [];
     //  ┌─┐┌─┐┌┬┐┬ ┬┌─┐┬─┐  ┌─┐┌┐┌┌┬┐  ┌─┐┬─┐┌─┐┌─┐┌─┐┌─┐┌─┐  ┌─┐┬  ┌─┐┌─┐┌┬┐  ┌─┐┌─┐┬  ┌┬┐┌─┐┌┬┐┌─┐
     //  │ ┬├─┤ │ ├─┤├┤ ├┬┘  ├─┤│││ ││  ├─┘├┬┘│ ││  ├┤ └─┐└─┐  ├┤ │  ├┤ ├┤  │   ├─┤├─┘│   ││├─┤ │ ├─┤
     //  └─┘┴ ┴ ┴ ┴ ┴└─┘┴└─  ┴ ┴┘└┘─┴┘  ┴  ┴└─└─┘└─┘└─┘└─┘└─┘  └  ┴─┘└─┘└─┘ ┴   ┴ ┴┴  ┴  ─┴┘┴ ┴ ┴ ┴ ┴
@@ -66,9 +66,9 @@ module.exports = {
     // Create an array to store the versions of compliant microsoft office software.Add the versions of compliant Microsoft office installs to an array.
     let compliantMicrosoftOfficeVersions = [];
     for(let $vulnInstall of allKnownExistingVulnInstalls) {
-      existingVulnInstallsByHostAndVulnIDs[`${$vulnInstall.fleetApid}|${$vulnInstall.vulnerability}|${$vulnInstall.host}`] = true;// « ex: {'140|56|2146':true, '135|2565|6729':true, ...}
-      // For every unresolved vulnerability, store the database ID of the VulnerabilityInstall record with a unique string as the key.
       if($vulnInstall.uninstalledAt === 0) {
+        // For every unresolved vulnerability, store the database ID of the VulnerabilityInstall record with a unique string as the key.
+        existingVulnInstallsByHostAndVulnIDs[`${$vulnInstall.fleetApid}|${$vulnInstall.vulnerability}|${$vulnInstall.host}`] = true;// « ex: {'140|56|2146':true, '135|2565|6729':true, ...}
         // We'll add the same unique string that we use for the existingVulnInstallsByHostAndVulnIDs dictionary as the key, but the values will be the database ID of the VulnerabilityInstall.
         missingVulnInstallsByIds[`${$vulnInstall.fleetApid}|${$vulnInstall.vulnerability}|${$vulnInstall.host}`] = $vulnInstall.id;// « ex {'123615|1998|4': 8323, '123615|1998|8': 8324, '123615|1998|9': 8325, ...}
       }
@@ -121,7 +121,7 @@ module.exports = {
       if(!os.version) {
         // If an operating system returned in the /os_versions API response is missing a version, we'll log a warning, but we'll still create a record for it with 'N/A' set as the version.
         // This is so we are still able to create host records for hosts with this operating system installed. (The ID of an operating system record is a required value for host records)
-        sails.log.warn(`An operating system (name: ${os.name}) returned in the response from the /os_versions endpoint is missing a 'version'. This operating system will be reported as having "N/A" as the version. Operating system without a version:`, os);
+        loggedWarningsFromThisScriptRun.push(`An operating system (name: ${os.name}) returned in the response from the /os_versions endpoint is missing a 'version'. This operating system will be reported as having "N/A" as the version. Operating system without a version:`, os);
         os.version = 'N/A';// Note: This does not affect how we match hosts to operating system records.
       }
       let osToReport = {
@@ -299,7 +299,7 @@ module.exports = {
             .retry(['requestFailed', {name: 'TimeoutError'}])
             .tolerate({raw:{statusCode: 404}} , (error)=>{
               // If the hosts API returns a 404 response for a software item that was returned from in the list of vulnerable software, we'll log a warning and remove this software from the list of software.
-              sails.log.warn(`When sending a request to the '/api/v1/fleet/hosts' API endpoint to get a filtered array of hosts with ${ware.name} ${ware.version} installed (software ID: ${ware.id}), the Fleet instance returned a 404 response when we expected it to return an array of ${ware.hosts_count} host(s).\n Response from Fleet instance: ${error.raw.body}`);
+              loggedWarningsFromThisScriptRun.push(`When processing vulnerable software, a request to the '/hosts' endpoint to get a filtered array of hosts with ${ware.name} ${ware.version} installed (software ID: ${ware.id}), the Fleet instance returned a 404 response when we expected it to return an array of ${ware.hosts_count} host(s).\n Impact: If this vulnerable software was previously processed, the database record(s) for it will be marked as uninstalled. If it shows up in a future run of this script, a new database record will be created.`);
               vulnerableWaresWithNoHostInformation.push(ware);// Add this software to the vulnerableWaresWithNoHostInformation array, these will be removed before we create and update database records.
               return {};// Return an empty object. This will let the script continue without information about this software.
             });
@@ -496,6 +496,7 @@ module.exports = {
       }//∞
       if (dry) {
         sails.log.warn(`Dry run: ${potentialVulnInstalls.length} potential vulnerability installs are available and ${potentialVulnInstallUpdates.length} updates are available for existing vulnerability installs.`);
+        console.log(potentialVulnInstalls);
       } else {
         sails.log(`Detected ${potentialVulnInstalls.length} changes to software installations and ${potentialVulnInstallUpdates.length} updates are available for existing vulnerability installs.  Saving...`);
         let batchedPotentialVulnInstalls = _.chunk(potentialVulnInstalls, 1000);
@@ -590,7 +591,6 @@ module.exports = {
           });
         }
       }//∫
-
       let hostApidsBySoftwareVersionApid = {};
       await sails.helpers.flow.simultaneouslyForEach(criticalWares, async(ware)=>{
         // Get hosts with this version of software installed.
@@ -609,7 +609,7 @@ module.exports = {
             .retry(['requestFailed', {name: 'TimeoutError'}])
             .tolerate({raw:{statusCode: 404}} , (error)=>{
               // If the hosts API returns a 404 response for a software item that was returned from in the list of critical software, we'll log a warning and remove this software from the list of software.
-              sails.log.warn(`When sending a request to the '/api/v1/fleet/hosts' API endpoint to get a filtered array of hosts with ${ware.name} ${ware.version} installed (software ID: ${ware.id}), the Fleet instance returned a 404 response when we expected it to return an array of ${ware.hosts_count} host(s).\n Response from Fleet instance: ${error.raw.body}`);
+              loggedWarningsFromThisScriptRun.push(`When processing critical software, a request to the '/hosts' endpoint to get a filtered array of hosts with ${ware.name} ${ware.version} installed (software ID: ${ware.id}), the Fleet instance returned a 404 response when we expected it to return an array of ${ware.hosts_count} host(s).\n Impact: This software will be marked as uninstalled, and a new database record will be created if it shows up in a future run of this script.`);
               criticalWaresWithNoHostInformation.push(ware);// Add this software to the criticalWaresWithNoHostInformation array, these will be removed before we create and update database records.
               return {};// Return an empty object. This will let the script continue without information about this software.
             });
@@ -898,6 +898,12 @@ module.exports = {
       sails.log(`${vulnerabilityRecordIdsWithNoAssociatedRecords.length} vulnerabilities affecting previously-enrolled hosts were found. (Fleet did not return them in the API this time, and no associated Host or VulnerabilityInstall records were found.)`);
       await Vulnerability.destroy({id: {in: _.pluck(vulnerabilityRecordIdsWithNoAssociatedRecords, 'id')}});
     }
+    if(loggedWarningsFromThisScriptRun.length > 0) {
+      sails.log.warn(`During this run of the update-reports script ${loggedWarningsFromThisScriptRun.length} warning(s) were logged.`)
+      for(warning of loggedWarningsFromThisScriptRun){
+        sails.log.warn(warning);
+      }
+    }
     // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
     sails.log('Successfully completed scan and processing of',numVulnerableWaresProcessed,'vulnerable software items using Fleet.');
   }