From bdfcf646b75dffb72ccc985231adfe5e4e837598 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 7 Jun 2024 11:18:34 -0500 Subject: [PATCH] Vulnerability dashboard: batch `Host` record creation (#19595) Changes: - Updated the `update-reports` script to create new host records in batches. --- ee/vulnerability-dashboard/scripts/update-reports.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ee/vulnerability-dashboard/scripts/update-reports.js b/ee/vulnerability-dashboard/scripts/update-reports.js index 8190194d4400..92128905370d 100644 --- a/ee/vulnerability-dashboard/scripts/update-reports.js +++ b/ee/vulnerability-dashboard/scripts/update-reports.js @@ -386,7 +386,10 @@ module.exports = { sails.log.warn(`Dry run: ${hostRecordsToUpdate.length} hosts will be updated with new information. (Fleet returned them in the API.)`); } else { sails.log(`Creating ${newRecordsForUnrecognizedHosts.length} host records… `); - await Host.createEach(newRecordsForUnrecognizedHosts); + let batchedNewRecordsForUnrecognizedHosts = _.chunk(newRecordsForUnrecognizedHosts, 500); + for(let batch of batchedNewRecordsForUnrecognizedHosts){ + await Host.createEach(batch); + } for(let hostUpdate of hostRecordsToUpdate){ await Host.updateOne({id: hostUpdate.id}).set(_.omit(hostUpdate, 'id')); } @@ -696,7 +699,10 @@ module.exports = { totalNumberOfHostRecordsCreated += newRecordsForUnrecognizedHosts.length; totalNumberOfHostRecordsUpdated += hostRecordsToUpdate.length; sails.log.verbose(`Creating ${newRecordsForUnrecognizedHosts.length} new host records…`); - await Host.createEach(newRecordsForUnrecognizedHosts); + let batchedNewRecordsForUnrecognizedHosts = _.chunk(newRecordsForUnrecognizedHosts, 500); + for(let batch of batchedNewRecordsForUnrecognizedHosts){ + await Host.createEach(batch); + } sails.log.verbose(`Updating ${hostRecordsToUpdate.length} host records…`); for(let hostUpdate of hostRecordsToUpdate){ await Host.updateOne({id: hostUpdate.id}).set(_.omit(hostUpdate, 'id'));