fix: switch additional cases from distinct to aggregate

forwardemail · Nov 12, 2024 · e0dc69c · e0dc69c
1 parent 446af27
commit e0dc69c
Show file tree

Hide file tree

Showing 2 changed files with 54 additions and 24 deletions.
diff --git a/jobs/check-smtp-queue-count.js b/jobs/check-smtp-queue-count.js
@@ -46,22 +46,37 @@ graceful.listen();
     // get list of all suspended domains
     // and recently blocked emails to exclude
     const now = new Date();
-    const [suspendedDomainIds, recentlyBlockedIds] = await Promise.all([
-      Domains.distinct('_id', {
-        is_smtp_suspended: true
-      }),
-      Emails.distinct('_id', {
-        updated_at: {
-          $gte: dayjs().subtract(1, 'hour').toDate(),
-          $lte: now
+    const [suspendedDomains, recentlyBlocked] = await Promise.all([
+      Domains.aggregate([
+        { $match: { is_smtp_suspended: true } },
+        { $group: { _id: '$_id' } }
+      ])
+        .allowDiskUse(true)
+        .exec(),
+      Emails.aggregate([
+        {
+          $match: {
+            updated_at: {
+              $gte: dayjs().subtract(1, 'hour').toDate(),
+              $lte: now
+            },
+            has_blocked_hashes: true,
+            blocked_hashes: {
+              $in: getBlockedHashes(env.SMTP_HOST)
+            }
+          }
         },
-        has_blocked_hashes: true,
-        blocked_hashes: {
-          $in: getBlockedHashes(env.SMTP_HOST)
+        {
+          $group: { _id: '$_id' }
         }
-      })
+      ])
+        .allowDiskUse(true)
+        .exec()
     ]);
 
+    const suspendedDomainIds = suspendedDomains.map((v) => v._id);
+    const recentlyBlockedIds = recentlyBlocked.map((v) => v._id);
+
     logger.info('%d suspended domain ids', suspendedDomainIds.length);
 
     logger.info('%d recently blocked ids', recentlyBlockedIds.length);

diff --git a/jobs/send-emails.js b/jobs/send-emails.js
@@ -96,22 +96,37 @@ async function sendEmails() {
   //
   // get list of all suspended domains
   // and recently blocked emails to exclude
-  const [suspendedDomainIds, recentlyBlockedIds] = await Promise.all([
-    Domains.distinct('_id', {
-      is_smtp_suspended: true
-    }),
-    Emails.distinct('_id', {
-      updated_at: {
-        $gte: dayjs().subtract(1, 'hour').toDate(),
-        $lte: now
+  const [suspendedDomains, recentlyBlocked] = await Promise.all([
+    Domains.aggregate([
+      { $match: { is_smtp_suspended: true } },
+      { $group: { _id: '$_id' } }
+    ])
+      .allowDiskUse(true)
+      .exec(),
+    Emails.aggregate([
+      {
+        $match: {
+          updated_at: {
+            $gte: dayjs().subtract(1, 'hour').toDate(),
+            $lte: now
+          },
+          has_blocked_hashes: true,
+          blocked_hashes: {
+            $in: getBlockedHashes(IP_ADDRESS)
+          }
+        }
       },
-      has_blocked_hashes: true,
-      blocked_hashes: {
-        $in: getBlockedHashes(IP_ADDRESS)
+      {
+        $group: { _id: '$_id' }
       }
-    })
+    ])
+      .allowDiskUse(true)
+      .exec()
   ]);
 
+  const suspendedDomainIds = suspendedDomains.map((v) => v._id);
+  const recentlyBlockedIds = recentlyBlocked.map((v) => v._id);
+
   logger.info('%d suspended domain ids', suspendedDomainIds.length);
 
   logger.info('%d recently blocked ids', recentlyBlockedIds.length);