fix: fixed SQLite too many variables issue (when trash check occurrin…

…g), fixed response.body.dump function exception (was undefined in some cases and throwing error), ensure auto vacuum set up properly with automatic emails to customers alerting them once started and once complete (once per week, but should not occur more than once since this is a permanent setting pragma on the db), fixed issue with worker thread exception related to node lib curl package not supporting worker threads, added additional yahoo domains to list for historical purposes, ensured temp_store is set to 1 (which ensures that SQLite WSS server does not run out of memory)

app/models/domains.js

@@ -1358,7 +1358,11 @@ async function verifySMTP(domain, resolver, purgeCache = true) {
             retries: 1
           });
           // consume body
-          if (!response?.signal?.aborted) await response.body.dump();
+          if (
+            !response?.signal?.aborted &&
+            typeof response?.body?.dump === 'function'
+          )
+            await response.body.dump();
         },
         { concurrency }
       );
@@ -1572,7 +1576,11 @@ async function getVerificationResults(domain, resolver, purgeCache = false) {
             retries: 1
           });
           // consume body
-          if (!response?.signal?.aborted) await response.body.dump();
+          if (
+            !response?.signal?.aborted &&
+            typeof response?.body?.dump === 'function'
+          )
+            await response.body.dump();
         },
         { concurrency }
       );

config/phrases.js

@@ -20,6 +20,13 @@ for (const key of Object.keys(statuses.message)) {
 }
 
 module.exports = {
+  IMAP_VACUUM_STARTED_SUBJECT:
+    'Mailbox temporarily unavailable for automatic optimization',
+  IMAP_VACUUM_STARTED_MESSAGE:
+    'Your database is being automatically optimized via SQLite VACUUM in order to ensure "auto_vacuum=FULL" pragma is set properly on your mailbox.  This operation will take roughly 1 minute per GB of storage you currently use on this alias.  It is necessary to perform this operation in order to keep your database size optimized over time as you read and write from your mailbox.  Once completed you will receive an email confirmation.',
+  IMAP_VACUUM_COMPLETE_SUBJECT: 'Mailbox optimization completed',
+  IMAP_VACUUM_COMPLETE_MESSAGE:
+    'Your database optimization process has been completed.  The "auto_vacuum=FULL" pragma has been set properly on your SQLite database mailbox and over time your database file size will be optimized.',
   INVALID_BYTES:
     'Bytes were invalid, must be a string such as "1 GB" or "100 MB".',
   ALIAS_QUOTA_EXCEEDS_DOMAIN:

helpers/get-apn-certs.js

@@ -19,6 +19,7 @@ const crypto = require('node:crypto');
 const fs = require('node:fs');
 const path = require('node:path');
 const { Buffer } = require('node:buffer');
+const { isMainThread } = require('node:worker_threads');
 
 const X509 = require('@peculiar/x509');
 const _ = require('lodash');
@@ -34,7 +35,7 @@ const logger = require('#helpers/logger');
 
 // <https://github.com/JCMais/node-libcurl/issues/414>
 let Curl;
-if (config.env !== 'test') Curl = require('node-libcurl').Curl;
+if (config.env !== 'test' && isMainThread) Curl = require('node-libcurl').Curl;
 
 X509.cryptoProvider.set(crypto);
 

helpers/get-database.js

@@ -13,11 +13,10 @@ const isSANB = require('is-string-and-not-blank');
 const mongoose = require('mongoose');
 const ms = require('ms');
 const pRetry = require('p-retry');
-const pify = require('pify');
+const parseErr = require('parse-err');
 const { Builder } = require('json-sql');
 const { boolean } = require('boolean');
 
-const parseErr = require('parse-err');
 const Aliases = require('#models/aliases');
 const Attachments = require('#models/attachments');
 const CalendarEvents = require('#models/calendar-events');
@@ -30,16 +29,14 @@ const config = require('#config');
 const email = require('#helpers/email');
 const env = require('#config/env');
 const getPathToDatabase = require('#helpers/get-path-to-database');
+const i18n = require('#helpers/i18n');
 const isRetryableError = require('#helpers/is-retryable-error');
 const isValidPassword = require('#helpers/is-valid-password');
 const logger = require('#helpers/logger');
 const migrateSchema = require('#helpers/migrate-schema');
-const onExpunge = require('#helpers/imap/on-expunge');
 const setupPragma = require('#helpers/setup-pragma');
 const { decrypt } = require('#helpers/encrypt-decrypt');
 
-const onExpungePromise = pify(onExpunge, { multiArgs: true });
-
 const builder = new Builder();
 
 const HOSTNAME = os.hostname();
@@ -599,19 +596,22 @@ async function getDatabase(
     let folderCheck = !instance.server;
     let trashCheck = !instance.server;
     let threadCheck = !instance.server;
+    let vacuumCheck = !instance.server;
 
     if (instance.client && instance.server) {
       try {
         const results = await instance.client.mget([
           `migrate_check:${session.user.alias_id}`,
           `folder_check:${session.user.alias_id}`,
           `trash_check:${session.user.alias_id}`,
-          `thread_check:${session.user.alias_id}`
+          `thread_check:${session.user.alias_id}`,
+          `vacuum_check:${session.user.alias_id}`
         ]);
         migrateCheck = boolean(results[0]);
         folderCheck = boolean(results[1]);
         trashCheck = boolean(results[2]);
         threadCheck = boolean(results[3]);
+        vacuumCheck = boolean(results[4]);
       } catch (err) {
         logger.fatal(err);
       }
@@ -765,7 +765,7 @@ async function getDatabase(
           throw new TypeError('Trash folder(s) do not exist');
 
         const sql = builder.build({
-          type: 'update',
+          type: 'remove',
           table: 'Messages',
           condition: {
             $or: [
@@ -785,28 +785,20 @@ async function getDatabase(
                 rdate: {
                   $lte: dayjs().subtract(30, 'days').toDate().getTime()
                 }
+              },
+              {
+                mailbox: {
+                  $in: mailboxes.map((m) => m._id.toString())
+                },
+                undeleted: 0
               }
             ]
-          },
-          modifier: {
-            $set: {
-              undeleted: false
-            }
           }
         });
 
         db.prepare(sql.query).run(sql.values);
 
-        await Promise.all(
-          mailboxes.map((mailbox) =>
-            onExpungePromise.call(
-              instance,
-              mailbox._id.toString(),
-              { silent: true },
-              session
-            )
-          )
-        );
+        // TODO: wss broadcast changes here to connected clients
 
         await instance.client.set(
           `trash_check:${session.user.alias_id}`,
@@ -855,8 +847,86 @@ async function getDatabase(
       }
     }
 
-    if (!migrateCheck || !folderCheck || !trashCheck || !threadCheck) {
+    if (
+      !migrateCheck ||
+      !folderCheck ||
+      !trashCheck ||
+      !threadCheck ||
+      !vacuumCheck
+    ) {
       try {
+        //
+        // Ensure that auto vacuum is enabled
+        // (otherwise we email the user that this operation is taking place)
+        //
+        const hasAutoVacuum = db.pragma('auto_vacuum', { simple: true }) === 1;
+        if (!hasAutoVacuum) {
+          // get latest from cache in case another connection started a vacuum
+          vacuumCheck = boolean(
+            await instance.client.get(`vacuum_check:${session.user.alias_id}`)
+          );
+          if (!vacuumCheck) {
+            // only once per week should we attempt this
+            await instance.client.set(
+              `vacuum_check:${session.user.alias_id}`,
+              true,
+              'PX',
+              ms('7d')
+            );
+
+            //
+            // email user it's taking place
+            //
+            email({
+              template: 'alert',
+              message: {
+                to: session.user.username,
+                bcc: config.email.message.from,
+                subject: i18n.translate(
+                  'IMAP_VACUUM_STARTED_SUBJECT',
+                  session.user.locale
+                )
+              },
+              locals: {
+                message: i18n.translate(
+                  'IMAP_VACUUM_STARTED_MESSAGE',
+                  session.user.locale
+                ),
+                locale: session.user.locale
+              }
+            })
+              .then()
+              .catch((err) => logger.fatal(err));
+
+            db.pragma('auto_vacuum=FULL');
+            db.prepare('VACUUM').run();
+
+            //
+            // email user once it's complete
+            //
+            email({
+              template: 'alert',
+              message: {
+                to: session.user.username,
+                bcc: config.email.message.from,
+                subject: i18n.translate(
+                  'IMAP_VACUUM_COMPLETE_SUBJECT',
+                  session.user.locale
+                )
+              },
+              locals: {
+                message: i18n.translate(
+                  'IMAP_VACUUM_COMPLETE_MESSAGE',
+                  session.user.locale
+                ),
+                locale: session.user.locale
+              }
+            })
+              .then()
+              .catch((err) => logger.fatal(err));
+          }
+        }
+
         //
         // All applications should run "PRAGMA optimize;" after a schema change,
         // especially after one or more CREATE INDEX statements.

helpers/is-arbitrary.js

@@ -34,9 +34,22 @@ const REGEX_SYSADMIN_SUBJECT = new RE2(
 const YAHOO_DOMAINS = new Set([
   'yahoo.com',
   'aol.com',
+  'cox.net',
+  'ymail.com',
+  'yahoo.it',
+  'rogers.com',
+  'yahoo.es',
+  'yahoo.co.nz',
+  'netscape.net',
+  'aim.com',
+  'epix.net',
+  'yahoo.dk',
+  'yahoo.com.ph',
+  'yahoo.com.hk',
   'verizon.net',
   'yahoo.de',
   'yahoo.ca',
+  'yahoo.gr',
   'rocketmail.com',
   'yahoo.com.mx',
   'yahoo.co.in',

helpers/process-email.js

@@ -1293,7 +1293,11 @@ async function processEmail({ email, port = 25, resolver, client }) {
           retries: 1
         });
         // consume body
-        if (!response?.signal?.aborted) await response.body.dump();
+        if (
+          !response?.signal?.aborted &&
+          typeof response?.body?.dump === 'function'
+        )
+          await response.body.dump();
       })
         .then()
         .catch(async (err) => {

helpers/refine-and-log-error.js

@@ -121,9 +121,6 @@ function refineAndLogError(err, session, isIMAP = false, instance) {
         err.imapResponse = 'UNAVAILABLE';
     }
 
-    // TODO: we could arbitrarily render alerts if we updated wildduck
-    // else if (!err.imapResponse) err.imapResponse = 'ALERT';
-
     //
     // NOTE: do not set `err.response` here since WildDuck uses it internally
     //       (e.g. NO or BAD must be value of err.response for commands like AUTHENTICATE PLAIN

helpers/setup-pragma.js

@@ -151,6 +151,10 @@ async function setupPragma(db, session, cipher = 'chacha20') {
   // <https://github.com/m4heshd/better-sqlite3-multiple-ciphers/blob/master/docs/api.md#loadextensionpath-entrypoint---this>
   // db.loadExtension(...);
 
+  // ensure we don't use memory and instead use disk for tmp storage
+  // (otherwise 10 GB sqlite file will take up +10 GB memory)
+  db.pragma('temp_store=1;');
+
   //
   // NOTE: if we don't set this then we get the following error for VACUUM commands:
   //

helpers/validate-domain.js

@@ -26,7 +26,8 @@ function validateDomain(domain, domainName) {
       i18n.translate(
         'EMAIL_SMTP_GLOBAL_NOT_PERMITTED',
         i18n.config.defaultLocale
-      )
+      ),
+      { responseCode: 535, ignoreHook: true }
     );
 
   //
@@ -49,7 +50,8 @@ function validateDomain(domain, domainName) {
     )
   )
     throw new SMTPError(
-      i18n.translate('PAST_DUE_OR_INVALID_ADMIN', i18n.config.defaultLocale)
+      i18n.translate('PAST_DUE_OR_INVALID_ADMIN', i18n.config.defaultLocale),
+      { responseCode: 535, ignoreHook: true }
     );
 }
 

helpers/worker.js

@@ -851,6 +851,10 @@ async function backup(payload) {
     await logger.fatal(err, { payload });
   }
 
+  //
+  // NOTE: this was commented out because auto_vacuum wasn't enabled properly
+  //
+  /*
   //
   // NOTE: if the SQLite file is 2x larger than the backup, then we
   //       should run a VACUUM since auto vacuum isn't optimal
@@ -911,6 +915,7 @@ async function backup(payload) {
       await logger.fatal(_err, { payload });
     }
   }
+  */
 
   // always do cleanup in case of errors
   if (tmp && backup) {

locales/ar.json

@@ -10340,5 +10340,9 @@
   "Alias password change (rekey) for <span class=\"notranslate\">%s</span> is complete": "تم الانتهاء من تغيير كلمة المرور (إعادة التشفير) للاسم <span class=\"notranslate\">%s</span>",
   "Alias password change (rekey) is now complete.  You can now log in to IMAP, POP3, and CalDAV servers with the new password for <span class=\"notranslate font-weight-bold text-monospace\">%s</span>.": "تم الآن الانتهاء من تغيير كلمة مرور الاسم المستعار (إعادة التشفير). يمكنك الآن تسجيل الدخول إلى خوادم IMAP وPOP3 وCalDAV باستخدام كلمة المرور الجديدة لـ <span class=\"notranslate font-weight-bold text-monospace\">%s</span> .",
   "Alias password change (rekey) for <span class=\"notranslate\">%s</span> has failed due to an error": "فشلت عملية تغيير كلمة المرور المستعارة (إعادة التشفير) لـ <span class=\"notranslate\">%s</span> بسبب خطأ",
-  "<p>The alias password change (rekey) for <span class=\"notranslate text-monospace font-weight-bold\">%s</span> has failed and we have been alerted.</p><p>You may proceed to retry if necessary, and we may email you soon to provide help if necessary.</p><p>The error received during the rekey process was:</p><pre><code>%s</code></pre>": "<p style=\";text-align:right;direction:rtl\">فشلت عملية تغيير كلمة المرور المستعارة (إعادة التشفير) لـ <span class=\"notranslate text-monospace font-weight-bold\">%s</span> وتم تنبيهنا بذلك.</p><p style=\";text-align:right;direction:rtl\"> يمكنك إعادة المحاولة إذا لزم الأمر، وقد نرسل إليك بريدًا إلكترونيًا قريبًا لتقديم المساعدة إذا لزم الأمر.</p><p style=\";text-align:right;direction:rtl\"> الخطأ الذي تم تلقيه أثناء عملية إعادة المفتاح كان:</p><pre style=\";text-align:right;direction:rtl\"> <code>%s</code></pre>"
+  "<p>The alias password change (rekey) for <span class=\"notranslate text-monospace font-weight-bold\">%s</span> has failed and we have been alerted.</p><p>You may proceed to retry if necessary, and we may email you soon to provide help if necessary.</p><p>The error received during the rekey process was:</p><pre><code>%s</code></pre>": "<p style=\";text-align:right;direction:rtl\">فشلت عملية تغيير كلمة المرور المستعارة (إعادة التشفير) لـ <span class=\"notranslate text-monospace font-weight-bold\">%s</span> وتم تنبيهنا بذلك.</p><p style=\";text-align:right;direction:rtl\"> يمكنك إعادة المحاولة إذا لزم الأمر، وقد نرسل إليك بريدًا إلكترونيًا قريبًا لتقديم المساعدة إذا لزم الأمر.</p><p style=\";text-align:right;direction:rtl\"> الخطأ الذي تم تلقيه أثناء عملية إعادة المفتاح كان:</p><pre style=\";text-align:right;direction:rtl\"> <code>%s</code></pre>",
+  "Mailbox temporarily unavailable for automatic optimization": "صندوق البريد غير متاح مؤقتًا للتحسين التلقائي",
+  "Your database is being automatically optimized via SQLite VACUUM in order to ensure \"auto_vacuum=FULL\" pragma is set properly on your mailbox.  This operation will take roughly 1 minute per GB of storage you currently use on this alias.  It is necessary to perform this operation in order to keep your database size optimized over time as you read and write from your mailbox.  Once completed you will receive an email confirmation.": "يتم تحسين قاعدة البيانات الخاصة بك تلقائيًا عبر SQLite VACUUM لضمان ضبط pragma \"auto_vacuum=FULL\" بشكل صحيح على صندوق البريد الخاص بك. ستستغرق هذه العملية دقيقة واحدة تقريبًا لكل جيجابايت من مساحة التخزين التي تستخدمها حاليًا على هذا الاسم المستعار. من الضروري إجراء هذه العملية للحفاظ على تحسين حجم قاعدة البيانات الخاصة بك بمرور الوقت أثناء القراءة والكتابة من صندوق البريد الخاص بك. بمجرد الانتهاء، ستتلقى رسالة تأكيد عبر البريد الإلكتروني.",
+  "Mailbox optimization completed": "تم الانتهاء من تحسين صندوق البريد",
+  "Your database optimization process has been completed.  The \"auto_vacuum=FULL\" pragma has been set properly on your SQLite database mailbox and over time your database file size will be optimized.": "لقد اكتملت عملية تحسين قاعدة البيانات الخاصة بك. تم ضبط pragma \"auto_vacuum=FULL\" بشكل صحيح على صندوق بريد قاعدة بيانات SQLite الخاص بك، وبمرور الوقت سيتم تحسين حجم ملف قاعدة البيانات الخاصة بك."
 }