fix: fixed timestamp issue on sqlite mailbox cleanup, fixed migration…

… for CalendarEvents.uid (and others), synced locales, added back in-memory sqlite mapping

app/models/aliases.js

@@ -872,7 +872,7 @@ Aliases.statics.isOverQuota = async function (
   // log fatal error to admins (so they will get notified by email/text)
   if (isOverQuota)
     logger.fatal(
-      new TypeError(
+      new Error(
         `Alias ${alias.id} is over quota (${bytes(storageUsed + size)}/${bytes(
           maxQuotaPerAlias
         )})`

app/views/faq/index.md

@@ -146,20 +146,6 @@ Everything is done in-memory and [our source code is on GitHub](https://github.c
   <span>Less than 10 minutes</span>
 </div>
 
-<div class="alert my-3 alert-danger">
-  <i class="fa fa-stop-circle font-weight-bold"></i>
-  <strong class="font-weight-bold">
-    Enhanced Privacy Protection:
-  </strong>
-  <span>
-    If you would like to hide your information from being publicly searchable over the Internet, then please go to <a class="alert-link" href="/my-account/domains" target="_blank">My Account <i class="fa fa-angle-right"></i> Domains</a> and upgrade your domain to a paid plan before starting this guide.
-    Publicly searchable information on free plans includes, but is not limited to: aliases, forwarded addresses, recipients, and advanced settings such as custom port forwarding.
-    If you would like to learn more about paid plans see our <a class="alert-link" rel="noopener noreferrer" href="/private-business-email">Pricing</a> page &ndash; otherwise keep reading!
-    All plans abide by our <a class="alert-link" href="/privacy">Privacy</a> policy of strictly not storing metadata nor emails.
-    We don't track you like other services do.
-  </span>
-</div>
-
 <div class="alert my-3 alert-success">
   <i class="fa fa-bullhorn font-weight-bold"></i>
   <strong class="font-weight-bold">

config/phrases.js

@@ -174,11 +174,11 @@ module.exports = {
   EMAIL_SMTP_ACCESS_DISABLED:
     '<p class="text-center text-danger">Your domain <span class="notranslate">%s</span> had its outbound SMTP access removed.</p>',
   EMAIL_NEWSLETTER_ACCESS_ENABLED_SUBJECT:
-    '<span class="notranslate">%s</span> approved for NEWSLETTER access',
+    '<span class="notranslate">%s</span> approved for newsletter access',
   EMAIL_NEWSLETTER_ACCESS_ENABLED_MESSAGE:
-    '<p class="text-center">Your domain <span class="notranslate">%s</span> was approved for NEWSLETTER access.</p><p class="text-center mb-0"><a class="btn btn-lg btn-danger" href="%s">Complete Setup</a></p>',
+    '<p class="text-center">Your domain <span class="notranslate">%s</span> was approved for newsletter access.</p><p class="text-center mb-0"><a class="btn btn-lg btn-danger" href="%s">Complete Setup</a></p>',
   EMAIL_NEWSLETTER_ACCESS_DISABLED:
-    '<p class="text-center text-danger">Your domain <span class="notranslate">%s</span> had its NEWSLETTER access removed.</p>',
+    '<p class="text-center text-danger">Your domain <span class="notranslate">%s</span> had its newsletter access removed.</p>',
   EMAIL_SMTP_ACCESS_REQUIRED:
     'Domain is not approved for outbound SMTP access, please <a class="font-weight-bold" href="/help">contact us</a>.',
   ENVELOPE_FROM_MISSING:

helpers/get-database.js

@@ -486,25 +486,24 @@ async function getDatabase(
     ) {
       db = instance.databaseMap.get(alias.id);
       session.db = db;
-      return db;
-    }
-
-    db = new Database(dbFilePath, {
-      readonly,
-      fileMustExist: readonly,
-      timeout: config.busyTimeout,
-      // <https://github.com/WiseLibs/better-sqlite3/issues/217#issuecomment-456535384>
-      verbose: env.AXE_SILENT ? null : console.log
-    });
+    } else {
+      db = new Database(dbFilePath, {
+        readonly,
+        fileMustExist: readonly,
+        timeout: config.busyTimeout,
+        // <https://github.com/WiseLibs/better-sqlite3/issues/217#issuecomment-456535384>
+        verbose: env.AXE_SILENT ? null : console.log
+      });
 
-    // store in-memory open connection
-    if (instance.databaseMap) instance.databaseMap.set(alias.id, db);
+      // store in-memory open connection
+      if (instance.databaseMap) instance.databaseMap.set(alias.id, db);
 
-    await setupPragma(db, session); // takes about 30ms
+      await setupPragma(db, session); // takes about 30ms
 
-    // assigns to session so we can easily re-use
-    // (also used in allocateConnection in IMAP notifier)
-    session.db = db;
+      // assigns to session so we can easily re-use
+      // (also used in allocateConnection in IMAP notifier)
+      session.db = db;
+    }
 
     // if it is readonly then return early
     if (readonly) {
@@ -546,6 +545,7 @@ async function getDatabase(
           'PX',
           ms('1d')
         );
+
         //
         // NOTE: if we change schema on db then we
         //       need to stop sqlite server then
@@ -700,15 +700,15 @@ async function getDatabase(
                   },
                   exp: 1,
                   rdate: {
-                    $lte: Date.now()
+                    $lte: new Date().toISOString()
                   }
                 },
                 {
                   mailbox: {
                     $in: mailboxes.map((m) => m._id.toString())
                   },
                   rdate: {
-                    $lte: dayjs().subtract(30, 'days').toDate().getTime()
+                    $lte: dayjs().subtract(30, 'days').toDate().toISOString()
                   }
                 },
                 {
@@ -770,6 +770,7 @@ async function getDatabase(
 
           const existingHashes = db.prepare(sql.query).pluck().all(sql.values);
 
+          // TODO: this is too slow, it took 1 hour in production
           db.transaction((hashes) => {
             for (const hash of hashes) {
               if (hashSet.has(hash)) continue;
@@ -782,6 +783,7 @@ async function getDatabase(
                   hash
                 }
               });
+
               db.prepare(sql.query).run(sql.values);
             }
           }).immediate(existingHashes);
@@ -855,11 +857,6 @@ async function getDatabase(
       }
     }
 
-    //
-    // TODO: delete orphaned attachments (those without messages that reference them)
-    //       (note this is unlikely as we already take care of this in EXPUNGE)
-    //
-
     return db;
   } catch (err) {
     // in case developers are connected to it in SQLiteStudio (this will cause a read/write error)

helpers/migrate-schema.js

@@ -215,7 +215,22 @@ async function migrateSchema(db, session, tables) {
       columnInfo.map((c) => c.name),
       columnInfo
     );
-    // TODO: drop other columns that we don't need (?)
+
+    //
+    // NOTE: this is a migration for columns removed (e.g. CalendarEvents.uid)
+    //       which previously caused NOT NULL constraint errors
+    //       (and we could have done a workaround to set a default value, e.g. '')
+    //       (however newer version of SQLite - which we use - support dropping columns)
+    //       <https://stackoverflow.com/a/66399224>
+    //
+    for (const key of Object.keys(columnInfoByKey)) {
+      const column = columnInfoByKey[key];
+      if (!column) continue; // safeguard
+      // ensure mapping exists, otherwise remove it
+      if (tables[table].mapping[key]) continue;
+      db.exec(`ALTER TABLE ${table} DROP COLUMN ${key}`);
+    }
+
     for (const key of Object.keys(tables[table].mapping)) {
       const column = columnInfoByKey[key];
       if (!column) {

helpers/on-connect.js

@@ -117,7 +117,9 @@ async function onConnect(session, fn) {
   // NOTE: we return early here because we do not want to limit concurrent connections from allowlisted values
   //
   //
-  if (session.isAllowlisted) return fn();
+  if (session.isAllowlisted) {
+    return fn();
+  }
 
   //
   // NOTE: until onConnect is available for IMAP and POP3 servers
@@ -126,8 +128,9 @@ async function onConnect(session, fn) {
   //       <https://github.com/nodemailer/wildduck/issues/721>
   //       (see this same comment in `helpers/on-auth.js`)
   //
-  if (this.server instanceof IMAPServer || this.server instanceof POP3Server)
+  if (this.server instanceof IMAPServer || this.server instanceof POP3Server) {
     return fn();
+  }
 
   // do not allow more than 10 concurrent connections using constructor
   try {
@@ -143,6 +146,7 @@ async function onConnect(session, fn) {
         `Too many concurrent connections from ${session.remoteAddress}`,
         { responseCode: 421, ignoreHook: true }
       );
+
     fn();
   } catch (err) {
     fn(refineAndLogError(err, session, false, this));

helpers/setup-pragma.js

@@ -129,6 +129,7 @@ async function setupPragma(db, session, cipher = 'chacha20') {
     db.loadExtension(sqliteRegex.getLoadablePath());
   } catch (err) {
     // <https://github.com/asg017/sqlite-regex/issues/14>
+    err.isCodeBug = true;
     logger.fatal(err);
     if (err.message.includes('sqlite-regex-linux-arm64'))
       logger.error(

jobs/paypal/sync-paypal-subscription-payments.js

@@ -24,6 +24,8 @@ async function syncPayPalSubscriptionPayments() {
   //       download the entire TSV/CSV file and then run steps like here:
   //
   //       <https://github.com/paypal/PayPal-REST-API-issues/issues/5>
+  //       (NOTE: PayPal for some reason completely disabled/deleted all issues...)
+  //       (but here's a snapshot from Wayback <https://web.archive.org/web/20201019010837/https://github.com/paypal/PayPal-REST-API-issues/issues/5>)
   //
   const paypalCustomers = await Users.find({
     $or: [