diff --git a/app/controllers/api/v1/stripe.js b/app/controllers/api/v1/stripe.js index 456e90c17f..397bf15b60 100644 --- a/app/controllers/api/v1/stripe.js +++ b/app/controllers/api/v1/stripe.js @@ -12,13 +12,15 @@ const humanize = require('humanize-string'); const isSANB = require('is-string-and-not-blank'); const ms = require('ms'); const parseErr = require('parse-err'); +const pMapSeries = require('p-map-series'); const titleize = require('titleize'); const { Users, Domains } = require('#models'); const config = require('#config'); const env = require('#config/env'); -const syncStripePaymentIntent = require('#helpers/sync-stripe-payment-intent'); const emailHelper = require('#helpers/email'); +const logger = require('#helpers/logger'); +const syncStripePaymentIntent = require('#helpers/sync-stripe-payment-intent'); const stripe = new Stripe(env.STRIPE_SECRET_KEY); const { STRIPE_PRODUCTS } = config.payments; @@ -30,6 +32,107 @@ async function processEvent(ctx, event) { // // switch (event.type) { + // + // NOTE: due to unprecedented Stripe credit card fraud (which Stripe has refused to help mitigate) + // we've implemented our own logic here to prevent fraud (user's doing client-side attacks with generated numbers) + // + // + // prevent fraud by checking for users with 5+ failed charges in < 30 days + // with zero verified domains on their account and/or unverified email address + // ban user and notify admins, and refund all other charges from them + // + case 'charge.failed': { + // exit early if it wasn't a charge failure + if (event?.data?.object?.object !== 'charge') break; + if (typeof event?.data?.object?.customer !== 'string') + throw new Error('Charge did not have customer'); + const user = await Users.findOne({ + [config.userFields.stripeCustomerID]: event.data.object.customer + }); + if (!user) throw new Error('User did not exist for customer'); + // + const charges = await stripe.charges.list({ + customer: event.data.object.customer, + created: { + gte: dayjs().subtract(1, 'month').unix() // only search last 30 days to prevent false positives + } + }); + + const filtered = charges.data.filter( + (d) => d.status === 'failed' && d.failure_code === 'card_declined' + ); + + // if not more than 5 then return early + if (filtered.length < 5) break; + + // TODO: we may want to use payment methods count here too instead of just failed charges + // (see `jobs/stripe/fraud-check.js` which uses this approach on a recurring basis) + + // if user had verified domains then alert admins + // otherwise ban the user and refund all their payments + const count = await Domains.countDocuments({ + members: { + $elemMatch: { + user: user._id, + group: 'admin' + } + }, + plan: { $in: ['enhanced_protection', 'team'] }, + has_txt_record: true + }); + + const subject = `${user.email} - ${event.data.object.customer} - ${filtered.length} declined charges and ${count} verified domains`; + + emailHelper({ + template: 'alert', + message: { + to: config.email.message.from, + subject: `${ + count > 0 + ? 'Potential Fraud to Investigate' + : 'Banned User for Fraud Alert' + }: ${subject}` + }, + locals: { + message: `

Review Stripe Customer

` + } + }) + .then() + .catch((err) => logger.fatal(err)); + + if (count === 0) { + user.is_banned = true; + await user.save(); + + const [charges, subscriptions] = await Promise.all([ + stripe.charges.list({ + customer: event.data.object.customer + }), + stripe.subscriptions.list({ + customer: event.data.object.customer + }) + ]); + + // refund all payments as fraudulent + if (charges?.data?.length > 0) + await pMapSeries(charges.data, async (charge) => { + if (charge.status !== 'succeeded' || charge.paid !== true) return; + await stripe.refunds.create({ + charge: charge.id + }); + }); + + // cancel all subscriptions + if (subscriptions?.data?.length > 0) + await pMapSeries(subscriptions.data, async (subscription) => { + if (subscription.status !== 'canceled') return; + await stripe.subscriptions.cancel(subscription.id); + }); + } + + break; + } + // create or update existing payment // (we may also want to upgrade plan; e.g. in case redirect does not occur) // (also need to ensure no conflicts with redirect) @@ -387,18 +490,37 @@ async function processEvent(ctx, event) { // event.data.object is a subscription object if (event.data.object.object !== 'subscription') throw new Error('Event object was not a subscription'); - const subscription = event.data.object; - if (['active', 'trialing'].includes(subscription.status)) + if (['active', 'trialing'].includes(event.data.object.status)) await Users.findOneAndUpdate( { - [config.userFields.stripeCustomerID]: subscription.customer + [config.userFields.stripeCustomerID]: event.data.object.customer }, { $set: { - [config.userFields.stripeSubscriptionID]: subscription.id + [config.userFields.stripeSubscriptionID]: event.data.object.id } } ); + // if user had more than one subscription then notify admins by email + const subscriptions = await stripe.subscriptions.list({ + customer: event.data.object.customer + }); + const filtered = subscriptions.filter((s) => s.status !== 'canceled'); + if (filtered.length > 1) { + emailHelper({ + template: 'alert', + message: { + to: config.email.message.from, + subject: `Multiple Subscriptions Detected: ${event.data.object.customer}` + }, + locals: { + message: `

Review Stripe Customer

` + } + }) + .then() + .catch((err) => logger.fatal(err)); + } + break; } diff --git a/app/models/users.js b/app/models/users.js index 1466a3acc8..c1f0010b42 100644 --- a/app/models/users.js +++ b/app/models/users.js @@ -490,10 +490,6 @@ Users.pre('validate', async function (next) { Users.pre('save', async function (next) { const user = this; - // arbitrary block due to stripe spam unresolved in november 2024 - if (typeof user.email === 'string' && user.email.startsWith('hbrzi')) - return next(new Error('Try again later')); - // If user has a paid plan then consider their email verified if (user.plan !== 'free') user[config.userFields.hasVerifiedEmail] = true; diff --git a/helpers/get-bounce-info.js b/helpers/get-bounce-info.js index 2de5a06b61..c072f44bbb 100644 --- a/helpers/get-bounce-info.js +++ b/helpers/get-bounce-info.js @@ -47,7 +47,7 @@ function getBounceInfo(err) { if ( bounceInfo.message === 'Unknown' || (bounceInfo.action === 'reject' && - ['blocklist', 'policy', 'message', 'block', 'other'].includes( + ['blocklist', 'envelope', 'policy', 'message', 'block', 'other'].includes( bounceInfo.category )) ) { @@ -68,7 +68,9 @@ function getBounceInfo(err) { // } - if (response.includes('Connection dropped due to SocketError')) { + if (response.includes('Comcast block for spam')) { + bounceInfo.category = 'blocklist'; + } else if (response.includes('Connection dropped due to SocketError')) { // modify message to include URL for debugging err.message += ' ; Resolve this issue by visiting https://learn.microsoft.com/en-us/exchange/troubleshoot/email-delivery/send-receive-emails-socketerror#cause ;'; diff --git a/helpers/parse-payload.js b/helpers/parse-payload.js index 76c552def1..bef5f0cd9e 100644 --- a/helpers/parse-payload.js +++ b/helpers/parse-payload.js @@ -22,7 +22,7 @@ const isFQDN = require('is-fqdn'); const isSANB = require('is-string-and-not-blank'); const mongoose = require('mongoose'); const ms = require('ms'); -const pEvent = require('p-event'); +// const pEvent = require('p-event'); const pMap = require('p-map'); const parseErr = require('parse-err'); const pify = require('pify'); @@ -889,6 +889,81 @@ async function parsePayload(data, ws) { )} was available` ); + // we should only use in-memory database is if was connected (IMAP session open) + if ( + this.databaseMap && + this.databaseMap.has(session.user.alias_id) && + this.databaseMap.get(session.user.alias_id).open === true + ) + session.db = this.databaseMap.get(session.user.alias_id); + + if (session.db) { + try { + // since we use onAppend it re-uses addEntries + // which notifies all connected imap users via EXISTS + await onAppendPromise.call( + this, + 'INBOX', + [], + _.isDate(payload.date) + ? payload.date + : new Date(payload.date), + payload.raw, + { + user: { + ...session.user + // NOTE: we don't have the password since we're using in-memory mapping + // password: user.password + }, + db: session.db, + remoteAddress: payload.remoteAddress, + resolvedRootClientHostname: + payload.resolvedRootClientHostname, + resolvedClientHostname: payload.resolvedClientHostname, + allowlistValue: payload.allowlistValue, + + // don't emit wss.broadcast + selected: false, + + // don't append duplicate messages + checkForExisting: true + } + ); + + // + // increase rate limiting size and count + // + try { + await increaseRateLimiting( + this.client, + date, + sender, + root, + byteLength + ); + } catch (err) { + err.isCodeBug = true; + err.payload = _.omit(payload, 'raw'); + logger.fatal(err); + } + } catch (_err) { + // in order to ensure tmp write still occurs + delete session.db; + + const err = Array.isArray(_err) ? _err[0] : _err; + if (isRetryableError(err)) { + err.isCodeBug = true; + err.payload = _.omit(payload, 'raw'); + logger.error(err); + } else { + err.isCodeBug = true; + err.payload = _.omit(payload, 'raw'); + logger.error(err); + } + } + } + + /* // // attempt to get in-memory password from IMAP servers // @@ -970,11 +1045,13 @@ async function parsePayload(data, ws) { logger.error(err); } } + */ // // fallback to writing to temporary database storage // - if (fallback) { + // if (fallback) + if (!session.db) { const tmpDb = await getTemporaryDatabase.call(this, session); let err; diff --git a/helpers/worker.js b/helpers/worker.js index 5e98f73072..974dffd048 100644 --- a/helpers/worker.js +++ b/helpers/worker.js @@ -717,7 +717,6 @@ async function backup(payload) { stream.end(); } - archive.finalize(); archive.on('warning', (err) => { logger.warn(err); }); @@ -725,6 +724,7 @@ async function backup(payload) { archive.once('error', reject); archive.once('end', resolve); }); + archive.finalize(); break; } @@ -787,7 +787,6 @@ async function backup(payload) { } } - archive.finalize(); archive.on('warning', (err) => { logger.warn(err); }); @@ -795,6 +794,7 @@ async function backup(payload) { archive.on('error', reject); archive.on('end', resolve); }); + archive.finalize(); break; } // No default diff --git a/imap-server.js b/imap-server.js index 4a9de2ccf2..acc60e0615 100644 --- a/imap-server.js +++ b/imap-server.js @@ -24,7 +24,6 @@ const pRetry = require('p-retry'); const pWaitFor = require('p-wait-for'); const pify = require('pify'); const ms = require('ms'); -const safeStringify = require('fast-safe-stringify'); const { IMAPServer } = require('wildduck/imap-core'); const Aliases = require('#models/aliases'); @@ -270,7 +269,7 @@ class IMAP { this.subscriber.on('message', async (channel, id) => { if ( - channel !== 'sqlite_auth_request' && + // channel !== 'sqlite_auth_request' && channel !== 'sqlite_auth_reset' && channel !== 'pgp_reload' ) @@ -290,6 +289,7 @@ class IMAP { return; } + /* if (channel === 'sqlite_auth_request') { for (const connection of this.server.connections) { if (connection?.session?.user?.alias_id === id) { @@ -304,6 +304,7 @@ class IMAP { return; } + */ if (channel === 'pgp_reload') { const alias = await Aliases.findOne({ id }) @@ -374,13 +375,13 @@ class IMAP { } async listen(port = env.IMAP_PORT, host = '::', ...args) { - this.subscriber.subscribe('sqlite_auth_request'); + // this.subscriber.subscribe('sqlite_auth_request'); this.subscriber.subscribe('sqlite_auth_reset'); await pify(this.server.listen).bind(this.server)(port, host, ...args); } async close() { - this.subscriber.unsubscribe('sqlite_auth_request'); + // this.subscriber.unsubscribe('sqlite_auth_request'); this.subscriber.unsubscribe('sqlite_auth_reset'); await pify(this.server.close).bind(this.server)(); } diff --git a/jobs/check-smtp-frozen-queue.js b/jobs/check-smtp-frozen-queue.js index 96bacdac56..2d41842394 100644 --- a/jobs/check-smtp-frozen-queue.js +++ b/jobs/check-smtp-frozen-queue.js @@ -45,22 +45,35 @@ graceful.listen(); // get list of all suspended domains // and recently blocked emails to exclude const now = new Date(); - const [suspendedDomainIds, recentlyBlockedIds] = await Promise.all([ - Domains.distinct('_id', { - is_smtp_suspended: true - }), - Emails.distinct('_id', { - updated_at: { - $gte: dayjs().subtract(1, 'hour').toDate(), - $lte: now + let [suspendedDomainIds, recentlyBlockedIds] = await Promise.all([ + Domains.aggregate([ + { $match: { is_smtp_suspended: true } }, + { $group: { _id: '$_id' } } + ]) + .allowDiskUse(true) + .exec(), + Emails.aggregate([ + { + $match: { + updated_at: { + $gte: dayjs().subtract(1, 'hour').toDate(), + $lte: now + }, + has_blocked_hashes: true, + blocked_hashes: { + $in: getBlockedHashes(env.SMTP_HOST) + } + } }, - has_blocked_hashes: true, - blocked_hashes: { - $in: getBlockedHashes(env.SMTP_HOST) - } - }) + { $group: { _id: '$_id' } } + ]) + .allowDiskUse(true) + .exec() ]); + suspendedDomainIds = suspendedDomainIds.map((v) => v._id); + recentlyBlockedIds = recentlyBlockedIds.map((v) => v._id); + logger.info('%d suspended domain ids', suspendedDomainIds.length); logger.info('%d recently blocked ids', recentlyBlockedIds.length); @@ -84,7 +97,13 @@ graceful.listen(); } }; - const ids = await Emails.distinct('id', query); + let ids = await Emails.aggregate([ + { $match: query }, + { $group: { _id: '$id' } } + ]) + .allowDiskUse(true) + .exec(); + ids = ids.map((v) => v.id); // if no ids then return early if (ids.length === 0) { @@ -97,12 +116,21 @@ graceful.listen(); await setTimeout(ms('1m')); // check if ids is the same - const newIds = await Emails.distinct('id', { - ...query, - date: { - $lte: new Date() - } - }); + let newIds = await Emails.aggregate([ + { + $match: { + ...query, + date: { + $lte: new Date() + } + } + }, + { $group: { _id: '$id' } } + ]) + .allowDiskUse(true) + .exec(); + + newIds = newIds.map((v) => v.id); // if no ids then return early if (newIds.length === 0) { diff --git a/jobs/stripe/check-subscription-accuracy.js b/jobs/stripe/check-subscription-accuracy.js index 9fda953135..f52d3700a5 100644 --- a/jobs/stripe/check-subscription-accuracy.js +++ b/jobs/stripe/check-subscription-accuracy.js @@ -47,6 +47,8 @@ async function mapper(customer) { if (!user) return; + if (user.is_banned) return; + // if emails did not match if (user.email !== customer.email) { logger.info( @@ -74,7 +76,7 @@ async function mapper(customer) { if (activeSubscriptions.has_more || trialingSubscriptions.has_more) { const err = new TypeError( - 'Subscriptions has_more issue - this should not have pagination' + `Subscriptions has_more issue - this should not have pagination ${customer.email} (${customer.id})` ); err.isCodeBug = true; err.customer = customer; diff --git a/jobs/stripe/fraud-check.js b/jobs/stripe/fraud-check.js new file mode 100644 index 0000000000..da180c75ea --- /dev/null +++ b/jobs/stripe/fraud-check.js @@ -0,0 +1,69 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const Stripe = require('stripe'); +const pMap = require('p-map'); + +const getAllStripeCustomers = require('./get-all-stripe-customers'); + +const Domains = require('#models/domains'); +const Users = require('#models/users'); +const config = require('#config'); +const emailHelper = require('#helpers/email'); +const env = require('#config/env'); +const logger = require('#helpers/logger'); + +const stripe = new Stripe(env.STRIPE_SECRET_KEY); + +async function mapper(customer) { + // check if user has more than 5 payment methods + // and if the user is not currently banned in system + const user = await Users.findOne({ + [config.userFields.stripeCustomerID]: customer.id + }); + + if (!user) return; + if (user.is_banned) return; + + // check for # of verified domains and # stripe payment methods + const [count, paymentMethods] = await Promise.all([ + Domains.countDocuments({ + members: { + $elemMatch: { + user: user._id, + group: 'admin' + } + }, + plan: { $in: ['enhanced_protection', 'team'] }, + has_txt_record: true + }), + stripe.customers.listPaymentMethods(customer.id) + ]); + + if (count === 0 && paymentMethods.data.length >= 3) { + emailHelper({ + template: 'alert', + message: { + to: config.email.message.from, + subject: `Potential Payment Method Fraud: ${user.email} - ${customer.id} - ${paymentMethods.data.length} payment methods` + }, + locals: { + message: `

Review Stripe Customer

` + } + }) + .then() + .catch((err) => logger.fatal(err)); + } +} + +async function fraudCheck() { + logger.info('Fetching Stripe customers'); + const customers = await getAllStripeCustomers(); + logger.info(`Started checking ${customers.length} Stripe customers`); + await pMap(customers, mapper, { concurrency: config.concurrency }); + logger.info(`Finished checking ${customers.length} Stripe customers`); +} + +module.exports = fraudCheck; diff --git a/jobs/stripe/index.js b/jobs/stripe/index.js index e92ce0104a..2614e2fcd8 100644 --- a/jobs/stripe/index.js +++ b/jobs/stripe/index.js @@ -18,6 +18,7 @@ const parseErr = require('parse-err'); const mongoose = require('mongoose'); const syncStripePayments = require('./sync-stripe-payments'); +const fraudCheck = require('./fraud-check'); const checkSubscriptionAccuracy = require('./check-subscription-accuracy'); const config = require('#config'); @@ -43,6 +44,27 @@ graceful.listen(); (async () => { await setupMongoose(logger); + // check for stripe fraud + try { + await fraudCheck(); + } catch (err) { + await logger.error(err); + await emailHelper({ + template: 'alert', + message: { + to: config.email.message.from, + subject: 'Error with job for Stripe fraud check' + }, + locals: { + message: `
${JSON.stringify(
+          parseErr(err),
+          null,
+          2
+        )}
` + } + }); + } + // // get all stripe customers and check for // users with multiple active subscriptions diff --git a/jobs/sync-paid-alias-allowlist.js b/jobs/sync-paid-alias-allowlist.js index 7e684283ba..7c162f5185 100644 --- a/jobs/sync-paid-alias-allowlist.js +++ b/jobs/sync-paid-alias-allowlist.js @@ -265,6 +265,7 @@ graceful.listen(); } // email admins regarding this specific domain + /* if (list.length > 0) await emailHelper({ template: 'alert', @@ -280,6 +281,7 @@ graceful.listen(); )}` } }); + */ } } @@ -295,6 +297,7 @@ graceful.listen(); } // email admins regarding this specific domain + /* if (list.length > 0) { await emailHelper({ template: 'alert', @@ -320,6 +323,7 @@ graceful.listen(); await p.exec(); } + */ } if (set.size === 0) continue; diff --git a/routes/web/my-account.js b/routes/web/my-account.js index b76402e8d7..62e4fe97a6 100644 --- a/routes/web/my-account.js +++ b/routes/web/my-account.js @@ -129,10 +129,9 @@ router rateLimit(100, 'create domain billing'), web.myAccount.createDomainBilling ) - // rate limit to 3 years of receipts due to PDF downloading .get( '/billing/:reference', - rateLimit(36, 'retrieve receipt'), + rateLimit(90, 'retrieve receipt'), web.myAccount.retrieveReceipt ) .get('/domains', paginate.middleware(10, 50), web.myAccount.listDomains) diff --git a/scripts/convert-sqlite-to-eml.js b/scripts/convert-sqlite-to-eml.js index 85f5c8d2a5..eb2f215ef5 100644 --- a/scripts/convert-sqlite-to-eml.js +++ b/scripts/convert-sqlite-to-eml.js @@ -192,7 +192,6 @@ const instance = { } } - archive.finalize(); archive.on('warning', (err) => { logger.warn(err); }); @@ -200,6 +199,7 @@ const instance = { archive.on('error', reject); archive.on('end', resolve); }); + archive.finalize(); console.log('tmp', tmp); })();