email_scrape_main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Subject Line There are no responses to the specified emails search criteria Local Supplier for Baseball Apparel in the inbox with a filter date of 3/4/2024\n",
      "Current Subject Line You have 74 email responses to the specified search criteria Local Supplier for Baseball Apparel in the outbox with a filter date of 3/4/2024\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\samuel.taylor\\OneDrive - Green Dot Public Schools\\Desktop\\Git_Directory\\CP\\CustomPlanet_Work\\Email_Scraper\\Scraping_Emails\\modules\\db_operations_aws.py:24: UserWarning: pandas only supports SQLAlchemy connectable (engine/connection) or database string URI or sqlite3 DBAPI2 connection. Other DBAPI2 objects are not tested. Please consider using SQLAlchemy.\n",
      "  df_SQL = pd.read_sql_query(query, con=self.conn)\n",
      "c:\\Users\\samuel.taylor\\OneDrive - Green Dot Public Schools\\Desktop\\Git_Directory\\CP\\CustomPlanet_Work\\Email_Scraper\\Scraping_Emails\\modules\\db_operations_aws.py:24: UserWarning: pandas only supports SQLAlchemy connectable (engine/connection) or database string URI or sqlite3 DBAPI2 connection. Other DBAPI2 objects are not tested. Please consider using SQLAlchemy.\n",
      "  df_SQL = pd.read_sql_query(query, con=self.conn)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "74 new emails appended to email_history\n",
      "Thread is an empty frame\n"
     ]
    }
   ],
   "source": [
    "#2000 emails sent to email_history took 5 minutes\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import logging\n",
    "import warnings\n",
    "import os\n",
    "import pandas as pd\n",
    "from config import *\n",
    "from Scraping_Emails.modules.scrapes import scrape\n",
    "from Scraping_Emails.modules.db_operations_aws import DatabaseConnector\n",
    "from Sending_Emails.modules.sends import EmailConfig\n",
    "warnings.filterwarnings(\"ignore\", category=UserWarning, module=\"pandas\")\n",
    "\n",
    "#create log dir and file\n",
    "logpath_creation = os.getcwd() + '\\\\Logs'\n",
    "if not os.path.exists(logpath_creation):\n",
    "    os.makedirs(logpath_creation)\n",
    "\n",
    "logging.basicConfig(filename= logpath_creation + '\\\\Email_Scraper.log', level=logging.INFO,\n",
    "                    format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', force=True)\n",
    "\n",
    "# ------------------------------------packaging process into final function----------------------------------------\n",
    "\n",
    "def process(subject_line, email_address, email_pass, start_date, campaign_name, sport):\n",
    "\n",
    "    #create the msgs via scrape, then iterate through list of messages, and lastly cleanse frame\n",
    "\n",
    "    #First thing to occur everytime\n",
    "    logging.info('\\n\\nNew logging instance')\n",
    " \n",
    "    msgs_inbox = scrape.scrape_msgs_outbox_or_inbox('inbox', subject_line, email_address, email_pass, start_date, 'Current Subject Line')\n",
    "    inbox = scrape.create_msg_frame(msgs_inbox)\n",
    "    inbox = scrape.cleanse_frame(inbox, 'inbox')\n",
    "    inbox.name = 'inbox'\n",
    "\n",
    "    msgs_outbox = scrape.scrape_msgs_outbox_or_inbox('outbox', subject_line, email_address, email_pass, start_date, 'Current Subject Line')\n",
    "    outbox = scrape.create_msg_frame(msgs_outbox)\n",
    "    outbox = scrape.cleanse_frame(outbox, 'outbox')\n",
    "    outbox.name = 'outbox'\n",
    "        \n",
    "    if inbox.empty == True:\n",
    "        thread = pd.DataFrame()\n",
    "        pass\n",
    "    else: #this is a left merge on outbox\n",
    "        thread = scrape.piece_together(outbox, inbox)\n",
    "        thread = scrape.assign_sentiment(thread)\n",
    "       \n",
    "    # #If message was accidentally triggered more than once\n",
    "    thread = thread.drop_duplicates(subset = ['subject', 'to'], keep='last')\n",
    "    thread.reset_index(drop = True, inplace = True)\n",
    "    outbox = scrape.map_reply_outbox(thread, outbox)\n",
    "\n",
    "    #Optional column identifiers to add in to the DB\n",
    "    outbox['email_campaign_tag'] = campaign_name\n",
    "    outbox['sport'] = sport    #frame   #table_name\n",
    "\n",
    "    return(inbox, outbox, thread)\n",
    "\n",
    "\n",
    "# -------------------------------------------\n",
    "\n",
    "#Get inbox, outbox, and create thread with given subject line back to given date\n",
    "inbox, outbox, thread= process(email_config.email_subject_line, email_config.EMAIL_ADDRESS_FROM, email_config.EMAIL_PASS, email_config.filter_date, email_config.email_campaign_name, email_config.sport)\n",
    "\n",
    "# ----------Send outbox\n",
    "\n",
    "if outbox.empty != True:\n",
    "    db_connector_email_history = DatabaseConnector(email_config.server, email_config.db_user, email_config.db_pass, email_config.database, 'email_history')\n",
    "    db_connector_email_history.send(outbox, 'email_history')\n",
    "else:\n",
    "    logging.info('Outbox is an empty frame')\n",
    "    print('Outbox is an empty frame')\n",
    "\n",
    "# ----------Send thread\n",
    "\n",
    "if thread.empty != True:\n",
    "    db_connector_thread = DatabaseConnector(email_config.server, email_config.db_user, email_config.db_pass, email_config.database, 'thread')\n",
    "    db_connector_thread.send(thread, 'thread')\n",
    "    pre_existing_thread_updates = db_connector_thread.update_reply_thread(thread)\n",
    "else:\n",
    "    logging.info('Thread is an empty frame')\n",
    "    print('Thread is an empty frame')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}