google-alert-package.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from bs4 import BeautifulSoup\n",
    "import time\n",
    "from collections import defaultdict\n",
    "import pandas as pd\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GoogleAlertCrawler:\n",
    "    def __init__(self, keywords:list, driver_path:str, output_path:str='./google_alert.csv', sleep_time=1):\n",
    "        self.keywords = keywords\n",
    "        self.driver_path = driver_path\n",
    "        self.output_path = output_path\n",
    "        self.url = 'https://www.google.com/alerts?hl=zh-tw#1:1'\n",
    "        self.data = None\n",
    "        self.sleep_time = sleep_time\n",
    "\n",
    "    def get_data(self):\n",
    "        data_collection = defaultdict(list)\n",
    "\n",
    "        # 開啟web driver & get google alert url\n",
    "        chrome_options = Options()  \n",
    "        chrome_options.add_argument(\"--headless\")\n",
    "        driver = webdriver.Chrome(executable_path=self.driver_path, chrome_options=chrome_options)\n",
    "        driver.get(self.url)\n",
    "\n",
    "        for k in self.keywords:\n",
    "            # 取得關鍵字輸入框位置 & 輸入關鍵字\n",
    "            search_bar = driver.find_element_by_xpath('//*[@id=\"query_div\"]/input')\n",
    "            search_bar.clear()\n",
    "            search_bar.send_keys(k)\n",
    "            # 等待讀取網頁資訊使用\n",
    "            time.sleep(self.sleep_time)\n",
    "\n",
    "            # 取得網路資料（preview set)\n",
    "            preview_set = driver.find_elements_by_xpath('//*[@id=\"preview_results\"]/ul/li')\n",
    "            for preview in preview_set:\n",
    "                news, news_count = self.parse_html(preview.get_attribute('innerHTML'))\n",
    "                data_collection['關鍵字'].extend([k for i in range(news_count)])\n",
    "                data_collection['網頁標題'].extend(news['網頁標題'])\n",
    "                data_collection['網頁連結'].extend(news['網頁連結'])\n",
    "                data_collection['抓取日期'].extend([datetime.now().strftime(\"%Y-%m-%d\") for i in range(news_count)])\n",
    "                \n",
    "        self.data = data_collection\n",
    "        return self.data\n",
    "    \n",
    "    def parse_html(self, html_source:str):\n",
    "        data = defaultdict(list)\n",
    "        soup = BeautifulSoup(html_source, 'html.parser')\n",
    "        counter = 0\n",
    "    \n",
    "        # 解析每筆資料的標題與連結\n",
    "        for l in soup.find_all('li', class_='result'):\n",
    "            counter += 1\n",
    "            data['資料來源'].append(soup.h3.text)\n",
    "            if len(l.a.text.replace(' ', '')) > 0:\n",
    "                data['網頁標題'].append(l.a.text)\n",
    "                data['網頁連結'].append(l.a.attrs['href'])\n",
    "            else:\n",
    "                data['網頁標題'].append(l.h4.a.text)\n",
    "                data['網頁連結'].append(l.h4.a.attrs['href'])\n",
    "        return data, counter\n",
    "\n",
    "    def output_csv(self):\n",
    "        df = pd.DataFrame.from_dict(self.data)\n",
    "        df.to_csv(self.output_path ,index=False)\n",
    "        print(f'{self.output_path} 存取完成!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# config\n",
    "driver = './chromedriver'\n",
    "keywords = ['蔡英文', '馬英九']\n",
    "sleep = 2\n",
    "\n",
    "# create instance\n",
    "google_alert = GoogleAlertCrawler(keywords, driver, sleep_time=sleep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "//anaconda3/envs/analytic/lib/python3.7/site-packages/ipykernel_launcher.py:16: DeprecationWarning: use options instead of chrome_options\n",
      "  app.launch_new_instance()\n"
     ]
    }
   ],
   "source": [
    "data = google_alert.get_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./google_alert.csv 存取完成!\n"
     ]
    }
   ],
   "source": [
    "google_alert.output_csv()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "analytic",
   "language": "python",
   "name": "analytic"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}