Udemy Grabber.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Udemy Grabber"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from requests.utils import urlparse\n",
    "from bs4 import BeautifulSoup, NavigableString\n",
    "from bs4.element import Tag\n",
    "import pandas as pd\n",
    "from random import randint\n",
    "from time import sleep\n",
    "import re\n",
    "from datetime import datetime, timedelta\n",
    "from urllib.parse import parse_qs, unquote\n",
    "import json\n",
    "from Udemy_Auth import GIST_TOKEN, LOGIN, PASS\n",
    "import glob\n",
    "import os\n",
    "from numpy import datetime64, timedelta64\n",
    "import asyncio\n",
    "from aiohttp import ClientSession\n",
    "from Udemy_Grabber_Objects import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)',\n",
    "           'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n",
    "           'Accept-Encoding': 'gzip, deflate',\n",
    "           'Upgrade-Insecure-Requests': '1',\n",
    "           'Accept-Language': 'de;q=0.8',\n",
    "          }\n",
    "\n",
    "sess = requests.Session()\n",
    "sess.headers.update(headers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gather Courses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grab(url: str, timeout=40):\n",
    "    return BeautifulSoup(sess.get(url, timeout=timeout).text, 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "async def fetch(url, session):\n",
    "    async with session.get(url) as response:\n",
    "        resp = await response.text()\n",
    "        return BeautifulSoup(resp, 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "async def run(urls):\n",
    "    tasks = []\n",
    "    async with ClientSession(headers=headers) as session:\n",
    "        tasks = (asyncio.ensure_future(fetch(url, session)) for url in urls)\n",
    "        return await asyncio.gather(*tasks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## jucktion.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = ['https://www.jucktion.com/forum/udemy-coupon/{}0'.format(i) for i in range(0, 7, 2)]\n",
    "jucktion = await run(urls)\n",
    "\n",
    "links = []\n",
    "for soup in jucktion:\n",
    "    for link in soup.find_all('td', class_='subject windowbg2'):\n",
    "        l = link.find('a', href=True)['href']\n",
    "        links.append(l[:l.find('?PHP')])\n",
    "\n",
    "courses = await run(links)\n",
    "jucktion_courses = set([course.find('a', class_='bbc_link')['href'] for course in courses if course.find('a', class_='bbc_link')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 60 courses.\n"
     ]
    }
   ],
   "source": [
    "print('Found {} courses.'.format(len(jucktion_courses)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bez-postovneho.cz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = ['https://bez-postovneho.cz/?page={}'.format(i) for i in range(1, 6)]\n",
    "bez = await run(urls)\n",
    "\n",
    "links = []\n",
    "for soup in bez:\n",
    "    tags = soup.find_all(href=re.compile(\"free-udemy-course/\"))\n",
    "    links += [link['href'] for link in tags]\n",
    "\n",
    "bez_courses = set()\n",
    "courses = await run(set(links))\n",
    "for course in courses:\n",
    "    url = course.find('a', class_='btn btn-success', href=True)\n",
    "    if url and url['href'].find('couponCode') > -1:\n",
    "        clean = unquote(url['href'])\n",
    "        bez_courses.add(clean[clean.find('https://www.udemy.com/course'):])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1 courses.\n"
     ]
    }
   ],
   "source": [
    "print('Found {} courses.'.format(len(bez_courses)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Smartbro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "smartbro_courses = set()\n",
    "soup = grab('https://smartybro.com/category/udemy-coupon-100-off/')\n",
    "links = soup.find_all('a', href=re.compile(\"https://smartybro.com/202\"))\n",
    "urls = set([link['href'][:link['href'].rfind('/')] for link in links])\n",
    "\n",
    "courses = await run(urls)\n",
    "for soup in courses:\n",
    "    tag = soup.find('a', href=re.compile('udemy.com/course/[\\S]+/?couponCode='))\n",
    "    if tag:\n",
    "        l = tag['href']\n",
    "        smartbro_courses.add(l[:l.find('&')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 15 courses.\n"
     ]
    }
   ],
   "source": [
    "print('Found {} courses.'.format(len(smartbro_courses)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## couponscorpion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = ['https://couponscorpion.com/page/{}/'.format(i) for i in range(1, 10)]\n",
    "scorpion = await run(urls)\n",
    "\n",
    "links = []\n",
    "for soup in scorpion:\n",
    "    for article in soup.find_all('div', 'info_in_dealgrid'):\n",
    "        if article.find('span', 'grid_onsale').text == '100% OFF':\n",
    "            links.append(article.find('a', href=True, class_=False)['href'])\n",
    "\n",
    "scorpion_courses = set()\n",
    "regex = re.compile(\"[\\w]{20,}%3D&s=[\\w]+&n=[\\w]+\")\n",
    "courses = await run(links)\n",
    "for soup in courses:\n",
    "    for script in soup.find_all('script', {'type':False, 'data-cfasync':False, 'src':False}):\n",
    "        token = re.findall(regex, script.text)\n",
    "        if token:\n",
    "            try:\n",
    "                url = sess.get('https://couponscorpion.com/scripts/udemy/out.php?go=' + token[0]).url\n",
    "                coupon = parse_qs(urlparse(url).query)['couponCode']\n",
    "                scorpion_courses.add(url[:url.find('?')] + '?couponCode=' + str(coupon[0]))\n",
    "            except:\n",
    "                print('error getting', url, '\\nTrying again.')\n",
    "                url = sess.get('https://couponscorpion.com/scripts/udemy/out.php?go=' + token[0]).url\n",
    "                coupon = parse_qs(urlparse(url).query)['couponCode']\n",
    "                scorpion_courses.add(url[:url.find('?')] + '?couponCode=' + str(coupon[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 28 courses.\n"
     ]
    }
   ],
   "source": [
    "print('Found {} courses.'.format(len(scorpion_courses)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## discudemy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = ['https://www.discudemy.com/all/{}'.format(i) for i in range(1, 6)]\n",
    "discudemy = await run(urls)\n",
    "\n",
    "links = []\n",
    "for soup in discudemy:\n",
    "    for div in soup.find_all('div', 'content'):\n",
    "        if div.find('div', 'header')and div.find('div', 'meta').text.find('->') > 0:\n",
    "            links.append(div.find('a', href=True)['href'])\n",
    "\n",
    "regex = re.compile(\"/[\\w\\d\\-]+/\")\n",
    "courses = await run([regex.sub('/go/', link) for link in links])\n",
    "discudemy_courses = set(soup.find('a', href=True, class_=False, target='_blank')['href'] for soup in courses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 69 courses.\n"
     ]
    }
   ],
   "source": [
    "print('Found {} courses.'.format(len(discudemy_courses)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## oodemy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = grab('https://www.schoobly.com/search?updated-max=2025-02-28T21:09:00%2B05:30&max-results=30&reverse-paginate=true')\n",
    "oodemy = set([tag['href'] for tag in soup.find_all('a', 'read-more', href=True)])\n",
    "soup = grab(soup.find('a', 'blog-pager-older-link')['href'])\n",
    "oodemy = oodemy.union([tag['href'] for tag in soup.find_all('a', 'read-more', href=True)])\n",
    "\n",
    "oodemy_courses = set()\n",
    "courses = await run(oodemy)\n",
    "for soup in courses:\n",
    "    link = soup.find('a', 'read-more', target=True, href=True)\n",
    "    if link and link['href'].find('couponCode') > -1:\n",
    "        oodemy_courses.add(link['href'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 32 courses.\n"
     ]
    }
   ],
   "source": [
    "print('Found {} courses.'.format(len(oodemy_courses)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## freebiesglobal\n",
    "We have to split the async requests."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = ['https://freebiesglobal.com/dealstore/udemy/page/{}'.format(i) for i in range(1, 5)]\n",
    "freebies = [grab(url) for url in urls]\n",
    "links = [span.next.next.next['href'] for soup in freebies for span in soup.find_all('span', 'grid_onsale')]\n",
    "\n",
    "freebies_courses = []\n",
    "counter = [int(len(links)/2), len(links)]\n",
    "for c, i in enumerate(counter):\n",
    "    courses = await run(links[max(0, i-counter[c-1]-1):i])\n",
    "    freebies_courses += [course.find('a', 're_track_btn', href=True)['href'] for course in courses if course.find('a', 're_track_btn', href=True)]\n",
    "freebies_courses = set(freebies_courses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 94 courses.\n"
     ]
    }
   ],
   "source": [
    "print('Found {} courses.'.format(len(freebies_courses)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Google"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.google.com/search?q=allinurl:+couponcode+site:udemy.com/course&lr=&safe=strict&hl=de&tbs=qdr:w&filter=0&biw=1920&bih=966'\n",
    "soup = grab(url)\n",
    "google = [unquote(a['href'][7:a['href'].find('&sa')]) for a in soup.find_all('a', href=True) if a['href'].find('udemy.com/course/') > -1]\n",
    "soup = grab(url + '&start=10')\n",
    "google += [unquote(a['href'][7:a['href'].find('&sa')]) for a in soup.find_all('a', href=True) if a['href'].find('udemy.com/course/') > -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 16 courses.\n"
     ]
    }
   ],
   "source": [
    "print('Found {} courses.'.format(len(set(google))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge Courses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "udemy = scorpion_courses | Reddit().courses | bez_courses | Freesamples().courses \\\n",
    "        | discudemy_courses | Facebook().courses | smartbro_courses | Dsmenders().courses | \\\n",
    "        jucktion_courses | oodemy_courses | freebies_courses | set(google)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyse\n",
    "Find courses on Udemy, verify that they are free but not permanently, and crawl metadata."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_of_files = glob.glob('./udemy-courses/*.csv')\n",
    "file_day_before = list_of_files[-1]\n",
    "file_old_courses = list_of_files[-2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "day_before = pd.read_csv(file_day_before)\n",
    "old_courses = pd.read_csv(file_old_courses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "parsed 150 courses\r"
     ]
    }
   ],
   "source": [
    "parser = UdemyParser(udemy)\n",
    "await parser.parse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = parser.df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find valuable Courses with high rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_num_reviews = 9\n",
    "min_avg_rating = 3.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "instructor_block = set(['Srinidhi Ranganathan', 'Learn Tech Plus', 'ABD Muhammad',\n",
    "                        'Bardha I, MBA', 'Pronoy Kumar', 'Mohammad Mostafizur Rahaman',\n",
    "                        'Shokal Shah', 'Masud Rana', 'Muhammad Saju Ahmed', 'Marufa Yasmin',])\n",
    "mask = [not bool(instructor_block & set(instr)) for instr in list(df.instructor)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "valuable = df[((df.language=='Deutsch') | ((df.rating > min_avg_rating) & \\\n",
    "              (df.num_ratings > min_num_reviews))) & mask].sort_values(by=['language', 'topic_0', 'topic_1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>link</th>\n",
       "      <th>rating</th>\n",
       "      <th>num_ratings</th>\n",
       "      <th>language</th>\n",
       "      <th>duration</th>\n",
       "      <th>topic_0</th>\n",
       "      <th>topic_1</th>\n",
       "      <th>topic_2</th>\n",
       "      <th>expiring</th>\n",
       "      <th>instructor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Kundengewinnung und Kundenbindung mit Wowing</td>\n",
       "      <td>https://www.udemy.com/course/kundengewinnung-u...</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>Deutsch</td>\n",
       "      <td>2 total Stunden</td>\n",
       "      <td>Marketing</td>\n",
       "      <td>Branding</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2 days 11:00:00</td>\n",
       "      <td>{Sebastian Glöckner}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>Agile Project Management: Scrum Step by Step w...</td>\n",
       "      <td>https://www.udemy.com/course/scrum-master-trai...</td>\n",
       "      <td>4.20208</td>\n",
       "      <td>2746</td>\n",
       "      <td>English</td>\n",
       "      <td>2 total Stunden</td>\n",
       "      <td>Business</td>\n",
       "      <td>Management</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0 days 00:00:00</td>\n",
       "      <td>{Paul Ashun}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>AMAZON MASTERY - How To Become An Amazon Best ...</td>\n",
       "      <td>https://www.udemy.com/course/amazon-best-selli...</td>\n",
       "      <td>3.65000</td>\n",
       "      <td>43</td>\n",
       "      <td>English</td>\n",
       "      <td>2,5 total Stunden</td>\n",
       "      <td>Business</td>\n",
       "      <td>Medien</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1 days 14:00:00</td>\n",
       "      <td>{Ricky Shetty, Scott Paton, Scott Alex}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>The HR professional’s guide to cybersecurity</td>\n",
       "      <td>https://www.udemy.com/course/the-hr-profession...</td>\n",
       "      <td>3.17857</td>\n",
       "      <td>15</td>\n",
       "      <td>English</td>\n",
       "      <td>1 total Stunde</td>\n",
       "      <td>Business</td>\n",
       "      <td>Personalwesen</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2 days 04:00:00</td>\n",
       "      <td>{Teo Dvali}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>Dropshipping with Wordpress: Create a Dropship...</td>\n",
       "      <td>https://www.udemy.com/course/dropshipping-with...</td>\n",
       "      <td>3.85000</td>\n",
       "      <td>62</td>\n",
       "      <td>English</td>\n",
       "      <td>2,5 total Stunden</td>\n",
       "      <td>Business</td>\n",
       "      <td>Selbstständigkeit</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0 days 13:00:00</td>\n",
       "      <td>{MindQuest Academy, Scott Paton, Scott Alex}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>C# ile Web Kazıma (Web Scraping &amp; Web Crawling...</td>\n",
       "      <td>https://www.udemy.com/course/c-ile-web-kazma-w...</td>\n",
       "      <td>3.95000</td>\n",
       "      <td>13</td>\n",
       "      <td>Türkçe</td>\n",
       "      <td>3 total Stunden</td>\n",
       "      <td>IT &amp; Software</td>\n",
       "      <td>Sonstiges</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1 days 02:00:00</td>\n",
       "      <td>{Halil AKCIL}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>Golang Programlama Dili : Standart Kütüphane</td>\n",
       "      <td>https://www.udemy.com/course/golang-programlam...</td>\n",
       "      <td>3.90000</td>\n",
       "      <td>35</td>\n",
       "      <td>Türkçe</td>\n",
       "      <td>10,5 total Stunden</td>\n",
       "      <td>Programmierung</td>\n",
       "      <td>Programmiersprachen</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0 days 04:00:00</td>\n",
       "      <td>{Cihan Özhan}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>Golang Programlama Dili</td>\n",
       "      <td>https://www.udemy.com/course/go-programlama-di...</td>\n",
       "      <td>4.18750</td>\n",
       "      <td>210</td>\n",
       "      <td>Türkçe</td>\n",
       "      <td>17 total Stunden</td>\n",
       "      <td>Programmierung</td>\n",
       "      <td>Softwareentwicklung</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0 days 04:00:00</td>\n",
       "      <td>{Cihan Özhan}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>Golang ile Web API Programlama</td>\n",
       "      <td>https://www.udemy.com/course/golang-restful-ap...</td>\n",
       "      <td>4.50000</td>\n",
       "      <td>60</td>\n",
       "      <td>Türkçe</td>\n",
       "      <td>18 total Stunden</td>\n",
       "      <td>Programmierung</td>\n",
       "      <td>Webentwicklung</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0 days 04:00:00</td>\n",
       "      <td>{Cihan Özhan}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>Complete YouTube Training 2019 in Hindi/urdu -...</td>\n",
       "      <td>https://www.udemy.com/course/complete-youtube-...</td>\n",
       "      <td>3.80000</td>\n",
       "      <td>18</td>\n",
       "      <td>हिन्दी</td>\n",
       "      <td>16 total Stunden</td>\n",
       "      <td>Marketing</td>\n",
       "      <td>Branding</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2 days 05:00:00</td>\n",
       "      <td>{Sarfaraz abro}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>77 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 title  \\\n",
       "11        Kundengewinnung und Kundenbindung mit Wowing   \n",
       "86   Agile Project Management: Scrum Step by Step w...   \n",
       "18   AMAZON MASTERY - How To Become An Amazon Best ...   \n",
       "28        The HR professional’s guide to cybersecurity   \n",
       "96   Dropshipping with Wordpress: Create a Dropship...   \n",
       "..                                                 ...   \n",
       "83   C# ile Web Kazıma (Web Scraping & Web Crawling...   \n",
       "61        Golang Programlama Dili : Standart Kütüphane   \n",
       "70                             Golang Programlama Dili   \n",
       "136                     Golang ile Web API Programlama   \n",
       "106  Complete YouTube Training 2019 in Hindi/urdu -...   \n",
       "\n",
       "                                                  link   rating num_ratings  \\\n",
       "11   https://www.udemy.com/course/kundengewinnung-u...  0.00000           0   \n",
       "86   https://www.udemy.com/course/scrum-master-trai...  4.20208        2746   \n",
       "18   https://www.udemy.com/course/amazon-best-selli...  3.65000          43   \n",
       "28   https://www.udemy.com/course/the-hr-profession...  3.17857          15   \n",
       "96   https://www.udemy.com/course/dropshipping-with...  3.85000          62   \n",
       "..                                                 ...      ...         ...   \n",
       "83   https://www.udemy.com/course/c-ile-web-kazma-w...  3.95000          13   \n",
       "61   https://www.udemy.com/course/golang-programlam...  3.90000          35   \n",
       "70   https://www.udemy.com/course/go-programlama-di...  4.18750         210   \n",
       "136  https://www.udemy.com/course/golang-restful-ap...  4.50000          60   \n",
       "106  https://www.udemy.com/course/complete-youtube-...  3.80000          18   \n",
       "\n",
       "    language            duration         topic_0              topic_1  \\\n",
       "11   Deutsch     2 total Stunden       Marketing             Branding   \n",
       "86   English     2 total Stunden        Business           Management   \n",
       "18   English   2,5 total Stunden        Business               Medien   \n",
       "28   English      1 total Stunde        Business        Personalwesen   \n",
       "96   English   2,5 total Stunden        Business    Selbstständigkeit   \n",
       "..       ...                 ...             ...                  ...   \n",
       "83    Türkçe     3 total Stunden   IT & Software            Sonstiges   \n",
       "61    Türkçe  10,5 total Stunden  Programmierung  Programmiersprachen   \n",
       "70    Türkçe    17 total Stunden  Programmierung  Softwareentwicklung   \n",
       "136   Türkçe    18 total Stunden  Programmierung       Webentwicklung   \n",
       "106   हिन्दी    16 total Stunden       Marketing             Branding   \n",
       "\n",
       "     topic_2        expiring                                    instructor  \n",
       "11       NaN 2 days 11:00:00                          {Sebastian Glöckner}  \n",
       "86       NaN 0 days 00:00:00                                  {Paul Ashun}  \n",
       "18       NaN 1 days 14:00:00       {Ricky Shetty, Scott Paton, Scott Alex}  \n",
       "28       NaN 2 days 04:00:00                                   {Teo Dvali}  \n",
       "96       NaN 0 days 13:00:00  {MindQuest Academy, Scott Paton, Scott Alex}  \n",
       "..       ...             ...                                           ...  \n",
       "83       NaN 1 days 02:00:00                                 {Halil AKCIL}  \n",
       "61       NaN 0 days 04:00:00                                 {Cihan Özhan}  \n",
       "70       NaN 0 days 04:00:00                                 {Cihan Özhan}  \n",
       "136      NaN 0 days 04:00:00                                 {Cihan Özhan}  \n",
       "106      NaN 2 days 05:00:00                               {Sarfaraz abro}  \n",
       "\n",
       "[77 rows x 11 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valuable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = datetime.now().strftime('%Y-%m-%d %H-%M')\n",
    "valuable.to_csv('./udemy-courses/' + path + '.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "yesterdays_courses = [course.title for course in day_before.itertuples() if not course.title in list(old_courses.title)]\n",
    "new_courses = [course.title for course in valuable.itertuples() if not course.title in list(day_before.title)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>link</th>\n",
       "      <th>rating</th>\n",
       "      <th>num_ratings</th>\n",
       "      <th>language</th>\n",
       "      <th>duration</th>\n",
       "      <th>topic_0</th>\n",
       "      <th>topic_1</th>\n",
       "      <th>topic_2</th>\n",
       "      <th>expiring</th>\n",
       "      <th>instructor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Kundengewinnung und Kundenbindung mit Wowing</td>\n",
       "      <td>https://www.udemy.com/course/kundengewinnung-u...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Deutsch</td>\n",
       "      <td>2 total Stunden</td>\n",
       "      <td>Marketing</td>\n",
       "      <td>Branding</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2 days 11:00:00</td>\n",
       "      <td>{Sebastian Glöckner}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           title  \\\n",
       "11  Kundengewinnung und Kundenbindung mit Wowing   \n",
       "\n",
       "                                                 link  rating num_ratings  \\\n",
       "11  https://www.udemy.com/course/kundengewinnung-u...     0.0           0   \n",
       "\n",
       "   language         duration    topic_0   topic_1  topic_2        expiring  \\\n",
       "11  Deutsch  2 total Stunden  Marketing  Branding      NaN 2 days 11:00:00   \n",
       "\n",
       "              instructor  \n",
       "11  {Sebastian Glöckner}  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.language == 'Deutsch']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150 kostenlose Kurse.\n",
      "77 wertvolle Kurse.\n",
      "47 neue Kurse\n"
     ]
    }
   ],
   "source": [
    "print('{} kostenlose Kurse.'.format(len(df)))\n",
    "print('{} wertvolle Kurse.'.format(len(valuable)))\n",
    "print('{} neue Kurse'.format(len(new_courses)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "languages = valuable.language.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Deutsch' 'English' 'Español' 'Türkçe' 'हिन्दी']\n"
     ]
    }
   ],
   "source": [
    "print(languages)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Edit Gist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "GIST=\"https://api.github.com/gists/bc90582aadd7d466fef67aed524d886d\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "git_headers={'Authorization':'token %s'%GIST_TOKEN}\n",
    "params={'scope':'gist'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "content = '\\n'.join(s for s in valuable.link.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "payload={\"description\":\"Kostenlose und gut bewertete Kurse\",\n",
    "         \"public\":True,\n",
    "         \"files\":{\n",
    "             \"gistfile1.txt\":{\n",
    "                 \"content\":content}}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully patched gist.\n"
     ]
    }
   ],
   "source": [
    "res = requests.patch(GIST,\n",
    "                     headers=git_headers,\n",
    "                     params=params,\n",
    "                     data=json.dumps(payload))\n",
    "if res.ok:\n",
    "    print('Successfully patched gist.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Edit Deal\n",
    "* login to mydealz\n",
    "* manually open the deal and delete the old list\n",
    "* run script to create a new list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.firefox.options import Options\n",
    "from selenium.webdriver.common.keys import Keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "options = Options()\n",
    "#options.headless = True\n",
    "driver = webdriver.Firefox(options=options)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Login to mydealz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://www.mydealz.de/login')\n",
    "driver.find_element_by_id('login_form-identity').send_keys(LOGIN)\n",
    "driver.find_element_by_id('login_form-password').send_keys(PASS)\n",
    "driver.find_element_by_id('login_form-password').send_keys(Keys.ENTER)\n",
    "sleep(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://www.mydealz.de/deals/udemy-kostenlose-und-gut-bewertete-kurse-1533722/edit')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Manual step: delete old list and adjust description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "bold = driver.find_element_by_xpath(\"//button[@class='toolbar-btn toolbar-btn--bold space--mr-1']/span\")\n",
    "li = driver.find_element_by_xpath(\"//button[@class='toolbar-btn toolbar-btn--unorderedlist space--mr-1']/span\")\n",
    "link = driver.find_element_by_xpath(\"//button[@class='cept-linkBtn space--mr-1']/span\")\n",
    "editor = driver.find_element_by_xpath(\"//div[@class='redactor-box aGrid']/div\")\n",
    "editor.click()\n",
    "editor.send_keys(Keys.CONTROL, Keys.END)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang = {'Deutsch': 'Deutsch (ohne Qualitätsfilter)',\n",
    "        'English': 'Englisch',\n",
    "        'Español': 'Spanisch',\n",
    "        'العربية': 'Arabisch',\n",
    "        'Italiano': 'Italienisch',\n",
    "        'Türkçe': 'Türkisch',\n",
    "        'English (India)': 'Englisch (Indisch)',\n",
    "        'Русский': 'Russisch',\n",
    "        'Français': 'Französisch',\n",
    "        '中文': 'Chinesisch',\n",
    "        '日本語': 'Japanisch',\n",
    "        'Português': 'Portugiesisch',\n",
    "        '한국어': 'Koreanisch',\n",
    "        'हिन्दी': 'Hindi'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def list_courses(courses):\n",
    "    li.click()\n",
    "    for course in courses.itertuples():\n",
    "        if course.expiring < timedelta64(24,'h'):\n",
    "            editor.send_keys('⏳ ')\n",
    "        if course.title in yesterdays_courses:\n",
    "            editor.send_keys('*gestern ')\n",
    "        elif course.title in new_courses:\n",
    "            editor.send_keys('*neu ')\n",
    "        if course.rating > 4.5:\n",
    "            bold.click()\n",
    "        link.click()\n",
    "        driver.find_element_by_id('linkPopover-href').send_keys(course.link)\n",
    "        driver.find_element_by_id('linkPopover-content').send_keys(course.title)\n",
    "        driver.find_element_by_id('linkPopover-content').send_keys(Keys.RETURN)\n",
    "        if course.rating > 4.5:\n",
    "            bold.click()\n",
    "        rounded = float(round(course.rating * 2, 0)/2)\n",
    "        stars = '★' * int(rounded) + '✯' * bool(rounded % 1)\n",
    "        stars += '☆' * (5 - len(stars))\n",
    "        num_ratings = '({} Bewertungen)'.format(int(course.num_ratings))\n",
    "        length = course.duration.replace('total ', '').replace(' gesamt', '')\n",
    "        editor.send_keys(', ' + stars + ' ' + num_ratings + ', ' + length)\n",
    "\n",
    "        editor.send_keys(Keys.RETURN)\n",
    "        sleep(0.5)\n",
    "    editor.send_keys(Keys.RETURN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Programme\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:12: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  if sys.path[0] == '':\n"
     ]
    }
   ],
   "source": [
    "for language in languages:\n",
    "    bold.click()\n",
    "    editor.send_keys(lang[language])\n",
    "    bold.click()\n",
    "    editor.send_keys(Keys.ENTER)\n",
    "\n",
    "    val_lang = valuable[valuable.language == language]\n",
    "    main_topics = val_lang.topic_0.unique()\n",
    "    for main_topic in main_topics:\n",
    "        editor.send_keys(main_topic)\n",
    "        editor.send_keys(Keys.RETURN)\n",
    "        list_courses(val_lang[valuable.topic_0 == main_topic])\n",
    "    editor.send_keys(Keys.RETURN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Subscribe to Courses\n",
    "Login manually"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox(options=options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://www.udemy.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.udemy.com/course/windows-server-2012-r2-installation-and-configuration/?couponCode=67016E1E0247CD20D53E already subscribed.\n",
      "https://www.udemy.com/course/server-2016-hands-on-training/?couponCode=11AD3E28087B73CC0D9B already subscribed.\n",
      "https://www.udemy.com/course/microsoft-windows-server-2019-hands-on-training-part-i/?couponCode=2AC2D131259E473D3BC5 already subscribed.\n",
      "https://www.udemy.com/course/linux-made-easy-with-hands-on-training/?couponCode=4B1CEB85E07DECA880EC already subscribed.\n",
      "https://www.udemy.com/course/windows-server-2012r2-hands-on-training-part-ii/?couponCode=B8B5DDF7194931BF1E4F already subscribed.\n",
      "https://www.udemy.com/course/hacking-made-easy-with-hands-on-training/?couponCode=1AC6A6C7FAE3F5092315 already subscribed.\n",
      "https://www.udemy.com/course/ethical-hacking-capture-the-flag-walkthroughs/?couponCode=910C7FB0458866FECC31 already subscribed.\n",
      "https://www.udemy.com/course/ethical-hacking-capture-the-flag-walkthroughs-v2/?couponCode=5048CD2346D8BFFEC714 already subscribed.\n",
      "https://www.udemy.com/course/how-to-draw-the-head/?couponCode=FREEBIESGLOBAL already subscribed.\n",
      "https://www.udemy.com/course/how-to-write-amazon-affiliate-content-that-converts/?couponCode=CORONASKILLS already subscribed.\n",
      "https://www.udemy.com/course/advancing-business-english-grammar-meetings-and-more/?couponCode=21A33D4B3D8D6B5D2043 already subscribed.\n",
      "https://www.udemy.com/course/personal-emergency-planning/?couponCode=3145C8F3E98E134FDC44 already subscribed.\n",
      "https://www.udemy.com/course/react-native-and-redux-course-using-hooks/?couponCode=REACTNATIVEFREE already subscribed.\n",
      "https://www.udemy.com/course/python-for-network-engineers-bootcamp/?couponCode=5848D2EB6AFB49507D51 already subscribed.\n",
      "https://www.udemy.com/course/php-mysql-certification-course-for-beginners/?couponCode=EB560531DEF8536C89F1 already subscribed.\n",
      "https://www.udemy.com/course/html-javascript-bootstrap-certification-course/?couponCode=0ECA67B51A620FB29318 already subscribed.\n",
      "https://www.udemy.com/course/linux-debian-desde-cero/?couponCode=0340C920E9736C654107 already subscribed.\n",
      "https://www.udemy.com/course/tienda-virtual-de-software-en-wordpress-y-woocommerce/?couponCode=4AC4794252B3DB544FD3 already subscribed.\n",
      "https://www.udemy.com/course/facturacion-electronica-en-visual-basic-net-sunat-ubl-21/?couponCode=EE8D4571DF00754E0857 already subscribed.\n",
      "https://www.udemy.com/course/construccion-wordpress/?couponCode=E3C8EF5E687A5072E79F already subscribed.\n",
      "https://www.udemy.com/course/aprende-a-programar-con-go/?couponCode=9E23CB9F9676F2EF30A5 already subscribed.\n"
     ]
    }
   ],
   "source": [
    "new_links = [course.link for course in valuable.itertuples() if not course.title in list(day_before.title)]\n",
    "error_links = []\n",
    "for link in new_links:\n",
    "    driver.get(link)\n",
    "    try:\n",
    "        sleep(randint(4, 10))\n",
    "        btn = driver.find_element_by_xpath(\"//button[@data-purpose='buy-this-course-button']\")\n",
    "        if btn.text == 'Jetzt einschreiben':\n",
    "            btn.click()\n",
    "            sleep(6)\n",
    "            if driver.current_url.find('success') < 0:\n",
    "                error_links.append(link)\n",
    "        else:\n",
    "            print(link, 'already subscribed.')\n",
    "    except:\n",
    "        print(link, '# already subscribed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to subscribe to 16 courses.\n"
     ]
    }
   ],
   "source": [
    "print('Failed to subscribe to {} courses.'.format(len(error_links)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "error subscribing to https://www.udemy.com/course/12-proven-ways-to-turn-youtube-into-a-career/?couponCode=F8E3772C8BE15A356259\n",
      "error subscribing to https://www.udemy.com/course/entrepreneurship-accelerator-program/?couponCode=BBF6A4AB54B50A47DC6B\n",
      "error subscribing to https://www.udemy.com/course/deploy-spring-boot-to-azure/?couponCode=FOCUS-AND-LEARN\n",
      "error subscribing to https://www.udemy.com/course/deploy-java-spring-boot-to-aws-amazon-web-service/?couponCode=FOCUS-AND-LEARN\n",
      "error subscribing to https://www.udemy.com/course/learn-pivotal-cloud-foundry-pcf-deploying-spring-boot-apps/?couponCode=FOCUS-AND-LEARN\n",
      "error subscribing to https://www.udemy.com/course/devops-with-docker-kubernetes-and-azure-devops/?couponCode=FOCUS-AND-LEARN\n",
      "error subscribing to https://www.udemy.com/course/fatherhood/?couponCode=MARCH27FATHERHOOD\n",
      "error subscribing to https://www.udemy.com/course/full-stack-application-with-spring-boot-and-react/?couponCode=FOCUS-AND-LEARN\n",
      "error subscribing to https://www.udemy.com/course/microservices-with-spring-boot-and-spring-cloud/?couponCode=FOCUS-AND-LEARN\n",
      "error subscribing to https://www.udemy.com/course/full-stack-application-development-with-spring-boot-and-angular/?couponCode=FOCUS-AND-LEARN\n",
      "error subscribing to https://www.udemy.com/course/microservices-with-spring-boot-and-spring-cloud/?&couponCode=FOCUS-AND-LEARN\n",
      "error subscribing to https://www.udemy.com/course/microsoft-teams-and-onenote-essentials-for-educators/?couponCode=23MARCH2020\n",
      "error subscribing to https://www.udemy.com/course/mantenimiento-pc/?couponCode=07D27475F573DC18325A\n",
      "error subscribing to https://www.udemy.com/course/analista-de-help-desk/?couponCode=BFD78E149CB1C1682B5D\n",
      "error subscribing to https://www.udemy.com/course/conceptos-de-aws/?couponCode=E1F3AF17EE918BF2CB12\n"
     ]
    }
   ],
   "source": [
    "for link in error_links:\n",
    "    driver.get(link)\n",
    "    try:\n",
    "        sleep(randint(4, 10))\n",
    "        btn = driver.find_element_by_xpath(\"//button[@data-purpose='buy-this-course-button']\")\n",
    "        if btn.text == 'Jetzt einschreiben':\n",
    "            btn.click()\n",
    "            sleep(6)\n",
    "            if driver.current_url.find('success') < 0:\n",
    "                print('error subscribing to', link)\n",
    "        else:\n",
    "            print(link, 'already subscribed.')\n",
    "    except:\n",
    "        print(link, '# already subscribed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(this_is_an_error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = 'https://www.mydealz.de/?page={}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deals = []\n",
    "for i in range(1, 500):\n",
    "    soup = BeautifulSoup(sess.get(base.format(i)).text, 'lxml')\n",
    "    links = soup.find_all('a', 'cept-thread-image-link imgFrame imgFrame--noBorder thread-listImgCell', href=True)\n",
    "    deals += [link['href'] for link in links]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(columns=['title', 'published', 'time', 'hour', 'weekday', 'temperature', 'url'])\n",
    "for deal in set(deals):\n",
    "    soup = BeautifulSoup(sess.get(deal).text, 'lxml')\n",
    "    if soup.find('span', 'cept-vote-temp vote-temp vote-temp--burn'):\n",
    "        published = datetime.fromisoformat(soup.find('meta', {'property':'og:article:published_time'})['content'])\n",
    "        title = soup.find('meta', {'property':'og:title'})['content'][:-13]\n",
    "        temperature = int(soup.find('span', 'cept-vote-temp vote-temp vote-temp--burn').text.strip()[:-1])\n",
    "        time = published.time()\n",
    "        hour = time.hour\n",
    "        weekday = int(published.weekday())\n",
    "\n",
    "        data = {'title':title,\n",
    "                'published':published,\n",
    "                'time':time,\n",
    "                'hour':hour,\n",
    "                'weekday':weekday,\n",
    "                'temperature':temperature,\n",
    "                'url':deal}\n",
    "        df = df.append(data, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.weekday = pd.to_numeric(df.weekday)\n",
    "df.temperature = pd.to_numeric(df.temperature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby(['hour', 'weekday']).median().sort_values('temperature')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = ['nachts', 'morgens', 'nachmittags', 'abends']\n",
    "pd.cut(df.hour, range(0, 24, 6), right=False, labels=labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://ibmcoders.ibmviprewards.ibm.com/channels/266')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n",
      "oops...\n"
     ]
    }
   ],
   "source": [
    "for _ in range(300):\n",
    "    try:\n",
    "        driver.get('https://ibmcoders.ibmviprewards.ibm.com/channels/266')\n",
    "        driver.find_element_by_xpath(\"//div[@id='challenge-862']/div/div/div/div/a\").click()\n",
    "        sleep(1.5)\n",
    "        driver.find_element_by_xpath('//section[@class=\"pt-modal large available\"]').click()\n",
    "        sleep(3)\n",
    "        driver.find_element_by_xpath('//section[@class=\"pt-modal large available\"]').send_keys(Keys.DOWN)\n",
    "        driver.find_element_by_xpath('//section[@class=\"pt-modal large available\"]').send_keys(Keys.DOWN)\n",
    "        driver.find_element_by_xpath('//section[@class=\"pt-modal large available\"]').send_keys(Keys.DOWN)\n",
    "        driver.find_element_by_xpath('//section[@class=\"pt-modal large available\"]').send_keys(Keys.DOWN)\n",
    "        driver.find_element_by_xpath('//section[@class=\"pt-modal large available\"]').send_keys(Keys.DOWN)\n",
    "        driver.find_element_by_xpath('//section[@class=\"pt-modal large available\"]').send_keys(Keys.DOWN)\n",
    "        driver.find_element_by_xpath('//section[@class=\"pt-modal large available\"]').send_keys(Keys.DOWN)\n",
    "        driver.find_element_by_xpath(\"//div[@class='pt-form-actions']/div/div/input\").click()\n",
    "        sleep(0.5)\n",
    "        driver.find_element_by_xpath(\"//div[@class='dialog-header']/button\").click()\n",
    "    except:\n",
    "        print('oops...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://ibmcoders.ibmviprewards.ibm.com/channels/266')\n",
    "driver.find_element_by_xpath(\"//div[@id='challenge-862']/div/div/div/div/a\").click()\n",
    "sleep(1.5)\n",
    "driver.find_element_by_xpath('//section[@class=\"pt-modal large available\"]').click()\n",
    "sleep(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}