web_scraping_for_tabular_data_project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "ad0bfeca-6c6b-4c9f-aef2-0e1f26da26cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# IMPORTATION OF NECESSARY LIBRARIES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "1932bcaf-fefd-407e-8a73-bd7bc912c82f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- imports the Beautiful Soup Library for Parsing HTML Code.\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# -- imports the request library for making HTML requests to a website.\n",
    "\n",
    "import requests\n",
    "\n",
    "# -- imports the regex library for parsing of data\n",
    "import re\n",
    "\n",
    "# -- imports pandas library for creating and working with dataframe (s)\n",
    "import pandas as pd\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "2054000c-cf00-4e7a-986a-f62023e540e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "class WebsiteAccess:\n",
    "\n",
    "    \"\"\"Class Initialization\"\"\"\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    # url1 = \"https://en.wikipedia.org/wiki/List_of_universities_in_Nigeria\"\n",
    "    # url2 = \"https://en.wikipedia.org/wiki/List_of_largest_companies_in_the_United_States_by_revenue\"\n",
    "\n",
    "    # --- method for taking in the URL from the user, and returning the website reponse back to the user    \n",
    "    def website_response(self):\n",
    "    \n",
    "        url = input(\"Kindly enter the url of the website you want to scrape: \") # requests website link from the user\n",
    "    \n",
    "        # try-except block to handle any unexpected error that might occur\n",
    "    \n",
    "        try:\n",
    "            response = requests.get(url) # sends a request to website and gets a response, 200 means success\n",
    "            \n",
    "            response.raise_for_status() # returns a HTPPError if the response code was unsuccessful\n",
    "            \n",
    "        except requests.exceptions.RequestException as e:\n",
    "            print( f\"Failed to get response from the website: {e}\") # prints back the error that occurs\n",
    "            \n",
    "        # returns the website's response and raw html code in an easy to read hierarchical format    \n",
    "        return response\n",
    "    \n",
    "    \n",
    "    # PLEASE NOTE: check - https://developer.mozilla.org/en-US/docs/Web/HTTP/Status for meaning of other potential responses\n",
    "\n",
    "\n",
    "    # --- method to get the selected table using its index\n",
    "    \n",
    "    def table_html_code(self, table_index : int, soup):\n",
    "        \n",
    "        table = soup.find_all(\"table\") [table_index] # gets the html code for the selected table -- works for tables in any website\n",
    "        # table = soup.find_all(\"table\") # returns the total number of tables in the website -- only works for tables in wikipedia\n",
    "        return table # returns htmll code for current table\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "3b14034d-97d8-43f6-979a-c49fff815163",
   "metadata": {},
   "outputs": [],
   "source": [
    "# *** Definition of WebsiteScraping Class with Parent Class: WebsiteAcess\n",
    "\n",
    "class WebsiteScraping(WebsiteAccess):\n",
    "    \n",
    "    \"\"\"Class Initialization\"\"\"\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    #  --- method to get the name of the table\n",
    "   \n",
    "    def get_table_name(self, total_no_of_tables, soup, name_tag = \"h2\"):\n",
    "        \n",
    "        h2_tag = soup.find_all(name_tag, id=True) # gets the h2 hmtl tag for the current table using \"table_index\" together with its contents\n",
    "\n",
    "        # NOTE \"id = True\" above ensures that the h2 tag of the current table\n",
    "        table_name_lst = [title.text.strip(\"\\n\") for title in h2_tag] # loops through the content of the current table's h2 html tag and gets only text data type element while stripping off newline characters and stores it in a list\n",
    "        regex_pattern = r'[^a-zA-Z\\s]' # regex pattern to remove any other foreign text characters\n",
    "        table_name_lst = [re.sub(regex_pattern, '', title) for title in table_name_lst] # removes all foreign characters from the names using the regex_pattern\n",
    "        table_valid_names = table_name_lst[:(total_no_of_tables)] # gets the formatted text data element out of the list \"table_name_lst\" using list indexing \n",
    "        return table_valid_names # returns the list of valid table names in the html doc\n",
    "\n",
    "    # --- method to define the generator function\n",
    "    \n",
    "    def index_generator(self, table_valid_names_lst, total_no_of_tables, soup):\n",
    "        \n",
    "        for i in range(len(self.get_table_name(total_no_of_tables, soup))):\n",
    "            yield i # creates a generator (a kind of iterable) for the current index and resumes from there when called next\n",
    "            \n",
    "    # NOTE: Difference between yeild and return statement is that for a return statemnt, the function's state is discarded once call has been executed and subsequent call start\n",
    "    # from the beginning but for a yield statement, the function's is resumed from where it left off allowing it to produce values over multiple call.\n",
    "    # This was done to help return different index values for the tags since the tag_index() function was placed inside a for loop at the get_all_tables() function.\n",
    "    \n",
    "    \n",
    "    # --- method to define the function that will use the generator\n",
    "    \n",
    "    def tag_index(self,gen, total_no_of_tables, soup):\n",
    "        \n",
    "        # try-except block to get the next generator or print an error statement incase of any errors\n",
    "        try:\n",
    "            # Get the next index from the generator\n",
    "            index = next(gen) # retrieves the next index by advancing the generator to the next yield statement\n",
    "            # print(index)\n",
    "            # Access the list element using the index\n",
    "            return index \n",
    "            \n",
    "        except StopIteration:\n",
    "            print(\"No more items to iterate.\")    \n",
    "       \n",
    "        \n",
    "    # --- method to get all rows in the table\n",
    "    \n",
    "    def table_data(self, table_index : int, soup, header_title_tag : str = \"th\" ): \n",
    "        \n",
    "        table = self.table_html_code(table_index, soup) # calls the table_html_code method and returns table specified by index while assigning it to the \"table\" variable\n",
    "        # header_cells = table.find_all(header_title_tag) # gets all header_cells from the table\n",
    "        header_cells = table.find_all(header_title_tag) # gets all header_cells from the table\n",
    "\n",
    "        # ********************************************************************************************************************************\n",
    "        #  --- first option code to get only the string name of each header column e.g \"State\", \"Abbreviation\", \"Location\" etc\n",
    "        \n",
    "        # regex_pattern = r'<\\/?[a-zA-Z]+>|\\n' # regex pattern to remove opening or closing tags and newline characters\n",
    "        # replacement = \"\" # replacement string for the matched regex pattern\n",
    "        # header_title_names = [] # new list for the header_titles\n",
    "        \n",
    "        # # loops through each element of the header_cell list object and gets the only the String name of each column\n",
    "        # for column_name in header_cells:\n",
    "        #     # print(type(column_name)) # gets the type of each element in the header_cell list\n",
    "        #     column_name = str(column_name) # converts each element of the header_cell list from a beautifulsoup  tag to string object\n",
    "        #     column_name = re.sub(regex_pattern, replacement, column_name) # deletes the starting tags, ending tags and newline characters in each element of header_cells\n",
    "        #     header_title_names.append(column_name) # appends the string name of the column to the new header_title list\n",
    "        # *********************************************************************************************************************************\n",
    "\n",
    "        # --- alternative code to the previous one using list comprehension\n",
    "        \n",
    "        header_title_names = [title.text.strip(\"\\n\") for title in header_cells] # loops through all elements of header_cells list and gets only text data type element while stripping off newline characters and appending them to the header_title list\n",
    "        \n",
    "        # print(header_title) # prints the header_title list\n",
    "\n",
    "        \n",
    "        all_row_data = [header_title_names] # creates a list that contains the \"header_title_names\" in a list.\n",
    "        \n",
    "        \n",
    "        # --- code to get all the data for each row in the table the html raw code\n",
    "        \n",
    "        table_rows = table.find_all(\"tr\")[1:] # gets all the rows in the table excluding the header_title_row using list slicing\n",
    "        \n",
    "    \n",
    "        # loops through each element of the table_rows list object\n",
    "        for row in table_rows:\n",
    "            each_row = row.find_all(\"td\") # gets all td tag elements form each row\n",
    "            each_row_data = [data.text.strip(\"\\n\") for data in each_row] # gets only text data from the td tag elements of each row and strips off the newline character\n",
    "            # print(each_row_data) # prints the formatted row for each element row in table_rows\n",
    "        \n",
    "            all_row_data.append(each_row_data) # appends each row data to the list \"all_row_data\"\n",
    "    \n",
    "        return all_row_data # returns all the rows in the table in a list object \"all_row_data\"\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "bb0875e3-9885-43c6-9839-d34af2063fc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# *** Definition of WebsiteDataExtraction Class with Parent Class: WebsiteScraping\n",
    "\n",
    "class WebsiteDataExtraction(WebsiteScraping):\n",
    "    \"\"\"Class Initialization\"\"\"\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    # --- method to create dataframe\n",
    "    \n",
    "    def get_into_df(self, table_index : int, soup):\n",
    "        \n",
    "        # table_df = pd.DataFrame(columns = self.table_data(table_index, soup)) # creates a new data frame for the table using the \"table_data\" method -- only works for wikipedia tables\n",
    "        table_df = pd.DataFrame(self.table_data(table_index, soup)) # creates a new data frame for the table using the \"table_data\" method -- works for any tables\n",
    "        return table_df # returns the created dataframe\n",
    "    \n",
    "    # --- method to write dataframe to a csv file\n",
    "    \n",
    "    def convert_df2_csv(self, total_no_of_tables, table_index, tg_index: int, soup):\n",
    "        \n",
    "        table_name = self.get_table_name(total_no_of_tables, soup)[tg_index] # calls the \"get_table_name\" function to get the name of the current table        \n",
    "        # try-except block to handle any exceptions that may arise\n",
    "        try:\n",
    "            csv_file = self.get_into_df(table_index, soup).to_csv(f\"{table_name}.csv\", index = False) # writes to a csv file with table_name without an indexing column\n",
    "            return csv_file # returns the created csv file\n",
    "            \n",
    "        except PermissionError as pe:\n",
    "            print(\"Permission Error: The file you are trying to modify is currently in use. Kindly close it or use another file\") # prints this out incase a Permission Error is raised\n",
    "            \n",
    "        except Exception as e:\n",
    "            print(f\"An Error Occured: {e}\") # prints this incase any other error is raised.\n",
    "\n",
    "    # --- method to write all available tables in the website to csv files.\n",
    "    \n",
    "    def get_all_tables(self):\n",
    "        \n",
    "        response = self.website_response() # method call for the website's response and assigns the result to the \"response\" variable\n",
    "        soup = BeautifulSoup(response.text, \"html\") # takes the raw response and returns it in html format\n",
    "        # total_no_of_tables = len(soup.find_all(\"table\", class_ = \"wikitable sortable\")) # returns the total number of tables in the website -- only works for wikipedia tables\n",
    "        total_no_of_tables = len(soup.find_all(\"table\")) # returns the total number of tables in the website -- works for any table\n",
    "        gen = self.index_generator(self.get_table_name(total_no_of_tables, soup), total_no_of_tables, soup) # gets generator object for the next valid tag index\n",
    "\n",
    "        # for loop to iterate through all tables and name only tables that are not empty\n",
    "        for table_index in range(total_no_of_tables):\n",
    "            # conditional statement to check if table is empty\n",
    "            if self.get_into_df(table_index, soup).empty != True:\n",
    "                tg_index = self.tag_index(gen,total_no_of_tables, soup) # gets the next valid index if current table is not empty\n",
    "                # print(tg_index)\n",
    "                self.convert_df2_csv(total_no_of_tables, table_index, tg_index, soup) # writes the current table dataframe to a csv file\n",
    "                         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "42fc54d3-7237-4272-a49d-673c9daabe40",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Kindly enter the url of the website you want to scrape:  https://en.wikipedia.org/wiki/List_of_universities_in_Nigeria\n"
     ]
    }
   ],
   "source": [
    "# creates an instance of the WebsiteDataExtraction class \n",
    "extraction = WebsiteDataExtraction()\n",
    "\n",
    "# calls the get_all_tables() method\n",
    "extraction.get_all_tables()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c10c342c-43e6-43c7-87df-e56e4e96c986",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89912ed7-a5ab-43e8-b534-e8841dce4edd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}