Python Data Science Toolbox part 1 and 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#################### NESTED INNER FUNCTIONS ######################\n",
    "# Define echo\n",
    "def echo(n):\n",
    "    \"\"\"Return the inner_echo function.\"\"\"\n",
    "    # Define inner_echo\n",
    "    def inner_echo(word1):\n",
    "        \"\"\"Concatenate n copies of word1.\"\"\"\n",
    "        echo_word = word1 * n\n",
    "        return echo_word\n",
    "    # Return inner_echo\n",
    "    return inner_echo\n",
    "# Call echo: twice\n",
    "twice = echo(2)\n",
    "# Call echo: thrice\n",
    "thrice = echo(3)\n",
    "print(echo(4)('hulu '))                 ## Note how inner functions can have 2 arguments afteer one another\n",
    "# Call twice() and thrice() then print\n",
    "print(twice('hello'), thrice('hello'))\n",
    "\n",
    "\n",
    "\n",
    "#################REDUCE FUNCTION USED WITH LAMBDA################\n",
    "# Import reduce from functools\n",
    "from functools import reduce\n",
    "# Create a list of strings: stark\n",
    "stark = ['robb', 'sansa', 'arya', 'brandon', 'rickon']\n",
    "# Use reduce() to apply a lambda function over stark: result\n",
    "result = reduce(lambda item1, item2: item1+item2, stark)\n",
    "# Print the result\n",
    "print(result)\n",
    "\n",
    "\n",
    "\n",
    "################## USING ADVANCED LIST COMPREHENSIONS ###################\n",
    "# Create a list of strings: fellowship\n",
    "fellowship = ['frodo', 'samwise', 'merry', 'aragorn', 'legolas', 'boromir', 'gimli']\n",
    "# Create list comprehension: new_fellowship\n",
    "new_fellowship = [member for member in fellowship if len(member)>=7]\n",
    "# Print the new list\n",
    "print(new_fellowship)            ##print if member name is 7 or greater character\n",
    "\n",
    "\n",
    "##In the following we are using both if and else in the comprehension and printing a '' or the name itself\n",
    "\n",
    "# Create a list of strings: fellowship\n",
    "fellowship = ['frodo', 'samwise', 'merry', 'aragorn', 'legolas', 'boromir', 'gimli']\n",
    "# Create list comprehension: new_fellowship\n",
    "new_fellowship = [member if len(member)>=7 else '' for member in fellowship]\n",
    "# Print the new list\n",
    "print(new_fellowship)\n",
    "\n",
    "\n",
    "\n",
    "#################### DICTIONARY COMPREHENSIONS ##########################\n",
    "# Create a list of strings: fellowship\n",
    "fellowship = ['frodo', 'samwise', 'merry', 'aragorn', 'legolas', 'boromir', 'gimli']\n",
    "# Create dict comprehension: new_fellowship\n",
    "new_fellowship = {member: len(member) for member in fellowship}\n",
    "# Print the new list\n",
    "print(new_fellowship)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "##################### GENERATOR OBJECTS ######################\n",
    "\n",
    "\"\"\"Returns the 0-29 numbers. Notice how next(result) is used to get the objects\"\"\"\n",
    "\n",
    "# Create generator object: result\n",
    "result = (num for num in range(30))\n",
    "# Print the first 5 values\n",
    "print(next(result))\n",
    "print(next(result))\n",
    "print(next(result))\n",
    "print(next(result))\n",
    "print(next(result))\n",
    "# Print the rest of the values\n",
    "for value in result:\n",
    "    print(value)\n",
    "    \n",
    "    \n",
    "\"\"\"Generating the lengths in the strings\"\"\"\n",
    "# Create a list of strings: lannister\n",
    "lannister = ['cersei', 'jaime', 'tywin', 'tyrion', 'joffrey']\n",
    "# Create a generator object: lengths\n",
    "lengths = (len(person) for person in lannister)\n",
    "# Iterate over and print the values in lengths\n",
    "for value in lengths:\n",
    "    print(value)\n",
    "\n",
    "    \n",
    "\n",
    "\"\"\"GENERATOR FUNCTION ITSELF\"\"\"\n",
    "# Create a list of strings\n",
    "lannister = ['cersei', 'jaime', 'tywin', 'tyrion', 'joffrey']\n",
    "# Define generator function get_lengths\n",
    "def get_lengths(input_list):\n",
    "    # Yield the length of a string\n",
    "    for person in input_list:\n",
    "        yield len(person)\n",
    "# Print the values generated by get_lengths()\n",
    "for value in get_lengths(lannister):\n",
    "    print(value)\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "################ PUTTING ALL THE SKILLS TOGETHER####################\n",
    "\"\"\"Extracted the time from tweet column\"\"\"\n",
    "# Extract the created_at column from df: tweet_time\n",
    "tweet_time = df['created_at']\n",
    "# Extract the clock time: tweet_clock_time\n",
    "tweet_clock_time = [entry[11:19] for entry in tweet_time]\n",
    "# Print the extracted times\n",
    "print(tweet_clock_time)\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Conditional comprehension if last 2 are '19'\"\"\"\n",
    "# Extract the created_at column from df: tweet_time\n",
    "tweet_time = df['created_at']\n",
    "# Extract the clock time: tweet_clock_time\n",
    "tweet_clock_time = [entry[11:19] for entry in tweet_time if entry[17:19] == '19']\n",
    "# Print the extracted times\n",
    "print(tweet_clock_time)\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Zipping and then transforming the zip to dictionary\"\"\"\n",
    "# Zip lists: zipped_lists\n",
    "zipped_lists = zip(feature_names,row_vals)\n",
    "# Create a dictionary: rs_dict\n",
    "rs_dict = dict(zipped_lists)\n",
    "# Print the dictionary\n",
    "print(rs_dict)\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"After the custom function list_of_dicts{we zipped and converted to dictionary} we created a list of dictionaries\"\"\"\n",
    "# Print the first two lists in row_lists\n",
    "print(row_lists[0])\n",
    "print(row_lists[1])\n",
    "# Turn list of lists into list of dicts: list_of_dicts\n",
    "list_of_dicts = [lists2dict(feature_names,sublist) for sublist in row_lists]\n",
    "# Print the first two dictionaries in list_of_dicts\n",
    "print(list_of_dicts[0])\n",
    "print(list_of_dicts[1])\n",
    "\n",
    "\n",
    "\"\"\"Turning the above to a dataframe\"\"\"\n",
    "# Import the pandas package\n",
    "import pandas as pd\n",
    "# Turn list of lists into list of dicts: list_of_dicts\n",
    "list_of_dicts = [lists2dict(feature_names, sublist) for sublist in row_lists]\n",
    "# Turn list of dicts into a DataFrame: df\n",
    "df = pd.DataFrame(list_of_dicts)\n",
    "# Print the head of the DataFrame\n",
    "df.head()\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Reading 1000 lines from csv and seeing how many countries count is there in a column\"\"\"\n",
    "# Open a connection to the file\n",
    "with open('world_dev_ind.csv') as file:\n",
    "    # Skip the column names\n",
    "    file.readline()\n",
    "    # Initialize an empty dictionary: counts_dict\n",
    "    counts_dict = {}\n",
    "    # Process only the first 1000 rows\n",
    "    for j in range(1000):\n",
    "        # Split the current line into a list: line\n",
    "        line = file.readline().split(',')\n",
    "        # Get the value for the first column: first_col\n",
    "        first_col = line[0]\n",
    "        # If the column value is in the dict, increment its value\n",
    "        if first_col in counts_dict.keys():\n",
    "            counts_dict[first_col] += 1\n",
    "        # Else, add to the dict and set value to 1\n",
    "        else:\n",
    "            counts_dict[first_col] = 1\n",
    "# Print the resulting dictionary\n",
    "print(counts_dict)\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Reading from a large file using generator object to return one line at a time\"\"\"\n",
    "# Define read_large_file()\n",
    "def read_large_file(file_object):\n",
    "    \"\"\"A generator function to read a large file lazily.\"\"\"\n",
    "    # Loop indefinitely until the end of the file\n",
    "    while True:\n",
    "        # Read a line from the file: data\n",
    "        data = file_object.readline()\n",
    "        # Break if this is the end of the file\n",
    "        if not data:\n",
    "            break\n",
    "        # Yield the line of data\n",
    "        yield data\n",
    "# Open a connection to the file\n",
    "with open('world_dev_ind.csv') as file:\n",
    "    # Create a generator object for the file: gen_file\n",
    "    gen_file = read_large_file(file)\n",
    "    # Print the first three lines of the file\n",
    "    print(next(gen_file))\n",
    "    print(next(gen_file))\n",
    "    print(next(gen_file))\n",
    "    \n",
    "    \n",
    "    \n",
    "\"\"\"Then we read full file using the above generator\"\"\"\n",
    "# Initialize an empty dictionary: counts_dict\n",
    "counts_dict = {}\n",
    "# Open a connection to the file\n",
    "with open('world_dev_ind.csv') as file:\n",
    "    # Iterate over the generator from read_large_file()\n",
    "    for line in read_large_file(file):\n",
    "        row = line.split(',')\n",
    "        first_col = row[0]\n",
    "        if first_col in counts_dict.keys():\n",
    "            counts_dict[first_col] += 1\n",
    "        else:\n",
    "            counts_dict[first_col] = 1\n",
    "# Print            \n",
    "print(counts_dict)\n",
    "\n",
    "\n",
    "\n",
    "################### SUMMARY OF ALL ABOVE ITEMS PRACTISE#################\n",
    "\"\"\"breaking csv to chunks and reading them with next()\"\"\"\n",
    "# Import the pandas package\n",
    "import pandas as pd\n",
    "# Initialize reader object: df_reader\n",
    "df_reader = pd.read_csv('ind_pop.csv', chunksize=10)\n",
    "# Print two chunks\n",
    "print(next(df_reader))\n",
    "print(next(df_reader))\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Loading data chunks and then zipping it only if a colummn value equals something\"\"\"\n",
    "# Initialize reader object: urb_pop_reader\n",
    "urb_pop_reader = pd.read_csv('ind_pop_data.csv', chunksize=1000)\n",
    "# Get the first DataFrame chunk: df_urb_pop\n",
    "df_urb_pop = next(urb_pop_reader)\n",
    "# Check out the head of the DataFrame\n",
    "print(df_urb_pop.head())\n",
    "# Check out specific country: df_pop_ceb\n",
    "df_pop_ceb = df_urb_pop[df_urb_pop['CountryCode']=='CEB']\n",
    "# Zip DataFrame columns of interest: pops\n",
    "pops = zip(df_pop_ceb['Total Population'], df_pop_ceb['Urban population (% of total)'])\n",
    "# Turn zip object into list: pops_list\n",
    "pops_list = list(pops)\n",
    "# Print pops_list\n",
    "print(pops_list)\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Zipped, created a new column and then plotted\"\"\"\n",
    "# Code from previous exercise\n",
    "urb_pop_reader = pd.read_csv('ind_pop_data.csv', chunksize=1000)\n",
    "df_urb_pop = next(urb_pop_reader)\n",
    "df_pop_ceb = df_urb_pop[df_urb_pop['CountryCode'] == 'CEB']\n",
    "pops = zip(df_pop_ceb['Total Population'], \n",
    "           df_pop_ceb['Urban population (% of total)'])\n",
    "pops_list = list(pops)\n",
    "# Use list comprehension to create new DataFrame column 'Total Urban Population'\n",
    "df_pop_ceb['Total Urban Population'] = [int(popz[0]*popz[1]*0.01) for popz in pops_list]\n",
    "# Plot urban population data\n",
    "df_pop_ceb.plot(kind='scatter', x='Year', y='Total Urban Population')\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Now doing the above for full dataset. Notice appending data to dataframe and initializing an empty one\"\"\"\n",
    "# Initialize reader object: urb_pop_reader\n",
    "urb_pop_reader = pd.read_csv('ind_pop_data.csv', chunksize=1000)\n",
    "# Initialize empty DataFrame: data\n",
    "data = pd.DataFrame()\n",
    "# Iterate over each DataFrame chunk\n",
    "for df_urb_pop in urb_pop_reader:\n",
    "    # Check out specific country: df_pop_ceb\n",
    "    df_pop_ceb = df_urb_pop[df_urb_pop['CountryCode'] == 'CEB']\n",
    "    # Zip DataFrame columns of interest: pops\n",
    "    pops = zip(df_pop_ceb['Total Population'],\n",
    "                df_pop_ceb['Urban population (% of total)'])\n",
    "    # Turn zip object into list: pops_list\n",
    "    pops_list = list(pops)\n",
    "    # Use list comprehension to create new DataFrame column 'Total Urban Population'\n",
    "    df_pop_ceb['Total Urban Population'] = [int(tup[0] * tup[1] * 0.01) for tup in pops_list]\n",
    "    # Append DataFrame chunk to data: data\n",
    "    data = data.append(df_pop_ceb)\n",
    "# Plot urban population data\n",
    "data.plot(kind='scatter', x='Year', y='Total Urban Population')\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Putting all the above together\"\"\"\n",
    "# Define plot_pop()\n",
    "def plot_pop(filename, country_code):\n",
    "    # Initialize reader object: urb_pop_reader\n",
    "    urb_pop_reader = pd.read_csv(filename, chunksize=1000)\n",
    "    # Initialize empty DataFrame: data\n",
    "    data = pd.DataFrame()\n",
    "    # Iterate over each DataFrame chunk\n",
    "    for df_urb_pop in urb_pop_reader:\n",
    "        # Check out specific country: df_pop_ceb\n",
    "        df_pop_ceb = df_urb_pop[df_urb_pop['CountryCode'] == country_code]\n",
    "        # Zip DataFrame columns of interest: pops\n",
    "        pops = zip(df_pop_ceb['Total Population'],\n",
    "                    df_pop_ceb['Urban population (% of total)'])\n",
    "        # Turn zip object into list: pops_list\n",
    "        pops_list = list(pops)\n",
    "        # Use list comprehension to create new DataFrame column 'Total Urban Population'\n",
    "        df_pop_ceb['Total Urban Population'] = [int(tup[0] * tup[1] * 0.01) for tup in pops_list]\n",
    "        # Append DataFrame chunk to data: data\n",
    "        data = data.append(df_pop_ceb)\n",
    "    # Plot urban population data\n",
    "    data.plot(kind='scatter', x='Year', y='Total Urban Population')\n",
    "    plt.show()\n",
    "# Set the filename: fn\n",
    "fn = 'ind_pop_data.csv'\n",
    "# Call plot_pop for country code 'CEB'\n",
    "plot_pop('ind_pop_data.csv','CEB')\n",
    "# Call plot_pop for country code 'ARB'\n",
    "plot_pop('ind_pop_data.csv','ARB')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}