Pandas Foundations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "################ NUMPY AND PANDAS WORK TOGETHER ##################\n",
    "\"\"\"Notice how Pandas and Numpy are working together. Notice the log10() method\"\"\"\n",
    "\n",
    "# Import numpy\n",
    "import numpy as np\n",
    "# Create array of DataFrame values: np_vals\n",
    "np_vals = df.values\n",
    "# Create new array of base 10 logarithm values: np_vals_log10\n",
    "np_vals_log10 = np.log10(np_vals)\n",
    "# Create array of new DataFrame by passing df to np.log10(): df_log10\n",
    "df_log10 = np.log10(df)\n",
    "# Print original and new data containers\n",
    "[print(x, 'has type', type(eval(x))) for x in ['np_vals', 'np_vals_log10', 'df', 'df_log10']]\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"First make a big list, then zip and make it list and then dictify it and then convert to dataframe\"\"\"\n",
    "# Zip the 2 lists together into one list of (key,value) tuples: zipped\n",
    "zipped = list(zip(list_keys, list_values))\n",
    "# Inspect the list using print()\n",
    "print(zipped)\n",
    "# Build a dictionary with the zipped list: data\n",
    "data = dict(zipped)\n",
    "# Build and inspect a DataFrame from the dictionary: df\n",
    "df = pd.DataFrame(data)\n",
    "print(df)\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Assigning column names\"\"\"\n",
    "# Build a list of labels: list_labels\n",
    "list_labels = ['year','artist','song','chart weeks']\n",
    "# Assign the list of labels to the columns attribute: df.columns\n",
    "df.columns = list_labels\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Adding the state column by Broadcasting\"\"\"\n",
    "# Make a string with the value 'PA': state\n",
    "state = 'PA'\n",
    "# Construct a dictionary: data\n",
    "data = {'state':state, 'city':cities}\n",
    "# Construct a DataFrame from dictionary data: df\n",
    "df = pd.DataFrame(data)\n",
    "# Print the DataFrame\n",
    "print(df)\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Reading a file in and changing header\"\"\"\n",
    "# Read in the file: df1\n",
    "df1 = pd.read_csv(data_file)\n",
    "# Create a list of the new column labels: new_labels\n",
    "new_labels = ['year', 'population']\n",
    "# Read in the file, specifying the header and names parameters: df2\n",
    "df2 = pd.read_csv(data_file, header=0, names=new_labels)\n",
    "# Print both the DataFrames\n",
    "print(df1)\n",
    "print(df2)\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Reading a file with comments and delimiters and then saving them\"\"\"\n",
    "# Read the raw file as-is: df1\n",
    "df1 = pd.read_csv(file_messy)\n",
    "# Print the output of df1.head()\n",
    "print(df1.head())\n",
    "# Read in the file with the correct parameters: df2\n",
    "df2 = pd.read_csv(file_messy, delimiter=' ', header=3, comment='#')\n",
    "# Print the output of df2.head()\n",
    "print(df2.head())\n",
    "# Save the cleaned up DataFrame to a CSV file without the index\n",
    "df2.to_csv(file_clean, index=False)\n",
    "# Save the cleaned up DataFrame to an excel file without the index\n",
    "df2.to_excel('file_clean.xlsx', index=False)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "######################## PLOTTING IN PANDAS #########################\n",
    "\"\"\"Plotting a dataframe and labelling\"\"\"\n",
    "# Create a plot with color='red'\n",
    "df.plot(color='red')\n",
    "# Add a title\n",
    "plt.title('Temperature in Austin')\n",
    "# Specify the x-axis label\n",
    "plt.xlabel('Hours since midnight August 1, 2010')\n",
    "# Specify the y-axis label\n",
    "plt.ylabel('Temperature (degrees F)')\n",
    "# Display the plot\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Plotting all together, then each in subplot, then only certain columns or column\"\"\"\n",
    "# Plot all columns (default)\n",
    "df.plot()\n",
    "plt.show()\n",
    "# Plot all columns as subplots\n",
    "df.plot(subplots=True)\n",
    "plt.show()\n",
    "# Plot just the Dew Point data\n",
    "column_list1 = ['Dew Point (deg F)']\n",
    "df[column_list1].plot()\n",
    "plt.show()\n",
    "# Plot the Dew Point and Temperature data, but not the Pressure data\n",
    "column_list2 = ['Temperature (deg F)','Dew Point (deg F)']\n",
    "df[column_list2].plot()\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Below we are plotting with 2 items in y-axis and one in x-axis | Also adding title\"\"\"\n",
    "# Create a list of y-axis column names: y_columns\n",
    "y_columns = ['AAPL','IBM']\n",
    "# Generate a line plot\n",
    "df.plot(x='Month', y=y_columns)\n",
    "# Add the title\n",
    "plt.title('Monthly stock prices')\n",
    "# Add the y-axis label\n",
    "plt.ylabel('Price ($US)')\n",
    "# Display the plot\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Plotting scatter with s argument that specifies size of each dot by data\"\"\"\n",
    "# Generate a scatter plot\n",
    "df.plot(kind='scatter', x='hp', y='mpg', s=sizes)\n",
    "# Add the title\n",
    "plt.title('Fuel efficiency vs Horse-power')\n",
    "# Add the x-axis label\n",
    "plt.xlabel('Horse-power')\n",
    "# Add the y-axis label\n",
    "plt.ylabel('Fuel efficiency (mpg)')\n",
    "# Display the plot\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Plotting box plots, each in it's own plot, Notice how df[cols].plot is used\"\"\"\n",
    "# Make a list of the column names to be plotted: cols\n",
    "cols = ['weight','mpg']\n",
    "# Generate the box plots\n",
    "df[cols].plot(kind='box',subplots=True)\n",
    "# Display the plot\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"PDF and CDF are plotted below\"\"\"\n",
    "# This formats the plots such that they appear on separate rows\n",
    "fig, axes = plt.subplots(nrows=2, ncols=1)\n",
    "# Plot the PDF\n",
    "df.fraction.plot(ax=axes[0], kind='hist', normed=True, bins=30, range=(0,.3))\n",
    "plt.show()\n",
    "# Plot the CDF\n",
    "df.fraction.plot(ax=axes[1], kind='hist', normed=True, cumulative=True, bins=30, range=(0,.3))\n",
    "plt.show()\n",
    "\n",
    "\n",
    "######################## EXPLORATORY DATA ANALYSIS ########################\n",
    "\"\"\"A lot of min, max and mean. Notice how mean is calculated for each row! and then plotted\"\"\"\n",
    "# Print the minimum value of the Engineering column\n",
    "print(min(df['Engineering']))\n",
    "# Print the maximum value of the Engineering column\n",
    "print(max(df['Engineering']))\n",
    "# Construct the mean percentage per year: mean\n",
    "mean = df.mean(axis='columns')\n",
    "# Plot the average percentage per year\n",
    "mean.plot()\n",
    "# Display the plot\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Making a box plot\"\"\"\n",
    "# Print summary statistics of the fare column with .describe()\n",
    "print(df['fare'].describe())\n",
    "# Generate a box plot of the fare column\n",
    "df['fare'].plot(kind='box')\n",
    "# Show the plot\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Getting particular quantiles and boxplotting only certain years\"\"\"\n",
    "# Print the number of countries reported in 2015\n",
    "print(df['2015'].count())\n",
    "# Print the 5th and 95th percentiles\n",
    "print(df.quantile([0.05,0.95]))\n",
    "# Generate a box plot\n",
    "years = ['1800','1850','1900','1950','2000']\n",
    "df[years].plot(kind='box')\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\"\"\"Rows with a particular column value\"\"\"\n",
    "df[df['origin']=='Asia']\n",
    "\n",
    "\n",
    "\"\"\"Plotting selective data in 3 separate box-plots\"\"\"\n",
    "# Display the box plots on 3 separate rows and 1 column\n",
    "fig, axes = plt.subplots(nrows=3, ncols=1)\n",
    "# Generate a box plot of the fare prices for the First passenger class\n",
    "titanic.loc[titanic['pclass'] == 1].plot(ax=axes[0], y='fare', kind='box')\n",
    "# Generate a box plot of the fare prices for the Second passenger class\n",
    "titanic.loc[titanic['pclass'] == 2].plot(ax=axes[1], y='fare', kind='box')\n",
    "# Generate a box plot of the fare prices for the Third passenger class\n",
    "titanic.loc[titanic['pclass'] ==3].plot(ax=axes[2], y='fare', kind='box')\n",
    "# Display the plot\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "###################### DATE FORMATTING ################################\n",
    "\n",
    "\n",
    "\"\"\"Here we are indexing with date\"\"\"\n",
    "df3 = pd.read_csv(filename, index_col='Date', parse_dates=True)\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Converting a date to a date format defined using a string and then making\n",
    "a series out of it and merging to a dataframe while making it the index \"\"\"\n",
    "# Prepare a format string: time_format\n",
    "time_format = '%Y-%m-%d %H:%M'\n",
    "# Convert date_list into a datetime object: my_datetimes\n",
    "my_datetimes = pd.to_datetime(date_list, format=time_format)\n",
    "# Construct a pandas Series using temperature_list and my_datetimes: time_series\n",
    "time_series = pd.Series(temperature_list, index=my_datetimes)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Partial string indexing and slicing : Extracting data for certain slices of time\"\"\"\n",
    "# Extract the hour from 9pm to 10pm on '2010-10-11': ts1\n",
    "ts1 = ts0.loc['2010-10-11 21:00:00':'2010-10-11 22:00:00']\n",
    "# Extract '2010-07-04' from ts0: ts2\n",
    "ts2 = ts0.loc['2010-07-04']\n",
    "# Extract data from '2010-12-15' to '2010-12-31': ts3\n",
    "ts3 = ts0.loc['2010-12-15':'2010-12-31']\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"REINDEXING THE INDEX : Reindexing is useful in preparation for adding or \n",
    "otherwise combining two time series data sets. To reindex the data, \n",
    "we provide a new index and ask pandas to try and match the old data to the new index. \n",
    "If data is unavailable for one of the new index dates or times, \n",
    "you must tell pandas how to fill it in. Otherwise, pandas will fill with NaN by default. || Notice the\n",
    "method attribute of how to fill index if value is not there\"\"\"\n",
    "# Reindex without fill method: ts3\n",
    "ts3 = ts2.reindex(ts1.index)\n",
    "# Reindex with fill method, using forward fill: ts4\n",
    "ts4 = ts2.reindex(ts1.index, method='ffill')\n",
    "# Combine ts1 + ts2: sum12\n",
    "sum12 = ts1+ts2\n",
    "# Combine ts1 + ts3: sum13\n",
    "sum13 = ts1+ts3\n",
    "# Combine ts1 + ts4: sum14\n",
    "sum14 = ts1+ts4\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Pandas provides methods for resampling time series data. When downsampling or upsampling,\n",
    "the syntax is similar, but the methods called are different. Both use the concept of 'method chaining' \n",
    "- df.method1().method2().method3() - to direct the output from one method call to the input of the next, \n",
    "and so on, as a sequence of operations, one feeding into the next.\n",
    "For example, if you have hourly data, and just need daily data, \n",
    "pandas will not guess how to throw out the 23 of 24 points. \n",
    "You must specify this in the method. One approach, for instance, could be to take the mean, \n",
    "as in df.resample('D').mean().\"\"\"\n",
    "# Downsample to 6 hour data and aggregate by mean: df1\n",
    "df1 = df['Temperature'].resample('6H').mean()\n",
    "# Downsample to daily data and count the number of data points: df2\n",
    "df2 = df['Temperature'].resample('D').count()\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Here we are selecting only a certain stretch of data and then resampling it. Notice how\n",
    "only 'Temperature' column is selected only for Auust and February\"\"\"\n",
    "# Extract temperature data for August: august\n",
    "august = df.loc['2010-8-1':'2010-8-31','Temperature']\n",
    "# Downsample to obtain only the daily highest temperatures in August: august_highs\n",
    "august_highs = august.resample('D').max()\n",
    "# Extract temperature data for February: february\n",
    "february =df.loc['2010-2-1':'2010-2-28','Temperature']\n",
    "# Downsample to obtain the daily lowest temperatures in February: february_lows\n",
    "february_lows = february.resample('D').min()\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"GYAN : Rolling means (or moving averages) are generally used to smooth out short-term fluctuations \n",
    "in time series data and highlight long-term trends.To use the .rolling() method, you must always use method chaining, \n",
    "first calling .rolling() and then chaining an aggregation method after it. \n",
    "For example, with a Series hourly_data, hourly_data.rolling(window=24).mean() \n",
    "would compute new values for each hourly point, based on a 24-hour window stretching out behind each point. \n",
    "The frequency of the output data is the same: it is still hourly. \n",
    "Such an operation is useful for smoothing time series data.\n",
    "NOTICE : Notice how there is a differnt way to slice out data section\"\"\"\n",
    "# Extract data from 2010-Aug-01 to 2010-Aug-15: unsmoothed\n",
    "unsmoothed = df['Temperature']['2010-8-1':'2010-8-15']\n",
    "# Apply a rolling mean with a 24 hour window: smoothed\n",
    "smoothed = unsmoothed.rolling(window=24).mean()\n",
    "# Create a new DataFrame with columns smoothed and unsmoothed: august\n",
    "august = pd.DataFrame({'smoothed':smoothed, 'unsmoothed':unsmoothed})\n",
    "# Plot both smoothed and unsmoothed data using august.plot().\n",
    "august.plot()\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"First we get only the daily highs. Then smooth it using rolling mean to a 7 day period\n",
    "NOTICE : window is the attribute to smooth out. that 7 will be day or hour depending upon \n",
    "what initially was there\"\"\"\n",
    "# Extract the August 2010 data: august\n",
    "august = df['Temperature']['2010-8-1':'2010-8-31']\n",
    "# Resample to daily data, aggregating by max: daily_highs\n",
    "daily_highs = august.resample('D').max()\n",
    "# Use a rolling 7-day window with method chaining to smooth the daily high temperatures in August\n",
    "daily_highs_smoothed = daily_highs.rolling(window=7).mean()\n",
    "print(daily_highs_smoothed)\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"We used method chaining and string manipulation to select only Dallas flights. Then resampled\"\"\"\n",
    "# Strip extra whitespace from the column names: df.columns\n",
    "df.columns = df.columns.str.strip()\n",
    "# Extract data for which the destination airport is Dallas: dallas\n",
    "dallas = df['Destination Airport'].str.contains('DAL')\n",
    "# Compute the total number of Dallas departures each day: daily_departures\n",
    "daily_departures = dallas.resample('D').sum()\n",
    "# Generate the summary statistics for daily Dallas departures: stats\n",
    "stats = daily_departures.describe()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Reassigning index and doing interpolation to sensibly fill out missing values\"\"\"\n",
    "# Reset the index of ts2 to ts1, and then use linear interpolation to fill in the NaNs: ts2_interp\n",
    "ts2_interp = ts2.reindex(ts1.index).interpolate(how='linear')\n",
    "# Compute the absolute difference of ts1 and ts2_interp: differences \n",
    "differences = np.abs(ts1-ts2_interp)\n",
    "# Generate and print summary statistics of the differences\n",
    "print(differences.describe())\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Selecting only LAX containing rows using index 'mask'. Note that 'mask' yields a true false column.\n",
    "Then using to_datetime to create a series.Then localizing it to a different timezone after standardizing it to one\"\"\"\n",
    "# Build a Boolean mask to filter out all the 'LAX' departure flights: mask\n",
    "mask = df['Destination Airport'] == 'LAX'\n",
    "# Use the mask to subset the data: la\n",
    "la = df[mask]\n",
    "# Combine two columns of data to create a datetime series: times_tz_none \n",
    "times_tz_none = pd.to_datetime( la['Date (MM/DD/YYYY)'] + ' ' + la['Wheels-off Time'] )\n",
    "# Localize the time to US/Central: times_tz_central\n",
    "times_tz_central = times_tz_none.dt.tz_localize('US/Central')\n",
    "# Convert the datetimes from US/Central to US/Pacific\n",
    "times_tz_pacific = times_tz_central.dt.tz_convert('US/Pacific')\n",
    "\n",
    "\n",
    "\n",
    "######################## PLOTTING USING PANDAS###############################\n",
    "\"\"\"Setting index and then plotting\"\"\"\n",
    "# Plot the raw data before setting the datetime index\n",
    "df.plot()\n",
    "plt.show()\n",
    "# Convert the 'Date' column into a collection of datetime objects: df.Date\n",
    "df.Date = pd.to_datetime(df.Date)\n",
    "# Set the index to be the converted 'Date' column\n",
    "df.set_index('Date', inplace=True)\n",
    "# Re-plot the DataFrame to see that the axis is now datetime aware!\n",
    "df.plot()\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Plotting different slices of timeranges. \n",
    "NOTICE : How 1 month is sliced\"\"\"\n",
    "# Plot the summer data\n",
    "df.Temperature['2010-Jun':'2010-Aug'].plot()\n",
    "plt.show()\n",
    "plt.clf()\n",
    "# Plot the one week data\n",
    "df.Temperature['2010-06-10':'2010-06-17'].plot()\n",
    "plt.show()\n",
    "plt.clf()\n",
    "\n",
    "\n",
    "\n",
    "########################## CASE STUDY ##################################\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Reading in datafile and then Rereading to make sure headers are consistent\"\"\"\n",
    "# Import pandas\n",
    "import pandas as pd\n",
    "# Read in the data file: df\n",
    "df = pd.read_csv(data_file)\n",
    "# Print the output of df.head()\n",
    "print(df.head())\n",
    "# Read in the data file with header=None: df_headers\n",
    "df_headers = pd.read_csv(data_file, header=None)\n",
    "# Print the output of df_headers.head()\n",
    "print(df_headers.head())\n",
    "\"\"\"Below we are : splitting data columns that had comma separation ||\n",
    "assigning column names ||\n",
    "dropping columns that we dont want ||\"\"\"\n",
    "# Split on the comma to create a list: column_labels_list\n",
    "column_labels_list = column_labels.split(',')\n",
    "# Assign the new column labels to the DataFrame: df.columns\n",
    "df.columns = column_labels_list\n",
    "# Remove the appropriate columns: df_dropped\n",
    "df_dropped = df.drop(list_to_drop, axis='columns')\n",
    "# Print the output of df_dropped.head()\n",
    "print(df_dropped.head())\n",
    "\"\"\"In order to use the full power of pandas time series, you must construct a DatetimeIndex. \n",
    "To do so, it is necessary to clean and transform the date and time columns.\n",
    "Here we are doing the following : Converting the date to string. Note how we do it ||\n",
    "We are padding the time with zeroes. Note how it is done ||\n",
    "Then we concat the date na dtime strings ||\n",
    "After concat we apply the datetime conversion and also mention the format ||\n",
    "Then we set that column as the index\"\"\"\n",
    "# Convert the date column to string: df_dropped['date']\n",
    "df_dropped['date'] = df_dropped['date'].astype(str)\n",
    "# Pad leading zeros to the Time column: df_dropped['Time']\n",
    "df_dropped['Time'] = df_dropped['Time'].apply(lambda x:'{:0>4}'.format(x))\n",
    "# Concatenate the new date and Time columns: date_string\n",
    "date_string = df_dropped['date']+df_dropped['Time']\n",
    "# Convert the date_string Series to datetime: date_times\n",
    "date_times = pd.to_datetime(date_string, format='%Y%m%d%H%M')\n",
    "# Set the index to be the new date_times container: df_clean\n",
    "df_clean = df_dropped.set_index(date_times)\n",
    "# Print the output of df_clean.head()\n",
    "print(df_clean.head())\n",
    "\"\"\"Here we clean the data specially for columns where there are non-numeric values instead of numeric\n",
    "First print a certain time interval with .loc || then convert all values to numeric or NaN ||\n",
    "Note how errors(values that are non-numeric) are coerced to be NaN\"\"\"\n",
    "# Print the dry_bulb_faren temperature between 8 AM and 9 AM on June 20, 2011\n",
    "print(df_clean.loc['2011-6-20 08:00:00':'2011-6-20 09:00:00', 'dry_bulb_faren'])\n",
    "# Convert the dry_bulb_faren column to numeric values: df_clean['dry_bulb_faren']\n",
    "df_clean['dry_bulb_faren'] = pd.to_numeric(df_clean['dry_bulb_faren'], errors='coerce')\n",
    "# Print the transformed dry_bulb_faren temperature between 8 AM and 9 AM on June 20, 2011\n",
    "print(df_clean.loc['2011-6-20 08:00:00':'2011-6-20 09:00:00', 'dry_bulb_faren'])\n",
    "# Convert the wind_speed and dew_point_faren columns to numeric values\n",
    "df_clean['wind_speed'] = pd.to_numeric(df_clean['wind_speed'], errors='coerce')\n",
    "df_clean['dew_point_faren'] = pd.to_numeric(df_clean['dew_point_faren'], errors='coerce')\n",
    "\n",
    "\"\"\"Below we are selecting some slices of time data and then doing EDA on them\"\"\"\n",
    "# Print the median of the dry_bulb_faren column\n",
    "print(df_clean['dry_bulb_faren'].median())\n",
    "# Print the median of the dry_bulb_faren column for the time range '2011-Apr':'2011-Jun'\n",
    "print(df_clean.loc['2011-Apr':'2011-Jun', 'dry_bulb_faren'].median())\n",
    "# Print the median of the dry_bulb_faren column for the month of January\n",
    "print(df_clean.loc['2011-Jan':'2011-Jan', 'dry_bulb_faren'].median())\n",
    "\n",
    "\"\"\"Below we are downsampling and getting mean || Then converting to numpy array ||\n",
    "Then notice how we are resetting index and getting a particular column ||\n",
    "Oddly after that we are subtracting ndarray and Series\"\"\"\n",
    "# Downsample df_clean by day and aggregate by mean: daily_mean_2011\n",
    "daily_mean_2011 = df_clean.resample('D').mean()\n",
    "# Extract the dry_bulb_faren column from daily_mean_2011 using .values: daily_temp_2011\n",
    "daily_temp_2011 = daily_mean_2011['dry_bulb_faren'].values\n",
    "# Downsample df_climate by day and aggregate by mean: daily_climate\n",
    "daily_climate = df_climate.resample('D').mean()\n",
    "# Extract the Temperature column from daily_climate using .reset_index(): daily_temp_climate\n",
    "daily_temp_climate = daily_climate.reset_index()['Temperature']\n",
    "# Compute the difference between the two arrays and print the mean difference\n",
    "difference = daily_temp_2011 - daily_temp_climate\n",
    "print(difference.mean())\n",
    "\n",
    "\"\"\"We are selecting sunny and then overcast days. Notice how odd that == is not working for ovc and we have to use string\"\"\"\n",
    "# Select days that are sunny: sunny\n",
    "sunny = df_clean.loc[df_clean['sky_condition']=='CLR']\n",
    "# Select days that are overcast: overcast\n",
    "overcast = df_clean.loc[df_clean['sky_condition'].str.contains('OVC')]\n",
    "# Resample sunny and overcast, aggregating by maximum daily temperature\n",
    "sunny_daily_max = sunny.resample('D').max()\n",
    "overcast_daily_max = overcast.resample('D').max()\n",
    "# Print the difference between the mean of sunny_daily_max and overcast_daily_max\n",
    "print(sunny_daily_max.mean() - overcast_daily_max.mean())\n",
    "\n",
    "\"\"\"Resampling weekly and then plotting in separate plots.\n",
    "Notice how we need to pass a list with [[]] for 2 columns and not a single[]. Notice the correlation coefficient .corr()\"\"\"\n",
    "# Import matplotlib.pyplot as plt\n",
    "import matplotlib.pyplot as plt\n",
    "# Select the visibility and dry_bulb_faren columns and resample them: weekly_mean\n",
    "weekly_mean = df_clean[['visibility','dry_bulb_faren']].resample('W').mean()\n",
    "# Print the output of weekly_mean.corr()\n",
    "print(weekly_mean.corr())\n",
    "# Plot weekly_mean with subplots=True\n",
    "weekly_mean.plot(subplots=True)\n",
    "plt.show()\n",
    "\n",
    "\"\"\"Below we are doing a boolean series and then resampling it. The we are getting sunny hours and total hours. Box plot\"\"\"\n",
    "# Create a Boolean Series for sunny days: sunny\n",
    "sunny = df_clean['sky_condition']=='CLR'\n",
    "# Resample the Boolean Series by day and compute the sum: sunny_hours\n",
    "sunny_hours = sunny.resample('D').sum()\n",
    "# Resample the Boolean Series by day and compute the count: total_hours\n",
    "total_hours = sunny.resample('D').count()\n",
    "# Divide sunny_hours by total_hours: sunny_fraction\n",
    "sunny_fraction = sunny_hours / total_hours\n",
    "# Make a box plot of sunny_fraction\n",
    "sunny_fraction.plot(kind='box')\n",
    "plt.show()\n",
    "\"\"\"Notice Here we are resampling by month and aggreegating by max for 2 columns in 1 go. The plottin hist\"\"\"\n",
    "# Resample dew_point_faren and dry_bulb_faren by Month, aggregating the maximum values: monthly_max\n",
    "monthly_max = df_clean[['dew_point_faren','dry_bulb_faren']].resample('M').max()\n",
    "# Generate a histogram with bins=8, alpha=0.5, subplots=True\n",
    "monthly_max.plot(kind='hist',bins=8,alpha=0.5,subplots=True)\n",
    "# Show the plot\n",
    "plt.show()\n",
    "\"\"\"Extracting max from 2 columns | extracting daywise sampled(from hourly) temps by getting only max for each day |\n",
    "choosing only those of column that exceed a threshold | plotting a histogram with that\"\"\"\n",
    "# Extract the maximum temperature in August 2010 from df_climate: august_max\n",
    "august_max = df_climate.loc['2010-Aug','Temperature'].max()\n",
    "print(august_max)\n",
    "# Resample August 2011 temps in df_clean by day & aggregate the max value: august_2011\n",
    "august_2011 = df_clean.loc['2011-Aug','dry_bulb_faren'].resample('D').max()\n",
    "# Filter for days in august_2011 where the value exceeds august_max: august_2011_high\n",
    "august_2011_high = august_2011.loc[august_2011 > august_max]\n",
    "# Construct a CDF of august_2011_high\n",
    "august_2011_high.plot(kind='hist', normed=True, cumulative=True, bins=25)\n",
    "# Display the plot\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}