ProgramInfoAndCreation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LOADED\n"
     ]
    }
   ],
   "source": [
    "from googlefinance import getNews\n",
    "from os.path import isfile, join\n",
    "from yahoo_finance import Share\n",
    "from nyse_dates_prds import *\n",
    "from os import listdir\n",
    "import cPickle as pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "import urllib\n",
    "\n",
    "print \"LOADED\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LOADED\n"
     ]
    }
   ],
   "source": [
    "def print_helper_stocks():\n",
    "    \"\"\"\n",
    "    Print the helper Indexes/Funds/ETFs that are added to every company indicator dataframes \n",
    "    during the prediction process due to informational knowledge you can gain by following\n",
    "    these helper stocks.\n",
    "    \"\"\"\n",
    "    print \"Stocks used to help track movement include...\"\n",
    "    attrs      = ['^GSPC', '^DJI', '^IXIC', 'USO', 'GLD', 'SPY']    \n",
    "    names      = ['S&P 500 Index', 'Dow Jones Index', 'Nasdaq Index', 'U.S. Oil Fund',\n",
    "                  'SPDR Gold Shares', 'SPDR S&P 500 ETF']\n",
    "    \n",
    "    # Put the stock symbols(attrs), and the stock names in a dataframe together\n",
    "    df         = pd.DataFrame([attrs, names]).T\n",
    "    df.columns = ['Stock Tickers', 'Ticker Names']\n",
    "    df         = df.set_index('Stock Tickers')\n",
    "    # Print the dataframe\n",
    "    print df, '\\n'\n",
    "    return\n",
    "\n",
    "def get_stock_info(hlc):\n",
    "    \"\"\"\n",
    "    Retrieve information of the available stocks you can predict for. For the real-time \n",
    "    calculating portion of the project, you can calculate indicators for more than 50 \n",
    "    companies at once. Call this function to learn many attributes of the stock, like \n",
    "    price, avg volume, etc.. We do this by using a yahoo finance function that \n",
    "    internally calls a yahoo finance server. We don't use this server during real-time \n",
    "    calculation because of a 15-minute delay but it's fine for just simply learning \n",
    "    about stocks we might want to try.\n",
    "    \"\"\"\n",
    "    stock_info_dict = {}\n",
    "    stock_nm, stock_tk = [], []\n",
    "    # The 46 stocks used in this project\n",
    "    stock_names   = {'BPOP':'Popular Inc.',\n",
    "                     'FITB':'Fifth Third Bancorp', \n",
    "                     'HBAN':'Huntington Bancshares Inc.',\n",
    "                     'CMCSA':'Comcast Corp.', \n",
    "                     'EBAY':'Ebay Inc.',\n",
    "                     'AAPL':'Apple Inc.', \n",
    "                     'AMAT':'Applied Materials Inc.',\n",
    "                     'BRCD':'Brocade Communications Systems Inc.',\n",
    "                     'CSCO':'Cisco Systems Inc.', \n",
    "                     'GOOG':'Google Inc.',\n",
    "                     'INTC':'Intel Corp.',\n",
    "                     'LVLT':'Level 3 Communications Inc.', \n",
    "                     'MSFT':'Microsoft Corp.',\n",
    "                     'MU':'Micron Technology Inc.',\n",
    "                     'NVDA':'NVIDIA Corp.', \n",
    "                     'ORCL':'Oracle Corp.',\n",
    "                     'QCOM':'QUALCOMM Inc.',\n",
    "                     'SIRI':'Sirius XM Holdings Inc.', \n",
    "                     'WIN':'Windstream Holdings Inc.',\n",
    "                     'YHOO':'Yahoo! Inc.',\n",
    "                     'BHP':'BHP Billiton Limited', \n",
    "                     'BP':'British Petroleum Plc',\n",
    "                     'RIO':'Rio Tinto Plc',\n",
    "                     'XOM':'Exxon Mobil Corp.', \n",
    "                     'GE':'General Electric Company',\n",
    "                     'F':'Ford Motor Company',\n",
    "                     'MO':'Altria Group Inc.', \n",
    "                     'XRX':'Xerox Corp.',\n",
    "                     'GS':'Goldman Sachs Group Inc.',\n",
    "                     'JPM':'JPMorgan Chase & Co.', \n",
    "                     'LYG':'Lloyds Banking Group Plc',\n",
    "                     'MS':'Morgan Stanley',\n",
    "                     'RF':'Regions Financial Corp.', \n",
    "                     'USB':'U.S. Bancorp',\n",
    "                     'WFC':'Wells Fargo & Co.',\n",
    "                     'MRK':'Merck & Co. Inc.', \n",
    "                     'PFE':'Pfizer Inc.',\n",
    "                     'LMT':'Lockheed Martin Corp.',\n",
    "                     'MGM':'MGM Resorts International', \n",
    "                     'AMD':'Advanced Micro Devices Inc.',\n",
    "                     'GLW':'Corning Inc.',\n",
    "                     'HPQ':'HP Inc.', \n",
    "                     'S':'Sprint Corp.',\n",
    "                     'T':'AT&T Inc.'}\n",
    "    \n",
    "    # For each stock you are using\n",
    "    for each in hlc.keys():\n",
    "        # If it's in our symbol/name dictionary, append there symbol/nmaes to lists\n",
    "        if each in stock_names.keys():\n",
    "            stock_tk.append(each)\n",
    "            stock_nm.append(stock_names[each])\n",
    "    \n",
    "    print \"Stocks available to track/predict...\"\n",
    "    name_df = pd.DataFrame([stock_tk, stock_nm]).T\n",
    "    name_df.columns = ['Abreviations', 'Stock Names']\n",
    "    name_df = name_df.set_index('Abreviations')\n",
    "    print name_df, '\\n'\n",
    "    print \"For information on certain stock, type: \\n\\t print info['stock abreviation'] \\n\"\n",
    "    print \"Example: print info['AAPL']\"\n",
    "    \n",
    "    # For each stock your using, gets all of its relevent data and put it in a dictionary\n",
    "    # entry\n",
    "    for name, name2 in zip(stock_tk, stock_nm):\n",
    "        try:\n",
    "            comp  = Share(name)\n",
    "            comp.refresh()\n",
    "            \n",
    "            price       = comp.get_price()\n",
    "            avg_vol     = comp.get_avg_daily_volume()\n",
    "            stk_exc     = comp.get_stock_exchange()\n",
    "            mark_cap    = comp.get_market_cap()\n",
    "            book_val    = comp.get_book_value()\n",
    "            ebit        = comp.get_ebitda()\n",
    "            div_share   = comp.get_dividend_share()\n",
    "            div_yield   = comp.get_dividend_yield()\n",
    "            earn_shares = comp.get_earnings_share()\n",
    "            yr_hi       = comp.get_year_high()\n",
    "            yr_lo       = comp.get_year_low()\n",
    "            avg50       = comp.get_50day_moving_avg()\n",
    "            avg200      = comp.get_200day_moving_avg()\n",
    "            pe_rat      = comp.get_price_earnings_ratio()\n",
    "            pe_growth   = comp.get_price_earnings_growth_ratio()\n",
    "            short_rat   = comp.get_short_ratio()\n",
    "\n",
    "            # Names of the stock attributes youre retrieving\n",
    "            df_index   = ['Stock Price:', 'Average Daily Volume:', 'Stock Exchange:', \n",
    "                          'Market Cap:', 'Book Value:', 'EBITDA:', 'Dividend Share:', \n",
    "                          'Dividend Yield:', 'Earnings Share:', 'Year High:', \n",
    "                          'Year Low:', 'Moving 50 Day Average:', \n",
    "                          'Moving 200 Day Average:', 'Price Earnings Ratio:', \n",
    "                          'Price Earnings Growth Ratio:','Short Ratio']\n",
    "            index_vals = [price, avg_vol, stk_exc, mark_cap, book_val,\n",
    "                          ebit, div_share, div_yield, earn_shares, yr_hi,\n",
    "                          yr_lo, avg50, avg200, pe_rat, pe_growth,\n",
    "                          short_rat]\n",
    "\n",
    "            # Combine the list of names with its respective values and put it in the dictionry\n",
    "            stock_info_dict[name] = pd.DataFrame(index_vals, index = df_index, columns=[name2])\n",
    "        except:\n",
    "            print name, name2\n",
    "            pass\n",
    "    print \"DONE\"\n",
    "    return stock_info_dict\n",
    "\n",
    "def get_parameter_choices():\n",
    "    \"\"\"\n",
    "    There are 12 different indicators that we use for prediction, but for each different \n",
    "    indicator, we have up to 32 different period length values, which is essentially how\n",
    "    far back the data uses to create the indicator value, so some use 10 minutes of previous\n",
    "    stock data, while others use up to 125 stock days worth of data to calculate. \n",
    "    \n",
    "    The real-time calculation will calculate all of indicators with all of the period lengths\n",
    "    but for the prediction process, you need to choose a subset of these to use because the more\n",
    "    attributes you use, the longer it takes to predict, and soon becomes to long to be useful. I\n",
    "    found a good number to be 230 attributes. Now we use our 6 helper stocks, plus the stock we're\n",
    "    are predicting for, so for every indicator/period we choose, that's equal to 7 attributes\n",
    "    because each stock uses it to help the prediction. \n",
    "    \n",
    "    So try to keep the number of different indicator/periods you choose to be around 40 unless\n",
    "    you have the computing power/memory to do it. I'm running 12GB of memory, with a i7-4510U\n",
    "    so if you have better specs you can use more indicators.\n",
    "    \"\"\"\n",
    "    attribute_choices = {}\n",
    "\n",
    "    attributes      = ['rets', 'per', 'bol', 'vol', 'aro', 'mac', \n",
    "                       'mac2', 'sma', 'kdo', 'mom', 'adx', 'rsi']\n",
    "    attr_names      = ['Returns', 'Price Earnings Ratio', 'Bollinger Bands',\n",
    "                       'Volatility', 'Aroon', 'Moving Average Convergence Divergence',\n",
    "                       'Custom MACD', 'Simple Moving Average', 'Stochastic Oscillators',\n",
    "                       'Momentum', 'Average Directional Index', 'Relative Strength Index']\n",
    "\n",
    "    day_lengths     = ['1day',  '2day',  '3day',  '5day', \n",
    "                      '8day',  '10day', '12day', '14day', \n",
    "                      '16day', '20day', '25day', '30day', \n",
    "                      '40day', '50day', '80day', '125day']\n",
    "    \n",
    "    day_lengths3    = ['1&2day','2&3day','3&5day','5&8day',\n",
    "                       '8&10day','10&12day','12&14day','14&16day',\n",
    "                       '16&20day','20&25day','25&30day','30&40day',\n",
    "                       '40&50day','50&80day']\n",
    "    day_lengths4    = ['1&2&4day','2&3&8day','3&5&10day','5&8&12day',\n",
    "                       '8&10&14day','10&12&16day','12&14&20day','14&16&25day',\n",
    "                       '16&20&30day','20&25&40day','25&30&50day','30&40&80day',\n",
    "                       '40&50&125day']\n",
    "    \n",
    "    day_lengths5    = ['1&4day','2&8day','3&10day','5&12day',\n",
    "                       '8&14day','10&16day','12&20day','14&25day',\n",
    "                       '16&30day','20&40day','25&50day','30&80day',\n",
    "                       '40&125day']\n",
    "\n",
    "    minute_lengths1 = ['10min',  '14min',  '16min',  '18min',  \n",
    "                      '20min',  '25min',  '30min',  '40min',  \n",
    "                      '50min',  '75min',  '100min', '125min', \n",
    "                      '150min', '200min', '250min', '300min']\n",
    "    \n",
    "    minute_lengths2 = ['1min','2min','3min','4min','5min'] + minute_lengths1\n",
    "    \n",
    "    minute_lengths3 = ['8&10min','10&14min','14&16min','16&18min',\n",
    "                       '18&20min','20&25min','25&30min','30&40min',\n",
    "                       '40&50min','50&75min','75&100min','100&125min',\n",
    "                       '125&150min','150&200min','200&250min','250&300min',\n",
    "                       '300&350min','350min&1day']\n",
    "    \n",
    "    minute_lengths4 = ['6&8&14min','8&10&16min','10&14&18min','14&16&20min',\n",
    "                       '16&18&25min','18&20&30min','20&25&40min','25&30&50min',\n",
    "                       '30&40&75min','40&50&100min','50&75&125min','75&100&150min',\n",
    "                       '100&125&200min','125&150&250min','150&200&300min','200&250&350min',\n",
    "                       '250&300&1day','300&350min&2day','350min&1&3day']\n",
    "    \n",
    "    minute_lengths5 = ['6&14min','8&16min','10&18min','14&20min',\n",
    "                       '16&25min','18&30min','20&40min','25&50min',\n",
    "                       '30&75min','40&100min','50&125min','75&150min',\n",
    "                       '100&200min','125&250min','150&300min','200&350min',\n",
    "                       '250min&1day','300min&2day','350min&3day']\n",
    "\n",
    "    minute_lengths2 = ['1min','2min','3min','4min','5min'] + minute_lengths1\n",
    "\n",
    "    # Combine the minutes with the days\n",
    "    indicator_lengths1 = minute_lengths1 + day_lengths\n",
    "    indicator_lengths2 = minute_lengths2 + day_lengths[:11]\n",
    "    indicator_lengths3 = minute_lengths3 + day_lengths3\n",
    "    indicator_lengths4 = minute_lengths4 + day_lengths4\n",
    "    indicator_lengths5 = minute_lengths5 + day_lengths5\n",
    "\n",
    "    new_df = pd.DataFrame()\n",
    "    # The first 6 indicators in the list have different indicator lengths\n",
    "    # while the rest of the indicators all have the same\n",
    "    lst = ['rsi','mac','mac2','vol','kdo','rets','ALL OTHERS']\n",
    "    for key in lst:\n",
    "        if key in ['rsi', 'vol']:\n",
    "            df = pd.DataFrame(indicator_lengths5, columns=[key])\n",
    "        elif key in ['mac', 'mac2']:\n",
    "            df = pd.DataFrame(indicator_lengths4, columns=[key])\n",
    "        elif key == 'kdo':\n",
    "            df = pd.DataFrame(indicator_lengths3, columns=[key])\n",
    "        elif key == 'rets':\n",
    "            df = pd.DataFrame(indicator_lengths2, columns=[key])\n",
    "        else:\n",
    "            df = pd.DataFrame(indicator_lengths1,  columns=[key])\n",
    "        new_df = pd.concat([new_df, df], axis=1)\n",
    "    \n",
    "    names = pd.DataFrame([attributes, attr_names]).T\n",
    "    names.columns = ['Abreviations','Indicators']\n",
    "    names = names.set_index('Abreviations')\n",
    "    print \"Indicators available to use, with the \"\n",
    "    print names, '\\n'\n",
    "    print new_df\n",
    "    return\n",
    "\n",
    "def create_indicator_list(choice, rets=None, per=None, bol=None, vol=None, aro=None, mac=None, \n",
    "                          mac2=None, sma=None, kdo=None, mom=None, adx=None, rsi=None):\n",
    "    \"\"\"\n",
    "    This parses your parameter choices you made for indicators. You feed in a string with your\n",
    "    number choices, based on the number row from the dataframe shown in the parameter choices\n",
    "    function.\n",
    "    \"\"\"\n",
    "    indicators = []\n",
    "    stocks = ['^GSPC','^IXIC','^DJI','GLD','USO','SPY',choice]\n",
    "    lst1  = [rets, per, bol, vol, aro, mac, mac2, sma, kdo, mom, adx, rsi]\n",
    "    lst2 = ['rets','per','bol','vol','aro','mac','mac2','sma','kdo','mom','adx','rsi']\n",
    "    lst3 = [per, bol, vol, aro, mac, mac2, sma, kdo, mom, adx, rsi]\n",
    "    lst4 = ['per','bol','vol','aro','mac','mac2','sma','kdo','mom','adx','rsi']\n",
    "    nums = ['0','1','2','3','4','5','6','7','8','9']\n",
    "    \n",
    "    for stock in stocks:\n",
    "        name_lst     = []\n",
    "        name_nums    = []\n",
    "        through_char = False\n",
    "        double       = False\n",
    "        \n",
    "        if stock != choice:\n",
    "            ind_lst1 = lst1\n",
    "            ind_lst2 = lst2\n",
    "        else:\n",
    "            ind_lst1 = lst3\n",
    "            ind_lst2 = lst4\n",
    "            \n",
    "        for key, key2 in zip(ind_lst1, ind_lst2):\n",
    "            if key != None:\n",
    "                while key != '':\n",
    "                    if through_char == False:\n",
    "                        next_char = key[0]\n",
    "                        if next_char == '-':\n",
    "                            through_char = True\n",
    "                        elif next_char == ',':\n",
    "                            tt = 0\n",
    "                        elif next_char == ' ':\n",
    "                            print \"No spaces allowed in parameters\"\n",
    "                            break\n",
    "                        else:\n",
    "                            try:\n",
    "                                if key[1] != '-':\n",
    "                                    if key[1] in nums:\n",
    "                                        if int(key[:2]) < 32:\n",
    "                                            try:\n",
    "                                                if key[2] == '-':\n",
    "                                                    num_start = int(key[:2])\n",
    "                                                    double = True\n",
    "                                                else:\n",
    "                                                    if int(key[:2]) not in name_nums:\n",
    "                                                        name_lst.append(key[:2])\n",
    "                                                        name_nums.append(int(key[:2]))\n",
    "                                                    double = True\n",
    "                                            except:\n",
    "                                                if int(key[:2]) not in name_nums:\n",
    "                                                    name_lst.append(key[:2])\n",
    "                                                    name_nums.append(int(key[:2]))\n",
    "                                                double = True\n",
    "                                        else:\n",
    "                                            print \"Number incorrect:\"\n",
    "                                            print \"Make sure number is between 0 and 31\"\n",
    "                                            break\n",
    "                                    else:\n",
    "                                        if int(key[0]) >= 0:\n",
    "                                            if int(key[0]) not in name_nums:\n",
    "                                                name_lst.append(key[0])\n",
    "                                                name_nums.append(int(key[0]))\n",
    "                                        else:\n",
    "                                            print \"Number incorrect:\"\n",
    "                                            print \"Make sure number is between 0 and 31\"\n",
    "                                            break\n",
    "\n",
    "                                else:\n",
    "                                    num_start = int(key[0])\n",
    "\n",
    "                            except:\n",
    "                                if int(key[0]) not in name_nums:\n",
    "                                    name_lst.append(key[0])\n",
    "                                    name_nums.append(int(key[0]))\n",
    "                                pass\n",
    "\n",
    "                        if double == False:\n",
    "                            key = key[1:]\n",
    "                        else:\n",
    "                            key = key[2:]\n",
    "                            double = False\n",
    "\n",
    "                    else:\n",
    "                        try:\n",
    "                            if key[1] not in nums:\n",
    "                                next_char = int(key[0])\n",
    "                                if next_char > num_start:\n",
    "                                    comb = xrange(num_start, int(next_char)+1)\n",
    "                                    for each in comb:\n",
    "                                        if int(each) not in name_nums:\n",
    "                                            name_lst.append(str(each))\n",
    "                                            name_nums.append(int(each))\n",
    "                                else:\n",
    "                                    print \"x-y range incorrect:\"\n",
    "                                    print \"Make sure y > x and y < 32 and x > 0\"\n",
    "                                    break\n",
    "                            else:\n",
    "                                next_char = int(key[:2])\n",
    "                                double = True\n",
    "                                if next_char > num_start and next_char < 32:\n",
    "                                    comb = xrange(num_start, int(next_char)+1)\n",
    "                                    for each in comb:\n",
    "                                        if int(each) not in name_nums:\n",
    "                                            name_lst.append(str(each))\n",
    "                                            name_nums.append(int(each))\n",
    "                                else:\n",
    "                                    print \"x-y range incorrect:\"\n",
    "                                    print \"Make sure y > x and y < 32 and x > 0\"\n",
    "                                    break\n",
    "                        except:\n",
    "                            next_char = int(key[0])\n",
    "                            if next_char > num_start:\n",
    "                                comb = xrange(num_start, int(next_char)+1)\n",
    "                                for each in comb:\n",
    "                                    if int(each) not in name_nums:\n",
    "                                        name_lst.append(str(each))\n",
    "                                        name_nums.append(int(each))\n",
    "                            else:\n",
    "                                print \"x-y range incorrect:\"\n",
    "                                print \"Make sure y > x and y < 32 and x > 0\"\n",
    "                                break\n",
    "\n",
    "                        if double == False:\n",
    "                            key = key[1:]\n",
    "                        else:\n",
    "                            key = key[2:]\n",
    "                        double = False\n",
    "                        through_char = False \n",
    "                \n",
    "                for number in name_lst:\n",
    "                    indicators.append(stock+'_'+key2+number)\n",
    "    return indicators\n",
    "\n",
    "def create_columnname_dict(hlc):\n",
    "    \"\"\"\n",
    "    This creates a column name list for each company in your highlowclose dictionary\n",
    "    that is used in the real-time calculation.\n",
    "    \"\"\"\n",
    "    # This dictionary contains the names of stocks that have different Yahoo server\n",
    "    # symbol names vs Google server symbol names, the key is the yahoo name.\n",
    "    tickers2      = {'USO':'NYSEARCA:USO','GLD':'NYSEARCA:GLD',\n",
    "                     'SPY':'NYSEARCA:SPY','^DJI':'INDEXDJX:.DJI',\n",
    "                     '^GSPC':'INDEXSP:.INX','^IXIC':'INDEXNASDAQ:.IXIC', \n",
    "                     'LMT':'NYSE:LMT'}\n",
    "    # List of indicator abreviations\n",
    "    indicator_lst = ['rsi', 'vol', 'sma', 'cci', 'per', 'mom', 'bol', \n",
    "                     'aro', 'mac', 'mactwo', 'adx', 'kdo', 'rets']\n",
    "    nm_dict = {}\n",
    "    # For each stock youre using\n",
    "    for name in hlc.keys():\n",
    "        # If the stock is one of the different named stocks, use the google symbol\n",
    "        if name in tickers2.keys():\n",
    "            name2 = tickers2[name]\n",
    "        else:\n",
    "            name2 = name\n",
    "            \n",
    "        indicator_nms = []\n",
    "        # For each indicator abreviation\n",
    "        for ind in indicator_lst:\n",
    "            # For each of the 32 periods for each indicator\n",
    "            for d in range(32):\n",
    "                # The indicator name is combination of stock name\n",
    "                indicator_nms.append(name+'_'+indd+str(d))\n",
    "        nm_dict[name2] = indicator_nms\n",
    "    \n",
    "    # Dump the indicator name dictionary to a file\n",
    "    opp = open('Pickles/columnnames.pickle','wb')\n",
    "    pickle.dump(nm_dict, opp)\n",
    "    opp.close()\n",
    "    return\n",
    "\n",
    "def company_news(newstickers):\n",
    "    \"\"\"\n",
    "    Use Google's server and use https://github.com/hongtaocai/googlefinance module to retrieve\n",
    "    the real-time stock news data for each company stock symbol fed to it. This retrieves all\n",
    "    of the news articles related to that stock from present back up to 2 years ago.\n",
    "    \"\"\"\n",
    "    tickernewslist = {}\n",
    "    \n",
    "    for each in newstickers:\n",
    "        tickernewslist[each] = getNews(each)  \n",
    "    return tickernewslist\n",
    "\n",
    "def create_news_dict(hlc):\n",
    "    \"\"\"\n",
    "    We use the google finance function that calls the Google Finance servers for news articles\n",
    "    for every company we provide it. They archive articles having to do with the stocks from \n",
    "    current to several years ago. We currently aren't using the news for prediction but are in\n",
    "    the process of creating a NLP component to help our RTNN component.\n",
    "    \"\"\"\n",
    "    tickers       = []\n",
    "    tickers2      = {'USO':'NYSEARCA:USO','GLD':'NYSEARCA:GLD',\n",
    "                     'SPY':'NYSEARCA:SPY','^DJI':'INDEXDJX:.DJI',\n",
    "                     '^GSPC':'INDEXSP:.INX','^IXIC':'INDEXNASDAQ:.IXIC', \n",
    "                     'LMT':'NYSE:LMT'}\n",
    "    for each in hlc.keys():\n",
    "        if each in tickers2.keys():\n",
    "            tickers.append(tickers2[each])\n",
    "        else:\n",
    "            tickers.append(each)\n",
    "            \n",
    "    hist_news = {}\n",
    "    last_date = {}\n",
    "\n",
    "    # Call our company news function passing a list of stock symbols, it will return a\n",
    "    # dictionary where the keys are the symbol names, and the values are lists of \n",
    "    # dictionaries where each dictionary is a seperate article with the url, and other\n",
    "    # metadata within it.\n",
    "    todays_news = company_news(tickers)\n",
    "    for tick in tickers:\n",
    "        count = 0\n",
    "        news  = {}\n",
    "\n",
    "        # For each stock, go through each dictionary containing seperate articles, and\n",
    "        # get our date, article title, and url from it.\n",
    "        for value in todays_news[tick]:\n",
    "            d     = value['d']\n",
    "            title = value['t']\n",
    "            url   = value['u']\n",
    "\n",
    "            # The date is either in string mm/dd/yy format, in which we strip it from \n",
    "            # its string format and turn it into date format, or its in a string \n",
    "            # x-number of hours since present format, in which we just use today's date\n",
    "            # as the date value\n",
    "            try:\n",
    "                date  = datetime.datetime.strptime(d, '%b %d, %Y').date()\n",
    "            except:\n",
    "                date  = datetime.date.today()\n",
    "                pass\n",
    "\n",
    "            # Take these attributes as a dictionary and store in our updated news dictionary\n",
    "            # since we only want these attributes and not the other data with it.\n",
    "            news[count]  = {'Date':date, 'Title':title, 'URL':url}\n",
    "            # Find what the latest article date is for each stock, so it makes it easier\n",
    "            # to update the news dictionary every day to only retrieve articles past that\n",
    "            # latest date.\n",
    "            if count == 0: \n",
    "                latest = date\n",
    "            else:\n",
    "                if date > latest:\n",
    "                    latest = date\n",
    "            count += 1\n",
    "\n",
    "        last_date[tick] = latest\n",
    "        hist_news[tick] = news\n",
    "\n",
    "    # Each news dictionary has a latest date associated with it\n",
    "    news_lst = [hist_news, last_date]\n",
    "    # Dump the news dictionary to a file\n",
    "    opp = open('Pickles/newsdict.pickle','wb')\n",
    "    pickle.dump(news_lst, opp)\n",
    "    opp.close()\n",
    "    return\n",
    "\n",
    "def create_hlc_dict():\n",
    "    \"\"\"\n",
    "    Download at least the base helper stocks, plus at least one other and store them in the a \n",
    "    directory called HLC/ in your path. This will put them all into a dictionary that is used \n",
    "    heavily throughout the program. These are precalculated highs/lows/closes/typical values \n",
    "    for each company.\n",
    "    \"\"\"\n",
    "    highlowclose = {}\n",
    "    mypath       = 'HLC/'\n",
    "    # Grab all the file names within the HLC/ directory\n",
    "    onlyfiles    = [f for f in listdir(mypath) if isfile(join(mypath, f))]\n",
    "\n",
    "    # For each file within the HLC/ directory, open it and store it as a dictionary\n",
    "    # key/value pair in the highlowclose dictionary, where the stock symbol is the key\n",
    "    # and the value is the dataframe\n",
    "    for fl in onlyfiles:\n",
    "        opp = open(mypath+fl, 'rb')\n",
    "        highlowclose[fl[:fl.find('.')]] = pickle.load(opp)\n",
    "        opp.close()\n",
    "    \n",
    "    #Uncomment these next 3 lines when you're ready to create your dictionary.\n",
    "    #It's commented out for personal implementation.\n",
    "    #opp = open('Pickles/pickleadjustedintracomplete.pickle','wb')\n",
    "    #pickle.dump(highlowclose, opp)\n",
    "    #opp.close()\n",
    "    return highlowclose\n",
    "\n",
    "def get_prev_10d1min_intraday(hlc):\n",
    "    \"\"\"\n",
    "    Added to allow you to make sure the data from our other source is accurate.\n",
    "    This grabs the last 10 days of intraday data from Google which is a more\n",
    "    reliable source but not used because they only store 10 days of data.\n",
    "    \"\"\"\n",
    "\n",
    "    tickers    = {'^DJI':'.DJI','^GSPC':'.INX','^IXIC':'IXIC'}\n",
    "\n",
    "    pages = {}\n",
    "    # For each stock youre using\n",
    "    for each in hlc.keys():\n",
    "        # If the stock is one of the different named symbols, use the google symbol\n",
    "        if each in tickers.keys():\n",
    "            name = tickers[each]\n",
    "        else:\n",
    "            name = each\n",
    "        # Grab that stocks last 10 days(p=10d) worth of intraday data at 1 minute \n",
    "        # intervals(i=60) of closing prices and store it in a dictionary entry\n",
    "        page = 'http://www.google.com/finance/getprices?i=60&p=10d&f=d,c&df=cpct&q='+name\n",
    "        pages[each] = urllib.urlopen(page).read()\n",
    "\n",
    "    complete_list = {}\n",
    "    day_list      = []\n",
    "\n",
    "    # Convert the data entries to the correct format\n",
    "    for each in hlc.keys():\n",
    "        current = pages[each]\n",
    "        start = current.find('\\na')+2\n",
    "        end = len(current)\n",
    "        test = True\n",
    "        df_list = []\n",
    "\n",
    "        while test == True:\n",
    "            date = current.find(',',start)\n",
    "            price = current.find('\\n',date)\n",
    "            df_list.append(float(current[date+1:price]))\n",
    "\n",
    "            if price != end-1:\n",
    "                if current[price+1] == 'a':\n",
    "                    day_list.append(df_list)\n",
    "                    df_list = []\n",
    "                start = price+2    \n",
    "            else:\n",
    "                test = False\n",
    "                complete_list[each] = day_list\n",
    "                day_list = []        \n",
    "    return complete_list\n",
    "\n",
    "def get_base_index(hlc):\n",
    "    \"\"\"\n",
    "    Find the missing dates that one or more dataframes have but another doesn't\n",
    "    so we can match up data and find what dates are missing and drop the dates\n",
    "    from the base index. A date is considered bad if it's missing more than\n",
    "    half the day's worth of data. Then return that base index.\n",
    "    \"\"\"\n",
    "    count = 0\n",
    "    bases = ['^IXIC','^GSPC','^DJI','GLD','USO','SPY']\n",
    "    complete_bad_dates = []\n",
    "    new_dfs = {}\n",
    "\n",
    "    # For each symbol in the bases list, get its associated highlowclose dictionary\n",
    "    # dataframe data and get the number of records it has, and find the one with\n",
    "    # the least amount of entries.\n",
    "    for key in bases:\n",
    "        if count == 0:\n",
    "            length     = len(hlc[key])\n",
    "            length_key = key\n",
    "        else:\n",
    "            next_length = len(hlc[key])\n",
    "            if next_length < length:\n",
    "                length     = next_length\n",
    "                length_key = key\n",
    "        count += 1\n",
    "\n",
    "        # Get the indexes for two of the dataframes so we can compare the indexes\n",
    "        for key2 in bases:\n",
    "            if key != key2:\n",
    "                val   = hlc[key].index\n",
    "                val2  = hlc[key2].index\n",
    "\n",
    "                # Find the differences between the indexes\n",
    "                diffs = val.difference(val2)\n",
    "\n",
    "                # For each dataframe, we often will have a few mismatching stock\n",
    "                # minutes every few days but we are looking for ones where we have\n",
    "                # more than 190 stock minutes missing from a certain day, in which \n",
    "                # case we append that date to our bad_dates list which gives us a\n",
    "                # list of days that are missing data\n",
    "                dates, bad_dates = {}, []\n",
    "                for datetime in diffs:\n",
    "                    date = str(datetime)[:10]\n",
    "\n",
    "                    if date not in dates.keys():\n",
    "                        dates[date] = 1\n",
    "                    elif dates[date] < 190:\n",
    "                        dates[date] += 1\n",
    "                    else:\n",
    "                        if date not in bad_dates:\n",
    "                            bad_dates.append(date)\n",
    "\n",
    "                for bad in bad_dates:\n",
    "                    if bad not in complete_bad_dates:\n",
    "                        complete_bad_dates.append(bad)\n",
    "\n",
    "    # Remove the bad dates from the 6 base dataframes\n",
    "    for key in bases:\n",
    "        cpy_df = hlc[key].copy(deep=True)\n",
    "        for dt in complete_bad_dates:\n",
    "            try: \n",
    "                nums   = cpy_df.index.get_loc(dt)\n",
    "                cpy_df = cpy_df.drop(cpy_df.index[nums.start:nums.stop])\n",
    "            except:\n",
    "                pass\n",
    "        new_dfs[key] = cpy_df\n",
    "    \n",
    "    # Use our dataframe with the least entries and then remove the bad dates from it\n",
    "    # so we can get our base index to use to compare with the rest of the stocks\n",
    "    cpy_df = hlc[length_key].copy(deep=True)\n",
    "    for dt in complete_bad_dates:\n",
    "        try: \n",
    "            nums   = cpy_df.index.get_loc(dt)\n",
    "            cpy_df = cpy_df.drop(cpy_df.index[nums.start:nums.stop])\n",
    "        except:\n",
    "            pass\n",
    "    \n",
    "    return cpy_df.index, bases, new_dfs, hlc\n",
    "\n",
    "def get_bad_comp_dates(hlc, bases, base, new_dfs):\n",
    "    \"\"\"\n",
    "    Take the base index, and compare to each companies indexes to find which dates\n",
    "    can be removed to match up the data better. A days data is considered bad if\n",
    "    more than half the days data is missing. Then return a dictionary with the bad\n",
    "    dates for each company as well as the new highlowclose dataframe without the \n",
    "    bad dates.\n",
    "    \"\"\"\n",
    "    indexes   = {}\n",
    "    date_dict = {}\n",
    "    for key, val in hlc.iteritems():\n",
    "        if key not in bases:\n",
    "            indexes[key] = val.index\n",
    "    \n",
    "    # Compare each stocks index with the base index passed in\n",
    "    for key, val in indexes.itervalues():\n",
    "        diffs1 = val.difference(base)\n",
    "        diffs2 = base.difference(val)\n",
    "        diffs  = [diffs1, diffs2]\n",
    "\n",
    "        # For each dataframe, we often will have a few mismatching stock\n",
    "        # minutes every few days but we are looking for ones where we have\n",
    "        # more than 190 stock minutes missing from a certain day, in which \n",
    "        # case we append that date to our bad_dates list which gives us a\n",
    "        # list of days that are missing data\n",
    "        bad_dates = []\n",
    "        for lst in diffs:\n",
    "            dates = {}\n",
    "            for datetime in lst:\n",
    "                date = str(datetime)[:10]\n",
    "            \n",
    "                if date not in dates.keys():\n",
    "                    dates[date] = 1\n",
    "                elif dates[date] < 190:\n",
    "                    dates[date] += 1\n",
    "                else:\n",
    "                    if date not in bad_dates:\n",
    "                        bad_dates.append(date)\n",
    "        \n",
    "        # Remove the bad dates from each of our stocks\n",
    "        cpy_df = hlc[key].copy(deep=True)\n",
    "        for dt in bad_dates:\n",
    "            try: \n",
    "                nums   = cpy_df.index.get_loc(dt)\n",
    "                cpy_df = cpy_df.drop(cpy_df.index[nums.start:nums.stop])\n",
    "            except:\n",
    "                pass\n",
    "        date_lst[key] = bad_dates\n",
    "        new_dfs[key]  = cpy_df\n",
    "\n",
    "    return date_dict, new_dfs\n",
    "\n",
    "print \"LOADED\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Create your highlowclose dictionary first with create_hlc_dict(), then you'll\n",
    "take that dictionary and use it to get information about each company that you\n",
    "can retrieve by typing first:\n",
    "    print hlc.keys()\n",
    "\n",
    "which gets you the list of stock tickers you need to access the info dictionary.\n",
    "You'll then be able to type:\n",
    "    print info['TICKER YOU WANT TO LOOK AT'] \n",
    "    \n",
    "where 'TICKER YOU WANT TO LOOK AT' is replaced with a ticker like 'AAPL'\n",
    "\n",
    "After these dictionaries are created, the print_helper_stocks() function is \n",
    "called which prints the helper etf/index/stock that help in prediction but\n",
    "you don't actually predict for.\n",
    "\n",
    "After that, get_parameter_choices() is called which prints the 13 different\n",
    "indicators that are available to you, as well as the 32 different period\n",
    "lengths for each of these indicators that you can choose to create the \n",
    "indicator prediction list. This list will contain the indicators you believe\n",
    "will be most informational. If you're not sure, there is an already created\n",
    "indicator list for you in the Pickles/ directory called final_lst222, which\n",
    "contains set of 222 company/indicator/periods attributes that I found to be\n",
    "useful.\n",
    "\n",
    "The next function is called is create_columnname_dict() which simply creates\n",
    "the full indicator list for each company and stores a list for each company.\n",
    "You don't need to worry to much about this as you won't directly need to use\n",
    "it. \n",
    "\n",
    "Finally, create_news_dict() is called which retrieves news articles for each\n",
    "stock/index/etf from current up to 2 years ago, and puts the urls/dates in a \n",
    "list. This is not used currently but will be integrated into the project in\n",
    "future updates so as to utilize NLP machine learning in combination w/ our\n",
    "recurrent learning.\n",
    "\"\"\"\n",
    "#hlc     = create_hlc_dict()\n",
    "#info    = get_stock_info(hlc)\n",
    "\n",
    "#print_helper_stocks()\n",
    "#get_parameter_choices()\n",
    "#create_columnname_dict(hlc)\n",
    "#create_news_dict(hlc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "If you choose and don't have advanced stock knowledge, you can use the pre-made indicator list\n",
    "that I found to be useful, but if you think you can create a more useful indicator list, then use\n",
    "this creation tool function below.\n",
    "\n",
    "Pre-made indicator list is called 'pre_made_indicatorlist.pickle'\n",
    "\n",
    "Once you have chosen your chosen stock to predict on, you'll now call your \n",
    "create_indicator_list() function. This will create a list filled with dataframe column names to \n",
    "be used as our input values for our ML function. To create you will enter your chosen stock \n",
    "ticker, followed by string values for each indicator periods chosen. \n",
    "\n",
    "For example, if you want to choose Apple as your chosen stock and you'd like to use period \n",
    "lengths 1-5,8 for pe ratio, and 0,2,6 for our volatility indicator, we'd type:\n",
    "    ind = create_indicator_list('AAPL', per='1-5,8', vol='0,2,6')\n",
    "\n",
    "You can use '-' to signify you want everything between those two values, and commas for \n",
    "individual numbers. Don't use spaces in the strings though and each indicator has a choice \n",
    "between 0 and 31 to choose from so look and the parameter choices and choice what you feel \n",
    "is best.\n",
    "\n",
    "Keep in mind there is 6 helper stocks plus your company choice, so 7 different stocks being \n",
    "used to predict so when you use 10 different indicator values, you're really using 42 and \n",
    "anything over about 250 indicator values starts to become unreasonably long to calculate, \n",
    "so try to keep it under that amount of indicators\n",
    "\"\"\"\n",
    "#ind = create_indicator_list('PUT STOCK TICKER HERE', rets=None, per=None, bol=None, \n",
    "#                            vol=None, aro=None, mac=None, mac2=None, sma=None, \n",
    "#                            kdo=None, mom=None, adx=None, rsi=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Many of the dataframes have missing data on certain dates, below is a list of\n",
    "dates that some of the base helper stocks have/not-have that the predicting\n",
    "stocks have/dont-have. These dates are dates where one or more dataframes has\n",
    "missing data for that day. To adjust for these missing dates, you can feed your\n",
    "highlowclose dictionary to the below two functions and it will return a new \n",
    "dataframe where the dates that have mismatching data will be removed from them.\n",
    "\n",
    "This is an alternative to simply using the ffill and bfill method when \n",
    "concatinating dataframes. Personally, I chose to keep the dates and just use\n",
    "these fill methods instead. The issue with removing the mismatching dates is\n",
    "you're taking away usable data even if others aren't correct. \n",
    "\n",
    "All in all, it's not a great choice either way because you'll have days where \n",
    "the data is missing, or the data is approximated and technically incorrect. \n",
    "This is something that happens when you get free data and considering it only\n",
    "amounts to about 1% of the data, it's not unmanagable. If you can find more\n",
    "reliable free intraday data, let me know.\n",
    "\n",
    "These are the dates that have missing data on most of dataframes but I reccomend\n",
    "calling the two functions below to get a dictionary with the dates that missing\n",
    "on one dataframe or another between the 6 base dataframes and the 1 chosen \n",
    "dataframe as they will be more accurate for any missing data since run as well.\n",
    "\n",
    "dates = ['2013-08-28', '2013-10-28', '2014-02-12', '2014-02-18', '2014-02-25',\n",
    "         '2014-10-02', '2014-10-06', '2014-10-08', '2014-10-09', '2014-10-13', \n",
    "         '2014-10-14', '2014-10-15', '2014-10-20', '2015-01-14', '2015-03-30', \n",
    "         '2015-04-21', '2015-05-05', '2015-05-18', '2015-06-08', '2015-07-08', \n",
    "         '2015-08-20', '2015-08-31', '2015-09-08', '2016-02-08', '2016-03-15', \n",
    "         '2016-03-21', '2016-03-22', '2016-04-13', '2016-06-15']\n",
    "\"\"\"\n",
    "#base_index, base_nms, new_dfs, hlc = get_base_index(hlc)\n",
    "#date_dict, new_hlc = get_bad_comp_dates(hlc, base_nms, base_index, new_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "If you would like to double check your highlowclose dictionary is accurate,\n",
    "call this function to grab Google's last 10 days of intraday data on these\n",
    "companies. However you can only retrieve up to 10 days ago, as they don't\n",
    "store intraday data for any longer than that.\n",
    "\"\"\"\n",
    "#compare = get_prev_10d1min_intraday(hlc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}