Topic modelling (2).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = open('C:/Users/ANKAN/TIMES OF INDIA/First/January.txt',encoding=\"utf8\").read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = []\n",
    "for i, line in enumerate(data.split(\"\\n\")):\n",
    "    content = line.split()\n",
    "    texts.append(\" \".join(content[1:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "DF = pandas.DataFrame()\n",
    "DF['text'] = texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>73668</th>\n",
       "      <td>have sent you a verification email. To verify,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46036</th>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70195</th>\n",
       "      <td>Comments ()+^ Back to Top</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14328</th>\n",
       "      <td>arrested for film protest, no show at Hind, Na...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86195</th>\n",
       "      <td>is scheduled to take place at Ranga Shankara o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111791</th>\n",
       "      <td>| Jan 31, 2019, 00:24 IST</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1251</th>\n",
       "      <td>After infrastructure, defence, realty and FMCG...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28296</th>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13902</th>\n",
       "      <td>Rajesh was the chief guest who inaugurated the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48762</th>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>144304 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text\n",
       "73668   have sent you a verification email. To verify,...\n",
       "46036                                                    \n",
       "70195                           Comments ()+^ Back to Top\n",
       "14328   arrested for film protest, no show at Hind, Na...\n",
       "86195   is scheduled to take place at Ranga Shankara o...\n",
       "...                                                   ...\n",
       "111791                          | Jan 31, 2019, 00:24 IST\n",
       "1251    After infrastructure, defence, realty and FMCG...\n",
       "28296                                                    \n",
       "13902   Rajesh was the chief guest who inaugurated the...\n",
       "48762                                                    \n",
       "\n",
       "[144304 rows x 1 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.utils import shuffle\n",
    "DF = shuffle(DF)\n",
    "DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "nan_value = float(\"NaN\")\n",
    "\n",
    "DF.replace(\"\", nan_value, inplace = True)\n",
    "\n",
    "DF.dropna(subset = [\"text\"], inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>73668</th>\n",
       "      <td>have sent you a verification email. To verify,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70195</th>\n",
       "      <td>Comments ()+^ Back to Top</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14328</th>\n",
       "      <td>arrested for film protest, no show at Hind, Na...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86195</th>\n",
       "      <td>is scheduled to take place at Ranga Shankara o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45687</th>\n",
       "      <td>held for sexual abuse of minor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20201</th>\n",
       "      <td>Seeking resurrection in the politically crucia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50704</th>\n",
       "      <td>IN WITH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111791</th>\n",
       "      <td>| Jan 31, 2019, 00:24 IST</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1251</th>\n",
       "      <td>After infrastructure, defence, realty and FMCG...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13902</th>\n",
       "      <td>Rajesh was the chief guest who inaugurated the...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>75115 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text\n",
       "73668   have sent you a verification email. To verify,...\n",
       "70195                           Comments ()+^ Back to Top\n",
       "14328   arrested for film protest, no show at Hind, Na...\n",
       "86195   is scheduled to take place at Ranga Shankara o...\n",
       "45687                      held for sexual abuse of minor\n",
       "...                                                   ...\n",
       "20201   Seeking resurrection in the politically crucia...\n",
       "50704                                             IN WITH\n",
       "111791                          | Jan 31, 2019, 00:24 IST\n",
       "1251    After infrastructure, defence, realty and FMCG...\n",
       "13902   Rajesh was the chief guest who inaugurated the...\n",
       "\n",
       "[75115 rows x 1 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73668    have sent you a verification email to verify j...\n",
       "70195                            comments ()+^ back to top\n",
       "14328    arrested for film protest no show at hind navi...\n",
       "86195    is scheduled to take place at ranga shankara o...\n",
       "45687                       held for sexual abuse of minor\n",
       "Name: text_processed, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the regular expression library\n",
    "import re\n",
    "# Remove punctuation\n",
    "DF['text_processed'] = DF['text'].map(lambda x: re.sub('[,\\.!?]', '', x))\n",
    "# Convert the titles to lowercase\n",
    "DF['text_processed'] = DF['text_processed'].map(lambda x: x.lower())\n",
    "# Print out the first rows of DF\n",
    "DF['text_processed'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>text_processed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>73668</th>\n",
       "      <td>have sent you a verification email. To verify,...</td>\n",
       "      <td>have sent you a verification email to verify j...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70195</th>\n",
       "      <td>Comments ()+^ Back to Top</td>\n",
       "      <td>comments ()+^ back to top</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14328</th>\n",
       "      <td>arrested for film protest, no show at Hind, Na...</td>\n",
       "      <td>arrested for film protest no show at hind navi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86195</th>\n",
       "      <td>is scheduled to take place at Ranga Shankara o...</td>\n",
       "      <td>is scheduled to take place at ranga shankara o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45687</th>\n",
       "      <td>held for sexual abuse of minor</td>\n",
       "      <td>held for sexual abuse of minor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20201</th>\n",
       "      <td>Seeking resurrection in the politically crucia...</td>\n",
       "      <td>seeking resurrection in the politically crucia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50704</th>\n",
       "      <td>IN WITH</td>\n",
       "      <td>in with</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111791</th>\n",
       "      <td>| Jan 31, 2019, 00:24 IST</td>\n",
       "      <td>| jan 31 2019 00:24 ist</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1251</th>\n",
       "      <td>After infrastructure, defence, realty and FMCG...</td>\n",
       "      <td>after infrastructure defence realty and fmcg t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13902</th>\n",
       "      <td>Rajesh was the chief guest who inaugurated the...</td>\n",
       "      <td>rajesh was the chief guest who inaugurated the...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>75115 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text  \\\n",
       "73668   have sent you a verification email. To verify,...   \n",
       "70195                           Comments ()+^ Back to Top   \n",
       "14328   arrested for film protest, no show at Hind, Na...   \n",
       "86195   is scheduled to take place at Ranga Shankara o...   \n",
       "45687                      held for sexual abuse of minor   \n",
       "...                                                   ...   \n",
       "20201   Seeking resurrection in the politically crucia...   \n",
       "50704                                             IN WITH   \n",
       "111791                          | Jan 31, 2019, 00:24 IST   \n",
       "1251    After infrastructure, defence, realty and FMCG...   \n",
       "13902   Rajesh was the chief guest who inaugurated the...   \n",
       "\n",
       "                                           text_processed  \n",
       "73668   have sent you a verification email to verify j...  \n",
       "70195                           comments ()+^ back to top  \n",
       "14328   arrested for film protest no show at hind navi...  \n",
       "86195   is scheduled to take place at ranga shankara o...  \n",
       "45687                      held for sexual abuse of minor  \n",
       "...                                                   ...  \n",
       "20201   seeking resurrection in the politically crucia...  \n",
       "50704                                             in with  \n",
       "111791                            | jan 31 2019 00:24 ist  \n",
       "1251    after infrastructure defence realty and fmcg t...  \n",
       "13902   rajesh was the chief guest who inaugurated the...  \n",
       "\n",
       "[75115 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the library with the CountVectorizer method\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_style('whitegrid')\n",
    "%matplotlib inline\n",
    "# Helper function\n",
    "def plot_10_most_common_words(count_data, count_vectorizer):\n",
    "    import matplotlib.pyplot as plt\n",
    "    words = count_vectorizer.get_feature_names()\n",
    "    total_counts = np.zeros(len(words))\n",
    "    for t in count_data:\n",
    "        total_counts+=t.toarray()[0]\n",
    "    \n",
    "    count_dict = (zip(words, total_counts))\n",
    "    count_dict = sorted(count_dict, key=lambda x:x[1], reverse=True)[0:100]\n",
    "    words = [w[0] for w in count_dict]\n",
    "    counts = [w[1] for w in count_dict]\n",
    "    x_pos = np.arange(len(words)) \n",
    "    \n",
    "    plt.figure(2, figsize=(15, 15/1.6180))\n",
    "    plt.subplot(title='10 most common words')\n",
    "    sns.set_context(\"notebook\", font_scale=1.25, rc={\"lines.linewidth\": 2.5})\n",
    "    sns.barplot(x_pos, counts, palette='husl')\n",
    "    plt.xticks(x_pos, words, rotation=90) \n",
    "    plt.xlabel('words')\n",
    "    plt.ylabel('counts')\n",
    "    plt.show()\n",
    "# Initialise the count vectorizer with the English stop words\n",
    "count_vectorizer = CountVectorizer(stop_words='english')\n",
    "# Fit and transform the processed titles\n",
    "count_data = count_vectorizer.fit_transform(DF['text_processed'])\n",
    "# Visualise the 10 most common words\n",
    "#plot_10_most_common_words(count_data, count_vectorizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topics found via LDA:\n",
      "\n",
      "Topic #0:\n",
      "court said government high case order state department issued act report justice filed committee action\n",
      "\n",
      "Topic #1:\n",
      "festival event day year kumbh city mela dance pm sankranti said new pongal guests time\n",
      "\n",
      "Topic #2:\n",
      "said road city residents corporation officials waste area civic work building municipal roads people garbage\n",
      "\n",
      "Topic #3:\n",
      "review bench justice said judge cji temple sc judges court ram ayodhya case apple hc\n",
      "\n",
      "Topic #4:\n",
      "searches state said power government strike protest liquor employees workers cows cow solar district electricity\n",
      "\n",
      "Topic #5:\n",
      "singh punjab said pakistan jail sad kumar sikh dera village border sahib ludhiana kaur mla\n",
      "\n",
      "Topic #6:\n",
      "said mlas congress government meeting bjp karnataka candidates january jd party minister channels exam kumaraswamy\n",
      "\n",
      "Topic #7:\n",
      "film says like people time life actor films year work just don love want lot\n",
      "\n",
      "Topic #8:\n",
      "said railway airport train passengers station flight trains hours officials delhi time express official flights\n",
      "\n",
      "Topic #9:\n",
      "2019 ist jan updated 00 07 06 08 09 04 10 05 12 11 13\n",
      "\n",
      "Topic #10:\n",
      "said bjp congress party minister government modi sabha state chief lok elections leader president leaders\n",
      "\n",
      "Topic #11:\n",
      "said hai khan rao best ki quota reservation reddy like ke strike ap nahi naidu\n",
      "\n",
      "Topic #12:\n",
      "government state said rs minister crore scheme development lakh district department chief centre education funds\n",
      "\n",
      "Topic #13:\n",
      "account existing login farmers said government loan loans gst bank rs farm lakh minister scheme\n",
      "\n",
      "Topic #14:\n",
      "3000 remaining forest tiger wedding wildlife animal reserve padma animals award wild day wear tigers\n",
      "\n",
      "Topic #15:\n",
      "project water land metro work construction said projects road nmc crore government river bridge development\n",
      "\n",
      "Topic #16:\n",
      "twitter facebookshare message link just sent follow verification verify email mutt seer camera swamiji swami\n",
      "\n",
      "Topic #17:\n",
      "said years food family year restaurants hotels city mumbai new hotel india january bars dance\n",
      "\n",
      "Topic #18:\n",
      "hospital said medical dr goa doctors photo patients treatment health hospitals blood surgery doctor patient\n",
      "\n",
      "Topic #19:\n",
      "music day art india event republic indian cultural culture exhibition city museum artists like parade\n",
      "\n",
      "Topic #20:\n",
      "comments community let personal help guidelines calling attacks work civil conversation posting follow indulge marking\n",
      "\n",
      "Topic #21:\n",
      "police said accused arrested case year old station woman house family complaint told registered cops\n",
      "\n",
      "Topic #22:\n",
      "health said flu cases swine plastic state year department city bus people patients bags reported\n",
      "\n",
      "Topic #23:\n",
      "university students college said colleges research department year institute education science student engineering teachers campus\n",
      "\n",
      "Topic #24:\n",
      "school students said schools children education people teachers class parents child time year help women\n",
      "\n",
      "Topic #25:\n",
      "women said temple year people sabarimala birds says bird food entry time like wall day\n",
      "\n",
      "Topic #26:\n",
      "rs crore said lakh water pay price farmers year tax market prices supply cost demand\n",
      "\n",
      "Topic #27:\n",
      "india team world game players cricket indian match sports games play won win year bccl\n",
      "\n",
      "Topic #28:\n",
      "said india company year new crore bank companies rs indian 2018 business growth data country\n",
      "\n",
      "Topic #29:\n",
      "traffic said city vehicles road air delhi temperature day cold degrees days police recorded january\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.simplefilter(\"ignore\", DeprecationWarning)\n",
    "# Load the LDA model from sk-learn\n",
    "from sklearn.decomposition import LatentDirichletAllocation as LDA\n",
    " \n",
    "# Helper function\n",
    "def print_topics(model, count_vectorizer, n_top_words):\n",
    "    words = count_vectorizer.get_feature_names()\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        print(\"\\nTopic #%d:\" % topic_idx)\n",
    "        print(\" \".join([words[i]\n",
    "                        for i in topic.argsort()[:-n_top_words - 1:-1]]))\n",
    "        \n",
    "# Tweak the two parameters below\n",
    "number_topics = 30\n",
    "number_words = 15\n",
    "# Create and fit the LDA model\n",
    "lda = LDA(n_components=number_topics, n_jobs=-1)\n",
    "lda.fit(count_data)\n",
    "# Print the topics found by the LDA model\n",
    "print(\"Topics found via LDA:\")\n",
    "print_topics(lda, count_vectorizer, number_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "doc_topic_dist = pd.DataFrame(lda.transform(count_data))\n",
    "#doc_topic_dist.to_csv('doc_topic_dist.csv', index=False)\n",
    "\n",
    "maxValueInd = doc_topic_dist.idxmax(axis = 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        16\n",
       "1        20\n",
       "2         2\n",
       "3         1\n",
       "4        21\n",
       "         ..\n",
       "75110    10\n",
       "75111     0\n",
       "75112     9\n",
       "75113    28\n",
       "75114    19\n",
       "Length: 75115, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxValueInd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75110</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75111</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75112</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75113</th>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75114</th>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>75115 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0\n",
       "0      16\n",
       "1      20\n",
       "2       2\n",
       "3       1\n",
       "4      21\n",
       "...    ..\n",
       "75110  10\n",
       "75111   0\n",
       "75112   9\n",
       "75113  28\n",
       "75114  19\n",
       "\n",
       "[75115 rows x 1 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxValueInd.to_csv('j_topics.csv')\n",
    "df = pd.read_csv('j_topics.csv')\n",
    "df = df.drop(columns=['Unnamed: 0'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>10.441323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20</td>\n",
       "      <td>9.042135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>8.486987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>7.102443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>6.360913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>13</td>\n",
       "      <td>5.082873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10</td>\n",
       "      <td>4.370632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>4</td>\n",
       "      <td>4.004526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>16</td>\n",
       "      <td>3.874060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2</td>\n",
       "      <td>3.846103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>14</td>\n",
       "      <td>3.316248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>28</td>\n",
       "      <td>3.045996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>27</td>\n",
       "      <td>3.004726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>24</td>\n",
       "      <td>2.496173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>29</td>\n",
       "      <td>2.223258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>3</td>\n",
       "      <td>2.193969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "      <td>2.027558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>26</td>\n",
       "      <td>1.975637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>8</td>\n",
       "      <td>1.903748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>23</td>\n",
       "      <td>1.855821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>12</td>\n",
       "      <td>1.737336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>19</td>\n",
       "      <td>1.682753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>25</td>\n",
       "      <td>1.522998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>6</td>\n",
       "      <td>1.433802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>18</td>\n",
       "      <td>1.396525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>22</td>\n",
       "      <td>1.383212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>11</td>\n",
       "      <td>1.203488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>15</td>\n",
       "      <td>1.106304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>17</td>\n",
       "      <td>0.941223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>5</td>\n",
       "      <td>0.937230</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0          0\n",
       "0            9  10.441323\n",
       "1           20   9.042135\n",
       "2            0   8.486987\n",
       "3           21   7.102443\n",
       "4            7   6.360913\n",
       "5           13   5.082873\n",
       "6           10   4.370632\n",
       "7            4   4.004526\n",
       "8           16   3.874060\n",
       "9            2   3.846103\n",
       "10          14   3.316248\n",
       "11          28   3.045996\n",
       "12          27   3.004726\n",
       "13          24   2.496173\n",
       "14          29   2.223258\n",
       "15           3   2.193969\n",
       "16           1   2.027558\n",
       "17          26   1.975637\n",
       "18           8   1.903748\n",
       "19          23   1.855821\n",
       "20          12   1.737336\n",
       "21          19   1.682753\n",
       "22          25   1.522998\n",
       "23           6   1.433802\n",
       "24          18   1.396525\n",
       "25          22   1.383212\n",
       "26          11   1.203488\n",
       "27          15   1.106304\n",
       "28          17   0.941223\n",
       "29           5   0.937230"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1=(df['0'].value_counts()/df['0'].count())*100\n",
    "df1.to_csv('freq_j.csv')\n",
    "df = pd.read_csv('freq_j.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}