LDA_Content.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\fazam\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\gensim\\utils.py:860: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\fazam\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gensim\n",
    "import nltk\n",
    "from gensim.utils import simple_preprocess\n",
    "from gensim.parsing.preprocessing import STOPWORDS\n",
    "from nltk.stem import WordNetLemmatizer, SnowballStemmer\n",
    "from nltk.stem.porter import *\n",
    "from gensim import corpora, models\n",
    "from datetime import datetime as dt\n",
    "np.random.seed(2019)\n",
    "nltk.download(\"wordnet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "article = pd.read_json(\"articles.json\")\n",
    "article = article.sort_values(by=\"publication_timestamp\")\n",
    "article['created_at']=article['publication_timestamp'].apply(lambda x: dt.fromtimestamp(x/1000)).dt.strftime('%Y-%m')\n",
    "date = article['created_at'].drop_duplicates().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "trimestering={}\n",
    "for i in date:\n",
    "    if date.index(i)==28:\n",
    "        trimestering[i] = int((date.index(i)-1)/4)\n",
    "    else:\n",
    "        trimestering[i]= int(date.index(i)/4)\n",
    "article['trimester']=article['created_at'].map(trimestering)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'2017-01': 0,\n",
       " '2017-02': 0,\n",
       " '2017-03': 0,\n",
       " '2017-04': 0,\n",
       " '2017-05': 1,\n",
       " '2017-06': 1,\n",
       " '2017-07': 1,\n",
       " '2017-08': 1,\n",
       " '2017-09': 2,\n",
       " '2017-10': 2,\n",
       " '2017-11': 2,\n",
       " '2017-12': 2,\n",
       " '2018-01': 3,\n",
       " '2018-02': 3,\n",
       " '2018-03': 3,\n",
       " '2018-04': 3,\n",
       " '2018-05': 4,\n",
       " '2018-06': 4,\n",
       " '2018-07': 4,\n",
       " '2018-08': 4,\n",
       " '2018-09': 5,\n",
       " '2018-10': 5,\n",
       " '2018-11': 5,\n",
       " '2018-12': 5,\n",
       " '2019-01': 6,\n",
       " '2019-02': 6,\n",
       " '2019-03': 6,\n",
       " '2019-04': 6,\n",
       " '2019-05': 6}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trimestering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dx = []\n",
    "for i in range(0,7):\n",
    "    x = article[article[\"trimester\"]==i]\n",
    "    dx.append(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_id</th>\n",
       "      <th>authors</th>\n",
       "      <th>contents</th>\n",
       "      <th>excerpt</th>\n",
       "      <th>fb_data</th>\n",
       "      <th>has_video</th>\n",
       "      <th>headline</th>\n",
       "      <th>image_link</th>\n",
       "      <th>keywords</th>\n",
       "      <th>li_data</th>\n",
       "      <th>...</th>\n",
       "      <th>max_velocity</th>\n",
       "      <th>pi_data</th>\n",
       "      <th>publication_timestamp</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>source</th>\n",
       "      <th>topics</th>\n",
       "      <th>tw_data</th>\n",
       "      <th>velocity</th>\n",
       "      <th>created_at</th>\n",
       "      <th>trimester</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>76509020-d093-11e6-8d6f-9f4c7b0f94de</td>\n",
       "      <td>[Corazon Miller]</td>\n",
       "      <td>United Airlines New Year's Day flight bound f...</td>\n",
       "      <td>A man was arrested after it was claimed he had...</td>\n",
       "      <td>{'total_engagement_count': 1737, 'likes': 0, '...</td>\n",
       "      <td>False</td>\n",
       "      <td>Passenger arrested after United Airlines fligh...</td>\n",
       "      <td>http://media.nzherald.co.nz/webcontent/image/j...</td>\n",
       "      <td>Business, Aviation, Airlines, Australia, Sydne...</td>\n",
       "      <td>{'li_count': 6}</td>\n",
       "      <td>...</td>\n",
       "      <td>97.729933</td>\n",
       "      <td>{'pi_count': 0}</td>\n",
       "      <td>1483324244514</td>\n",
       "      <td>0</td>\n",
       "      <td>{'publisher': 'nzherald.co.nz', 'link': 'http:...</td>\n",
       "      <td>[{'id': 2, 'name': 'News'}, {'id': 28, 'name':...</td>\n",
       "      <td>{'tw_count': 100}</td>\n",
       "      <td>0.115518</td>\n",
       "      <td>2017-01</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>b5c0ce50-d0a2-11e6-ab89-c11942898bd6</td>\n",
       "      <td>[Newstalk ZB Staff]</td>\n",
       "      <td>United Airlines New Year's Day flight bound f...</td>\n",
       "      <td>A United Airlines New Year's Day flight bound ...</td>\n",
       "      <td>{'total_engagement_count': 0, 'likes': 0, 'com...</td>\n",
       "      <td>False</td>\n",
       "      <td>United Airlines flight diverted to Auckland af...</td>\n",
       "      <td>http://www.newstalkzb.co.nz/media/16873988/uni...</td>\n",
       "      <td></td>\n",
       "      <td>{'li_count': 0}</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>{'pi_count': 0}</td>\n",
       "      <td>1483330793397</td>\n",
       "      <td>0</td>\n",
       "      <td>{'publisher': 'newstalkzb.co.nz', 'link': 'htt...</td>\n",
       "      <td>[{'id': 2, 'name': 'News'}]</td>\n",
       "      <td>{'tw_count': 0}</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2017-01</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>502a51e0-d0a9-11e6-ab89-c11942898bd6</td>\n",
       "      <td>[LEXI NELSON]</td>\n",
       "      <td>United Airlines New Year's Day flight set to t...</td>\n",
       "      <td>Unruly passenger goes on abusive, racist rant,...</td>\n",
       "      <td>{'total_engagement_count': 38, 'likes': 0, 'co...</td>\n",
       "      <td>False</td>\n",
       "      <td>United Airlines New Year's Day flight diverts ...</td>\n",
       "      <td>https://resources.stuff.co.nz/content/dam/imag...</td>\n",
       "      <td>Sport, Entertainment, Business, Motoring, Worl...</td>\n",
       "      <td>{'li_count': 0}</td>\n",
       "      <td>...</td>\n",
       "      <td>108.874289</td>\n",
       "      <td>{'pi_count': 0}</td>\n",
       "      <td>1483333629438</td>\n",
       "      <td>0</td>\n",
       "      <td>{'publisher': 'stuff.co.nz', 'link': 'http://s...</td>\n",
       "      <td>[{'id': 2, 'name': 'News'}, {'id': 443, 'name'...</td>\n",
       "      <td>{'tw_count': 40}</td>\n",
       "      <td>-4.779850</td>\n",
       "      <td>2017-01</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>61489c80-d0b7-11e6-919c-7789120ed8c4</td>\n",
       "      <td>None</td>\n",
       "      <td>You are not permitted to download, save or ema...</td>\n",
       "      <td>A man was arrested after an alleged \"racist ra...</td>\n",
       "      <td>{'total_engagement_count': 0, 'likes': 0, 'com...</td>\n",
       "      <td>False</td>\n",
       "      <td>Unruly passenger arrested in Auckland</td>\n",
       "      <td>https://www.odt.co.nz/sites/default/files/stor...</td>\n",
       "      <td></td>\n",
       "      <td>{'li_count': 0}</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>{'pi_count': 0}</td>\n",
       "      <td>1483339336000</td>\n",
       "      <td>0</td>\n",
       "      <td>{'publisher': 'odt.co.nz', 'link': 'https://od...</td>\n",
       "      <td>[{'id': 2, 'name': 'News'}]</td>\n",
       "      <td>{'tw_count': 0}</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2017-01</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>2ad85e20-d0c0-11e6-ab89-c11942898bd6</td>\n",
       "      <td>None</td>\n",
       "      <td>olice have arrested a badly behaved passenger ...</td>\n",
       "      <td>Police have arrested a badly behaved passenger...</td>\n",
       "      <td>{'total_engagement_count': 3, 'likes': 0, 'com...</td>\n",
       "      <td>False</td>\n",
       "      <td>Unruly passenger forces flight to divert</td>\n",
       "      <td>http://www.radionz.co.nz/assets/news_crops/180...</td>\n",
       "      <td>Radio New Zealand, Public Radio, News, Current...</td>\n",
       "      <td>{'li_count': 0}</td>\n",
       "      <td>...</td>\n",
       "      <td>0.649721</td>\n",
       "      <td>{'pi_count': 0}</td>\n",
       "      <td>1483343445250</td>\n",
       "      <td>-1</td>\n",
       "      <td>{'publisher': 'radionz.co.nz', 'link': 'http:/...</td>\n",
       "      <td>[{'id': 2, 'name': 'News'}, {'id': 668, 'name'...</td>\n",
       "      <td>{'tw_count': 0}</td>\n",
       "      <td>0.492976</td>\n",
       "      <td>2017-01</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                               article_id              authors  \\\n",
       "3    76509020-d093-11e6-8d6f-9f4c7b0f94de     [Corazon Miller]   \n",
       "142  b5c0ce50-d0a2-11e6-ab89-c11942898bd6  [Newstalk ZB Staff]   \n",
       "25   502a51e0-d0a9-11e6-ab89-c11942898bd6        [LEXI NELSON]   \n",
       "141  61489c80-d0b7-11e6-919c-7789120ed8c4                 None   \n",
       "64   2ad85e20-d0c0-11e6-ab89-c11942898bd6                 None   \n",
       "\n",
       "                                              contents  \\\n",
       "3     United Airlines New Year's Day flight bound f...   \n",
       "142   United Airlines New Year's Day flight bound f...   \n",
       "25   United Airlines New Year's Day flight set to t...   \n",
       "141  You are not permitted to download, save or ema...   \n",
       "64   olice have arrested a badly behaved passenger ...   \n",
       "\n",
       "                                               excerpt  \\\n",
       "3    A man was arrested after it was claimed he had...   \n",
       "142  A United Airlines New Year's Day flight bound ...   \n",
       "25   Unruly passenger goes on abusive, racist rant,...   \n",
       "141  A man was arrested after an alleged \"racist ra...   \n",
       "64   Police have arrested a badly behaved passenger...   \n",
       "\n",
       "                                               fb_data  has_video  \\\n",
       "3    {'total_engagement_count': 1737, 'likes': 0, '...      False   \n",
       "142  {'total_engagement_count': 0, 'likes': 0, 'com...      False   \n",
       "25   {'total_engagement_count': 38, 'likes': 0, 'co...      False   \n",
       "141  {'total_engagement_count': 0, 'likes': 0, 'com...      False   \n",
       "64   {'total_engagement_count': 3, 'likes': 0, 'com...      False   \n",
       "\n",
       "                                              headline  \\\n",
       "3    Passenger arrested after United Airlines fligh...   \n",
       "142  United Airlines flight diverted to Auckland af...   \n",
       "25   United Airlines New Year's Day flight diverts ...   \n",
       "141              Unruly passenger arrested in Auckland   \n",
       "64            Unruly passenger forces flight to divert   \n",
       "\n",
       "                                            image_link  \\\n",
       "3    http://media.nzherald.co.nz/webcontent/image/j...   \n",
       "142  http://www.newstalkzb.co.nz/media/16873988/uni...   \n",
       "25   https://resources.stuff.co.nz/content/dam/imag...   \n",
       "141  https://www.odt.co.nz/sites/default/files/stor...   \n",
       "64   http://www.radionz.co.nz/assets/news_crops/180...   \n",
       "\n",
       "                                              keywords          li_data  ...  \\\n",
       "3    Business, Aviation, Airlines, Australia, Sydne...  {'li_count': 6}  ...   \n",
       "142                                                     {'li_count': 0}  ...   \n",
       "25   Sport, Entertainment, Business, Motoring, Worl...  {'li_count': 0}  ...   \n",
       "141                                                     {'li_count': 0}  ...   \n",
       "64   Radio New Zealand, Public Radio, News, Current...  {'li_count': 0}  ...   \n",
       "\n",
       "    max_velocity          pi_data publication_timestamp  sentiment  \\\n",
       "3      97.729933  {'pi_count': 0}         1483324244514          0   \n",
       "142     0.000000  {'pi_count': 0}         1483330793397          0   \n",
       "25    108.874289  {'pi_count': 0}         1483333629438          0   \n",
       "141     0.000000  {'pi_count': 0}         1483339336000          0   \n",
       "64      0.649721  {'pi_count': 0}         1483343445250         -1   \n",
       "\n",
       "                                                source  \\\n",
       "3    {'publisher': 'nzherald.co.nz', 'link': 'http:...   \n",
       "142  {'publisher': 'newstalkzb.co.nz', 'link': 'htt...   \n",
       "25   {'publisher': 'stuff.co.nz', 'link': 'http://s...   \n",
       "141  {'publisher': 'odt.co.nz', 'link': 'https://od...   \n",
       "64   {'publisher': 'radionz.co.nz', 'link': 'http:/...   \n",
       "\n",
       "                                                topics            tw_data  \\\n",
       "3    [{'id': 2, 'name': 'News'}, {'id': 28, 'name':...  {'tw_count': 100}   \n",
       "142                        [{'id': 2, 'name': 'News'}]    {'tw_count': 0}   \n",
       "25   [{'id': 2, 'name': 'News'}, {'id': 443, 'name'...   {'tw_count': 40}   \n",
       "141                        [{'id': 2, 'name': 'News'}]    {'tw_count': 0}   \n",
       "64   [{'id': 2, 'name': 'News'}, {'id': 668, 'name'...    {'tw_count': 0}   \n",
       "\n",
       "     velocity  created_at trimester  \n",
       "3    0.115518     2017-01         0  \n",
       "142  0.000000     2017-01         0  \n",
       "25  -4.779850     2017-01         0  \n",
       "141  0.000000     2017-01         0  \n",
       "64   0.492976     2017-01         0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dx[0].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatize_stemming(text):\n",
    "    stemmer = SnowballStemmer(\"english\")\n",
    "    return stemmer.stem(WordNetLemmatizer().lemmatize(text, pos='v'))\n",
    "def preprocess(text):\n",
    "    result = []\n",
    "    for token in gensim.utils.simple_preprocess(text):\n",
    "        if token not in gensim.parsing.preprocessing.STOPWORDS and len(token) > 3:\n",
    "            result.append(lemmatize_stemming(token))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex=[]\n",
    "ex_pp=[]\n",
    "for i in range(0,7):\n",
    "    ex.append(pd.Series(dx[i][\"excerpt\"]))\n",
    "    ex_pp.append(ex[i].map(preprocess))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "co=[]\n",
    "co_pp=[]\n",
    "for i in range(0,7):\n",
    "    co.append(pd.Series(dx[i][\"contents\"]))\n",
    "    co_pp.append(co[i].map(preprocess))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionarye = gensim.corpora.Dictionary(ex_pp[0])\n",
    "\n",
    "dictionarye.filter_extremes(no_below=15, no_above=0.5, keep_n=100)\n",
    "\n",
    "bow_corpuse = [dictionarye.doc2bow(doc) for doc in ex_pp[0]]\n",
    "\n",
    "tfidfe = models.TfidfModel(bow_corpuse)\n",
    "corpus_tfidfe = tfidfe[bow_corpuse]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Coherence Score:  0.28792527305769644\n",
      "\n",
      "Coherence Score:  0.3587360515035209\n",
      "\n",
      "Coherence Score:  0.3268967724726705\n",
      "\n",
      "Coherence Score:  0.36008703089039595\n",
      "\n",
      "Coherence Score:  0.3416327324535744\n",
      "\n",
      "Coherence Score:  0.3688391050139477\n",
      "\n",
      "Coherence Score:  0.3878315523063306\n",
      "\n",
      "Coherence Score:  0.3952395943361595\n",
      "\n",
      "Coherence Score:  0.3802005820154778\n",
      "\n",
      "Coherence Score:  0.38312025460170185\n",
      "\n",
      "Coherence Score:  0.3655086117518152\n",
      "\n",
      "Coherence Score:  0.37351972273588513\n",
      "\n",
      "Coherence Score:  0.41200947607171245\n",
      "\n",
      "Coherence Score:  0.38948746561703856\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import CoherenceModel\n",
    "for i in range(1,15):\n",
    "    lda_model_tfidfe = gensim.models.LdaMulticore(corpus_tfidfe, num_topics=i, id2word=dictionarye, passes=2, workers=4)\n",
    "    coherence_model_lda = CoherenceModel(model=lda_model_tfidfe, texts=ex_pp[0], dictionary=dictionarye, coherence='c_v')\n",
    "    coherence_lda = coherence_model_lda.get_coherence()\n",
    "    print('\\nCoherence Score: ', coherence_lda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary= []\n",
    "bowcorpus= []\n",
    "tfidf=[]\n",
    "corpus_tfidf= []\n",
    "lda_model= []\n",
    "for i in range(0,7):   \n",
    "    dictionaryc = gensim.corpora.Dictionary(co_pp[i])\n",
    "\n",
    "    dictionaryc.filter_extremes(no_below=15, no_above=0.5, keep_n=100)\n",
    "\n",
    "    bow_corpusc = [dictionaryc.doc2bow(doc) for doc in co_pp[i]]\n",
    "\n",
    "    tfidfc = models.TfidfModel(bow_corpusc)\n",
    "    corpus_tfidfc = tfidfc[bow_corpusc]\n",
    "    lda_model_tfidfc= gensim.models.LdaMulticore(corpus_tfidfc, num_topics=6, id2word=dictionaryc, passes=2, workers=4)\n",
    "    \n",
    "    dictionary.append(dictionaryc)\n",
    "    bowcorpus.append(bow_corpusc)\n",
    "    tfidf.append(tfidfc)\n",
    "    corpus_tfidf.append(corpus_tfidfc)\n",
    "    lda_model.append(lda_model_tfidfc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'dictionary/dictionary-0.sav'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-14-5dd72d2d0903>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlda_model\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"lda_model/lda_model_{}.sav\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mco_pp\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'content/content_preprocessed-{}.sav'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m     \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdictionary\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'dictionary/dictionary-{}.sav'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m     \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbowcorpus\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'bowcorpus/bowcorpus-{}.sav'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtfidf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'tfidf/tfidf-{}.sav'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'dictionary/dictionary-0.sav'"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import json\n",
    "for i in range(0,7):\n",
    "    pickle.dump(lda_model[i], open(\"lda_model/lda_model_{}.sav\".format(i), 'wb'))\n",
    "    pickle.dump(co_pp[i], open('content/content_preprocessed-{}.sav'.format(i), 'wb'))\n",
    "    pickle.dump(dictionary[i], open('dictionary/dictionary-{}.sav'.format(i), 'wb'))\n",
    "    pickle.dump(bowcorpus[i], open('bowcorpus/bowcorpus-{}.sav'.format(i), 'wb'))\n",
    "    pickle.dump(tfidf[i], open('tfidf/tfidf-{}.sav'.format(i), 'wb'))\n",
    "    pickle.dump(corpus_tfidf[i], open('corpus/corpustfidf-{}.sav'.format(i), 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}