topic_compare.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare each lda_model generated on LDA_Content.ipynb."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\fazam\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\gensim\\utils.py:860: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from gensim import models\n",
    "import pickle\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=[]\n",
    "for i in range(0,7):\n",
    "    y= open(\"lda_model/lda_model_{}.sav\".format(i), 'rb')\n",
    "    x.append(pickle.load(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'munoz'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[i].print_topics()[0][1].split(\" + \")[0].split(\"*\")[1].replace('\"','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=[]\n",
    "for i in range(0,7):\n",
    "    p=[]\n",
    "    for j in range(0,6):\n",
    "        t=x[i].print_topics()[j][1]\n",
    "        q=t.split(\" + \")\n",
    "        r=[]\n",
    "        for k in q:\n",
    "            v = k.split('*')[1]\n",
    "            r.append(v.replace('\"',''))\n",
    "        p.append(r)\n",
    "    y.append(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['carrier', 'year']\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "n=[]\n",
    "w=[]\n",
    "wlist=defaultdict(int)\n",
    "for i in range(0,7):\n",
    "    n1=[]\n",
    "    w1=[]\n",
    "    for j in range(0,6):\n",
    "        n2=[]\n",
    "        w2=[]\n",
    "        for k in range(0,7):\n",
    "            n3=[]\n",
    "            w3=[]\n",
    "            for l in range(0,6):\n",
    "                counter=0\n",
    "                words =[]\n",
    "                for m in y[i][j]:\n",
    "                    wlist[m]+=1\n",
    "                    if m in y[k][l]:\n",
    "                        counter+=1\n",
    "                        words.append(m)\n",
    "                n3.append(counter)\n",
    "                w3.append(words)\n",
    "            n2.append(n3)\n",
    "            w2.append(w3)\n",
    "        n1.append(n2)\n",
    "        w1.append(w2)\n",
    "    n.append(n1)\n",
    "    w.append(w1)\n",
    "print(w[0][2][1][3])\n",
    "print(n[0][2][1][3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#o[i] contain similar topic between timeframe i and i+1\n",
    "o=[]\n",
    "for i in range(0,7):\n",
    "    o1=[]\n",
    "    for j in range(0,6):\n",
    "        highest=0\n",
    "        p_arr=[]\n",
    "        for k in range(0,7):\n",
    "            for l in range(0,6):\n",
    "                similar=n[i][j][k][l]\n",
    "                if(similar>highest and i!=k):\n",
    "                    highest= similar\n",
    "                    timeframe= k\n",
    "                    topic=l\n",
    "                    words=w[i][j][k][l]\n",
    "                    parameter=[highest, timeframe, topic, words]\n",
    "                    p_arr=[parameter]\n",
    "                elif(similar==highest and i!=k):\n",
    "                    highest= similar\n",
    "                    timeframe= k\n",
    "                    topic=l\n",
    "                    words=w[i][j][k][l]\n",
    "                    parameter=[highest, timeframe, topic, words]\n",
    "                    p_arr.append(parameter) \n",
    "        o1.append(p_arr)\n",
    "    o.append(o1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('airport', 714),\n",
       " ('seat', 714),\n",
       " ('travel', 546),\n",
       " ('servic', 462),\n",
       " ('plane', 462),\n",
       " ('report', 420),\n",
       " ('american', 420),\n",
       " ('custom', 378),\n",
       " ('chang', 378),\n",
       " ('offer', 336),\n",
       " ('carrier', 294),\n",
       " ('board', 294),\n",
       " ('attend', 294),\n",
       " ('chicago', 294),\n",
       " ('tell', 252),\n",
       " ('year', 252),\n",
       " ('fli', 252),\n",
       " ('aircraft', 252),\n",
       " ('offic', 252),\n",
       " ('incid', 252),\n",
       " ('intern', 252),\n",
       " ('delta', 252),\n",
       " ('land', 252),\n",
       " ('munoz', 210),\n",
       " ('plan', 210),\n",
       " ('post', 210),\n",
       " ('secur', 210),\n",
       " ('depart', 210),\n",
       " ('houston', 210),\n",
       " ('state', 210),\n",
       " ('oper', 210),\n",
       " ('famili', 210),\n",
       " ('boe', 210),\n",
       " ('crew', 168),\n",
       " ('time', 168),\n",
       " ('hour', 168),\n",
       " ('employe', 168),\n",
       " ('aviat', 168),\n",
       " ('million', 168),\n",
       " ('week', 126),\n",
       " ('press', 126),\n",
       " ('polici', 126),\n",
       " ('go', 126),\n",
       " ('overbook', 126),\n",
       " ('allow', 126),\n",
       " ('ticket', 126),\n",
       " ('leav', 126),\n",
       " ('citi', 126),\n",
       " ('compani', 126),\n",
       " ('share', 126),\n",
       " ('anim', 126),\n",
       " ('angel', 126),\n",
       " ('april', 84),\n",
       " ('tuesday', 84),\n",
       " ('issu', 84),\n",
       " ('forc', 84),\n",
       " ('presid', 84),\n",
       " ('expect', 84),\n",
       " ('busi', 84),\n",
       " ('like', 84),\n",
       " ('transport', 84),\n",
       " ('class', 84),\n",
       " ('know', 84),\n",
       " ('remov', 84),\n",
       " ('death', 84),\n",
       " ('pet', 84),\n",
       " ('francisco', 84),\n",
       " ('cabin', 84),\n",
       " ('york', 84),\n",
       " ('govern', 84),\n",
       " ('rout', 84),\n",
       " ('denver', 84),\n",
       " ('pilot', 84),\n",
       " ('check', 84),\n",
       " ('avail', 84),\n",
       " ('passeng', 84),\n",
       " ('deal', 84),\n",
       " ('news', 42),\n",
       " ('associ', 42),\n",
       " ('express', 42),\n",
       " ('respons', 42),\n",
       " ('viral', 42),\n",
       " ('twitter', 42),\n",
       " ('peopl', 42),\n",
       " ('gate', 42),\n",
       " ('louisvill', 42),\n",
       " ('doctor', 42),\n",
       " ('video', 42),\n",
       " ('polic', 42),\n",
       " ('apolog', 42),\n",
       " ('happen', 42),\n",
       " ('need', 42),\n",
       " ('execut', 42),\n",
       " ('hear', 42),\n",
       " ('question', 42),\n",
       " ('accord', 42),\n",
       " ('spokesman', 42),\n",
       " ('give', 42),\n",
       " ('cancel', 42),\n",
       " ('world', 42),\n",
       " ('drag', 42),\n",
       " ('right', 42),\n",
       " ('offici', 42),\n",
       " ('wednesday', 42),\n",
       " ('newark', 42),\n",
       " ('includ', 42),\n",
       " ('long', 42),\n",
       " ('thursday', 42),\n",
       " ('take', 42),\n",
       " ('overhead', 42),\n",
       " ('die', 42),\n",
       " ('investig', 42),\n",
       " ('support', 42),\n",
       " ('photo', 42),\n",
       " ('member', 42),\n",
       " ('program', 42),\n",
       " ('cargo', 42),\n",
       " ('stop', 42),\n",
       " ('washington', 42),\n",
       " ('administr', 42),\n",
       " ('author', 42),\n",
       " ('countri', 42),\n",
       " ('daili', 42),\n",
       " ('statement', 42),\n",
       " ('point', 42),\n",
       " ('line', 42),\n",
       " ('experi', 42),\n",
       " ('fee', 42),\n",
       " ('price', 42),\n",
       " ('increas', 42),\n",
       " ('inform', 42),\n",
       " ('arriv', 42),\n",
       " ('book', 42),\n",
       " ('provid', 42),\n",
       " ('close', 42),\n",
       " ('write', 42),\n",
       " ('friday', 42)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wlist_sorted=sorted(wlist.items(), key=lambda v: v[1],reverse=True)\n",
    "wlist_sorted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[2, 1, 3, ['carrier', 'year']],\n",
       " [2, 1, 5, ['fli', 'hour']],\n",
       " [2, 2, 2, ['time', 'hour']],\n",
       " [2, 2, 5, ['carrier', 'plan']],\n",
       " [2, 3, 5, ['go', 'fli']],\n",
       " [2, 4, 2, ['year', 'week']],\n",
       " [2, 5, 0, ['carrier', 'year']],\n",
       " [2, 6, 0, ['plan', 'hour']],\n",
       " [2, 6, 5, ['carrier', 'plan']]]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "o[0][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['seat',\n",
       " 'american',\n",
       " 'carrier',\n",
       " 'oper',\n",
       " 'state',\n",
       " 'aircraft',\n",
       " 'travel',\n",
       " 'offer',\n",
       " 'servic',\n",
       " 'delta']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[4][4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[[[4, 1, 4, ['munoz', 'offer', 'travel', 'custom']]],\n",
       "  [[3, 1, 4, ['munoz', 'chang', 'custom']]],\n",
       "  [[2, 1, 3, ['carrier', 'year']],\n",
       "   [2, 1, 5, ['fli', 'hour']],\n",
       "   [2, 2, 2, ['time', 'hour']],\n",
       "   [2, 2, 5, ['carrier', 'plan']],\n",
       "   [2, 3, 5, ['go', 'fli']],\n",
       "   [2, 4, 2, ['year', 'week']],\n",
       "   [2, 5, 0, ['carrier', 'year']],\n",
       "   [2, 6, 0, ['plan', 'hour']],\n",
       "   [2, 6, 5, ['carrier', 'plan']]],\n",
       "  [[3, 4, 1, ['travel', 'allow', 'chang']],\n",
       "   [3, 4, 3, ['travel', 'polici', 'chang']],\n",
       "   [3, 6, 2, ['travel', 'chang', 'employe']]],\n",
       "  [[4, 1, 3, ['post', 'aircraft', 'secur', 'report']]],\n",
       "  [[4, 1, 1, ['offic', 'aviat', 'depart', 'secur']],\n",
       "   [4, 2, 4, ['offic', 'aviat', 'depart', 'secur']]]],\n",
       " [[[4, 2, 3, ['seat', 'report', 'forc', 'incid']]],\n",
       "  [[5, 2, 4, ['depart', 'incid', 'secur', 'aviat', 'offic']]],\n",
       "  [[3, 6, 4, ['presid', 'chang', 'airport']]],\n",
       "  [[4, 0, 4, ['aircraft', 'secur', 'report', 'post']]],\n",
       "  [[6, 6, 2, ['custom', 'offer', 'seat', 'travel', 'busi', 'chang']]],\n",
       "  [[4, 3, 0, ['chicago', 'state', 'airport', 'fli']]]],\n",
       " [[[5, 4, 0, ['seat', 'custom', 'servic', 'airport', 'plane']]],\n",
       "  [[3, 1, 1, ['incid', 'depart', 'offic']],\n",
       "   [3, 6, 0, ['incid', 'plane', 'offic']]],\n",
       "  [[4, 5, 2, ['plane', 'seat', 'time', 'know']]],\n",
       "  [[6, 4, 0, ['report', 'intern', 'seat', 'airport', 'plane', 'board']]],\n",
       "  [[5, 1, 1, ['offic', 'incid', 'aviat', 'depart', 'secur']]],\n",
       "  [[4, 3, 4, ['compani', 'american', 'plan', 'delta']],\n",
       "   [4, 4, 4, ['carrier', 'american', 'travel', 'delta']],\n",
       "   [4, 5, 0, ['million', 'carrier', 'compani', 'share']]]],\n",
       " [[[4, 1, 5, ['chicago', 'state', 'fli', 'airport']]],\n",
       "  [[2, 4, 3, ['attend', 'famili']],\n",
       "   [2, 5, 1, ['attend', 'land']],\n",
       "   [2, 5, 2, ['land', 'plane']],\n",
       "   [2, 5, 4, ['land', 'plane']]],\n",
       "  [[3, 0, 0, ['travel', 'tuesday', 'report']],\n",
       "   [3, 2, 3, ['plane', 'board', 'report']],\n",
       "   [3, 4, 0, ['plane', 'board', 'report']],\n",
       "   [3, 4, 3, ['travel', 'famili', 'post']],\n",
       "   [3, 5, 2, ['land', 'plane', 'report']],\n",
       "   [3, 5, 4, ['land', 'plane', 'report']]],\n",
       "  [[5, 4, 0, ['servic', 'airport', 'board', 'intern', 'custom']]],\n",
       "  [[6, 4, 4, ['offer', 'state', 'seat', 'delta', 'servic', 'american']]],\n",
       "  [[2, 0, 2, ['fli', 'go']],\n",
       "   [2, 2, 5, ['american', 'delta']],\n",
       "   [2, 4, 4, ['american', 'delta']],\n",
       "   [2, 5, 5, ['american', 'delta']]]],\n",
       " [[[6, 2, 3, ['airport', 'seat', 'plane', 'board', 'intern', 'report']]],\n",
       "  [[3, 0, 3, ['travel', 'chang', 'allow']],\n",
       "   [3, 1, 4, ['seat', 'travel', 'chang']],\n",
       "   [3, 1, 5, ['fli', 'houston', 'chicago']],\n",
       "   [3, 6, 2, ['seat', 'travel', 'chang']]],\n",
       "  [[3, 1, 5, ['houston', 'chicago', 'airport']],\n",
       "   [3, 2, 0, ['houston', 'airport', 'servic']],\n",
       "   [3, 3, 0, ['famili', 'chicago', 'airport']],\n",
       "   [3, 5, 0, ['aircraft', 'year', 'servic']],\n",
       "   [3, 5, 1, ['chicago', 'airport', 'servic']],\n",
       "   [3, 6, 5, ['aircraft', 'airport', 'servic']]],\n",
       "  [[3, 0, 3, ['polici', 'travel', 'chang']],\n",
       "   [3, 3, 2, ['famili', 'travel', 'post']]],\n",
       "  [[6, 3, 4, ['seat', 'american', 'state', 'offer', 'servic', 'delta']]],\n",
       "  [[2, 0, 4, ['leav', 'airport']],\n",
       "   [2, 1, 5, ['state', 'airport']],\n",
       "   [2, 3, 0, ['state', 'airport']],\n",
       "   [2, 3, 4, ['state', 'airport']],\n",
       "   [2, 5, 0, ['oper', 'rout']],\n",
       "   [2, 6, 5, ['oper', 'airport']]]],\n",
       " [[[4, 2, 5, ['million', 'carrier', 'share', 'compani']],\n",
       "   [4, 4, 4, ['oper', 'servic', 'carrier', 'aircraft']],\n",
       "   [4, 6, 5, ['oper', 'servic', 'carrier', 'aircraft']]],\n",
       "  [[5, 4, 0, ['airport', 'seat', 'chicago', 'servic', 'intern']]],\n",
       "  [[4, 2, 2, ['plane', 'time', 'seat', 'know']],\n",
       "   [4, 6, 0, ['plane', 'tell', 'american', 'seat']]],\n",
       "  [[3, 0, 0, ['custom', 'offer', 'travel']],\n",
       "   [3, 1, 4, ['custom', 'offer', 'travel']],\n",
       "   [3, 3, 3, ['attend', 'board', 'custom']],\n",
       "   [3, 4, 4, ['delta', 'offer', 'travel']],\n",
       "   [3, 6, 2, ['custom', 'offer', 'travel']]],\n",
       "  [[5, 4, 0, ['airport', 'plane', 'report', 'intern', 'servic']]],\n",
       "  [[3, 3, 4, ['seat', 'delta', 'american']],\n",
       "   [3, 4, 4, ['seat', 'delta', 'american']],\n",
       "   [3, 6, 0, ['boe', 'seat', 'american']]]],\n",
       " [[[4, 1, 1, ['incid', 'seat', 'tell', 'offic']],\n",
       "   [4, 2, 3, ['incid', 'seat', 'plane', 'offic']],\n",
       "   [4, 5, 2, ['seat', 'plane', 'tell', 'american']]],\n",
       "  [[3, 5, 4, ['land', 'airport', 'boe']]],\n",
       "  [[6, 1, 4, ['custom', 'travel', 'seat', 'busi', 'offer', 'chang']]],\n",
       "  [[3, 1, 4, ['offer', 'seat', 'travel']],\n",
       "   [3, 2, 5, ['share', 'million', 'travel']],\n",
       "   [3, 4, 4, ['offer', 'seat', 'travel']],\n",
       "   [3, 5, 0, ['year', 'share', 'million']]],\n",
       "  [[3, 1, 2, ['chang', 'airport', 'presid']],\n",
       "   [3, 3, 3, ['chang', 'airport', 'intern']]],\n",
       "  [[5, 4, 4, ['seat', 'aircraft', 'oper', 'carrier', 'servic']]]]]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}