Job Postings preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.cluster import KMeans\n",
    "import json\n",
    "from os import listdir\n",
    "import glob\n",
    "from scipy import spatial\n",
    "import spacy\n",
    "#from sklearn.metrics.pairwise import cosine_similarity\n",
    "#from genism.models import Word2Vec\n",
    "#print(os.listdir(\"../input\"))\n",
    "#print(os.listdir(\"..\"))\n",
    "#import spacy\n",
    "#from spacy import displacy\n",
    "#from topia.termextract import extract\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pass a third parameter(flag) as 1 in the match_profile() in order to get your recommendations!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_uuid": "c4827ed5e92d47eaa059096e4edea8afcd2a6e06"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Tejas\\Anaconda3\\lib\\site-packages\\scipy\\spatial\\distance.py:644: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>advertiserurl</th>\n",
       "      <th>company</th>\n",
       "      <th>employmenttype_jobstatus</th>\n",
       "      <th>jobdescription</th>\n",
       "      <th>jobid</th>\n",
       "      <th>joblocation_address</th>\n",
       "      <th>jobtitle</th>\n",
       "      <th>postdate</th>\n",
       "      <th>shift</th>\n",
       "      <th>site_name</th>\n",
       "      <th>skills</th>\n",
       "      <th>uniq_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5874</th>\n",
       "      <td>https://www.dice.com/jobs/detail/Python-Develo...</td>\n",
       "      <td>ICONMA</td>\n",
       "      <td>Full Time, full time</td>\n",
       "      <td>Python Developer  Location: Houston, TXDuratio...</td>\n",
       "      <td>Dice Id : iconma</td>\n",
       "      <td>Houston, TX</td>\n",
       "      <td>Python Developer</td>\n",
       "      <td>1 week ago</td>\n",
       "      <td>Telecommuting not available|Travel not required</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4+ years working with Python, Django, PostgreS...</td>\n",
       "      <td>3bb898bbeca95b26bffbb8860b194d2c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3299</th>\n",
       "      <td>https://www.dice.com/jobs/detail/Full%2526%252...</td>\n",
       "      <td>Cypress Group</td>\n",
       "      <td>Full Time, Permanent</td>\n",
       "      <td>My client is a world-class media and entertain...</td>\n",
       "      <td>Dice Id : 10121728</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>Full-Stack Python Developer</td>\n",
       "      <td>3 weeks ago</td>\n",
       "      <td>Telecommuting not available|Travel not required</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Linux, Python, REST, AWS,</td>\n",
       "      <td>39d56fc47d8fec2c2baa9e6e06142181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7358</th>\n",
       "      <td>https://www.dice.com/jobs/detail/Full-Stack-Ja...</td>\n",
       "      <td>SolTech, Inc</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>SOLTECH Staffing SolutionsAt SOLTECH, we work ...</td>\n",
       "      <td>Dice Id : 10107031</td>\n",
       "      <td>Norcross, GA</td>\n",
       "      <td>Full Stack Javascript Engineer</td>\n",
       "      <td>1 week ago</td>\n",
       "      <td>Telecommuting not available|Travel required to...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Node, React, Javascript, Amazon Web Services, ...</td>\n",
       "      <td>d34bcd54a2b8fb7c3cf539a4f24c2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11702</th>\n",
       "      <td>https://www.dice.com/jobs/detail/Full-Stack-De...</td>\n",
       "      <td>Saicon Consultants Inc.</td>\n",
       "      <td>Contract W2, 12 Months</td>\n",
       "      <td>Proficient in RESTful web servicesProficient i...</td>\n",
       "      <td>Dice Id : saicon</td>\n",
       "      <td>Pleasanton, CA</td>\n",
       "      <td>Full Stack Developer</td>\n",
       "      <td>10 hours ago</td>\n",
       "      <td>Telecommuting not available|Travel not required</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Full Stack Developer</td>\n",
       "      <td>90d755eb5b34547c31978ce7a6eb519f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4009</th>\n",
       "      <td>https://www.dice.com/jobs/detail/System-Admini...</td>\n",
       "      <td>Samsung SDS America Inc</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Responsibilities:Responsible for deployment an...</td>\n",
       "      <td>Dice Id : 10265997</td>\n",
       "      <td>Ridgefield Park, NJ</td>\n",
       "      <td>System Administrator</td>\n",
       "      <td>4 weeks ago</td>\n",
       "      <td>Telecommuting not available|Travel not required</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AIX Linux Unix IBM Redhat VMWare ESXi Kernel T...</td>\n",
       "      <td>563b1f0ed315a503a4e71c3a002c8a9e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8515</th>\n",
       "      <td>https://www.dice.com/jobs/detail/System-Admini...</td>\n",
       "      <td>Strivector</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>“The best way to predict the future is to crea...</td>\n",
       "      <td>Dice Id : 10482237</td>\n",
       "      <td>Austin, TX</td>\n",
       "      <td>System Administrator</td>\n",
       "      <td>1 day ago</td>\n",
       "      <td>Telecommuting not available|Travel not required</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Linux and Windows Administration, Networking, ...</td>\n",
       "      <td>4a62b34021ee7f93571127d315c89f91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9981</th>\n",
       "      <td>https://www.dice.com/jobs/detail/Application-A...</td>\n",
       "      <td>Aria Systems</td>\n",
       "      <td>Full Time, PerYear</td>\n",
       "      <td>Title: Application AdministratorDepartment: Pr...</td>\n",
       "      <td>Dice Id : 10195482</td>\n",
       "      <td>Broomall, PA</td>\n",
       "      <td>Application Administrator</td>\n",
       "      <td>1 week ago</td>\n",
       "      <td>Telecommuting not available|Travel not required</td>\n",
       "      <td>NaN</td>\n",
       "      <td>perl, linux, system administration, DevOps, SQ...</td>\n",
       "      <td>52039b90fc1bdcfb076acd1812cf60e9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17522</th>\n",
       "      <td>https://www.dice.com/jobs/detail/Application-A...</td>\n",
       "      <td>Aria Systems</td>\n",
       "      <td>Full Time, PerYear</td>\n",
       "      <td>Title: Application AdministratorDepartment: Pr...</td>\n",
       "      <td>Dice Id : 10195482</td>\n",
       "      <td>Broomall, PA</td>\n",
       "      <td>Application Administrator</td>\n",
       "      <td>3 days ago</td>\n",
       "      <td>Telecommuting not available|Travel not required</td>\n",
       "      <td>NaN</td>\n",
       "      <td>perl, linux, system administration, DevOps, SQ...</td>\n",
       "      <td>27206a40dd73d701e33a91a30e6f26eb</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18190</th>\n",
       "      <td>https://www.dice.com/jobs/detail/System-Admini...</td>\n",
       "      <td>Strivector</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>“The best way to predict the future is to crea...</td>\n",
       "      <td>Dice Id : 10482237</td>\n",
       "      <td>Austin, TX</td>\n",
       "      <td>System Administrator</td>\n",
       "      <td>4 days ago</td>\n",
       "      <td>Telecommuting not available|Travel not required</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Linux and Windows Administration, Networking, ...</td>\n",
       "      <td>7417555dd385b839206562c3586f9991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21051</th>\n",
       "      <td>https://www.dice.com/jobs/detail/Full-Stack-En...</td>\n",
       "      <td>Universal Software Corporation</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>We are seeking a Full Stack Engineeer to join ...</td>\n",
       "      <td>Dice Id : uscma</td>\n",
       "      <td>Mountain View, CA</td>\n",
       "      <td>Full Stack Engineer</td>\n",
       "      <td>1 week ago</td>\n",
       "      <td>Telecommuting not available|Travel not required</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Full Stack Python</td>\n",
       "      <td>ed25e881bec334c636fc3edaba3272b2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           advertiserurl  \\\n",
       "5874   https://www.dice.com/jobs/detail/Python-Develo...   \n",
       "3299   https://www.dice.com/jobs/detail/Full%2526%252...   \n",
       "7358   https://www.dice.com/jobs/detail/Full-Stack-Ja...   \n",
       "11702  https://www.dice.com/jobs/detail/Full-Stack-De...   \n",
       "4009   https://www.dice.com/jobs/detail/System-Admini...   \n",
       "8515   https://www.dice.com/jobs/detail/System-Admini...   \n",
       "9981   https://www.dice.com/jobs/detail/Application-A...   \n",
       "17522  https://www.dice.com/jobs/detail/Application-A...   \n",
       "18190  https://www.dice.com/jobs/detail/System-Admini...   \n",
       "21051  https://www.dice.com/jobs/detail/Full-Stack-En...   \n",
       "\n",
       "                              company employmenttype_jobstatus  \\\n",
       "5874                           ICONMA     Full Time, full time   \n",
       "3299                    Cypress Group     Full Time, Permanent   \n",
       "7358                     SolTech, Inc                Full Time   \n",
       "11702         Saicon Consultants Inc.   Contract W2, 12 Months   \n",
       "4009          Samsung SDS America Inc                Full Time   \n",
       "8515                       Strivector                Full Time   \n",
       "9981                     Aria Systems       Full Time, PerYear   \n",
       "17522                    Aria Systems       Full Time, PerYear   \n",
       "18190                      Strivector                Full Time   \n",
       "21051  Universal Software Corporation                Full Time   \n",
       "\n",
       "                                          jobdescription               jobid  \\\n",
       "5874   Python Developer  Location: Houston, TXDuratio...    Dice Id : iconma   \n",
       "3299   My client is a world-class media and entertain...  Dice Id : 10121728   \n",
       "7358   SOLTECH Staffing SolutionsAt SOLTECH, we work ...  Dice Id : 10107031   \n",
       "11702  Proficient in RESTful web servicesProficient i...    Dice Id : saicon   \n",
       "4009   Responsibilities:Responsible for deployment an...  Dice Id : 10265997   \n",
       "8515   “The best way to predict the future is to crea...  Dice Id : 10482237   \n",
       "9981   Title: Application AdministratorDepartment: Pr...  Dice Id : 10195482   \n",
       "17522  Title: Application AdministratorDepartment: Pr...  Dice Id : 10195482   \n",
       "18190  “The best way to predict the future is to crea...  Dice Id : 10482237   \n",
       "21051  We are seeking a Full Stack Engineeer to join ...     Dice Id : uscma   \n",
       "\n",
       "       joblocation_address                        jobtitle      postdate  \\\n",
       "5874           Houston, TX                Python Developer    1 week ago   \n",
       "3299          New York, NY     Full-Stack Python Developer   3 weeks ago   \n",
       "7358          Norcross, GA  Full Stack Javascript Engineer    1 week ago   \n",
       "11702       Pleasanton, CA            Full Stack Developer  10 hours ago   \n",
       "4009   Ridgefield Park, NJ            System Administrator   4 weeks ago   \n",
       "8515            Austin, TX            System Administrator     1 day ago   \n",
       "9981          Broomall, PA       Application Administrator    1 week ago   \n",
       "17522         Broomall, PA       Application Administrator    3 days ago   \n",
       "18190           Austin, TX            System Administrator    4 days ago   \n",
       "21051    Mountain View, CA             Full Stack Engineer    1 week ago   \n",
       "\n",
       "                                                   shift site_name  \\\n",
       "5874     Telecommuting not available|Travel not required       NaN   \n",
       "3299     Telecommuting not available|Travel not required       NaN   \n",
       "7358   Telecommuting not available|Travel required to...       NaN   \n",
       "11702    Telecommuting not available|Travel not required       NaN   \n",
       "4009     Telecommuting not available|Travel not required       NaN   \n",
       "8515     Telecommuting not available|Travel not required       NaN   \n",
       "9981     Telecommuting not available|Travel not required       NaN   \n",
       "17522    Telecommuting not available|Travel not required       NaN   \n",
       "18190    Telecommuting not available|Travel not required       NaN   \n",
       "21051    Telecommuting not available|Travel not required       NaN   \n",
       "\n",
       "                                                  skills  \\\n",
       "5874   4+ years working with Python, Django, PostgreS...   \n",
       "3299                           Linux, Python, REST, AWS,   \n",
       "7358   Node, React, Javascript, Amazon Web Services, ...   \n",
       "11702                               Full Stack Developer   \n",
       "4009   AIX Linux Unix IBM Redhat VMWare ESXi Kernel T...   \n",
       "8515   Linux and Windows Administration, Networking, ...   \n",
       "9981   perl, linux, system administration, DevOps, SQ...   \n",
       "17522  perl, linux, system administration, DevOps, SQ...   \n",
       "18190  Linux and Windows Administration, Networking, ...   \n",
       "21051                                  Full Stack Python   \n",
       "\n",
       "                                uniq_id  \n",
       "5874   3bb898bbeca95b26bffbb8860b194d2c  \n",
       "3299   39d56fc47d8fec2c2baa9e6e06142181  \n",
       "7358   d34bcd54a2b8fb7c3cf539a4f24c2007  \n",
       "11702  90d755eb5b34547c31978ce7a6eb519f  \n",
       "4009   563b1f0ed315a503a4e71c3a002c8a9e  \n",
       "8515   4a62b34021ee7f93571127d315c89f91  \n",
       "9981   52039b90fc1bdcfb076acd1812cf60e9  \n",
       "17522  27206a40dd73d701e33a91a30e6f26eb  \n",
       "18190  7417555dd385b839206562c3586f9991  \n",
       "21051  ed25e881bec334c636fc3edaba3272b2  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def cosine_similarity(arr1,arr2):\n",
    "    ans=1- spatial.distance.cosine(arr1,arr2)\n",
    "    if(np.isnan(ans)):\n",
    "        return 0\n",
    "    else:\n",
    "        return ans\n",
    "class job_postings:    \n",
    "    def __init__(self,link):\n",
    "        self.df2=pd.read_csv(link)\n",
    "        self.training_range=int(len(self.df2.loc[:,'uniq_id']))\n",
    "    def check_threshold(threshold,ele):\n",
    "        if(ele[0]!=threshold[0][0] and abs(ele[1]-threshold[0][1])<0.03):\n",
    "            return True\n",
    "        else:\n",
    "            return False\n",
    "    def categorize_jobs(self):\n",
    "        # #Predefined categories\n",
    "        #Compare similarities of word embeddings\n",
    "        nlp=spacy.load('en_core_web_lg')\n",
    "        job_id=self.df2.loc[:,'uniq_id'].tolist()[:self.training_range]\n",
    "        job_titles=self.df2.loc[:,'jobtitle'].tolist()[:self.training_range]\n",
    "        job_descriptions=self.df2.loc[:,'jobdescription'].tolist()[:self.training_range]\n",
    "        final_cat=pd.DataFrame(index=job_id)\n",
    "        #categories=['Network Engineer','Application Development','Big Data','Data Analyst','Software Developer','DevOps','Software Testing','Front End','Back End','Full Stack','Web Development','Information Security','Mobile developer','System Administrator','Business Analyst','Manager','Cloud']\n",
    "        categories=['Network Engineer','Full stack','QA/Test Developer','Enterprise application','DevOps','Mobile Developer','Back End','Database Administrator(DBA)','Front End','Game developer','System Administrator','Data Scientist','Business analyst','Sales professional','Product Manager','Information Security','Software Developer/Java Developer','Web Developer','Cloud Computing']\n",
    "        for category in categories:\n",
    "            final_cat[category]=np.nan\n",
    "        for job_t_d in list(zip(job_id,job_titles,job_descriptions)):\n",
    "            id_job=job_t_d[0]\n",
    "            job_i=job_t_d[1]\n",
    "            job_d=job_t_d[2]\n",
    "            job_title=nlp(job_i.lower())\n",
    "            job_description=nlp(job_d.lower())\n",
    "            match_cat_title=dict()\n",
    "            match_cat_description=dict()\n",
    "            for category in categories:\n",
    "                word=nlp(category.lower())\n",
    "                match_cat_title[category]=job_title.similarity(word)\n",
    "                match_cat_description[category]=job_description.similarity(word)\n",
    "            match_cat_title=sorted(match_cat_title.items(),key=lambda x:x[1],reverse=True)\n",
    "            match_cat_description=sorted(match_cat_description.items(),key=lambda x:x[1],reverse=True)\n",
    "\n",
    "\n",
    "            #a represents max\n",
    "            #if(match_cat_title[0][1]>0.5 or match_cat_description[0][1]>0.5):\n",
    "            a=match_cat_title[0]\n",
    "            #print(a)\n",
    "            match_cat_description=list(filter(lambda x: self.check_threshold(match_cat_title,x),match_cat_description))\n",
    "            if(len(match_cat_description)!=0):\n",
    "                print(match_cat_description)\n",
    "                print(id_job)\n",
    "                #b=match_cat_description[0]\n",
    "                final_cat.loc[id_job,a[0]]=1\n",
    "                match_cat_description.extend([(match_cat_title[0][0],1)])\n",
    "                sum_proportion=sum([x[1] for x in match_cat_description])\n",
    "                for ele in match_cat_description:\n",
    "                    final_cat.loc[id_job,ele[0]]=ele[1]/sum_proportion\n",
    "            else:\n",
    "                print(id_job)\n",
    "                final_cat.loc[id_job,a[0]]=1\n",
    "        return final_cat\n",
    "    def clean_skills(self):\n",
    "        extracted_skills=dict()\n",
    "        job_skills=np.asarray(self.df2.loc[:,\"skills\"])\n",
    "        for i in range(self.training_range):\n",
    "            #print(i)\n",
    "            #Method 1: Manual pre-processing\n",
    "            job_id=self.df2.iloc[i,-1]\n",
    "            #Method 2:Using NLTK\n",
    "            tokenizer=nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "            #print(job_skills[i])\n",
    "            if(pd.isnull(job_skills[i])):\n",
    "                continue\n",
    "            stopwords_list=stopwords.words(\"english\")\n",
    "            tokens=re.split(\"|\".join([\",\",\" and\",\"/\",\" AND\",\" or\",\" OR\",\";\"]),job_skills[i])\n",
    "            tokens=list(set(tokens))\n",
    "            extracted_skills[job_id]=[]\n",
    "            extracted_skills[job_id].extend(tokens)\n",
    "        return extracted_skills\n",
    "    def extract_skills(self,extracted_skills):\n",
    "        df_languages=pd.read_excel('./data/job_profile/languages.xlsx')\n",
    "        df_frameworks=pd.read_csv(\"./data/job_profile/frameworks.csv\")\n",
    "        df_database=pd.read_csv(\"./data/job_profile/database.csv\")\n",
    "        df_os=pd.read_csv(\"./data/job_profile/operating_systems.csv\")\n",
    "        df_plat=pd.read_csv(\"./data/job_profile/platforms.csv\")\n",
    "        frameworks=df_frameworks.iloc[:,1].tolist()\n",
    "        frameworks=[x.lower().strip() for x in frameworks]\n",
    "        #frameworks=[str(x).split(\",\")[0] for x in df_frameworks.iloc[:,1]]\n",
    "        languages=list(df_languages.iloc[:,0])\n",
    "        languages=[x.lower().strip() for x in languages]\n",
    "        #frameworks=[x.lower().strip().split('\\t')[0] for x in frameworks]\n",
    "        databases=df_database.iloc[:,0].tolist()\n",
    "        databases=[x.lower().strip() for x in databases]\n",
    "        op_systems=df_os.iloc[:,0].tolist()\n",
    "        op_systems=[x.lower().strip() for x in op_systems]\n",
    "        platforms=df_plat.iloc[:,1].tolist()\n",
    "        #print(platforms)\n",
    "        platforms=[x.lower().strip() for x in platforms]\n",
    "        #print(frameworks)\n",
    "        new_extracted=dict()\n",
    "        for ele in extracted_skills.keys():\n",
    "            final_lang=''\n",
    "            final_frame=''\n",
    "            final_others=''\n",
    "            final_database=''\n",
    "            final_plat=''\n",
    "            final_os=''\n",
    "            #print(extracted_skills[ele])\n",
    "            for skill in extracted_skills[ele]:\n",
    "                skill_base=skill.lower().strip()\n",
    "                #print(skill_base)\n",
    "                if(skill_base in languages):\n",
    "                    if(final_lang==''):\n",
    "                        final_lang=skill_base\n",
    "                    else:\n",
    "                        final_lang=final_lang+\",\"+skill_base\n",
    "                elif(skill_base in frameworks):\n",
    "                    if(final_frame==''):\n",
    "                        final_frame=skill_base\n",
    "                    else:\n",
    "                        final_frame=final_frame+\",\"+skill_base\n",
    "                elif(skill_base in databases):\n",
    "                    if(final_database==''):\n",
    "                        final_database=skill_base\n",
    "                    else:\n",
    "                        final_database=final_database+\",\"+skill_base\n",
    "                elif(skill_base in op_systems):\n",
    "                    if(final_os==''):\n",
    "                        final_os=skill_base\n",
    "                    else:\n",
    "                        final_os=final_os+\",\"+skill_base\n",
    "                elif(skill_base in platforms):\n",
    "                    if(final_plat==''):\n",
    "                        final_plat=skill_base\n",
    "                    else:\n",
    "                        final_plat=final_plat+\",\"+skill_base\n",
    "                else:\n",
    "                    if(final_others==''):\n",
    "                        final_others=skill_base\n",
    "                    else:\n",
    "                        final_others=final_others+\",\"+skill_base\n",
    "            new_extracted[ele]=[final_lang,final_frame,final_database,final_os,final_plat,final_others]\n",
    "        print((list(new_extracted.items()))[:100])\n",
    "        for ele,describe in list(zip(self.df2.loc[:,'uniq_id'],self.df2.loc[:,'jobdescription'].tolist()))[:self.training_range]:\n",
    "            doc=nlp(describe)\n",
    "            final_lang=''\n",
    "            final_frame=''\n",
    "            final_others=''\n",
    "            final_database=''\n",
    "            final_plat=''\n",
    "            final_os=''\n",
    "            for ent in doc.ents:\n",
    "                word=ent.text\n",
    "                word=word.lower().strip()\n",
    "                if(word in languages and word not in final_lang and word not in new_extracted[ele][0].split(\",\")):\n",
    "                    if(final_lang==''):\n",
    "                        final_lang=word\n",
    "                    else:\n",
    "                        final_lang=final_lang+\",\"+word\n",
    "                elif(word in frameworks and word not in final_frame and word not in new_extracted[ele][1].split(\",\")):\n",
    "                    if(final_frame==''):\n",
    "                        final_frame=word\n",
    "                    else:\n",
    "                        final_frame=final_frame+\",\"+word\n",
    "                elif(word in databases and word not in final_database and word not in new_extracted[ele][2].split(\",\")):\n",
    "                    if(final_database==''):\n",
    "                        final_database=word\n",
    "                    else:\n",
    "                        final_database=final_database+\",\"+word\n",
    "                elif(word in op_systems and word not in final_os and word not in new_extracted[ele][3].split(\",\")):\n",
    "                    if(final_os==''):\n",
    "                        final_os=word\n",
    "                    else:\n",
    "                        final_os=final_os+\",\"+word\n",
    "                elif(word in platforms and word not in final_plat and word not in new_extracted[ele][4].split(\",\")):\n",
    "                    if(final_plat==''):\n",
    "                        final_plat=word\n",
    "                    else:\n",
    "                        final_plat=final_plat+\",\"+word\n",
    "                else:\n",
    "                    if(final_others==''):\n",
    "                        final_others=word\n",
    "                    else:\n",
    "                        final_others=final_others+\",\"+word\n",
    "            if(final_lang!=''):\n",
    "                new_extracted[ele][0]+=\",\"+final_lang\n",
    "            if(final_frame!=''):\n",
    "                new_extracted[ele][1]+=\",\"+final_frame\n",
    "            if(final_database!=''):\n",
    "                new_extracted[ele][2]+=\",\"+final_database\n",
    "            if(final_os!=''):\n",
    "                new_extracted[ele][3]+=\",\"+final_os\n",
    "            if(final_plat!=''):\n",
    "                new_extracted[ele][4]+=\",\"+final_plat\n",
    "            if(final_others!=''):\n",
    "                new_extracted[ele][5]+=\",\"+final_others\n",
    "            #new_extracted[ele]=[final_lang,final_frame,final_database,final_os,final_plat,final_others]\n",
    "        extracted_skills_df=pd.DataFrame.from_dict(new_extracted,orient='index',columns=['Language','Framework','Database','OS','Platform','Others'])\n",
    "        return extracted_skills_df\n",
    "    def create_job_profile(self,extracted_skills_df,domain_df):\n",
    "        job_id=extracted_skills_df.index.tolist()\n",
    "        languages_df=pd.DataFrame(index=job_id)\n",
    "        platforms_df=pd.DataFrame(index=job_id)\n",
    "        frameworks_df=pd.DataFrame(index=job_id)\n",
    "        databases_df=pd.DataFrame(index=job_id)\n",
    "        \n",
    "        for job,lang,frame,plat,datab in list(zip(job_id,extracted_skills_df.loc[:,'Language'].tolist(),extracted_skills_df.loc[:,'Framework'].tolist(),extracted_skills_df.loc[:,'Platform'].tolist(),extracted_skills_df.loc[:,'Database'].tolist())):\n",
    "            #Languages\n",
    "            l=lang.split(\",\")\n",
    "            if(lang!=np.nan or lang!=''):\n",
    "                for ele in l:\n",
    "                    if(ele==''):\n",
    "                        continue\n",
    "                    if(ele not in languages_df.columns):\n",
    "                        #languages.append(ele)\n",
    "                        languages_df[ele]=np.nan\n",
    "                    languages_df.loc[job,ele]=1\n",
    "            \n",
    "            #Frameworks\n",
    "            l=frame.split(\",\")\n",
    "            if(frame!=np.nan or frame!=''):\n",
    "                for ele in l:\n",
    "                    if(ele==''):\n",
    "                        continue\n",
    "                    if(ele not in frameworks_df.columns):\n",
    "                        #languages.append(ele)\n",
    "                        frameworks_df[ele]=np.nan\n",
    "                    frameworks_df.loc[job,ele]=1\n",
    "\n",
    "            #Platforms\n",
    "            l=plat.split(\",\")\n",
    "            if(plat!=np.nan or plat!=''):\n",
    "                for ele in l:\n",
    "                    if(ele==''):\n",
    "                        continue\n",
    "                    if(ele not in platforms_df.columns):\n",
    "                        #languages.append(ele)\n",
    "                        platforms_df[ele]=np.nan\n",
    "                    platforms_df.loc[job,ele]=1\n",
    "            \n",
    "            #Databases\n",
    "            l=datab.split(\",\")\n",
    "            if(datab!=np.nan or datab!=''):\n",
    "                for ele in l:\n",
    "                    if(ele==''):\n",
    "                        continue\n",
    "                    if(ele not in databases_df.columns):\n",
    "                        #languages.append(ele)\n",
    "                        databases_df[ele]=np.nan\n",
    "                    databases_df.loc[job,ele]=1\n",
    "        languages_df=languages_df.reindex_axis(sorted(languages_df.columns), axis=1)\n",
    "        frameworks_df=frameworks_df.reindex_axis(sorted(frameworks_df.columns), axis=1)\n",
    "        platforms_df=platforms_df.reindex_axis(sorted(platforms_df.columns), axis=1)\n",
    "        databases_df=databases_df.reindex_axis(sorted(databases_df.columns), axis=1)\n",
    "        domain_df=domain_df.reindex_axis(sorted(domain_df.columns), axis=1)\n",
    "        \n",
    "        languages_df.index.name=frameworks_df.index.name=platforms_df.index.name=databases_df.index.name=domain_df.index.name='uniq_id'\n",
    "        languages_df.to_csv(\"./data/job_profile/languages_job_profile.csv\")\n",
    "        frameworks_df.to_csv(\"./data/job_profile/frameworks_job_profile.csv\")\n",
    "        platforms_df.to_csv(\"./data/job_profile/platforms_job_profile.csv\")\n",
    "        databases_df.to_csv(\"./data/job_profile/databases_job_profile.csv\")\n",
    "        domain_df.to_csv(\"./data/job_profile/domain_job_profile.csv\")\n",
    "        print(languages_df.columns)\n",
    "        \n",
    "    def clean_common_profile(self,df_user,df_job,flag):\n",
    "        #Shift .net from languages to frameworks\n",
    "        if(flag=='Language'):\n",
    "            print(df_job.columns.tolist())\n",
    "            #bash and bash/shell\n",
    "            count=0\n",
    "            for ele in df_user.loc[:,'bash/shell']:\n",
    "                if(ele==1.0):\n",
    "                    df_user.ix[count,'bash']=1.0\n",
    "                count=count+1\n",
    "            df_user=df_user.drop('bash/shell',axis=1)\n",
    "            count=0\n",
    "            for ele in df_job.loc[:,'bash/shell']:\n",
    "                if(ele==1.0):\n",
    "                    df_job.ix[count,'bash']=1.0\n",
    "                count=count+1\n",
    "            df_job=df_job.drop('bash/shell',axis=1)\n",
    "\n",
    "        if(flag=='Framework'):\n",
    "            print(df_user.columns.tolist())\n",
    "            count=0\n",
    "            for ele in df_user.loc[:,'nodejs']:\n",
    "                if(ele==1.0):\n",
    "                    df_user.ix[count,'node.js']=1.0\n",
    "                count=count+1\n",
    "            df_user=df_user.drop('nodejs',axis=1)\n",
    "            count=0\n",
    "            for ele in df_job.loc[:,'nodejs']:\n",
    "                if(ele==1.0):\n",
    "                    df_job.ix[count,'node.js']=1.0\n",
    "                count=count+1\n",
    "            df_job=df_job.drop('nodejs',axis=1)\n",
    "            \n",
    "            count=0\n",
    "            for ele in df_user.loc[:,'angularjs']:\n",
    "                if(ele==1.0):\n",
    "                    df_user.ix[count,'angular']=1.0\n",
    "                count=count+1\n",
    "            df_user=df_user.drop('angularjs',axis=1)\n",
    "            count=0\n",
    "            for ele in df_job.loc[:,'angularjs']:\n",
    "                if(ele==1.0):\n",
    "                    df_job.ix[count,'angular']=1.0\n",
    "                count=count+1\n",
    "            df_job=df_job.drop('angularjs',axis=1)\n",
    "            \n",
    "        if(flag=='Platform'):\n",
    "            print(df_user.columns.tolist())\n",
    "        if(flag=='Database'):\n",
    "            print(df_user.columns.tolist())\n",
    "            count=0\n",
    "            for ele in df_user.loc[:,'microsoft sql server']:\n",
    "                if(ele==1.0):\n",
    "                    df_user.ix[count,'sql server']=1.0\n",
    "                count=count+1\n",
    "            df_user=df_user.drop('microsoft sql server',axis=1)\n",
    "            count=0\n",
    "            for ele in df_job.loc[:,'microsoft sql server']:\n",
    "                if(ele==1.0):\n",
    "                    df_job.ix[count,'sql server']=1.0\n",
    "                count=count+1\n",
    "            df_job=df_job.drop('microsoft sql server',axis=1)\n",
    "        return df_user,df_job\n",
    "\n",
    "    #Input is two dataframes    \n",
    "    def create_common_profile(self,job_profile_path,user_profile_path,output_path,flag=0):\n",
    "        if(flag==0):\n",
    "            #Domain\n",
    "            userprofile=pd.read_csv(user_profile_path+\"DevType.csv\",index_col='Respondent')\n",
    "            jobprofile=pd.read_csv(job_profile_path+\"domain_job_profile.csv\",index_col='Unnamed: 0')\n",
    "            print(\"Read from file\")\n",
    "            print(jobprofile.index)\n",
    "            #jobprofile=jobprofile.reset_index()\n",
    "            #userprofile=userprofile.reset_index()\n",
    "            userprofile.drop('Unnamed: 0', axis=1, inplace=True)\n",
    "            jobprofile.drop('uniq_id', axis=1, inplace=True)\n",
    "            jobprofile.index.name='uniq_id'\n",
    "            print(\"index 2in domain\")\n",
    "            print(jobprofile.index)\n",
    "            #print(jobprofile.loc[:,'uniq_id'])\n",
    "            userprofile.rename(columns={'Product manager':'Product Manager','Back-end developer':'Back End','C-suite executive (CEO, CTO, etc.)':'C-suite executive','Data scientist or machine learning specialist':'Data Scientist','Database administrator':'Database Administrator(DBA)','Mobile developer':'Mobile Developer','Desktop or enterprise applications developer':'Enterprise application','DevOps specialist':'DevOps','Front-end developer':'Front End','Full-stack developer':'Full stack','Marketing or sales professional':'Sales professional','QA or test developer':'QA/Test Developer','System administrator':'System Administrator','Game or graphics developer':'Game developer'},inplace=True)\n",
    "            jobprofile.rename(columns={'Business analyst':'Data or business analyst'},inplace=True)\n",
    "            print(userprofile.columns)\n",
    "            print(jobprofile.columns)\n",
    "            print(\"index in domain\")\n",
    "            print(jobprofile.index)\n",
    "            #Present in userprofile but not in jobprofile\n",
    "            a=list(set(userprofile.columns)-set(jobprofile.columns))\n",
    "            print(a)\n",
    "            for i in a:\n",
    "                if(i!='Respondent'):\n",
    "                    jobprofile[i]=0\n",
    "            b=list(set(jobprofile.columns)-set(userprofile.columns))\n",
    "            print(b)\n",
    "            for i in b:\n",
    "                if(i!='uniq_id'):\n",
    "                    userprofile[i]=0\n",
    "            #userprofile=userprofile.set_index('Respondent')\n",
    "            #jobprofile=jobprofile.set_index('uniq_id')\n",
    "            userprofile=userprofile[sorted(userprofile.columns.tolist())]\n",
    "            jobprofile=jobprofile[sorted(jobprofile.columns.tolist())]\n",
    "            #Exclude \n",
    "\n",
    "            print(userprofile.columns==jobprofile.columns)\n",
    "\n",
    "            print(userprofile.columns)\n",
    "            print(jobprofile.columns)\n",
    "            userprofile=userprofile[userprofile.columns.tolist()]\n",
    "            jobprofile=jobprofile[jobprofile.columns.tolist()]\n",
    "            userprofile.to_csv(output_path+\"domain_user_profile.csv\")\n",
    "            jobprofile.to_csv(output_path+\"domain_job_profile.csv\")\n",
    "\n",
    "            #Languages\n",
    "            df_user=pd.read_csv(user_profile_path+\"LanguageWorkedWith.csv\",index_col='Respondent')\n",
    "            df_job=pd.read_csv(job_profile_path+\"languages_job_profile.csv\",index_col=0)\n",
    "            df_job.index.name='uniq_id'\n",
    "            print(\"index is\")\n",
    "            print(df_job.index)\n",
    "            print(df_user.columns)\n",
    "            print(df_job.columns)\n",
    "            df_user.drop('Unnamed: 0', axis=1, inplace=True)\n",
    "            #df_job.drop('Unnamed: 0', axis=1, inplace=True)\n",
    "            df_job.rename(columns={'visual basic .net':'vb.net'},inplace=True)\n",
    "            df_user.columns=list(map(lambda x:x.lower(),df_user.columns))\n",
    "            df_job.columns=list(map(lambda x:x.lower(),df_job.columns))\n",
    "            columns_to_add=[]\n",
    "            a=list(set(df_user.columns)-(set(df_job.columns)))\n",
    "            print(a)\n",
    "            for i in a:\n",
    "                if(i!='Respondent'):\n",
    "                    df_job[i]=0        \n",
    "            b=list(set(df_job.columns)-set(df_user.columns))\n",
    "            print(b)\n",
    "            for i in b:\n",
    "                if(i!='uniq_id'):\n",
    "                    df_user[i]=0\n",
    "            print(df_job.index)        \n",
    "            df_user=df_user[sorted(df_user.columns.tolist())]\n",
    "            df_job=df_job[sorted(df_job.columns.tolist())]\n",
    "            #df_user=userprofile.reindex_axis(sorted(df_user.columns), axis=1)\n",
    "            #df_job=jobprofile.reindex_axis(sorted(df_job.columns), axis=1)\n",
    "            print(\"index 2\")\n",
    "            print(df_job.index)\n",
    "            print(len(set(df_user.columns).intersection(df_job.columns)),len(df_user.columns))\n",
    "            df_user,df_job=self.clean_common_profile(df_user,df_job,'Language')\n",
    "            print(\"language is\")\n",
    "            print(df_job.index[0])\n",
    "            print(df_job.loc[df_job.index[0],:])\n",
    "            df_user.to_csv(output_path+\"languages_profile_user.csv\")\n",
    "            df_job.to_csv(output_path+\"languages_profile_job.csv\")\n",
    "\n",
    "            #Frameworks\n",
    "            df_user=pd.read_csv(user_profile_path+\"FrameworkWorkedWith.csv\",index_col='Respondent')\n",
    "            df_job=pd.read_csv(job_profile_path+\"frameworks_job_profile.csv\",index_col=0) \n",
    "            df_job.index.name='uniq_id'\n",
    "            print(df_user.columns)\n",
    "            print(df_job.columns)\n",
    "            df_user.drop('Unnamed: 0', axis=1, inplace=True)\n",
    "            #df_job.drop('Unnamed: 0', axis=1, inplace=True)\n",
    "            #df_job.rename(columns={'visual basic .net':'vb.net'},inplace=True)\n",
    "            df_user.columns=list(map(lambda x:x.lower(),df_user.columns))\n",
    "            df_job.columns=list(map(lambda x:x.lower(),df_job.columns))\n",
    "\n",
    "            a=list(set(df_user.columns)-(set(df_job.columns)))\n",
    "            print(a)\n",
    "            for i in a:\n",
    "                if(i!='Respondent'):\n",
    "                    df_job[i]=0        \n",
    "            b=list(set(df_job.columns)-set(df_user.columns))\n",
    "            print(b)\n",
    "            for i in b:\n",
    "                if(i!='uniq_id'):\n",
    "                    df_user[i]=0\n",
    "            #userprofile=userprofile.reindex_axis(sorted(userprofile.columns), axis=1)\n",
    "            #jobprofile=jobprofile.reindex_axis(sorted(jobprofile.columns), axis=1)\n",
    "            df_user=df_user[sorted(df_user.columns.tolist())]\n",
    "            df_job=df_job[sorted(df_job.columns.tolist())]\n",
    "\n",
    "            print(len(set(df_user.columns).intersection(df_job.columns)),len(df_user.columns))\n",
    "            df_user,df_job=self.clean_common_profile(df_user,df_job,'Framework')   \n",
    "            df_user.to_csv(output_path+\"frameworks_profile_user.csv\")\n",
    "            df_job.to_csv(output_path+\"frameworks_profile_job.csv\")\n",
    "\n",
    "            #Platforms\n",
    "            df_user=pd.read_csv(user_profile_path+\"PlatformWorkedWith.csv\",index_col='Respondent')\n",
    "            df_job=pd.read_csv(job_profile_path+\"platforms_job_profile.csv\",index_col=0) \n",
    "            print(df_user.columns)\n",
    "            df_job.index.name='uniq_id'\n",
    "            print(df_job.columns)\n",
    "            df_user.drop('Unnamed: 0', axis=1, inplace=True)\n",
    "            #df_job.drop('Unnamed: 0', axis=1, inplace=True)\n",
    "            #df_job.rename(columns={'visual basic .net':'vb.net'},inplace=True)\n",
    "            df_user.columns=list(map(lambda x:x.lower(),df_user.columns))\n",
    "            df_job.columns=list(map(lambda x:x.lower(),df_job.columns))\n",
    "\n",
    "            a=list(set(df_user.columns)-(set(df_job.columns)))\n",
    "            print(a)\n",
    "            for i in a:\n",
    "                if(i!='Respondent'):\n",
    "                    df_job[i]=0\n",
    "            b=list(set(df_job.columns)-set(df_user.columns))\n",
    "            print(b)\n",
    "            for i in b:\n",
    "                if(i!='uniq_id'):\n",
    "                    df_user[i]=0\n",
    "            df_user=df_user[sorted(df_user.columns.tolist())]\n",
    "            df_job=df_job[sorted(df_job.columns.tolist())]\n",
    "\n",
    "            print(len(set(df_user.columns).intersection(df_job.columns)),len(df_user.columns))\n",
    "            df_user,df_job=self.clean_common_profile(df_user,df_job,'Platform')        \n",
    "            df_user.to_csv(output_path+\"platforms_profile_user.csv\")\n",
    "            df_job.to_csv(output_path+\"platforms_profile_job.csv\")\n",
    "\n",
    "            #Databases\n",
    "            df_user=pd.read_csv(user_profile_path+\"DatabaseWorkedWith.csv\",index_col='Respondent')\n",
    "            df_job=pd.read_csv(job_profile_path+\"databases_job_profile.csv\",index_col=0) \n",
    "            df_job.index.name='uniq_id'\n",
    "            print(df_user.columns)\n",
    "            print(df_job.columns)\n",
    "            df_user.drop('Unnamed: 0', axis=1, inplace=True)\n",
    "            #df_job.drop('Unnamed: 0', axis=1, inplace=True)\n",
    "            #df_job.rename(columns={'visual basic .net':'vb.net'},inplace=True)\n",
    "            df_user.columns=list(map(lambda x:x.lower(),df_user.columns))\n",
    "            df_job.columns=list(map(lambda x:x.lower(),df_job.columns))\n",
    "\n",
    "            a=list(set(df_user.columns)-(set(df_job.columns)))\n",
    "            print(a)\n",
    "            for i in a:\n",
    "                if(i!='Respondent'):\n",
    "                    df_job[i]=0\n",
    "            b=list(set(df_job.columns)-set(df_user.columns))\n",
    "            print(b)\n",
    "            for i in b:\n",
    "                if(i!='uniq_id'):\n",
    "                    df_user[i]=0\n",
    "            df_user=df_user[sorted(df_user.columns.tolist())]\n",
    "            df_job=df_job[sorted(df_job.columns.tolist())]\n",
    "\n",
    "            print(len(set(df_user.columns).intersection(df_job.columns)),len(df_user.columns))\n",
    "            df_user,df_job=self.clean_common_profile(df_user,df_job,'Database')        \n",
    "            df_user.to_csv(output_path+\"databases_profile_user.csv\")\n",
    "            df_job.to_csv(output_path+\"databases_profile_job.csv\")\n",
    "        #flag indicates that a new user profile\n",
    "    def match_profile(self,input_path,user_id,flag=0):\n",
    "        #Match a given user_id with all jobs in the database\n",
    "        \n",
    "        #Check if user id exists\n",
    "        df=pd.read_csv(input_path+\"domain_user_profile.csv\",index_col='Respondent')\n",
    "        #print(df.columns)\n",
    "        matches=dict()\n",
    "        if(flag==0):\n",
    "            if(user_id in df.index):\n",
    "                userdomain=df.loc[user_id,:]\n",
    "                #print(userdomain)\n",
    "                #If it does, retrieve the user profile from input_path\n",
    "                df=pd.read_csv(input_path+\"languages_profile_user.csv\",index_col='Respondent')\n",
    "                userlanguages=df.loc[user_id,:]\n",
    "\n",
    "                df=pd.read_csv(input_path+\"frameworks_profile_user.csv\",index_col='Respondent')\n",
    "                userframeworks=df.loc[user_id,:]\n",
    "\n",
    "                df=pd.read_csv(input_path+\"platforms_profile_user.csv\",index_col='Respondent')\n",
    "                userplatforms=df.loc[user_id,:]\n",
    "\n",
    "                df=pd.read_csv(input_path+\"databases_profile_user.csv\",index_col='Respondent')\n",
    "                userdatabases=df.loc[user_id,:]\n",
    "\n",
    "                userdomain=np.asarray(userdomain.fillna(0))\n",
    "                userlanguages=np.asarray(userlanguages.fillna(0))\n",
    "                userframeworks=np.asarray(userframeworks.fillna(0))\n",
    "                userplatforms=np.asarray(userplatforms.fillna(0))\n",
    "                userdatabases=np.asarray(userdatabases.fillna(0))\n",
    "                #print(userdomain)\n",
    "            else:\n",
    "                print(\"error! user id not in Dataset\")\n",
    "            #If it doesn't,take user profile as input\n",
    "        else:\n",
    "\n",
    "            print(\"New user!Enter details..\")\n",
    "            name=input(\"Enter full name\")\n",
    "            skills=input(\"Enter skills(comma separated). These are programming languages, frameworks,platforms or databases you have experience with\").split(\",\")\n",
    "            domains=''\n",
    "            flag=1\n",
    "            while(1):\n",
    "                print(\"Enter domain(s) of interest separated by commas(Names are case sensitive). Should be one of the following:\")\n",
    "                for i in df.columns:\n",
    "                    print(i,end=\",\")\n",
    "                domains=input().split(\",\")\n",
    "                for domain in domains:\n",
    "                    if(domain not in df.columns):\n",
    "                        flag=0\n",
    "                        break\n",
    "                if(flag==1):\n",
    "                    break\n",
    "                else:\n",
    "                    print(\"Please enter valid domain\")\n",
    "            #domains=list(map(lambda x:x.lower(),domains))\n",
    "            skills=list(map(lambda x:x.lower(),skills))                \n",
    "\n",
    "            userdomain=pd.DataFrame(columns=df.columns)\n",
    "            dictionary=dict()\n",
    "            for domain in domains:\n",
    "                dictionary[domain]=1.0\n",
    "            userdomain=userdomain.append(dictionary,ignore_index=True)\n",
    "\n",
    "\n",
    "            df=pd.read_csv(input_path+\"languages_profile_user.csv\",index_col='Respondent')\n",
    "            userlanguages=pd.DataFrame(columns=df.columns)\n",
    "            dictionary=dict()\n",
    "            for skill in skills:\n",
    "                if(skill in df.columns):\n",
    "                    dictionary[skill]=1.0\n",
    "            userlanguages=userlanguages.append(dictionary,ignore_index=True)\n",
    "\n",
    "            df=pd.read_csv(input_path+\"frameworks_profile_user.csv\",index_col='Respondent')\n",
    "            userframeworks=pd.DataFrame(columns=df.columns)\n",
    "            dictionary=dict()\n",
    "            for skill in skills:\n",
    "                if(skill in df.columns):\n",
    "                    dictionary[skill]=1.0\n",
    "            userframeworks=userframeworks.append(dictionary,ignore_index=True)\n",
    "\n",
    "            df=pd.read_csv(input_path+\"platforms_profile_user.csv\",index_col='Respondent')\n",
    "            userplatforms=pd.DataFrame(columns=df.columns)                \n",
    "            dictionary=dict()\n",
    "            for skill in skills:\n",
    "                if(skill in df.columns):\n",
    "                    dictionary[skill]=1.0\n",
    "            userplatforms=userplatforms.append(dictionary,ignore_index=True)\n",
    "\n",
    "            df=pd.read_csv(input_path+\"databases_profile_user.csv\",index_col='Respondent')\n",
    "            userdatabases=pd.DataFrame(columns=df.columns)               \n",
    "            dictionary=dict()\n",
    "            for skill in skills:\n",
    "                if(skill in df.columns):\n",
    "                    dictionary[skill]=1.0\n",
    "            userdatabases=userdatabases.append(dictionary,ignore_index=True)\n",
    "            #print(userdomain)\n",
    "            userdomain=np.asarray(userdomain.iloc[0,:].fillna(0))\n",
    "            userlanguages=np.asarray(userlanguages.iloc[0,:].fillna(0))\n",
    "            userframeworks=np.asarray(userframeworks.iloc[0,:].fillna(0))\n",
    "            userplatforms=np.asarray(userplatforms.iloc[0,:].fillna(0))\n",
    "            userdatabases=np.asarray(userdatabases.iloc[0,:].fillna(0))\n",
    "                \n",
    "        jobdomain=pd.read_csv(input_path+\"domain_job_profile.csv\",index_col='uniq_id')\n",
    "        joblanguages=pd.read_csv(input_path+'languages_profile_job.csv',index_col='uniq_id')\n",
    "        jobframeworks=pd.read_csv(input_path+'frameworks_profile_job.csv',index_col='uniq_id')\n",
    "        jobplatforms=pd.read_csv(input_path+'platforms_profile_job.csv',index_col='uniq_id')\n",
    "        jobdatabases=pd.read_csv(input_path+'databases_profile_job.csv',index_col='uniq_id')\n",
    "        #print(len(jobdomain.index),len(joblanguages.index))\n",
    "        for i in jobdomain.index:\n",
    "            #print(i)\n",
    "            domain=jobdomain.loc[i,:].fillna(0)\n",
    "            language=joblanguages.loc[i,:].fillna(0)\n",
    "            framework=jobframeworks.loc[i,:].fillna(0)\n",
    "            platform=jobplatforms.loc[i,:].fillna(0)\n",
    "            database=jobdatabases.loc[i,:].fillna(0)\n",
    "            job_id=str(i)\n",
    "            domain=np.asarray(domain)\n",
    "            language=np.asarray(language)\n",
    "            framework=np.asarray(framework)\n",
    "            platform=np.asarray(platform)\n",
    "            database=np.asarray(database)\n",
    "            #print(len(domain),len(userdomain))\n",
    "            score=(0.7*cosine_similarity(domain,userdomain))+(0.3*(cosine_similarity(language,userlanguages)+cosine_similarity(framework,userframeworks)+cosine_similarity(platform,userplatforms)+cosine_similarity(database,userdatabases)))\n",
    "            matches[job_id]=score\n",
    "            score=(0.7*cosine_similarity(domain,userdomain))+(0.3*(cosine_similarity(language,userlanguages)+cosine_similarity(framework,userframeworks)+cosine_similarity(platform,userplatforms)+cosine_similarity(database,userdatabases)))\n",
    "            #Initializing job profiles for later access\n",
    "            self.job_domain=domain\n",
    "            self.job_language=language\n",
    "            self.job_framework=framework\n",
    "            self.job_platform=platform\n",
    "            self.job_database=database\n",
    "            \n",
    "            self.user_domain=userdomain\n",
    "            self.user_language=userlanguages\n",
    "            self.user_framework=userframeworks\n",
    "            self.user_platform=userplatforms\n",
    "            self.user_database=userdatabases\n",
    "        matches=sorted(matches.items(),key=lambda x:x[1],reverse=True)\n",
    "        \n",
    "        recommendations=matches[:10]\n",
    "        #print(\"recommendations are\")\n",
    "        #print(recommendations)\n",
    "        rows=pd.DataFrame(columns=self.df2.columns)\n",
    "        count=0\n",
    "        for i in recommendations:\n",
    "            row=self.df2[self.df2['uniq_id']==i[0]]\n",
    "            #rows[count]=np.asarray(row.values.T.tolist()[0])\n",
    "            rows=rows.append(row.iloc[0])\n",
    "            count=count+1\n",
    "            #print(row)\n",
    "        return rows\n",
    "            \n",
    "\n",
    "obj=job_postings(\"./data/dice_com-job_us_sample.csv\")\n",
    "# final_cat=categorize_jobs()\n",
    "# final_cat.to_csv(\"./data/preprocessed_df.csv\")\n",
    "#extracted_skills=obj.clean_skills()\n",
    "#extracted_skills_df=obj.extract_skills(extracted_skills)\n",
    "#print(extracted_skills_df)\n",
    "#domain_df=pd.read_csv(\"./data/preprocessed_df.csv\")\n",
    "#obj.create_job_profile(extracted_skills_df,domain_df)\n",
    "#obj.create_common_profile(\"../data/job_profile/\",\"./data/user_profile/\",\"./data/\")\n",
    "\n",
    "#Path represents the location where final job and user profiles\n",
    "#df_user=pd.read_csv(\"./data/survey_results_public.csv\")\n",
    "#df_job=pd.read_csv(\"./data/dice_com-job_us_sample.csv\")\n",
    "\n",
    "#Pass a third parameter(flag) as 1 in order to get your recommendations!\n",
    "rows=obj.match_profile(\"./data/\",3)\n",
    "rows\n",
    "#rows\n",
    "# recommendations_1000=pd.DataFrame(columns=df_job.columns)\n",
    "# for ele in df_user.loc[:,'Respondent'].tolist()[:5000]:\n",
    "#     rows=obj.match_profile(\"./data/\",ele)\n",
    "#     recommendations_1000=recommendations_1000.append(rows.iloc[0,:],ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Tejas\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2717: DtypeWarning: Columns (8,12,13,14,15,16,50,51,52,53,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Hobby                                                                        Yes\n",
       "OpenSource                                                                    No\n",
       "Country                                                                    Italy\n",
       "Student                                                           Yes, full-time\n",
       "Employment                                Not employed, and not looking for work\n",
       "FormalEducation                Some college/university study without earning ...\n",
       "UndergradMajor                 Computer science, computer engineering, or sof...\n",
       "CompanySize                                                                  NaN\n",
       "DevType                                                                  Student\n",
       "YearsCoding                                                            3-5 years\n",
       "YearsCodingProf                                                              NaN\n",
       "JobSatisfaction                                                              NaN\n",
       "CareerSatisfaction                                                           NaN\n",
       "HopeFiveYears                                                                NaN\n",
       "JobSearchStatus                     I am not interested in new job opportunities\n",
       "LastNewJob                                                  I've never had a job\n",
       "AssessJob1                                                                     1\n",
       "AssessJob2                                                                     9\n",
       "AssessJob3                                                                     5\n",
       "AssessJob4                                                                     2\n",
       "AssessJob5                                                                     7\n",
       "AssessJob6                                                                     4\n",
       "AssessJob7                                                                     8\n",
       "AssessJob8                                                                     3\n",
       "AssessJob9                                                                    10\n",
       "AssessJob10                                                                    6\n",
       "AssessBenefits1                                                                2\n",
       "AssessBenefits2                                                                3\n",
       "AssessBenefits3                                                               11\n",
       "AssessBenefits4                                                                9\n",
       "                                                     ...                        \n",
       "EthicsResponsible                                                            NaN\n",
       "EthicalImplications                                                          NaN\n",
       "StackOverflowRecommend                                                       NaN\n",
       "StackOverflowVisit                                                           NaN\n",
       "StackOverflowHasAccount                                                      NaN\n",
       "StackOverflowParticipate                                                     NaN\n",
       "StackOverflowJobs                                                            NaN\n",
       "StackOverflowDevStory                                                        NaN\n",
       "StackOverflowJobsRecommend                                                   NaN\n",
       "StackOverflowConsiderMember                                                  NaN\n",
       "HypotheticalTools1                                                           NaN\n",
       "HypotheticalTools2                                                           NaN\n",
       "HypotheticalTools3                                                           NaN\n",
       "HypotheticalTools4                                                           NaN\n",
       "HypotheticalTools5                                                           NaN\n",
       "WakeTime                                                                     NaN\n",
       "HoursComputer                                                                NaN\n",
       "HoursOutside                                                                 NaN\n",
       "SkipMeals                                                                    NaN\n",
       "ErgonomicDevices                                                             NaN\n",
       "Exercise                                                                     NaN\n",
       "Gender                                                                       NaN\n",
       "SexualOrientation                                                            NaN\n",
       "EducationParents                                                             NaN\n",
       "RaceEthnicity                                                                NaN\n",
       "Age                                                                          NaN\n",
       "Dependents                                                                   NaN\n",
       "MilitaryUS                                                                   NaN\n",
       "SurveyTooLong                                                                NaN\n",
       "SurveyEasy                                                                   NaN\n",
       "Name: 2, Length: 128, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_user=pd.read_csv(\"./data/survey_results_public.csv\",index_col='Respondent')\n",
    "df_user.loc[2,:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "other methods and techniques attempted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                           advertiserurl  \\\n",
      "8193   https://www.dice.com/jobs/detail/Data-Scientis...   \n",
      "10053  https://www.dice.com/jobs/detail/DATA-SCIENTIS...   \n",
      "11722  https://www.dice.com/jobs/detail/Data-Scientis...   \n",
      "143    https://www.dice.com/jobs/detail/Lead%2526%252...   \n",
      "911    https://www.dice.com/jobs/detail/Lead-Data-Eng...   \n",
      "5271   https://www.dice.com/jobs/detail/Data-Architec...   \n",
      "8192   https://www.dice.com/jobs/detail/Data-Scientis...   \n",
      "8361   https://www.dice.com/jobs/detail/Dillard%2527s...   \n",
      "8643   https://www.dice.com/jobs/detail/Scientist-Cyb...   \n",
      "8911   https://www.dice.com/jobs/detail/Scientist-Cyb...   \n",
      "\n",
      "                         company  \\\n",
      "8193                 INSYS Group   \n",
      "10053      Quantig iBizlline Inc   \n",
      "11722  Intelliswift Software Inc   \n",
      "143             BayOne Solutions   \n",
      "911             InfoVision, Inc.   \n",
      "5271                 TRIAD Group   \n",
      "8192                 INSYS Group   \n",
      "8361                    Dillards   \n",
      "8643      CyberData Technologies   \n",
      "8911      CyberData Technologies   \n",
      "\n",
      "                                employmenttype_jobstatus  \\\n",
      "8193   Full Time, Contract Corp-To-Corp, Contract Ind...   \n",
      "10053  Contract Corp-To-Corp, Contract Independent, C...   \n",
      "11722  Full Time, Contract Corp-To-Corp, Contract Ind...   \n",
      "143                                            Full Time   \n",
      "911    Full Time, Contract Corp-To-Corp, Contract Ind...   \n",
      "5271                                     Full Time, perm   \n",
      "8192   Full Time, Contract Corp-To-Corp, Contract Ind...   \n",
      "8361                                Full Time, permanent   \n",
      "8643                                           Full Time   \n",
      "8911                                           Full Time   \n",
      "\n",
      "                                          jobdescription                jobid  \\\n",
      "8193   We are working with a prestigious Telecommunic...    Dice Id : insysus   \n",
      "10053  DATA SCIENTIST ROLE . Duration – 6 months with...   Dice Id : 10371829   \n",
      "11722  While applying please mention \"DY_Data_Science...   Dice Id : 10108150   \n",
      "143    Title: Lead Data Scientist (Applied Researcher...   Dice Id : 10494547   \n",
      "911    Hello  ,                                      ...     Dice Id : infotx   \n",
      "5271   Data ArchitectLocation: Seattle, WACompensatio...     Dice Id : TRIADW   \n",
      "8192   Insys is working directly with the Hiring Mana...    Dice Id : insysus   \n",
      "8361   DATA SCIENTISTThe Advanced Analytics and Strat...   Dice Id : 10138961   \n",
      "8643   CyberData Technologies, Inc., an established t...  Dice Id : RTX146efa   \n",
      "8911   CyberData Technologies, Inc., an established t...  Dice Id : RTX146efa   \n",
      "\n",
      "      joblocation_address                                        jobtitle  \\\n",
      "8193            Plano, TX                                  Data Scientist   \n",
      "10053    Philadelphia, PA                                DATA SCIENTIST .   \n",
      "11722       Sunnyvale, CA                                  Data Scientist   \n",
      "143     San Francisco, CA                          Lead/Sr Data Scientist   \n",
      "911           Waltham, MA                              Lead Data Engineer   \n",
      "5271          Seattle, WA                                  Data Architect   \n",
      "8192           Warren, NJ                                  Data Scientist   \n",
      "8361      Little Rock, AR  Dillard's Business Intelligence Data Scientist   \n",
      "8643         Sterling, VA                                       Scientist   \n",
      "8911         Sterling, VA                                       Scientist   \n",
      "\n",
      "           postdate                                            shift  \\\n",
      "8193      1 day ago  Telecommuting not available|Travel not required   \n",
      "10053    1 week ago  Telecommuting not available|Travel not required   \n",
      "11722  10 hours ago  Telecommuting not available|Travel not required   \n",
      "143     5 hours ago  Telecommuting not available|Travel not required   \n",
      "911     2 weeks ago  Telecommuting not available|Travel not required   \n",
      "5271     1 week ago  Telecommuting not available|Travel not required   \n",
      "8192      1 day ago  Telecommuting not available|Travel not required   \n",
      "8361      1 day ago  Telecommuting not available|Travel not required   \n",
      "8643    3 hours ago  Telecommuting not available|Travel not required   \n",
      "8911    3 hours ago  Telecommuting not available|Travel not required   \n",
      "\n",
      "          site_name                                             skills  \\\n",
      "8193            NaN                                     Data Scientist   \n",
      "10053           NaN  R (SparkR), Spark mllib, H20, Python , Data Sc...   \n",
      "11722           NaN  Data scientist, Statistics, R programming, Python   \n",
      "143    www.dice.com                   Machine Learning, Big Data, Java   \n",
      "911    www.dice.com  1+ years of experience engineering ingestion a...   \n",
      "5271            NaN  Data, SQL, ETL, AWS, MPP, Reshift, Terdata, Ve...   \n",
      "8192            NaN                   digital data science statistical   \n",
      "8361            NaN  data analytics, business intelligence, data mi...   \n",
      "8643            NaN  Scientist, Meteorology, thermodynamic principl...   \n",
      "8911            NaN  Scientist, Meteorology, thermodynamic principl...   \n",
      "\n",
      "                                uniq_id  \n",
      "8193   4534e8231eeaee03bdf626804bc51698  \n",
      "10053  ad372b0e20ed9fcc3511275df26fc6ef  \n",
      "11722  07835301c44a2d7acda29297321a35d1  \n",
      "143    a1e1bd1243cdd15a3219e0edc8a93131  \n",
      "911    41dc8a866b6351ff07abdd3bfa778d04  \n",
      "5271   33b9d97b3cfad10b6e038b4980aa8bd1  \n",
      "8192   056dbfe00756ac85618a02dd05b6916b  \n",
      "8361   9eb58eb1421f01b105803fd6ed7ab793  \n",
      "8643   b08b2ba58739c65dd8ac4aec76259c7c  \n",
      "8911   8be9971d84f98ba538f73e61d909d84e  \n"
     ]
    }
   ],
   "source": [
    "df2=pd.read_csv(\"../dice_com-job_us_sample.csv\")\n",
    "print(df2.head())\n",
    "jobs=[]\n",
    "for job_title  in df2.jobtitle:\n",
    "    if(job_title.lower() not in jobs):\n",
    "        jobs.append(job_title)\n",
    "#print(jobs)\n",
    "job_description=np.asarray(df2.loc[:,\"jobdescription\"])\n",
    "print(len(job_description[0:5]))\n",
    "#nlp=spacy.load('en')\n",
    "\n",
    "def remove_whitespace_entities(doc):\n",
    "    doc.ents=[x for x in doc.ents if not (x.text.isspace())]\n",
    "    return doc\n",
    "#nlp.add_pipe(remove_whitespace_entities,after='ner')\n",
    "#article=job_description[2]\n",
    "#doc=nlp(article)\n",
    "#article=[x for x in nlp(article) if not x.is_stop and x.pos!='PUNCT']\n",
    "#article=[x.lemma_ for x in article]\n",
    "\n",
    "#Using Named Entity Recognition\n",
    "# displacy.render(nlp(str(article)), jupyter=True, style='ent')\n",
    "# nlp(str(article)).ents\n",
    "\n",
    "#Comparing similarity of each word with a given set of words. If the similarity score is high, the word is related to technology. Hence, it should be considered.\n",
    "df_languages=pd.read_excel('./data/job_profile/languages.xlsx')\n",
    "df_frameworks=pd.read_excel(\"./data/job_profile/frameworks.xlsx\",header=None,error_bad_lines=False,delim_whitespace=True)\n",
    "experience_regex=['\\d+ years','\\d+ experience','']\n",
    "#print(df_frameworks)\n",
    "frame=[str(x).split(\",\")[0] for x in df_frameworks.iloc[:,0]]\n",
    "print(len(df_frameworks.columns))\n",
    "dictionary=list(df_languages.iloc[:,0])\n",
    "dictionary.extend(frame)\n",
    "print(dictionary)\n",
    "dictionary=[x.lower() for x in dictionary]\n",
    "extracted_jobs=dict()\n",
    "for i in range(len(job_description[:10000])):\n",
    "    job_id=df2.iloc[i,-1]\n",
    "    job=df2.iloc[i,3]\n",
    "    flag=0\n",
    "    for word in job.split(\" \"):\n",
    "        word=word.lower()\n",
    "        if word in dictionary:\n",
    "            flag=1\n",
    "            if job_id not in extracted_jobs.keys():\n",
    "                extracted_jobs[job_id]=[]\n",
    "            if word not in extracted_jobs[job_id]:\n",
    "                extracted_jobs[job_id].append(word)\n",
    "    if(flag==0):\n",
    "        print(job_id)\n",
    "print(extracted_jobs)\n",
    "print(len(extracted_jobs))        \n",
    "            \n",
    "#doc_given_text=nlp(u'computer science')\n",
    "#sample_word=nlp(u'Java')\n",
    "#words=[]\n",
    "#for ele in doc:\n",
    "#    if(doc_given_text.similarity(ele)>0.5):\n",
    "#        words.append(ele.text)\n",
    "#print(words)\n",
    "#print(doc_given_text.similarity(sample_word))\n",
    "#print(doc_given_text.similarity(doc[3]))\n",
    "#print(doc[3].vector)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "_uuid": "bc10bfb3c641c788952337123b05828afdf19d26"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>.net core</th>\n",
       "      <th>agile</th>\n",
       "      <th>angular</th>\n",
       "      <th>angularjs</th>\n",
       "      <th>asp.net mvc</th>\n",
       "      <th>aurelia</th>\n",
       "      <th>bottle</th>\n",
       "      <th>cakephp</th>\n",
       "      <th>cassandra</th>\n",
       "      <th>catalyst</th>\n",
       "      <th>...</th>\n",
       "      <th>vaadin</th>\n",
       "      <th>vanilla</th>\n",
       "      <th>vert.x</th>\n",
       "      <th>vue.js</th>\n",
       "      <th>wicket</th>\n",
       "      <th>xamarin</th>\n",
       "      <th>yarn</th>\n",
       "      <th>yii</th>\n",
       "      <th>zend</th>\n",
       "      <th>zope</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 76 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   .net core  agile  angular  angularjs  asp.net mvc  aurelia  bottle  \\\n",
       "0        1.0    NaN      NaN        NaN          NaN      NaN     NaN   \n",
       "\n",
       "   cakephp  cassandra  catalyst  ...   vaadin  vanilla  vert.x  vue.js  \\\n",
       "0      NaN        NaN       NaN  ...      NaN      NaN     NaN     NaN   \n",
       "\n",
       "   wicket  xamarin  yarn  yii  zend  zope  \n",
       "0     NaN      NaN   NaN  NaN   NaN   NaN  \n",
       "\n",
       "[1 rows x 76 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2=pd.read_csv(\"../input/us-technology-jobs-on-dicecom/dice_com-job_us_sample.csv\")\n",
    "print(df2.head())\n",
    "jobs=[]\n",
    "for job_title  in df2.jobtitle:\n",
    "    if(job_title.lower() not in jobs):\n",
    "        jobs.append(job_title)\n",
    "#print(jobs)\n",
    "job_skills=np.asarray(df2.loc[:,\"skills\"])\n",
    "print(len(job_description[0:5]))\n",
    "#nlp=spacy.load('en')\n",
    "\n",
    "def remove_whitespace_entities(doc):\n",
    "    doc.ents=[x for x in doc.ents if not (x.text.isspace())]\n",
    "    return doc\n",
    "#nlp.add_pipe(remove_whitespace_entities,after='ner')\n",
    "#article=job_description[2]\n",
    "#doc=nlp(article)\n",
    "#article=[x for x in nlp(article) if not x.is_stop and x.pos!='PUNCT']\n",
    "#article=[x.lemma_ for x in article]\n",
    "\n",
    "#Using Named Entity Recognition\n",
    "# displacy.render(nlp(str(article)), jupyter=True, style='ent')\n",
    "# nlp(str(article)).ents\n",
    "\n",
    "#Tokenizing words\n",
    "extracted_skills=dict()\n",
    "training_range=int(0.7*len(job_skills))\n",
    "for i in range(training_range):\n",
    "    #print(i)\n",
    "    #Method 1: Manual pre-processing\n",
    "    job_id=df2.iloc[i,-1]\n",
    "#     job=df2.iloc[i,-2]\n",
    "#     words=job_skills[i].split(\",\")\n",
    "#     if(words[0].lower()==\"(see job description)\"):\n",
    "#         continue\n",
    "#     #print(words)\n",
    "#     #print(len(words))\n",
    "#     for i in range(len(words)):\n",
    "#         #print(type(word))\n",
    "#         chunk=words[i].split(\"/\")\n",
    "#         words.remove(words[i])\n",
    "#         words.extend(chunk)\n",
    "            \n",
    "#     extracted_skills[job_id]=[]\n",
    "#     extracted_skills[job_id].extend(words)\n",
    "    #Method 2:Using NLTK\n",
    "    tokenizer=nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "    #print(job_skills[i])\n",
    "    if(pd.isnull(job_skills[i])):\n",
    "        continue\n",
    "    stopwords_list=stopwords.words(\"english\")\n",
    "    tokens=re.split(\"|\".join([\",\",\" and\",\"/\",\" AND\",\" or\",\" OR\",\";\"]),job_skills[i])\n",
    "    tokens=list(set(tokens))\n",
    "    #tokens=[x for x in tokens if x.isalpha()==True]\n",
    "    #tokens=tokenizer.tokenize(job_skills[i])\n",
    "    #print(tokens)\n",
    "    #tokens=nltk.word_tokenize(job_skills[i])\n",
    "    #print(stopwords)\n",
    "    #stopwords_list=stopwords\n",
    "    #words=[x for x in tokens if x not in stopwords_list]\n",
    "    #print(tokens)\n",
    "    extracted_skills[job_id]=[]\n",
    "    extracted_skills[job_id].extend(tokens)\n",
    "    #print(extracted_skills[job_id])\n",
    "print(extracted_skills)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "6ac15adc6a748dc3813694a054a1b2e7909a3ade"
   },
   "source": [
    "Part A: Building the content analyzer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "4403e95e5d0c3ebea3e178e9b9d03c72403c35e7",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Identifying various categories in job postings\n",
    "\n",
    "#Method1: Applying TF-IDF on the dataset\n",
    "count=0\n",
    "docs=[]\n",
    "for i in range(len(job_description[:100])):\n",
    "    #print(job_description[i])\n",
    "    if(job_description[i]==np.nan):\n",
    "        continue\n",
    "    doc=[x for x in job_description[i].split(\" \") if x not in stopwords_list]\n",
    "    docs.append(\" \".join(doc))\n",
    "print(len(docs))\n",
    "vectorizer=TfidfVectorizer(ngram_range=(1,2),max_df=0.6,max_features=50)\n",
    "response=vectorizer.fit_transform(docs)\n",
    "name_to_index=vectorizer.get_feature_names()\n",
    "response=response.toarray()\n",
    "scores=pd.DataFrame(data=response[:,:],index=range(len(response)),columns=name_to_index)\n",
    "print(scores)\n",
    "max_col_scores={}\n",
    "#print(scores.iloc[0,:])\n",
    "for col in range(len(scores.iloc[0,:])):\n",
    "    col_score=sum(scores.iloc[:,col])\n",
    "    max_col_scores[name_to_index[col]]=col_score\n",
    "max_col_scores=sorted(max_col_scores.items(),reverse=True,key=lambda x:x[1])[:50]\n",
    "print(max_col_scores)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "e3afbe12e20b0df23d430a016bb4bf2bacdcce76",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Method 2: Use a separate domain field in the dataset.\n",
    "#Use TF-DF on job titles\n",
    "def cluster_job_titles():\n",
    "    job_titles=df2.loc[:,'jobtitle'].tolist()\n",
    "    #Tokenization   \n",
    "    \n",
    "    docs=[]\n",
    "    for i in range(len(job_titles[:training_range])):\n",
    "        #print(job_description[i])\n",
    "        if(job_titles[i]==np.nan):\n",
    "            continue\n",
    "        doc=[x for x in job_description[i].split(\" \") if x not in stopwords_list]\n",
    "        docs.append(\" \".join(doc))\n",
    "    print(len(docs))\n",
    "    vectorizer=TfidfVectorizer(ngram_range=(1,2),max_df=1.0,max_features=50)\n",
    "    response=vectorizer.fit_transform(docs)\n",
    "    model=KMeans(n_clusters=10,init='k-means++')\n",
    "    model.fit(response)\n",
    "    labels=model.labels_\n",
    "    return labels\n",
    "#name_to_index=vectorizer.get_feature_names()\n",
    "#response=response.toarray()\n",
    "#scores=pd.DataFrame(data=response[:,:],index=range(len(response)),columns=name_to_index)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "5ef3577052cd750b7e49cf622164dd8f4ccee4d5",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# labels=cluster_job_titles()\n",
    "# cluster_mapping=dict()\n",
    "# for i in range(len(labels)):\n",
    "#     label=labels[i]\n",
    "#     if(label not in cluster_mapping.keys()):\n",
    "#         cluster_mapping[label]=[]\n",
    "#     cluster_mapping[label].append(job_titles[i])\n",
    "# print(cluster_mapping)\n",
    "# print(len(cluster_mapping))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "60298c92af63937bfd75c9b97213a56fe586f777",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# #Predefined categories\n",
    "# #Compare similarities of word embeddings\n",
    "# nlp=spacy.load('en_core_web_lg')\n",
    "# # jobs=df2.loc[:,'jobtitle'].tolist()[:50]\n",
    "# # jobs_descriptions=df2.loc[:,'jobdescription'].tolist()[:50]\n",
    "# # for job in jobs:\n",
    "# #     #print(job)\n",
    "# #     doc=nlp(job)\n",
    "# #     print(doc.ents)\n",
    "#     #print(\"Entity is\",ele.text,\"Label\",ele.label_)\n",
    "# def check_threshold(threshold,ele):\n",
    "#     if(ele[0]!=threshold[0][0] and abs(ele[1]-threshold[0][1])<0.03 and ele[1]>0.5):\n",
    "#         return True\n",
    "#     else:\n",
    "#         return False\n",
    "# def categorize(training_range):\n",
    "#     job_id=df2.loc[:,'uniq_id'].tolist()[:training_range]\n",
    "#     job_titles=df2.loc[:,'jobtitle'].tolist()[:training_range]\n",
    "#     job_descriptions=df2.loc[:,'jobdescription'].tolist()[:training_range]\n",
    "#     final_cat=pd.DataFrame(index=job_id)\n",
    "#     #categories=['Network Engineer','Application Development','Big Data','Data Analyst','Software Developer','DevOps','Software Testing','Front End','Back End','Full Stack','Web Development','Information Security','Mobile developer','System Administrator','Business Analyst','Manager','Cloud']\n",
    "#     categories=['Network Engineer','Full stack','QA/Test Developer','Enterprise application','DevOps','Mobile Developer','Back End','Database Administrator(DBA)','Front End','Game developer','System Administrator','Data Scientist','Business analyst','Sales professional','Product Manager','Information Security','Software Developer/Java Developer','Web Developer']\n",
    "#     for category in categories:\n",
    "#         final_cat[category]=np.nan\n",
    "#     for job_t_d in list(zip(job_id,job_titles,job_descriptions)):\n",
    "#         id_job=job_t_d[0]\n",
    "#         job_i=job_t_d[1]\n",
    "#         job_d=job_t_d[2]\n",
    "#         job_title=nlp(job_i.lower())\n",
    "#         job_description=nlp(job_d.lower())\n",
    "#         match_cat_title=dict()\n",
    "#         match_cat_description=dict()\n",
    "#         for category in categories:\n",
    "#             word=nlp(category.lower())\n",
    "#             match_cat_title[category]=job_title.similarity(word)\n",
    "#             match_cat_description[category]=job_description.similarity(word)\n",
    "#         match_cat_title=sorted(match_cat_title.items(),key=lambda x:x[1],reverse=True)\n",
    "#         match_cat_description=sorted(match_cat_description.items(),key=lambda x:x[1],reverse=True)\n",
    "        \n",
    "        \n",
    "#         #a represents max\n",
    "#         if(match_cat_title[0][1]>0.5 or match_cat_description[0][1]>0.5):\n",
    "#             a=match_cat_title[0]\n",
    "#             print(a)\n",
    "#             match_cat_description=list(filter(lambda x: check_threshold(match_cat_title,x),match_cat_description))\n",
    "#             if(len(match_cat_description)!=0):\n",
    "#                 print(match_cat_description)\n",
    "#                 print(id_job)\n",
    "#                 #b=match_cat_description[0]\n",
    "#                 final_cat.loc[id_job,a[0]]=1\n",
    "#                 match_cat_description.extend([(match_cat_title[0][0],1)])\n",
    "#                 sum_proportion=sum([x[1] for x in match_cat_description])\n",
    "#                 for ele in match_cat_description:\n",
    "#                     final_cat.loc[id_job,ele[0]]=ele[1]/sum_proportion\n",
    "#             else:\n",
    "#                 print(id_job)\n",
    "#                 final_cat.loc[id_job,a[0]]=1\n",
    "\n",
    "#         else:\n",
    "#             print(\"not considering\",job_i)\n",
    "#         #print(match_cat)\n",
    "#     return final_cat\n",
    "# training_range=int(0.7*len(df2.loc[:,'uniq_id']))\n",
    "# final_cat=categorize(training_range)\n",
    "\n",
    "# print(final_cat)\n",
    "# final_cat.to_csv(\"preprocessed_df.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "1e7658ea046e4f5876928a63aaeb9cebdb0bafbf",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Extracting years of experience\n",
    "nlp=spacy.load('en_core_web_lg')\n",
    "job_description=df2.loc[:,'jobdescription'].tolist()\n",
    "id_job=df2.loc[:,'uniq_id'].tolist()\n",
    "experience_regex=['\\d+ years \\w+ $\\.',r'\\d+ experience']\n",
    "matches=dict()\n",
    "entities=dict()\n",
    "for job_id,description in list(zip(id_job,job_description))[:10]:\n",
    "    #l=re.findall(r\"\\w* experience[\\S*\\s*]\\w*[.]\",description)\n",
    "    l=re.findall(r\"[^.]*experience[^.]*\\.\",description)\n",
    "    matches[job_id]=l    \n",
    "    for string in matches[job_id]:\n",
    "        print(string)\n",
    "        doc=nlp(string)\n",
    "        \n",
    "        for token in doc:\n",
    "            print(token.text,token.dep_,token.head.text)\n",
    "print(matches)\n",
    "#print(entities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "dc1c4c8814ec65c881476d2f8d83fa0e2319d74a",
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "5b24c1a19e23e3bde4d92103a08a65420bca1d14",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Identifying "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "ca2a373b2d99422ee8ba00f77883d3446dafa9c7"
   },
   "source": [
    "Part B: Profile Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "0cb0b32b64691045907a06bbe2b0a34f92ea8ff9",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Explicit/Implicit ways of identifying user preferences\n",
    "#1) Use a like/dislike method to indicate user preference:\n",
    "#Optimal method: Model based user-preference method to infer a score for each job(item) that the user has worked for in the past "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'positions': [{'title': 'Owner', 'is-current': True, 'company-name': 'AMBUS VENTURES'}], 'public-profile-url': '/pub/amark-a/39/565/222', 'location': 'Israel', 'first-name': 'Amark', 'num-connections': '13', 'last-name': 'A', 'industry': 'Wholesale'}, {'skills': ['Channel Management', 'Team Management', 'Key Account Management', 'People Management', 'Customer Relations'], 'positions': [{'title': 'Relationship Manager- WM', 'start-date': '2009-04-01', 'is-current': True, 'company-name': 'INGVysya Bank Ltd'}, {'title': 'Manager-Sales', 'end-date': '2009-03-01', 'start-date': '2008-05-01', 'company-name': 'Ikya Human Capital Solutions Pvt Ltd'}, {'title': 'Sales Manager', 'end-date': '2008-04-01', 'start-date': '2007-08-01', 'company-name': 'HDFC Bank ltd'}, {'title': 'Manager- Life insurance', 'end-date': '2007-08-01', 'start-date': '2003-08-01', 'company-name': 'ICICI Bank'}], 'public-profile-url': '/pub/amarnath-reddy-a/8/52a/280', 'location': 'Hyderabad Area, India', 'first-name': 'Amarnath Reddy', 'num-connections': '84', 'educations': [{'school-name': 'Jawaharlal Nehru Technological University', 'end-date': '2002-12-31', 'start-date': '1998-01-01', 'degree': 'B tech'}, {'school-name': 'Jayaprakash Narayan College Of Engineering', 'end-date': '2002-12-31', 'start-date': '1998-01-01'}], 'last-name': 'A', 'industry': 'Banking'}]\n"
     ]
    }
   ],
   "source": [
    "#LinkedIn Collab\n",
    "files=glob.glob(\"../scraped profiles/*.json\")\n",
    "#print(files)\n",
    "for file in files[:1]:\n",
    "    f=open(\"../scraped profiles/\"+file)\n",
    "    data=json.load(f)\n",
    "    print(data[0:2])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}