data_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8b92fe68",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install pdfminer.six\n",
    "import re\n",
    "import pandas as pd\n",
    "from pdfminer.high_level import extract_text\n",
    "from sentence_transformers import SentenceTransformer, util\n",
    "import torch, pandas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0689f4bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\DELL\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\DELL\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "# Download NLTK data\n",
    "nltk.download('punkt')  # Downloads the Punkt tokenizer models\n",
    "nltk.download('stopwords')  # Downloads the stopwords corpus\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a206a44d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"newdata.csv\")\n",
    "\n",
    "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n",
    "\n",
    "embed = []\n",
    "for jd in data[\"Description\"]:\n",
    "    embed.append(model.encode(jd, convert_to_tensor=True))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "264fb72c",
   "metadata": {},
   "outputs": [],
   "source": [
    "embed[0:3]\n",
    "data = pd.read_csv(\"data.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "940649fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"description_embedding\"] = embed\n",
    "\n",
    "data.to_csv(\"newdata.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c4245ada",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# sentences = [\"This is an example sentence\", \"Each sentence is converted\"]\n",
    "\n",
    "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n",
    "def get_similarity_text(jd, resume_embedding):\n",
    "    '''finding similarity between two texts'''\n",
    "    # calculating cosine similarity\n",
    "    jd_embedding = model.encode(jd, convert_to_tensor=True)\n",
    "\n",
    "    cos_scores = util.cos_sim(jd_embedding, resume_embedding)[0]\n",
    "    top_results = torch.topk(cos_scores, k=1)\n",
    "    score = top_results[0].numpy()[0]\n",
    "\n",
    "    return score\n",
    "\n",
    "resume = \"python javascript agile leadership scrum master computer vision generative ai git github mlops kubernetes mongodb firebase db pinecone mysql machine learning deep learning google cloud platform microsoft azure vector db docker flask fastapi professional experience dataknobs ml engineer june present automated key processes kreatewebsite major product enhance efficiency uplifted machine learning project predictive model precision made usable converting old written functions new one collaborate senior developers update website create new features open source contributor mlflow contributor august august contributed key functionality got merged administrator mlflow google cloud google cloud facilitator may july acquired proficiency docker mlops kubernetes kubernetes relevant projects sign language tutor march present used learning sign language fun interactive way chakla controller asphalt january january innovative racing game controlled unique physical interface round flat board blue square uses opencv computer vision techniques translate board movements game actions medsarthi january january helping seniors understand medications simple image upload voice enabled explanations education maharishi dayanand university rohtak bachelor computer science artificial intelligence rajokari institute technology dseu diploma information technology enabled service management\"\n",
    "\n",
    "# newdata = pandas.DataFrame()\n",
    "\n",
    "\n",
    "load_data = pandas.read_csv(\"newdata.csv\")\n",
    "resume_embedding = model.encode(resume, convert_to_tensor=True)   \n",
    "# print(get_similarity_text(jd,text))\n",
    "# resume_content = nlp(fetch_skills(\"Aman resume for y2.pdf\"))\n",
    "l = []\n",
    "for desc in load_data[\"Description\"]:\n",
    "    # desc_embed = np.array(desc.replace('\"',''))\n",
    "    # desc_embed = torch.tensor(desc_embed)\n",
    "    # print(desc_embed)\n",
    "    l.append(get_similarity_text(desc,resume_embedding))\n",
    "\n",
    "final = sorted(list(enumerate(l)), reverse=True, key=lambda x: x[1])[0:6]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "cc4cbcff",
   "metadata": {},
   "outputs": [],
   "source": [
    "load_data['Description'][4]\n",
    "from main import chat_completion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "adb4009a",
   "metadata": {},
   "outputs": [],
   "source": [
    "resume = \"python javascript agile leadership scrum master computer vision generative ai git github mlops kubernetes mongodb firebase db pinecone mysql machine learning deep learning google cloud platform microsoft azure vector db docker flask fastapi professional experience dataknobs ml engineer  administrator mlflow google cloud google cloud facilitator may july acquired proficiency docker mlops kubernetes innovative racing game controlled unique physical interface round flat board blue square uses opencv computer vision\"\n",
    "\n",
    "jd = \"Selected intern's day-to-day responsibilities include:\\n\\n1. Work on Java & Core Java\\n2. Work on object-oriented programming (OOP) concepts and patterns\\n3. Work on Java-based microservice architecture\\n4. Work on the project using Spring Java frameworks\\n5. Handle the service-oriented architecture/web services\\n\\nRequirements:\\n\\n1. Have some understanding of docker, and Gradle build tools\\n2. Understanding of the git version control system. AngularJS , Hibernate (Java) , Java , MySQL , Spring MVC\"\n",
    "\n",
    "ans = chat_completion(resume,jd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "8d0f0f9b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"JDMatch\":\"10%\",\"MissingKeywords\":[\"Java\",\"Core Java\",\"object-oriented programming\",\"OOP\",\"Java-based microservice architecture\",\"Spring Java frameworks\",\"service-oriented architecture\",\"web services\",\"docker\",\"Gradle build tools\",\"git version control system\",\"AngularJS\",\"Hibernate\",\"MySQL\",\"Spring MVC\"],\"Profile Summary\":\"The resume does not match the job description as it lacks experience in Java, Core Java, object-oriented programming, Spring Java frameworks, service-oriented architecture, web services, docker, Gradle build tools, git version control system, AngularJS, Hibernate, MySQL, and Spring MVC.\"}'"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ans = dict(ans)\n",
    "ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "868b371f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ff71242",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e8b3fd55",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import spacy\n",
    "\n",
    "load_data = pd.read_csv(\"data.csv\")\n",
    "load_data.reset_index(inplace=True)\n",
    "nlp = spacy.load(\"en_core_web_lg\")\n",
    "content = nlp(fetch_skills(\"Aman resume for y2.pdf\"))\n",
    "l = []\n",
    "for index, i in enumerate(load_data[\"Description\"]):\n",
    "    l.append(content.similarity(nlp(i)))\n",
    "final = sorted(list(enumerate(l)), reverse=True, key=lambda x: x[1])[0:6]\n",
    "for i, j in final:\n",
    "    print(load_data.iloc[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "7ccc1a16",
   "metadata": {},
   "outputs": [],
   "source": [
    "load_data[\"content\"] = load_data[\"Description\"] + load_data[\"JobTitles\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "4023ad85",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "cv=CountVectorizer(max_features=5000,stop_words=\"english\")\n",
    "vectors = cv.fit_transform(load_data[\"content\"]).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "4754a8a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "c036fe8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "6fa8de4d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.        , 0.17173031, 0.        , ..., 0.2362571 , 0.13328683,\n",
       "        0.12879861],\n",
       "       [0.17173031, 1.        , 0.        , ..., 0.16586995, 0.2211825 ,\n",
       "        0.3100325 ],\n",
       "       [0.        , 0.        , 1.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       ...,\n",
       "       [0.2362571 , 0.16586995, 0.        , ..., 1.        , 0.21222324,\n",
       "        0.20141487],\n",
       "       [0.13328683, 0.2211825 , 0.        , ..., 0.21222324, 1.        ,\n",
       "        0.37188078],\n",
       "       [0.12879861, 0.3100325 , 0.        , ..., 0.20141487, 0.37188078,\n",
       "        1.        ]])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity = cosine_similarity(vectors)\n",
    "similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "01aa07f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "final = sorted(list(enumerate(similarity[0])), reverse=True, key=lambda x: x[1]) [1:6]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "1ac5852f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "index                                                           3\n",
      "Company_Name                 ODN Digital Services Private Limited\n",
      "JobTitles                                        Research Analyst\n",
      "Skills                Business Research , Effective Communication\n",
      "Description     Selected intern's day-to-day responsibilities ...\n",
      "Stipend                                     ₹ 5,000-10,000 /month\n",
      "Links           https://internshala.com/internship/detail/rese...\n",
      "content         Selected intern's day-to-day responsibilities ...\n",
      "Name: 3, dtype: object\n",
      "index                                                         417\n",
      "Company_Name                           Invest4Edu Private Limited\n",
      "JobTitles                       Educational Strategy And Research\n",
      "Skills          Business Research , Data Analysis , Effective ...\n",
      "Description     Selected intern's day-to-day responsibilities ...\n",
      "Stipend                                            ₹ 6,000 /month\n",
      "Links           https://internshala.com/internship/detail/work...\n",
      "content         Selected intern's day-to-day responsibilities ...\n",
      "Name: 417, dtype: object\n",
      "index                                                         190\n",
      "Company_Name                                              AntWalk\n",
      "JobTitles                             Data Acquisition (US Shift)\n",
      "Skills          Lead Generation , MS-Excel , MS-Office , MS-Po...\n",
      "Description     We are seeking a motivated and detail-oriented...\n",
      "Stipend                                    ₹ 10,000-15,000 /month\n",
      "Links           https://internshala.com/internship/detail/work...\n",
      "content         We are seeking a motivated and detail-oriented...\n",
      "Name: 190, dtype: object\n",
      "index                                                         224\n",
      "Company_Name               LogicLoom IT Solutions Private Limited\n",
      "JobTitles                                         Market Research\n",
      "Skills          Data Analysis , Effective Communication , Engl...\n",
      "Description     Selected intern's day-to-day responsibilities ...\n",
      "Stipend                                            ₹ 5,000 /month\n",
      "Links           https://internshala.com/internship/detail/work...\n",
      "content         Selected intern's day-to-day responsibilities ...\n",
      "Name: 224, dtype: object\n",
      "index                                                         112\n",
      "Company_Name                                       Business Quant\n",
      "JobTitles                                         Equity Research\n",
      "Skills                                                   MS-Excel\n",
      "Description     Selected intern's day-to-day responsibilities ...\n",
      "Stipend                                            ₹ 3,000 /month\n",
      "Links           https://internshala.com/internship/detail/work...\n",
      "content         Selected intern's day-to-day responsibilities ...\n",
      "Name: 112, dtype: object\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "6db9ad3d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "index                                                         345\n",
      "Company_Name                    Printila Services Private Limited\n",
      "JobTitles                                     Flutter Development\n",
      "Skills          Android , Cloud Firestore , Dart , Firebase , ...\n",
      "Description     We are thrilled to offer an exciting opportuni...\n",
      "Stipend                                            ₹ 5,000 /month\n",
      "Links           https://internshala.com/internship/detail/work...\n",
      "Name: 345, dtype: object\n",
      "index                                                         148\n",
      "Company_Name                                      Vibhor  Kataria\n",
      "JobTitles                                          Data Analytics\n",
      "Skills                                  Data Analytics , MS-Excel\n",
      "Description     Are you passionate about data analytics and ex...\n",
      "Stipend                              ₹ 3,000 /month +  Incentives\n",
      "Links           https://internshala.com/internship/detail/work...\n",
      "Name: 148, dtype: object\n",
      "index                                                         151\n",
      "Company_Name                      Brihat Infotech Private Limited\n",
      "JobTitles                                     Flutter Development\n",
      "Skills          Android , Cloud Firestore , Dart , Firebase , ...\n",
      "Description     We are seeking a talented Flutter Development ...\n",
      "Stipend                                            ₹ 5,000 /month\n",
      "Links           https://internshala.com/internship/detail/work...\n",
      "Name: 151, dtype: object\n",
      "index                                                         421\n",
      "Company_Name                                      Village Company\n",
      "JobTitles                                          Data Analytics\n",
      "Skills          Data Analytics , Google Analytics , MS-Excel ,...\n",
      "Description     We are seeking a dynamic and motivated data an...\n",
      "Stipend                                      ₹ 5,000-6,000 /month\n",
      "Links           https://internshala.com/internship/detail/data...\n",
      "Name: 421, dtype: object\n",
      "index                                                         196\n",
      "Company_Name                                               PayPie\n",
      "JobTitles                                       Flutter Developer\n",
      "Skills                                   Firebase , Flutter , Git\n",
      "Description     Are you a talented Flutter Developer looking t...\n",
      "Stipend                                            ₹ 7,000 /month\n",
      "Links           https://internshala.com/internship/detail/work...\n",
      "Name: 196, dtype: object\n",
      "index                                                          22\n",
      "Company_Name                         Hex Wireless Private Limited\n",
      "JobTitles                                      Webflow Web Design\n",
      "Skills                    Design Thinking , Webflow , Wireframing\n",
      "Description     As a Webflow Web Design Intern at Hex Wireless...\n",
      "Stipend                                            ₹ 6,000 /month\n",
      "Links           https://internshala.com/internship/detail/work...\n",
      "Name: 22, dtype: object\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "27e07655",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['index', 'Company_Name', 'JobTitles', 'Skills', 'Description',\n",
       "       'Stipend', 'Links'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_data.columns"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}