feature_extraction_user_b.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['', 'C:\\\\Users\\\\keerthana\\\\Anaconda3\\\\python36.zip', 'C:\\\\Users\\\\keerthana\\\\Anaconda3\\\\DLLs', 'C:\\\\Users\\\\keerthana\\\\Anaconda3\\\\lib', 'C:\\\\Users\\\\keerthana\\\\Anaconda3', 'C:\\\\Users\\\\keerthana\\\\Anaconda3\\\\lib\\\\site-packages', 'C:\\\\Users\\\\keerthana\\\\Anaconda3\\\\lib\\\\site-packages\\\\Sphinx-1.5.1-py3.6.egg', 'C:\\\\Users\\\\keerthana\\\\Anaconda3\\\\lib\\\\site-packages\\\\win32', 'C:\\\\Users\\\\keerthana\\\\Anaconda3\\\\lib\\\\site-packages\\\\win32\\\\lib', 'C:\\\\Users\\\\keerthana\\\\Anaconda3\\\\lib\\\\site-packages\\\\Pythonwin', 'C:\\\\Users\\\\keerthana\\\\Anaconda3\\\\lib\\\\site-packages\\\\setuptools-27.2.0-py3.6.egg', 'C:\\\\Users\\\\keerthana\\\\Anaconda3\\\\lib\\\\site-packages\\\\IPython\\\\extensions', 'C:\\\\Users\\\\keerthana\\\\.ipython']\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import sys\n",
    "print(sys.path)\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\keerthana\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2717: DtypeWarning: Columns (8,12,13,14,15,16,50,51,52,53,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          Respondent    AssessJob1    AssessJob2    AssessJob3    AssessJob4  \\\n",
      "count   98855.000000  66985.000000  66985.000000  66985.000000  66985.000000   \n",
      "mean    50822.971635      6.397089      6.673524      5.906875      4.065791   \n",
      "std     29321.650410      2.788428      2.531202      2.642734      2.541196   \n",
      "min         1.000000      1.000000      1.000000      1.000000      1.000000   \n",
      "25%     25443.500000      4.000000      5.000000      4.000000      2.000000   \n",
      "50%     50823.000000      7.000000      7.000000      6.000000      4.000000   \n",
      "75%     76219.500000      9.000000      9.000000      8.000000      6.000000   \n",
      "max    101592.000000     10.000000     10.000000     10.000000     10.000000   \n",
      "\n",
      "         AssessJob5    AssessJob6    AssessJob7    AssessJob8    AssessJob9  \\\n",
      "count  66985.000000  66985.000000  66985.000000  66985.000000  66985.000000   \n",
      "mean       3.953243      4.407196      5.673181      4.225200      7.640009   \n",
      "std        2.520499      2.502069      2.923998      2.507411      2.407457   \n",
      "min        1.000000      1.000000      1.000000      1.000000      1.000000   \n",
      "25%        2.000000      2.000000      3.000000      2.000000      6.000000   \n",
      "50%        3.000000      4.000000      6.000000      4.000000      8.000000   \n",
      "75%        6.000000      6.000000      8.000000      6.000000     10.000000   \n",
      "max       10.000000     10.000000     10.000000     10.000000     10.000000   \n",
      "\n",
      "            ...        JobEmailPriorities6  JobEmailPriorities7  \\\n",
      "count       ...                46213.00000         46213.000000   \n",
      "mean        ...                    4.97425             4.836388   \n",
      "std         ...                    1.86063             1.659844   \n",
      "min         ...                    1.00000             1.000000   \n",
      "25%         ...                    4.00000             4.000000   \n",
      "50%         ...                    5.00000             5.000000   \n",
      "75%         ...                    7.00000             6.000000   \n",
      "max         ...                    7.00000             7.000000   \n",
      "\n",
      "       ConvertedSalary  AdsPriorities1  AdsPriorities2  AdsPriorities3  \\\n",
      "count     4.770200e+04    60479.000000    60479.000000    60479.000000   \n",
      "mean      9.578086e+04        2.726880        3.805784        3.340945   \n",
      "std       2.023482e+05        1.881078        1.821323        1.673485   \n",
      "min       0.000000e+00        1.000000        1.000000        1.000000   \n",
      "25%       2.384400e+04        1.000000        2.000000        2.000000   \n",
      "50%       5.507500e+04        2.000000        4.000000        3.000000   \n",
      "75%       9.300000e+04        4.000000        5.000000        5.000000   \n",
      "max       2.000000e+06        7.000000        7.000000        7.000000   \n",
      "\n",
      "       AdsPriorities4  AdsPriorities5  AdsPriorities6  AdsPriorities7  \n",
      "count    60479.000000    60479.000000    60479.000000    60479.000000  \n",
      "mean         3.782470        4.383604        5.138809        4.821459  \n",
      "std          1.844864        1.931746        1.853249        1.874895  \n",
      "min          1.000000        1.000000        1.000000        1.000000  \n",
      "25%          2.000000        3.000000        4.000000        3.000000  \n",
      "50%          4.000000        5.000000        6.000000        5.000000  \n",
      "75%          5.000000        6.000000        7.000000        7.000000  \n",
      "max          7.000000        7.000000        7.000000        7.000000  \n",
      "\n",
      "[8 rows x 42 columns]\n",
      "   Respondent Hobby OpenSource         Country         Student  \\\n",
      "0           1   Yes         No           Kenya              No   \n",
      "1           3   Yes        Yes  United Kingdom              No   \n",
      "2           4   Yes        Yes   United States              No   \n",
      "3           5    No         No   United States              No   \n",
      "4           7   Yes         No    South Africa  Yes, part-time   \n",
      "\n",
      "           Employment                                    FormalEducation  \\\n",
      "0  Employed part-time           Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
      "1  Employed full-time           Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
      "2  Employed full-time                                   Associate degree   \n",
      "3  Employed full-time           Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
      "4  Employed full-time  Some college/university study without earning ...   \n",
      "\n",
      "                                      UndergradMajor  \\\n",
      "0                          Mathematics or statistics   \n",
      "1  A natural science (ex. biology, chemistry, phy...   \n",
      "2  Computer science, computer engineering, or sof...   \n",
      "3  Computer science, computer engineering, or sof...   \n",
      "4  Computer science, computer engineering, or sof...   \n",
      "\n",
      "                CompanySize  \\\n",
      "0        20 to 99 employees   \n",
      "1  10,000 or more employees   \n",
      "2        20 to 99 employees   \n",
      "3      100 to 499 employees   \n",
      "4  10,000 or more employees   \n",
      "\n",
      "                                             DevType      ...        \\\n",
      "0                               Full-stack developer      ...         \n",
      "1  Database administrator;DevOps specialist;Full-...      ...         \n",
      "2           Engineering manager;Full-stack developer      ...         \n",
      "3                               Full-stack developer      ...         \n",
      "4  Data or business analyst;Desktop or enterprise...      ...         \n",
      "\n",
      "                     Exercise Gender         SexualOrientation  \\\n",
      "0        3 - 4 times per week   Male  Straight or heterosexual   \n",
      "1   Daily or almost every day   Male  Straight or heterosexual   \n",
      "2                         NaN    NaN                       NaN   \n",
      "3  I don't typically exercise   Male  Straight or heterosexual   \n",
      "4        3 - 4 times per week   Male  Straight or heterosexual   \n",
      "\n",
      "                                    EducationParents  \\\n",
      "0           Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
      "1           Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
      "2                                                NaN   \n",
      "3  Some college/university study without earning ...   \n",
      "4  Some college/university study without earning ...   \n",
      "\n",
      "                  RaceEthnicity                Age Dependents  MilitaryUS  \\\n",
      "0   Black or of African descent  25 - 34 years old        Yes         NaN   \n",
      "1  White or of European descent  35 - 44 years old        Yes         NaN   \n",
      "2                           NaN                NaN        NaN         NaN   \n",
      "3  White or of European descent  35 - 44 years old         No          No   \n",
      "4  White or of European descent  18 - 24 years old        Yes         NaN   \n",
      "\n",
      "                          SurveyTooLong     SurveyEasy  \n",
      "0  The survey was an appropriate length      Very easy  \n",
      "1  The survey was an appropriate length  Somewhat easy  \n",
      "2                                   NaN            NaN  \n",
      "3  The survey was an appropriate length  Somewhat easy  \n",
      "4  The survey was an appropriate length  Somewhat easy  \n",
      "\n",
      "[5 rows x 129 columns]\n",
      "Index(['Respondent', 'Hobby', 'OpenSource', 'Country', 'Student', 'Employment',\n",
      "       'FormalEducation', 'UndergradMajor', 'CompanySize', 'DevType',\n",
      "       ...\n",
      "       'Exercise', 'Gender', 'SexualOrientation', 'EducationParents',\n",
      "       'RaceEthnicity', 'Age', 'Dependents', 'MilitaryUS', 'SurveyTooLong',\n",
      "       'SurveyEasy'],\n",
      "      dtype='object', length=129)\n",
      "Respondent                     0.000000\n",
      "Hobby                          0.000000\n",
      "OpenSource                     0.000000\n",
      "Country                        0.004168\n",
      "Student                        0.039998\n",
      "Employment                     0.035749\n",
      "FormalEducation                0.042001\n",
      "UndergradMajor                 0.200486\n",
      "CompanySize                    0.276405\n",
      "DevType                        0.068353\n",
      "YearsCoding                    0.050781\n",
      "YearsCodingProf                0.211947\n",
      "JobSatisfaction                0.299216\n",
      "CareerSatisfaction             0.226099\n",
      "HopeFiveYears                  0.234050\n",
      "JobSearchStatus                0.195913\n",
      "LastNewJob                     0.201973\n",
      "AssessJob1                     0.322391\n",
      "AssessJob2                     0.322391\n",
      "AssessJob3                     0.322391\n",
      "AssessJob4                     0.322391\n",
      "AssessJob5                     0.322391\n",
      "AssessJob6                     0.322391\n",
      "AssessJob7                     0.322391\n",
      "AssessJob8                     0.322391\n",
      "AssessJob9                     0.322391\n",
      "AssessJob10                    0.322391\n",
      "AssessBenefits1                0.343301\n",
      "AssessBenefits2                0.343301\n",
      "AssessBenefits3                0.343301\n",
      "                                 ...   \n",
      "EthicsResponsible              0.347125\n",
      "EthicalImplications            0.298882\n",
      "StackOverflowRecommend         0.226251\n",
      "StackOverflowVisit             0.222993\n",
      "StackOverflowHasAccount        0.223196\n",
      "StackOverflowParticipate       0.334986\n",
      "StackOverflowJobs              0.238309\n",
      "StackOverflowDevStory          0.335623\n",
      "StackOverflowJobsRecommend     0.612392\n",
      "StackOverflowConsiderMember    0.231126\n",
      "HypotheticalTools1             0.291204\n",
      "HypotheticalTools2             0.291366\n",
      "HypotheticalTools3             0.291902\n",
      "HypotheticalTools4             0.292034\n",
      "HypotheticalTools5             0.291437\n",
      "WakeTime                       0.270184\n",
      "HoursComputer                  0.270315\n",
      "HoursOutside                   0.271418\n",
      "SkipMeals                      0.272207\n",
      "ErgonomicDevices               0.655475\n",
      "Exercise                       0.270568\n",
      "Gender                         0.347843\n",
      "SexualOrientation              0.395428\n",
      "EducationParents               0.374710\n",
      "RaceEthnicity                  0.418613\n",
      "Age                            0.346781\n",
      "Dependents                     0.366790\n",
      "MilitaryUS                     0.840362\n",
      "SurveyTooLong                  0.332952\n",
      "SurveyEasy                     0.333579\n",
      "dtype: float64\n",
      "Index(['Respondent', 'Hobby', 'OpenSource', 'Student', 'Employment',\n",
      "       'FormalEducation', 'UndergradMajor', 'DevType', 'YearsCoding',\n",
      "       'YearsCodingProf',\n",
      "       ...\n",
      "       'HoursOutside', 'SkipMeals', 'ErgonomicDevices', 'Exercise', 'Gender',\n",
      "       'SexualOrientation', 'EducationParents', 'Age', 'SurveyTooLong',\n",
      "       'SurveyEasy'],\n",
      "      dtype='object', length=110)\n",
      "                 Age  AssessBenefits1  AssessBenefits10  AssessBenefits11  \\\n",
      "0  25 - 34 years old              NaN               NaN               NaN   \n",
      "1  35 - 44 years old              1.0               2.0               8.0   \n",
      "2                NaN              NaN               NaN               NaN   \n",
      "3  35 - 44 years old              NaN               NaN               NaN   \n",
      "4  18 - 24 years old              1.0               9.0               6.0   \n",
      "\n",
      "   AssessBenefits2  AssessBenefits3  AssessBenefits4  AssessBenefits5  \\\n",
      "0              NaN              NaN              NaN              NaN   \n",
      "1              5.0              3.0              7.0             10.0   \n",
      "2              NaN              NaN              NaN              NaN   \n",
      "3              NaN              NaN              NaN              NaN   \n",
      "4             10.0              2.0              4.0              8.0   \n",
      "\n",
      "   AssessBenefits6  AssessBenefits7       ...         \\\n",
      "0              NaN              NaN       ...          \n",
      "1              4.0             11.0       ...          \n",
      "2              NaN              NaN       ...          \n",
      "3              NaN              NaN       ...          \n",
      "4              3.0             11.0       ...          \n",
      "\n",
      "                                     SelfTaughtTypes  \\\n",
      "0  The official documentation and/or standards fo...   \n",
      "1  The official documentation and/or standards fo...   \n",
      "2                                                NaN   \n",
      "3  The official documentation and/or standards fo...   \n",
      "4  The official documentation and/or standards fo...   \n",
      "\n",
      "          SexualOrientation             SkipMeals         Student  \\\n",
      "0  Straight or heterosexual                 Never              No   \n",
      "1  Straight or heterosexual                 Never              No   \n",
      "2                       NaN                   NaN              No   \n",
      "3  Straight or heterosexual  3 - 4 times per week              No   \n",
      "4  Straight or heterosexual                 Never  Yes, part-time   \n",
      "\n",
      "   TimeFullyProductive                                     UndergradMajor  \\\n",
      "0  One to three months                          Mathematics or statistics   \n",
      "1  One to three months  A natural science (ex. biology, chemistry, phy...   \n",
      "2                  NaN  Computer science, computer engineering, or sof...   \n",
      "3  Three to six months  Computer science, computer engineering, or sof...   \n",
      "4  Three to six months  Computer science, computer engineering, or sof...   \n",
      "\n",
      "      VersionControl                WakeTime       YearsCoding  \\\n",
      "0                Git  Between 5:00 - 6:00 AM         3-5 years   \n",
      "1     Git;Subversion  Between 6:01 - 7:00 AM  30 or more years   \n",
      "2                NaN                     NaN       24-26 years   \n",
      "3                Git  Between 6:01 - 7:00 AM       18-20 years   \n",
      "4  Zip file back-ups          Before 5:00 AM         6-8 years   \n",
      "\n",
      "   YearsCodingProf  \n",
      "0        3-5 years  \n",
      "1      18-20 years  \n",
      "2        6-8 years  \n",
      "3      12-14 years  \n",
      "4        0-2 years  \n",
      "\n",
      "[5 rows x 68 columns]\n",
      "0.29139955521018696\n",
      "ConvertedSalary            0.517455\n",
      "Salary                     0.488362\n",
      "FrameworkWorkedWith        0.477821\n",
      "TimeFullyProductive        0.475272\n",
      "FrameworkDesireNextYear    0.441424\n",
      "CommunicationTools         0.423701\n",
      "SelfTaughtTypes            0.419817\n",
      "DatabaseDesireNextYear     0.418360\n",
      "CurrencySymbol             0.406707\n",
      "Methodology                0.403358\n",
      "SexualOrientation          0.395428\n",
      "PlatformDesireNextYear     0.379364\n",
      "EducationParents           0.374710\n",
      "Gender                     0.347843\n",
      "Age                        0.346781\n",
      "AssessBenefits11           0.343311\n",
      "AssessBenefits2            0.343301\n",
      "AssessBenefits9            0.343301\n",
      "AssessBenefits10           0.343301\n",
      "AssessBenefits1            0.343301\n",
      "AssessBenefits7            0.343301\n",
      "AssessBenefits6            0.343301\n",
      "AssessBenefits5            0.343301\n",
      "AssessBenefits4            0.343301\n",
      "AssessBenefits3            0.343301\n",
      "AssessBenefits8            0.343301\n",
      "PlatformWorkedWith         0.332366\n",
      "DatabaseWorkedWith         0.329624\n",
      "AssessJob5                 0.322391\n",
      "AssessJob6                 0.322391\n",
      "                             ...   \n",
      "AssessJob10                0.322391\n",
      "AssessJob1                 0.322391\n",
      "EducationTypes             0.312528\n",
      "JobSatisfaction            0.299216\n",
      "EthicalImplications        0.298882\n",
      "SkipMeals                  0.272207\n",
      "HoursOutside               0.271418\n",
      "Exercise                   0.270568\n",
      "HoursComputer              0.270315\n",
      "WakeTime                   0.270184\n",
      "CheckInCode                0.268069\n",
      "LanguageDesireNextYear     0.259076\n",
      "VersionControl             0.248414\n",
      "IDE                        0.237287\n",
      "HopeFiveYears              0.234050\n",
      "OperatingSystem            0.229386\n",
      "NumberMonitors             0.227171\n",
      "CareerSatisfaction         0.226099\n",
      "YearsCodingProf            0.211947\n",
      "LanguageWorkedWith         0.207587\n",
      "LastNewJob                 0.201973\n",
      "UndergradMajor             0.200486\n",
      "DevType                    0.068353\n",
      "YearsCoding                0.050781\n",
      "FormalEducation            0.042001\n",
      "Student                    0.039998\n",
      "Employment                 0.035749\n",
      "Hobby                      0.000000\n",
      "OpenSource                 0.000000\n",
      "Respondent                 0.000000\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "df=pd.read_csv(\"./data/user_preprocessing/survey_results_public.csv\")\n",
    "print(df.describe())\n",
    "print(df.head())\n",
    "print(df.columns)\n",
    "\n",
    "n_row=df.shape[0]\n",
    "print(df.isnull().sum()/n_row)\n",
    "attr_to_drop=[]\n",
    "attr_to_drop.append(\"SurveyTooLong\")\n",
    "attr_to_drop.append('SurveyEasy')\n",
    "attr_to_drop.append(\"AdBlocker\")\n",
    "attr_to_drop.append(\"AdBlockerDisable\")\n",
    "attr_to_drop.append(\"AdBlockerReasons\")\n",
    "attr_to_drop.append(\"AdsAgreeDisagree1\")\n",
    "attr_to_drop.append(\"AdsAgreeDisagree2\")\n",
    "attr_to_drop.append(\"AdsAgreeDisagree3\")\n",
    "attr_to_drop.append(\"AdsActions\")\n",
    "attr_to_drop.append(\"AdsPriorities1\")\n",
    "attr_to_drop.append(\"AdsPriorities2\")\n",
    "attr_to_drop.append(\"AdsPriorities3\")\n",
    "attr_to_drop.append(\"AdsPriorities4\")\n",
    "attr_to_drop.append(\"AdsPriorities5\")\n",
    "attr_to_drop.append(\"AdsPriorities6\")\n",
    "attr_to_drop.append(\"AdsPriorities7\")\n",
    "attr_to_drop.append(\"AIDangerous\")\n",
    "attr_to_drop.append(\"AIInteresting\")\n",
    "attr_to_drop.append(\"AIResponsible\")\n",
    "attr_to_drop.append(\"AIFuture\")\n",
    "attr_to_drop.append(\"EthicsChoice\")\n",
    "attr_to_drop.append(\"EthicsReport\")\n",
    "attr_to_drop.append(\"EthicsResponsible\")\n",
    "attr_to_drop.append(\"EthicsImplications\")\n",
    "attr_to_drop.append(\"EthicsChoice\")\n",
    "attr_to_drop.append(\"StackOverflowRecommend\")\n",
    "attr_to_drop.append(\"StackOverflowVisit\")\n",
    "attr_to_drop.append(\"StackOverflowHasAccount\")\n",
    "attr_to_drop.append(\"StackOverflowParticipate\")\n",
    "attr_to_drop.append(\"StackOverflowJobs\")\n",
    "attr_to_drop.append(\"StackOverflowDevStory\")\n",
    "attr_to_drop.append(\"StackOverflowJobsRecommend\")\n",
    "attr_to_drop.append(\"StackOverflowConsiderMember\")\n",
    "attr_to_drop.append(\"HypotheticalTools1\")\n",
    "attr_to_drop.append(\"HypotheticalTools2\")\n",
    "attr_to_drop.append(\"HypotheticalTools3\")\n",
    "attr_to_drop.append(\"HypotheticalTools4\")\n",
    "attr_to_drop.append(\"HypotheticalTools5\")\n",
    "attr_to_drop.append(\"JobContactPriorities1\")\n",
    "attr_to_drop.append(\"JobContactPriorities2\")\n",
    "attr_to_drop.append(\"JobContactPriorities3\")\n",
    "attr_to_drop.append(\"JobContactPriorities4\")\n",
    "attr_to_drop.append(\"JobContactPriorities5\")\n",
    "attr_to_drop.append(\"TimeAfterBootcamp\")\n",
    "df=df.drop(df.columns[[126,3,8,15,50,51,53,62,63,64,123,125,43,44,45,46,47,48,49]],axis=1)\n",
    "print(df.columns)\n",
    "df=df[df.columns.difference(attr_to_drop)]\n",
    "print(df.head())\n",
    "attr_to_drop=[\"HackathonReasons\",\"ErgonomicDevices\"]\n",
    "df=df[df.columns.difference(attr_to_drop)]\n",
    "\n",
    "\n",
    "print(sum(df.isnull().sum())/float(np.prod(df.shape)))\n",
    "col_null=df.isnull().sum()/n_row\n",
    "print(col_null.sort_values(ascending=False))\n",
    "#print(sum(df.isnull().sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66\n",
      "54\n"
     ]
    }
   ],
   "source": [
    "#New set of attributes to be removed\n",
    "attr_to_drop=['Hobby','NumberMonitors','Salary','CheckInCode','WakeTime','TimeFullyProductive','SkipMeals','SexualOrientation','HoursOutside','Exercise','EthicalImplications','EducationParents']\n",
    "print(len(df.columns))\n",
    "df=df[df.columns.difference(attr_to_drop)]\n",
    "print(len(df.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#df.to_csv(\"removed2.csv\")\n",
    "import numpy as np\n",
    "df1 = df.replace(np.nan, '', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'React', 'Xamarin', 'Cordova', 'Node.js', 'Django', '.NET Core', 'TensorFlow', 'Hadoop', 'Torch/PyTorch', 'Spring', 'Spark', 'Angular'}\n",
      "{'React', 'Xamarin', 'Cordova', 'Node.js', 'Django', '.NET Core', 'TensorFlow', 'Hadoop', 'Torch/PyTorch', 'Spring', 'Spark', 'Angular'}\n",
      "{'Mac OS', 'Windows Phone', 'Linux', 'Serverless', 'Drupal', 'iOS', 'SharePoint', 'Gaming console', 'Windows Desktop or Server', 'IBM Cloud or Watson', 'Predix', 'Azure', 'Apple Watch or Apple TV', 'Heroku', 'Arduino', 'AWS', 'Android', 'Amazon Echo', 'Google Cloud Platform/App Engine', 'ESP8266', 'Raspberry Pi', 'Mainframe', 'Firebase', 'Salesforce', 'WordPress', 'Google Home'}\n",
      "{'Mac OS', 'Windows Phone', 'Linux', 'Serverless', 'Drupal', 'iOS', 'SharePoint', 'Gaming console', 'Windows Desktop or Server', 'IBM Cloud or Watson', 'Predix', 'Azure', 'Apple Watch or Apple TV', 'Heroku', 'Arduino', 'AWS', 'Android', 'Amazon Echo', 'Google Cloud Platform/App Engine', 'ESP8266', 'Raspberry Pi', 'Mainframe', 'Firebase', 'Salesforce', 'WordPress', 'Google Home'}\n",
      "{'Visual Basic 6', 'C++', 'TypeScript', 'Python', 'Groovy', 'Perl', 'Rust', 'Go', 'Erlang', 'VBA', 'Matlab', 'Objective-C', 'Java', 'Scala', 'Julia', 'Bash/Shell', 'C', 'Delphi/Object Pascal', 'Cobol', 'SQL', 'C#', 'Ocaml', 'JavaScript', 'Lua', 'F#', 'Haskell', 'Kotlin', 'HTML', 'CSS', 'PHP', 'Ruby', 'R', 'Swift', 'CoffeeScript', 'Assembly', 'Clojure', 'VB.NET', 'Hack'}\n",
      "{'Visual Basic 6', 'C++', 'TypeScript', 'Python', 'Groovy', 'Perl', 'Rust', 'Go', 'Erlang', 'VBA', 'Matlab', 'Objective-C', 'Java', 'Scala', 'Julia', 'Bash/Shell', 'C', 'Delphi/Object Pascal', 'Cobol', 'SQL', 'C#', 'Lua', 'JavaScript', 'Ocaml', 'F#', 'Haskell', 'Kotlin', 'HTML', 'CSS', 'PHP', 'Ruby', 'R', 'Swift', 'CoffeeScript', 'Assembly', 'Clojure', 'VB.NET', 'Hack'}\n",
      "{'Other wiki tool (Github, Google Sites, proprietary software, etc.)', 'Confluence', 'Facebook', 'Other chat system (IRC, proprietary software, etc.)', 'Office / productivity suite (Microsoft Office, Google Suite, etc.)', 'Jira', 'Slack', 'Trello', 'HipChat', 'Stack Overflow Enterprise', 'Google Hangouts/Chat'}\n",
      "{'MariaDB', 'PostgreSQL', 'Redis', 'Amazon DynamoDB', 'SQL Server', 'Cassandra', 'Google Cloud Storage', 'MongoDB', 'Elasticsearch', 'Amazon RDS/Aurora', 'Neo4j', 'IBM Db2', 'Apache Hive', 'Google BigQuery', 'SQLite', 'Oracle', 'Microsoft Azure (Tables, CosmosDB, SQL, etc)', 'MySQL', 'Memcached', 'Amazon Redshift', 'Apache HBase'}\n",
      "{'MariaDB', 'PostgreSQL', 'Redis', 'Amazon DynamoDB', 'SQL Server', 'Cassandra', 'Google Cloud Storage', 'MongoDB', 'Elasticsearch', 'Amazon RDS/Aurora', 'Neo4j', 'IBM Db2', 'Apache Hive', 'Google BigQuery', 'SQLite', 'Oracle', 'Microsoft Azure (Tables, CosmosDB, SQL, etc)', 'MySQL', 'Memcached', 'Amazon Redshift', 'Apache HBase'}\n",
      "{'Student', 'Embedded applications or devices developer', 'Full-stack developer', 'QA or test developer', 'Desktop or enterprise applications developer', 'DevOps specialist', 'Mobile developer', 'Back-end developer', 'Database administrator', 'Engineering manager', 'Front-end developer', 'Game or graphics developer', 'Designer', 'System administrator', 'Data scientist or machine learning specialist', 'Data or business analyst', 'Educator or academic researcher', 'Marketing or sales professional', 'C-suite executive (CEO, CTO, etc.)', 'Product manager'}\n"
     ]
    }
   ],
   "source": [
    "#find the unique values in each categorical attribute\n",
    "Frameworknextyear=list()\n",
    "for value in df1[\"FrameworkDesireNextYear\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            Frameworknextyear.append(i)\n",
    "sFNY=set(Frameworknextyear)\n",
    "FrameworkWorkedWith=list()\n",
    "for value in df1[\"FrameworkWorkedWith\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            FrameworkWorkedWith.append(i)\n",
    "sFW=set(FrameworkWorkedWith)\n",
    "PlatformDesireNextYear=list()\n",
    "for value in df1[\"PlatformDesireNextYear\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            PlatformDesireNextYear.append(i)\n",
    "sPDNY=set(PlatformDesireNextYear)\n",
    "PlatformWorkedWith=list()\n",
    "for value in df1[\"PlatformWorkedWith\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            PlatformWorkedWith.append(i)\n",
    "sPWW=set(PlatformWorkedWith)\n",
    "LanguageDesireNextYear=list()\n",
    "for value in df1[\"LanguageDesireNextYear\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            LanguageDesireNextYear.append(i)\n",
    "sLDNY=set(LanguageDesireNextYear)\n",
    "LanguageWorkedWith=list()\n",
    "for value in df1[\"LanguageWorkedWith\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            LanguageWorkedWith.append(i)\n",
    "sLWW=set(LanguageWorkedWith)\n",
    "         \n",
    "CommunicationTools=list()\n",
    "for value in df1[\"CommunicationTools\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            CommunicationTools.append(i)\n",
    "sCT=set(CommunicationTools)\n",
    "\n",
    "DatabaseWorkedWith=list()\n",
    "for value in df1[\"DatabaseWorkedWith\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            DatabaseWorkedWith.append(i)\n",
    "sDWW=set(DatabaseWorkedWith)\n",
    "\n",
    "DatabaseDesireNextYear=list()\n",
    "for value in df1[\"DatabaseDesireNextYear\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            DatabaseDesireNextYear.append(i)\n",
    "sDNY=set(DatabaseDesireNextYear)\n",
    "\n",
    "DevType=list()\n",
    "for value in df1[\"DevType\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            DevType.append(i)\n",
    "sDT=set(DevType)\n",
    "sFNY.remove(\"\")\n",
    "sFW.remove(\"\")\n",
    "sPDNY.remove(\"\")\n",
    "sPWW.remove(\"\")\n",
    "sLDNY.remove(\"\")\n",
    "sLWW.remove(\"\")\n",
    "sCT.remove(\"\")\n",
    "sDWW.remove(\"\")\n",
    "sDNY.remove(\"\")\n",
    "sDT.remove(\"\")\n",
    "print(sFNY)\n",
    "print(sFW)\n",
    "print(sPDNY)\n",
    "print(sPWW)\n",
    "print(sLDNY)\n",
    "print(sLWW)\n",
    "print(sCT)\n",
    "print(sDWW)\n",
    "print(sDNY)\n",
    "print(sDT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Extremely satisfied', 'Neither satisfied nor dissatisfied', '', 'Moderately dissatisfied', 'Slightly satisfied', 'Extremely dissatisfied', 'Moderately satisfied', 'Slightly dissatisfied'}\n"
     ]
    }
   ],
   "source": [
    "#converted categorical variables into numerical\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "print(set(df1['JobSatisfaction']))\n",
    "number=LabelEncoder()\n",
    "df1['CareerSatisfaction']=number.fit_transform(df1['CareerSatisfaction'])\n",
    "df1['Employment']=number.fit_transform(df1['Employment'])\n",
    "df1['FormalEducation']=number.fit_transform(df1['FormalEducation'])\n",
    "df1['Gender']=number.fit_transform(df1['Gender'])\n",
    "df1['HopeFiveYears']=number.fit_transform(df1['HopeFiveYears'])\n",
    "df1['HoursComputer']=number.fit_transform(df1['HoursComputer'])\n",
    "df1['JobSatisfaction']=number.fit_transform(df1['JobSatisfaction'])\n",
    "df1['OpenSource']=number.fit_transform(df1['OpenSource'])\n",
    "df1['OperatingSystem']=number.fit_transform(df1['OperatingSystem'])\n",
    "df1['Student']=number.fit_transform(df1['Student'])\n",
    "df1['UndergradMajor']=number.fit_transform(df1['UndergradMajor'])\n",
    "df1['YearsCoding']=number.fit_transform(df1['YearsCoding'])\n",
    "df1['YearsCodingProf']=number.fit_transform(df1['YearsCodingProf'])\n",
    "\n",
    "#df1.to_csv(\"./data/user_preprocessing/categoricalconverted.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mac OS\n",
      "Windows Phone\n",
      "Linux\n",
      "Serverless\n",
      "Drupal\n",
      "iOS\n",
      "SharePoint\n",
      "Gaming console\n",
      "Windows Desktop or Server\n",
      "IBM Cloud or Watson\n",
      "Predix\n",
      "Azure\n",
      "Apple Watch or Apple TV\n",
      "Heroku\n",
      "Arduino\n",
      "AWS\n",
      "Android\n",
      "Amazon Echo\n",
      "Google Cloud Platform/App Engine\n",
      "ESP8266\n",
      "Raspberry Pi\n",
      "Mainframe\n",
      "Firebase\n",
      "Salesforce\n",
      "WordPress\n",
      "Google Home\n",
      "12\n",
      "26\n"
     ]
    }
   ],
   "source": [
    "#sFNY.add('Respondent')\n",
    "#Create a separate dataframe based on one hot encoding for the attributes.\n",
    "# Framework=pd.DataFrame(columns=sFNY)\n",
    "# idx=0\n",
    "# Framework.insert(loc=idx,column='Respondent', value=[0])\n",
    "# #print(Framework.columns)\n",
    "# print(Framework.shape[0])\n",
    "# coldic=dict(zip(Framework.columns,range(0,13)))\n",
    "# print(coldic)\n",
    "# #print(df1.columns)\n",
    "# print(df1.shape[0])\n",
    "# for i in range(3):\n",
    "#     Framework = Framework.append({}, ignore_index=True)\n",
    "FrameworkWorkedWith = pd.DataFrame(df1['Respondent'])\n",
    "for i in sFW:\n",
    "    \n",
    "    FrameworkWorkedWith[i]=\"\"\n",
    "PlatformWorkedWith = pd.DataFrame(df1['Respondent'])\n",
    "for i in sPWW:\n",
    "    print(i)\n",
    "    PlatformWorkedWith[i]=\"\"\n",
    "LanguageWorkedWith = pd.DataFrame(df1['Respondent'])\n",
    "for i in sLWW:\n",
    "    LanguageWorkedWith[i]=\"\"\n",
    "DatabaseWorkedWith = pd.DataFrame(df1['Respondent'])\n",
    "for i in sDWW:\n",
    "    DatabaseWorkedWith[i]=\"\"\n",
    "DevType = pd.DataFrame(df1['Respondent'])\n",
    "for i in sDT:\n",
    "    DevType[i]=\"\"\n",
    "#framework.to_csv(\"framework.csv\")\n",
    "#framework.resize=[[98855,,len(sFNY)]]\n",
    "print(len(sFW))\n",
    "print(len(sPWW))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Respondent': 0, 'React': 1, 'Xamarin': 2, 'Cordova': 3, 'Node.js': 4, 'Django': 5, '.NET Core': 6, 'TensorFlow': 7, 'Hadoop': 8, 'Torch/PyTorch': 9, 'Spring': 10, 'Spark': 11, 'Angular': 12}\n",
      "                 Age AssessBenefits1 AssessBenefits10 AssessBenefits11  \\\n",
      "0  25 - 34 years old                                                     \n",
      "1  35 - 44 years old               1                2                8   \n",
      "2                                                                        \n",
      "3  35 - 44 years old                                                     \n",
      "4  18 - 24 years old               1                9                6   \n",
      "\n",
      "  AssessBenefits2 AssessBenefits3 AssessBenefits4 AssessBenefits5  \\\n",
      "0                                                                   \n",
      "1               5               3               7              10   \n",
      "2                                                                   \n",
      "3                                                                   \n",
      "4              10               2               4               8   \n",
      "\n",
      "  AssessBenefits6 AssessBenefits7       ...       OperatingSystem  \\\n",
      "0                                       ...                     2   \n",
      "1               4              11       ...                     2   \n",
      "2                                       ...                     0   \n",
      "3                                       ...                     4   \n",
      "4               3              11       ...                     4   \n",
      "\n",
      "              PlatformDesireNextYear                 PlatformWorkedWith  \\\n",
      "0           AWS;Azure;Linux;Firebase           AWS;Azure;Linux;Firebase   \n",
      "1                              Linux                              Linux   \n",
      "2                                                                         \n",
      "3                              Azure                              Azure   \n",
      "4  Arduino;Windows Desktop or Server  Arduino;Windows Desktop or Server   \n",
      "\n",
      "  Respondent                                    SelfTaughtTypes Student  \\\n",
      "0          1  The official documentation and/or standards fo...       1   \n",
      "1          3  The official documentation and/or standards fo...       1   \n",
      "2          4                                                          1   \n",
      "3          5  The official documentation and/or standards fo...       1   \n",
      "4          7  The official documentation and/or standards fo...       3   \n",
      "\n",
      "  UndergradMajor     VersionControl YearsCoding YearsCodingProf  \n",
      "0             11                Git           8               8  \n",
      "1              4     Git;Subversion           9               4  \n",
      "2              7                              6              10  \n",
      "3              7                Git           4               2  \n",
      "4              7  Zip file back-ups          10               1  \n",
      "\n",
      "[5 rows x 54 columns]\n",
      "780.4256777763367\n"
     ]
    }
   ],
   "source": [
    "# print(Framework.shape[0])\n",
    "# Framework.to_csv(\"hello1.csv\")\n",
    "\n",
    "\n",
    "time_start=time.time()\n",
    "coldic=dict(zip(FrameworkWorkedWith.columns,range(0,len(sFW)+1)))\n",
    "print(coldic)\n",
    "print(df1.head())\n",
    "for i in range(98855):\n",
    "    data=(df1.loc[i,'FrameworkWorkedWith']).split(';')\n",
    "    if(data[0]!=\"\"):\n",
    "       # print(data)\n",
    "        for value in data:\n",
    "            FrameworkWorkedWith.iloc[i,coldic[value]]=1\n",
    "time_end=time.time()\n",
    "print(time_end-time_start)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Respondent': 0, 'React': 1, 'Xamarin': 2, 'Cordova': 3, 'Node.js': 4, 'Django': 5, '.NET Core': 6, 'TensorFlow': 7, 'Hadoop': 8, 'Torch/PyTorch': 9, 'Spring': 10, 'Spark': 11, 'Angular': 12}\n",
      "                 Age AssessBenefits1 AssessBenefits10 AssessBenefits11  \\\n",
      "0  25 - 34 years old                                                     \n",
      "1  35 - 44 years old               1                2                8   \n",
      "2                                                                        \n",
      "3  35 - 44 years old                                                     \n",
      "4  18 - 24 years old               1                9                6   \n",
      "\n",
      "  AssessBenefits2 AssessBenefits3 AssessBenefits4 AssessBenefits5  \\\n",
      "0                                                                   \n",
      "1               5               3               7              10   \n",
      "2                                                                   \n",
      "3                                                                   \n",
      "4              10               2               4               8   \n",
      "\n",
      "  AssessBenefits6 AssessBenefits7       ...       OperatingSystem  \\\n",
      "0                                       ...                     2   \n",
      "1               4              11       ...                     2   \n",
      "2                                       ...                     0   \n",
      "3                                       ...                     4   \n",
      "4               3              11       ...                     4   \n",
      "\n",
      "              PlatformDesireNextYear                 PlatformWorkedWith  \\\n",
      "0           AWS;Azure;Linux;Firebase           AWS;Azure;Linux;Firebase   \n",
      "1                              Linux                              Linux   \n",
      "2                                                                         \n",
      "3                              Azure                              Azure   \n",
      "4  Arduino;Windows Desktop or Server  Arduino;Windows Desktop or Server   \n",
      "\n",
      "  Respondent                                    SelfTaughtTypes Student  \\\n",
      "0          1  The official documentation and/or standards fo...       1   \n",
      "1          3  The official documentation and/or standards fo...       1   \n",
      "2          4                                                          1   \n",
      "3          5  The official documentation and/or standards fo...       1   \n",
      "4          7  The official documentation and/or standards fo...       3   \n",
      "\n",
      "  UndergradMajor     VersionControl YearsCoding YearsCodingProf  \n",
      "0             11                Git           8               8  \n",
      "1              4     Git;Subversion           9               4  \n",
      "2              7                              6              10  \n",
      "3              7                Git           4               2  \n",
      "4              7  Zip file back-ups          10               1  \n",
      "\n",
      "[5 rows x 54 columns]\n",
      "1072.0754425525665\n"
     ]
    }
   ],
   "source": [
    "FrameworkDesireNextYear = pd.DataFrame(df1['Respondent'])\n",
    "for i in sFNY:\n",
    "    \n",
    "    FrameworkDesireNextYear[i]=\"\"\n",
    "time_start=time.time()\n",
    "coldic=dict(zip(FrameworkDesireNextYear.columns,range(0,len(sFNY)+1)))\n",
    "print(coldic)\n",
    "print(df1.head())\n",
    "for i in range(98855):\n",
    "    data=(df1.loc[i,'FrameworkDesireNextYear']).split(';')\n",
    "    if(data[0]!=\"\"):\n",
    "       # print(data)\n",
    "        for value in data:\n",
    "            FrameworkDesireNextYear.iloc[i,coldic[value]]=1\n",
    "time_end=time.time()\n",
    "print(time_end-time_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Respondent': 0, 'Mac OS': 1, 'Windows Phone': 2, 'Linux': 3, 'Serverless': 4, 'Drupal': 5, 'iOS': 6, 'SharePoint': 7, 'Gaming console': 8, 'Windows Desktop or Server': 9, 'IBM Cloud or Watson': 10, 'Predix': 11, 'Azure': 12, 'Apple Watch or Apple TV': 13, 'Heroku': 14, 'Arduino': 15, 'AWS': 16, 'Android': 17, 'Amazon Echo': 18, 'Google Cloud Platform/App Engine': 19, 'ESP8266': 20, 'Raspberry Pi': 21, 'Mainframe': 22, 'Firebase': 23, 'Salesforce': 24, 'WordPress': 25, 'Google Home': 26}\n",
      "                 Age AssessBenefits1 AssessBenefits10 AssessBenefits11  \\\n",
      "0  25 - 34 years old                                                     \n",
      "1  35 - 44 years old               1                2                8   \n",
      "2                                                                        \n",
      "3  35 - 44 years old                                                     \n",
      "4  18 - 24 years old               1                9                6   \n",
      "\n",
      "  AssessBenefits2 AssessBenefits3 AssessBenefits4 AssessBenefits5  \\\n",
      "0                                                                   \n",
      "1               5               3               7              10   \n",
      "2                                                                   \n",
      "3                                                                   \n",
      "4              10               2               4               8   \n",
      "\n",
      "  AssessBenefits6 AssessBenefits7       ...       OperatingSystem  \\\n",
      "0                                       ...                     2   \n",
      "1               4              11       ...                     2   \n",
      "2                                       ...                     0   \n",
      "3                                       ...                     4   \n",
      "4               3              11       ...                     4   \n",
      "\n",
      "              PlatformDesireNextYear                 PlatformWorkedWith  \\\n",
      "0           AWS;Azure;Linux;Firebase           AWS;Azure;Linux;Firebase   \n",
      "1                              Linux                              Linux   \n",
      "2                                                                         \n",
      "3                              Azure                              Azure   \n",
      "4  Arduino;Windows Desktop or Server  Arduino;Windows Desktop or Server   \n",
      "\n",
      "  Respondent                                    SelfTaughtTypes Student  \\\n",
      "0          1  The official documentation and/or standards fo...       1   \n",
      "1          3  The official documentation and/or standards fo...       1   \n",
      "2          4                                                          1   \n",
      "3          5  The official documentation and/or standards fo...       1   \n",
      "4          7  The official documentation and/or standards fo...       3   \n",
      "\n",
      "  UndergradMajor     VersionControl YearsCoding YearsCodingProf  \n",
      "0             11                Git           8               8  \n",
      "1              4     Git;Subversion           9               4  \n",
      "2              7                              6              10  \n",
      "3              7                Git           4               2  \n",
      "4              7  Zip file back-ups          10               1  \n",
      "\n",
      "[5 rows x 54 columns]\n",
      "1501.1580307483673\n"
     ]
    }
   ],
   "source": [
    "PlatformDesireNextYear = pd.DataFrame(df1['Respondent'])\n",
    "for i in sPDNY:\n",
    "    \n",
    "    PlatformDesireNextYear[i]=\"\"\n",
    "time_start=time.time()\n",
    "coldic=dict(zip(PlatformDesireNextYear.columns,range(0,len(sPDNY)+1)))\n",
    "print(coldic)\n",
    "print(df1.head())\n",
    "for i in range(98855):\n",
    "    data=(df1.loc[i,'PlatformDesireNextYear']).split(';')\n",
    "    if(data[0]!=\"\"):\n",
    "       # print(data)\n",
    "        for value in data:\n",
    "            PlatformDesireNextYear.iloc[i,coldic[value]]=1\n",
    "time_end=time.time()\n",
    "print(time_end-time_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Respondent': 0, 'Visual Basic 6': 1, 'C++': 2, 'TypeScript': 3, 'Python': 4, 'Groovy': 5, 'Perl': 6, 'Rust': 7, 'Go': 8, 'Erlang': 9, 'VBA': 10, 'Matlab': 11, 'Objective-C': 12, 'Java': 13, 'Scala': 14, 'Julia': 15, 'Bash/Shell': 16, 'C': 17, 'Delphi/Object Pascal': 18, 'Cobol': 19, 'SQL': 20, 'C#': 21, 'Ocaml': 22, 'JavaScript': 23, 'Lua': 24, 'F#': 25, 'Haskell': 26, 'Kotlin': 27, 'HTML': 28, 'CSS': 29, 'PHP': 30, 'Ruby': 31, 'R': 32, 'Swift': 33, 'CoffeeScript': 34, 'Assembly': 35, 'Clojure': 36, 'VB.NET': 37, 'Hack': 38}\n",
      "                 Age AssessBenefits1 AssessBenefits10 AssessBenefits11  \\\n",
      "0  25 - 34 years old                                                     \n",
      "1  35 - 44 years old               1                2                8   \n",
      "2                                                                        \n",
      "3  35 - 44 years old                                                     \n",
      "4  18 - 24 years old               1                9                6   \n",
      "\n",
      "  AssessBenefits2 AssessBenefits3 AssessBenefits4 AssessBenefits5  \\\n",
      "0                                                                   \n",
      "1               5               3               7              10   \n",
      "2                                                                   \n",
      "3                                                                   \n",
      "4              10               2               4               8   \n",
      "\n",
      "  AssessBenefits6 AssessBenefits7       ...       OperatingSystem  \\\n",
      "0                                       ...                     2   \n",
      "1               4              11       ...                     2   \n",
      "2                                       ...                     0   \n",
      "3                                       ...                     4   \n",
      "4               3              11       ...                     4   \n",
      "\n",
      "              PlatformDesireNextYear                 PlatformWorkedWith  \\\n",
      "0           AWS;Azure;Linux;Firebase           AWS;Azure;Linux;Firebase   \n",
      "1                              Linux                              Linux   \n",
      "2                                                                         \n",
      "3                              Azure                              Azure   \n",
      "4  Arduino;Windows Desktop or Server  Arduino;Windows Desktop or Server   \n",
      "\n",
      "  Respondent                                    SelfTaughtTypes Student  \\\n",
      "0          1  The official documentation and/or standards fo...       1   \n",
      "1          3  The official documentation and/or standards fo...       1   \n",
      "2          4                                                          1   \n",
      "3          5  The official documentation and/or standards fo...       1   \n",
      "4          7  The official documentation and/or standards fo...       3   \n",
      "\n",
      "  UndergradMajor     VersionControl YearsCoding YearsCodingProf  \n",
      "0             11                Git           8               8  \n",
      "1              4     Git;Subversion           9               4  \n",
      "2              7                              6              10  \n",
      "3              7                Git           4               2  \n",
      "4              7  Zip file back-ups          10               1  \n",
      "\n",
      "[5 rows x 54 columns]\n",
      "2735.35924077034\n"
     ]
    }
   ],
   "source": [
    "LanguageDesireNextYear = pd.DataFrame(df1['Respondent'])\n",
    "for i in sLDNY:\n",
    "    \n",
    "    LanguageDesireNextYear[i]=\"\"\n",
    "time_start=time.time()\n",
    "coldic=dict(zip(LanguageDesireNextYear.columns,range(0,len(sLDNY)+1)))\n",
    "print(coldic)\n",
    "print(df1.head())\n",
    "for i in range(98855):\n",
    "    data=(df1.loc[i,'LanguageDesireNextYear']).split(';')\n",
    "    if(data[0]!=\"\"):\n",
    "       # print(data)\n",
    "        for value in data:\n",
    "            LanguageDesireNextYear.iloc[i,coldic[value]]=1\n",
    "time_end=time.time()\n",
    "print(time_end-time_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Respondent': 0, 'MariaDB': 1, 'PostgreSQL': 2, 'Redis': 3, 'Amazon DynamoDB': 4, 'SQL Server': 5, 'Cassandra': 6, 'Google Cloud Storage': 7, 'MongoDB': 8, 'Elasticsearch': 9, 'Amazon RDS/Aurora': 10, 'Neo4j': 11, 'IBM Db2': 12, 'Apache Hive': 13, 'Google BigQuery': 14, 'SQLite': 15, 'Oracle': 16, 'Microsoft Azure (Tables, CosmosDB, SQL, etc)': 17, 'MySQL': 18, 'Memcached': 19, 'Amazon Redshift': 20, 'Apache HBase': 21}\n",
      "                 Age AssessBenefits1 AssessBenefits10 AssessBenefits11  \\\n",
      "0  25 - 34 years old                                                     \n",
      "1  35 - 44 years old               1                2                8   \n",
      "2                                                                        \n",
      "3  35 - 44 years old                                                     \n",
      "4  18 - 24 years old               1                9                6   \n",
      "\n",
      "  AssessBenefits2 AssessBenefits3 AssessBenefits4 AssessBenefits5  \\\n",
      "0                                                                   \n",
      "1               5               3               7              10   \n",
      "2                                                                   \n",
      "3                                                                   \n",
      "4              10               2               4               8   \n",
      "\n",
      "  AssessBenefits6 AssessBenefits7       ...       OperatingSystem  \\\n",
      "0                                       ...                     2   \n",
      "1               4              11       ...                     2   \n",
      "2                                       ...                     0   \n",
      "3                                       ...                     4   \n",
      "4               3              11       ...                     4   \n",
      "\n",
      "              PlatformDesireNextYear                 PlatformWorkedWith  \\\n",
      "0           AWS;Azure;Linux;Firebase           AWS;Azure;Linux;Firebase   \n",
      "1                              Linux                              Linux   \n",
      "2                                                                         \n",
      "3                              Azure                              Azure   \n",
      "4  Arduino;Windows Desktop or Server  Arduino;Windows Desktop or Server   \n",
      "\n",
      "  Respondent                                    SelfTaughtTypes Student  \\\n",
      "0          1  The official documentation and/or standards fo...       1   \n",
      "1          3  The official documentation and/or standards fo...       1   \n",
      "2          4                                                          1   \n",
      "3          5  The official documentation and/or standards fo...       1   \n",
      "4          7  The official documentation and/or standards fo...       3   \n",
      "\n",
      "  UndergradMajor     VersionControl YearsCoding YearsCodingProf  \n",
      "0             11                Git           8               8  \n",
      "1              4     Git;Subversion           9               4  \n",
      "2              7                              6              10  \n",
      "3              7                Git           4               2  \n",
      "4              7  Zip file back-ups          10               1  \n",
      "\n",
      "[5 rows x 54 columns]\n",
      "1350.2758250236511\n"
     ]
    }
   ],
   "source": [
    "DatabaseDesireNextYear = pd.DataFrame(df1['Respondent'])\n",
    "for i in sDNY:\n",
    "    \n",
    "    DatabaseDesireNextYear[i]=\"\"\n",
    "time_start=time.time()\n",
    "coldic=dict(zip(DatabaseDesireNextYear.columns,range(0,len(sDNY)+1)))\n",
    "print(coldic)\n",
    "print(df1.head())\n",
    "for i in range(98855):\n",
    "    data=(df1.loc[i,'DatabaseDesireNextYear']).split(';')\n",
    "    if(data[0]!=\"\"):\n",
    "       # print(data)\n",
    "        for value in data:\n",
    "            DatabaseDesireNextYear.iloc[i,coldic[value]]=1\n",
    "time_end=time.time()\n",
    "print(time_end-time_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# FrameworkDesireNextYear.to_csv(\"./data/user_preprocessing/FrameworkDesireNextYear.csv\")\n",
    "# PlatformDesireNextYear.to_csv(\"./data/user_preprocessing/PlatformDesireNextYear.csv\")\n",
    "# DatabaseDesireNextYear.to_csv(\"./data/user_preprocessing/DatabaseDesireNextYear.csv\")\n",
    "# LanguageDesireNextYear.to_csv(\"./data/user_preprocessing/LanguageDesireNextYear.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}