Primary_Predictive_Models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import needed packages\n",
    "\n",
    "from numpy import mean\n",
    "from numpy import std\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import pandas as pd\n",
    "import spacy\n",
    "import en_core_web_sm\n",
    "\n",
    "import re\n",
    "import string\n",
    "import os\n",
    "import sys\n",
    "import codecs\n",
    "import argparse\n",
    "import json\n",
    "import gzip\n",
    "import re\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "from numpy import mean\n",
    "from numpy import std\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "# Machine learning imports\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "import sklearn.metrics as metrics\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# NLP imports\n",
    "from spacy.lang.en import English"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#format the MyPersonality data\n",
    "\n",
    "df = pd.read_csv ('MyPersonalityData.csv')\n",
    "\n",
    "df = df.groupby('#AUTHID').agg({'cNEU':'first', \n",
    "                             'STATUS': ', '.join }).reset_index()\n",
    "\n",
    "data = df.values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#format the Essay data\n",
    "\n",
    "df = pd.read_csv ('Essays.csv')\n",
    "df = df.groupby('#AUTHID').agg({'cNEU':'first', \n",
    "                             str('TEXT'): ', '.join }).reset_index()\n",
    "\n",
    "essay_data = df.values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read from the data, returning parallel lists of documents and their labels\n",
    "\n",
    "def read_and_clean_lines(data):   \n",
    "    NeuScores = []\n",
    "    Statuses = []\n",
    "\n",
    "   \n",
    "    for line in data:\n",
    "        \n",
    "        NeuScore = line[1]\n",
    "        Status = line[2]\n",
    "        \n",
    "        clean_text = re.sub(r\"\\s+\",\" \",Status)\n",
    "        Statuses.append(clean_text)\n",
    "        NeuScores.append(NeuScore)\n",
    "        \n",
    "    #print(\"Read {} documents\".format(len(Statuses)))\n",
    "    #print(\"Read {} labels\".format(len(NeuScores)))\n",
    "    return Statuses,NeuScores\n",
    "\n",
    "def normalize_tokens(tokenlist):\n",
    "    # Input: list of tokens as strings,  e.g. ['I', ' ', 'saw', ' ', '@psresnik', ' ', 'on', ' ','Twitter']\n",
    "    # Output: list of tokens where\n",
    "    #   - All tokens are lowercased\n",
    "    #   - All tokens starting with a whitespace character have been filtered out\n",
    "    #   - All handles (tokens starting with @) have been filtered out\n",
    "    #   - Any underscores have been replaced with + (since we use _ as a special character in bigrams)\n",
    "\n",
    "    normalized_tokens = tokenlist # ASSIGNMENT: replace with your code\n",
    "    \n",
    "    return normalized_tokens\n",
    "\n",
    "def ngrams(tokens, n):\n",
    "    # Returns all ngrams of size n in sentence, where an ngram is itself a list of tokens\n",
    "    return [tokens[i:i+n] for i in range(len(tokens)-n+1)]\n",
    "\n",
    "def filter_punctuation_bigrams(ngrams):\n",
    "    # Input: assume ngrams is a list of ['token1','token2'] bigrams\n",
    "    # Removes ngrams like ['today','.'] where either token is a punctuation character\n",
    "    # Returns list with the items that were not removed\n",
    "    punct = string.punctuation\n",
    "    return [ngram   for ngram in ngrams   if ngram[0] not in punct and ngram[1] not in punct]\n",
    "\n",
    "def filter_stopword_bigrams(ngrams, stopwords):\n",
    "    # Input: assume ngrams is a list of ['token1','token2'] bigrams, stopwords is a set of words like 'the'\n",
    "    # Removes ngrams like ['in','the'] and ['senator','from'] where either word is a stopword\n",
    "    # Returns list with the items that were not removed\n",
    "    result = [ngram   for ngram in ngrams   if ngram[0] not in stopwords and ngram[1] not in stopwords]\n",
    "    return result\n",
    "\n",
    "\n",
    "# Split on whitespace, e.g. \"a    b_c  d\" returns tokens ['a','b_c','d']\n",
    "def whitespace_tokenizer(line):\n",
    "    return line.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read a set of stoplist words from data, assuming it contains one word per line\n",
    "# Return a python Set data structure (https://www.w3schools.com/python/python_sets.asp)\n",
    "\n",
    "def load_stopwords(filename):\n",
    "    stopwords = []\n",
    "    with codecs.open(filename, 'r', encoding='ascii', errors='ignore') as fp:\n",
    "        stopwords = fp.read().split('\\n')\n",
    "    return set(stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call sklearn's train_test_split function to split the dataset into training items/labels and test items/labels.  \n",
    "\n",
    "def split_training_set(lines, labels, test_size, random_seed=42):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(lines, labels, test_size=test_size, random_state=random_seed, stratify=labels)\n",
    "    #print(\"Training set label counts: {}\".format(Counter(y_train)))\n",
    "    #print(\"Test set     label counts: {}\".format(Counter(y_test)))\n",
    "    return X_train, X_test, y_train, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bagofwords \n",
    "\n",
    "def BOW(X, stopwords_arg, analyzefn=\"word\", range=(1,10)):\n",
    "    training_vectorizer = CountVectorizer(stop_words=stopwords_arg,\n",
    "                                          analyzer=analyzefn,\n",
    "                                          lowercase=True,\n",
    "                                          ngram_range=range)\n",
    "    \n",
    "    X_features = training_vectorizer.fit_transform(X)\n",
    "    return X_features, training_vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tf-idf\n",
    "\n",
    "def TF_IDF(X, stopwords_arg, analyzefn=\"word\", range=(1,4)):\n",
    "    \n",
    "    from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "  \n",
    "    training_vectorizer = TfidfVectorizer(min_df = 10, max_df = 0.1, stop_words=stopwords_arg,\n",
    "                                          analyzer=analyzefn,\n",
    "                                          lowercase=True,\n",
    "                                          ngram_range=range)\n",
    "    \n",
    "    X_features = training_vectorizer.fit_transform(X)\n",
    "  \n",
    "    return X_features, training_vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Linguistic Inquiry and Word Count\n",
    "\n",
    "def LIWC(X, range_low, range_high):\n",
    "    \n",
    "    import liwc\n",
    "    parse, category_names = liwc.load_token_parser('liwc2007_updates-DO_NOT_SHARE.dic')\n",
    "    \n",
    "    import re\n",
    "    from collections import Counter\n",
    "    \n",
    "    #tokenize the text\n",
    "    def tokenize(text):\n",
    "        for match in re.finditer(r'\\w+', text, re.UNICODE):\n",
    "            yield match.group(0)\n",
    "\n",
    "    data_dict = []\n",
    "   \n",
    "    for line in X:\n",
    "        line = line.lower()\n",
    "        tokens = tokenize(line)\n",
    "        \n",
    "        \n",
    "        c = Counter(category for token in tokens for category in parse(token))\n",
    "        \n",
    "        #use = ['future','feel','sad','cause','home','leisure','verb','tentat','they','posemo','negemo','death','adverb','hear','ingest','swear','percept','we','money','insight','nonfl','see','family','anger','auxverb','inhib','number','past','excl','assent','i','cogmech','anx','friend','bio','ipron','article','you','present','achieve']\n",
    "        \n",
    "        #for word in list(c):\n",
    "            #if word not in use:\n",
    "                #del c[word]\n",
    "        \n",
    "        \n",
    "        c = Counter({k: c for k, c in c.items() if c >= range_low})\n",
    "        c = Counter({k: c for k, c in c.items() if c <= range_high})\n",
    "            \n",
    "        d = dict(c)\n",
    "        \n",
    "        data_dict.append(c)\n",
    "        \n",
    "    # Create DictVectorizer object\n",
    "    dictvectorizer = DictVectorizer(sparse=False)\n",
    "\n",
    "    # Convert dictionary into feature matrix\n",
    "    features = dictvectorizer.fit_transform(data_dict)\n",
    "    \n",
    "    return features, dictvectorizer\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#empath\n",
    "\n",
    "def Empath(X):\n",
    "    \n",
    "    from empath import Empath\n",
    "    lexicon = Empath()\n",
    "    \n",
    "    import re\n",
    "    from collections import Counter\n",
    "\n",
    "\n",
    "    data_dict = []\n",
    "  \n",
    "    \n",
    "    for line in X:\n",
    "        \n",
    "        line = line.lower()\n",
    "        c = lexicon.analyze(line, normalize=True)\n",
    "        counts = dict(c)\n",
    "        \n",
    "        data_dict.append(counts)\n",
    "        \n",
    "    \n",
    "    # Create DictVectorizer object\n",
    "    dictvectorizer = DictVectorizer(sparse=False)\n",
    "\n",
    "    # Convert dictionary into feature matrix\n",
    "    features = dictvectorizer.fit_transform(data_dict)\n",
    "    \n",
    "    return features, dictvectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_most_informative_features(vectorizer, clf, n=20):\n",
    "    feature_names = vectorizer.get_feature_names()\n",
    "    coefs_with_fns = sorted(zip(clf.coef_[0], feature_names))\n",
    "    top = zip(coefs_with_fns[:n], coefs_with_fns[:-(n + 1):-1])\n",
    "    for (coef_1, fn_1), (coef_2, fn_2) in top:\n",
    "        print (\"\\t%.4f\\t%-15s\\t\\t%.4f\\t%-15s\" % (coef_1, fn_1, coef_2, fn_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################################################################################################################\n",
    "# Adaptation of the main logistic regression experiment from Assignment 2, to use cross-validation.\n",
    "###################################################################################################################\n",
    "def run_experiment(Classifier, Feature_Extraction_Method, input, essay_data, stopwords_file, test_size, num_folds, stratify, random_seed, range_low,range_high):\n",
    "\n",
    "    # Load stopwords\n",
    "    stop_words = load_stopwords(stopwords_file)\n",
    "\n",
    "    #Clean training/test data\n",
    "    X, y = read_and_clean_lines(input)\n",
    "    Len_X = len(X)\n",
    "  \n",
    "    #Clean essay data\n",
    "    A, b = read_and_clean_lines(essay_data)\n",
    "    Len_A = len(A)\n",
    "\n",
    "    #combine both MyPersonality and Essay data\n",
    "    Combined = X+A\n",
    "        \n",
    "    if Feature_Extraction_Method == \"BOW\":\n",
    "        \n",
    "        Features, training_vectorizer = BOW(Combined, stop_words, \"word\", range=(1,10))\n",
    "        Len_F = Features.getnnz()\n",
    "        X_features= Features[0:Len_X]\n",
    "        Essay_features = Features[Len_X:Len_F+1]\n",
    "\n",
    "    if Feature_Extraction_Method == \"TF_IDF\":\n",
    "    \n",
    "        Features, training_vectorizer = TF_IDF(Combined, stop_words, \"word\", range=(1,3))\n",
    "        Len_F = Features.getnnz()\n",
    "        X_features= Features[0:Len_X]\n",
    "        Essay_features = Features[Len_X:Len_F+1]\n",
    "      \n",
    "    if Feature_Extraction_Method == \"LIWC\":\n",
    "        \n",
    "        Features, training_vectorizer = LIWC(Combined, range_low, range_high)\n",
    "        Len_F = len(Features)\n",
    "        X_features= Features[0:Len_X]\n",
    "        Essay_features = Features[Len_X:Len_F+1]\n",
    "        \n",
    "    if Feature_Extraction_Method == \"Empath\":\n",
    "        \n",
    "        Features, training_vectorizer = Empath(Combined)\n",
    "        Len_F = len(Features)\n",
    "        X_features= Features[0:Len_X]\n",
    "        Essay_features = Features[Len_X:Len_F+1]\n",
    "      \n",
    "    \n",
    "    #split MyPeronality data\n",
    "    X_train, X_test, y_train, y_test = split_training_set(X_features, y, test_size=test_size)\n",
    "   \n",
    "    # Create a k-fold cross validation object.\n",
    "    if stratify == False:\n",
    "        cv = KFold(n_splits=num_folds, random_state=random_seed, shuffle=True)\n",
    "        \n",
    "\n",
    "    if stratify == True:\n",
    "        cv = StratifiedKFold(n_splits=num_folds, random_state=random_seed, shuffle=True)\n",
    "    \n",
    "    # create model\n",
    "    if Classifier == \"Logistic Regression\":\n",
    "        model = LogisticRegression(solver='liblinear')\n",
    "    \n",
    "    if Classifier == \"SVM\":\n",
    "        model = SVC(kernel='linear')\n",
    "        \n",
    "    if Classifier == \"DecisionTree\":\n",
    "        model = DecisionTreeClassifier()\n",
    "        \n",
    "    if Classifier == \"NB\":\n",
    "        model = MultinomialNB()\n",
    "    \n",
    "    \n",
    "    # evaluate model\n",
    "    train_scores = cross_val_score(model, X_train, y_train, scoring='accuracy', cv=cv, n_jobs=-1)\n",
    "  \n",
    "    print('TRAINING SET --- Mean: %.3f STD: %.3f' % (mean(train_scores), std(train_scores)))\n",
    "  \n",
    "    # fit model\n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    # make predictions on remaining test set\n",
    "    yhat = model.predict(X_test)\n",
    "    # evaluate predictions\n",
    "    accuracy = accuracy_score(y_test, yhat)\n",
    "    print('Accuracy of Test Set Predictions: %.3f' % (accuracy * 100))\n",
    "    \n",
    "    # make predictions on the essay dataset\n",
    "    xhat = model.predict(Essay_features)\n",
    "    # evaluate predictions\n",
    "    accuracy = accuracy_score(b, xhat)\n",
    "    print('Accuracy of Essay Data Predictions: %.3f' % (accuracy * 100))\n",
    "    \n",
    "    \n",
    "    clf = LogisticRegression(random_state=0).fit(X_train, y_train)\n",
    "    print(show_most_informative_features(training_vectorizer, clf, n=20))\n",
    "    \n",
    "    return accuracy\n",
    "    \n",
    "       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6044444444444445"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#best LIWC ranges for Logistic Regression\n",
    "run_experiment(\"Logistic Regression\", \"LIWC\", data, essay_data, \"mallet_en_stoplist.txt\", 0.1, 5, True, 3, 1, 15) \n",
    "\n",
    "run_experiment(\"Logistic Regression\", \"LIWC\", data, essay_data, \"mallet_en_stoplist.txt\", 0.1, 5, True, 3, 101, 1000)\n",
    "\n",
    "run_experiment(\"Logistic Regression\", \"LIWC\", data, essay_data, \"mallet_en_stoplist.txt\", 0.9, 5, True, 3, 1, 5) \n",
    "\n",
    "run_experiment(\"Logistic Regression\", \"LIWC\", data, essay_data, \"mallet_en_stoplist.txt\", 0.9, 5, True, 3, 791, 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAINING SET --- Mean: 0.591 STD: 0.031\n",
      "Accuracy of Test Set Predictions: 64.000\n",
      "Accuracy of Essay Data Predictions: 50.162\n",
      "\t-0.2714\tfuture         \t\t0.3484\tfamily         \n",
      "\t-0.2450\tthey           \t\t0.3089\tnonfl          \n",
      "\t-0.2305\thear           \t\t0.2904\tsee            \n",
      "\t-0.2268\tleisure        \t\t0.2477\tanger          \n",
      "\t-0.2034\thumans         \t\t0.2224\tnumber         \n",
      "\t-0.1468\tmoney          \t\t0.2179\texcl           \n",
      "\t-0.1457\tsad            \t\t0.1714\tanx            \n",
      "\t-0.1439\tingest         \t\t0.1292\tinhib          \n",
      "\t-0.1249\thome           \t\t0.0993\tipron          \n",
      "\t-0.1201\ttentat         \t\t0.0900\tposemo         \n",
      "\t-0.1150\tquant          \t\t0.0843\tfriend         \n",
      "\t-0.1115\tsexual         \t\t0.0659\tincl           \n",
      "\t-0.1007\tfeel           \t\t0.0600\tarticle        \n",
      "\t-0.0945\taffect         \t\t0.0461\tassent         \n",
      "\t-0.0896\tnegate         \t\t0.0458\tppron          \n",
      "\t-0.0780\tdeath          \t\t0.0445\tnegemo         \n",
      "\t-0.0741\twe             \t\t0.0436\tpresent        \n",
      "\t-0.0686\tcause          \t\t0.0417\tbody           \n",
      "\t-0.0573\ttime           \t\t0.0335\tinsight        \n",
      "\t-0.0530\tyou            \t\t0.0290\tcogmech        \n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\conoc\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAINING SET --- Mean: 0.609 STD: 0.041\n",
      "Accuracy of Test Set Predictions: 60.000\n",
      "Accuracy of Essay Data Predictions: 50.527\n",
      "\t-0.0573\tppron          \t\t0.1116\tpresent        \n",
      "\t-0.0509\tpronoun        \t\t0.0281\tverb           \n",
      "\t-0.0508\tpreps          \t\t0.0012\tfunct          \n",
      "\t-0.0452\tsocial         \t\t0.0000\tspace          \n",
      "\t-0.0394\ttime           \t\t0.0000\tipron          \n",
      "\t-0.0332\tauxverb        \t\t0.0000\ti              \n",
      "\t-0.0036\tcogmech        \t\t0.0000\tconj           \n",
      "\t-0.0010\trelativ        \t\t0.0000\tarticle        \n",
      "\t0.0000\tadverb         \t\t0.0000\taffect         \n",
      "\t0.0000\taffect         \t\t0.0000\tadverb         \n",
      "\t0.0000\tarticle        \t\t-0.0010\trelativ        \n",
      "\t0.0000\tconj           \t\t-0.0036\tcogmech        \n",
      "\t0.0000\ti              \t\t-0.0332\tauxverb        \n",
      "\t0.0000\tipron          \t\t-0.0394\ttime           \n",
      "\t0.0000\tspace          \t\t-0.0452\tsocial         \n",
      "\t0.0012\tfunct          \t\t-0.0508\tpreps          \n",
      "\t0.0281\tverb           \t\t-0.0509\tpronoun        \n",
      "\t0.1116\tpresent        \t\t-0.0573\tppron          \n",
      "None\n",
      "TRAINING SET --- Mean: 0.550 STD: 0.107\n",
      "Accuracy of Test Set Predictions: 62.222\n",
      "Accuracy of Essay Data Predictions: 49.878\n",
      "\t-0.3620\trelativ        \t\t0.4868\tpronoun        \n",
      "\t-0.2891\tverb           \t\t0.2728\ttime           \n",
      "\t-0.2815\tposemo         \t\t0.2657\tcogmech        \n",
      "\t-0.2555\tpreps          \t\t0.2366\tcertain        \n",
      "\t-0.2157\ttentat         \t\t0.2149\tanger          \n",
      "\t-0.1779\tppron          \t\t0.1523\tpast           \n",
      "\t-0.1553\tfuture         \t\t0.1426\tfriend         \n",
      "\t-0.1299\tsocial         \t\t0.1408\trelig          \n",
      "\t-0.1297\tarticle        \t\t0.1323\tquant          \n",
      "\t-0.1247\tdiscrep        \t\t0.1208\tshehe          \n",
      "\t-0.1101\tpercept        \t\t0.1040\tnumber         \n",
      "\t-0.1078\tleisure        \t\t0.1030\tnegemo         \n",
      "\t-0.1043\tachieve        \t\t0.0974\tswear          \n",
      "\t-0.1017\ti              \t\t0.0891\twork           \n",
      "\t-0.0977\tadverb         \t\t0.0864\tipron          \n",
      "\t-0.0936\tmotion         \t\t0.0778\tyou            \n",
      "\t-0.0893\tbody           \t\t0.0695\tauxverb        \n",
      "\t-0.0777\thumans         \t\t0.0692\texcl           \n",
      "\t-0.0731\twe             \t\t0.0689\tfamily         \n",
      "\t-0.0642\taffect         \t\t0.0612\tassent         \n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\conoc\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAINING SET --- Mean: 0.583 STD: 0.083\n",
      "Accuracy of Test Set Predictions: 63.111\n",
      "Accuracy of Essay Data Predictions: 50.891\n",
      "\t0.0000\tadverb         \t\t0.0207\tcogmech        \n",
      "\t0.0000\taffect         \t\t0.0203\tpronoun        \n",
      "\t0.0000\tarticle        \t\t0.0007\tfunct          \n",
      "\t0.0000\tauxverb        \t\t0.0000\tverb           \n",
      "\t0.0000\tbio            \t\t0.0000\ttime           \n",
      "\t0.0000\tconj           \t\t0.0000\tspace          \n",
      "\t0.0000\ti              \t\t0.0000\tsocial         \n",
      "\t0.0000\tipron          \t\t0.0000\trelativ        \n",
      "\t0.0000\tposemo         \t\t0.0000\tpresent        \n",
      "\t0.0000\tppron          \t\t0.0000\tpreps          \n",
      "\t0.0000\tpreps          \t\t0.0000\tppron          \n",
      "\t0.0000\tpresent        \t\t0.0000\tposemo         \n",
      "\t0.0000\trelativ        \t\t0.0000\tipron          \n",
      "\t0.0000\tsocial         \t\t0.0000\ti              \n",
      "\t0.0000\tspace          \t\t0.0000\tconj           \n",
      "\t0.0000\ttime           \t\t0.0000\tbio            \n",
      "\t0.0000\tverb           \t\t0.0000\tauxverb        \n",
      "\t0.0007\tfunct          \t\t0.0000\tarticle        \n",
      "\t0.0203\tpronoun        \t\t0.0000\taffect         \n",
      "\t0.0207\tcogmech        \t\t0.0000\tadverb         \n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5089141004862237"
      ]
     },
     "execution_count": 340,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#best LIWC ranges for SVM\n",
    "run_experiment(\"SVM\", \"LIWC\", data, essay_data, \"mallet_en_stoplist.txt\", 0.1, 10, True, 3, 1, 120) \n",
    "\n",
    "run_experiment(\"SVM\", \"LIWC\", data, essay_data, \"mallet_en_stoplist.txt\", 0.1, 10, True, 3, 251, 1000)\n",
    "\n",
    "#best LIWC ranges for SVM\n",
    "run_experiment(\"SVM\", \"LIWC\", data, essay_data, \"mallet_en_stoplist.txt\", 0.9, 10, True, 3, 1, 45) \n",
    "\n",
    "run_experiment(\"SVM\", \"LIWC\", data, essay_data, \"mallet_en_stoplist.txt\", 0.9, 10, True, 3, 216, 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t-0.2496\tgod            \t\t0.2983\tblack          \n",
      "\t-0.1885\tcanada         \t\t0.2518\tcavs           \n",
      "\t-0.1867\tbless          \t\t0.1322\tlong           \n",
      "\t-0.1867\tcanada fam     \t\t0.1306\tlong time      \n",
      "\t-0.1867\tcanada fam god \t\t0.1259\trock house     \n",
      "\t-0.1867\tcanada fam god bless\t\t0.1259\tcavs rock house\n",
      "\t-0.1867\tfam god        \t\t0.1259\tcavs rock      \n",
      "\t-0.1867\tfam god bless  \t\t0.1259\tcavs cavs rock house\n",
      "\t-0.1867\tgod bless      \t\t0.1259\tcavs cavs rock \n",
      "\t-0.1849\tfam            \t\t0.1259\tcavs cavs      \n",
      "\t-0.1697\tgive           \t\t0.1208\thates          \n",
      "\t-0.1631\thappy          \t\t0.1196\trock           \n",
      "\t-0.1563\tthings         \t\t0.1155\tmarea hates long time\n",
      "\t-0.1535\tlife           \t\t0.1155\tmarea hates long\n",
      "\t-0.1516\tmornings       \t\t0.1155\tmarea hates    \n",
      "\t-0.1477\tmagic          \t\t0.1155\tmarea          \n",
      "\t-0.1459\tmagic things   \t\t0.1155\thates long time\n",
      "\t-0.1459\tmagic things give\t\t0.1155\thates long     \n",
      "\t-0.1459\tmornings winter\t\t0.1155\tfacebook marea hates long time\n",
      "\t-0.1459\tmornings winter magic\t\t0.1155\tfacebook marea hates long\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.584"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_experiment(\"Logistic Regression\", \"BOW\", data, essay_data, \"mallet_en_stoplist.txt\", 0.5, 10, True, 3, 400,600) \n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}