modifiedNB_old.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2-NB Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 HOUSEHOLD IDENTIFIER\n",
      "1 ARE YOUR LIVING QUARTERS...\n",
      "2 IS THERE A TELEPHONE IN THIS \n",
      "3 FAMILY INCOME\n",
      "4 TOTAL NUMBER OF PERSONS LIVING \n",
      "5 REGION\n",
      "6 DIVISION\n",
      "7 INDIVIDUAL PRINCIPAL CITY\n",
      "8 PERSONS AGE  \n",
      "9 MARITAL STATUS \n",
      "10 SEX\n",
      "11 DID YOU EVER SERVE ON ACTIVE \n",
      "12 HIGHEST LEVEL OF SCHOOL \n",
      "13 RACE\n",
      "14 CHANGE IN HOUSEHOLD COMPOSITION\n",
      "15 HISPANIC OR NON-HISPANIC\n",
      "16 COUNTRY OF BIRTH\n",
      "17 CITIZENSHIP STATUS\n",
      "18 LAST WEEK, DID YOU DO ANY WORK\n",
      "19 LAST WEEK, DID YOU DO ANY\n",
      "20 DO YOU RECEIVE ANY PAYMENTS\n",
      "21 DISABILITY STATUS\n",
      "22 DO YOU CURRENTLY WANT A JOB, EITHER \n",
      "23 INDUSTRY AND OCCUPATION \n",
      "24 CLASS OF WORKER\n",
      "25 CLASS OF WORKER\n",
      "26 COW - PRIVATE OR GOVERNMENT\n",
      "27 PERIODICITY\n",
      "28 HOURLY/NONHOURLY STATUS\n",
      "29 ON THIS JOB, ARE YOU A MEMBER OF A \n",
      "30 ON THIS JOB ARE YOU COVERED BY A UNION \n",
      "31 PRESENCE OF OWN CHILDREN <18 YEARS \n",
      "32 Number of own children <18 years of age \n",
      "33 race-binary\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "file = 'cleaned5.csv'\n",
    "\n",
    "with open(file, 'r') as fi:\n",
    "    reader = csv.reader(fi)\n",
    "    headers = next(reader)\n",
    "    for i in range(len(headers)):\n",
    "        print(i, headers[i])     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prep: Reading the data in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(file, 'r') as fi:\n",
    "    reader = csv.reader(fi)\n",
    "    headers = next(reader)\n",
    "    \n",
    "    white = []\n",
    "    nonwhite = []\n",
    "    income_w = []\n",
    "    income_n = []\n",
    "    counts = [0 for i in range(27)]\n",
    "    \n",
    "    for line in reader:\n",
    "        \n",
    "        # remove id, income, race, and race_binary\n",
    "        data = []\n",
    "        for i in range(len(line)):\n",
    "            if i != 0 and i != 13 and i != 3 and i != 33:\n",
    "                data.append(int(line[i]))\n",
    "\n",
    "        # put in the correct racial category\n",
    "        if int(line[13]) == 1:\n",
    "            white.append(data)\n",
    "            income_w.append(int(line[3]))\n",
    "        else:\n",
    "            nonwhite.append(data)\n",
    "            income_n.append(int(line[3]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "104841\n",
      "26918\n"
     ]
    }
   ],
   "source": [
    "# something that might cause problems - there's WAY more data for white people\n",
    "print(len(white))\n",
    "print(len(nonwhite))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making the Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I am going to make the models using k-fold cross validation to try to improve the accuracy. Let's start with white people."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy rate:  0.6297567954220314\n",
      "accuracy rate:  0.6452689813048454\n",
      "accuracy rate:  0.6009156810377718\n",
      "accuracy rate:  0.581934376192293\n",
      "accuracy rate:  0.5891835177413202\n",
      "accuracy rate:  0.6288630293781\n",
      "accuracy rate:  0.5890881342998855\n",
      "accuracy rate:  0.5914727203357497\n",
      "accuracy rate:  0.6184662342617322\n",
      "accuracy rate:  0.584128195345288\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "# returns indexes into fields and labels\n",
    "def run_kfold(fields, labels):\n",
    "    kf = KFold(n_splits=10)\n",
    "    best = [], []\n",
    "    best_accuracy = 0\n",
    "\n",
    "    # train_index and test_index index into fields and labels\n",
    "    for train_index, test_index in kf.split(fields):\n",
    "        train_fields = [fields[i] for i in train_index]\n",
    "        train_labels = [labels[i] for i in train_index]\n",
    "        test_fields = [fields[i] for i in test_index]\n",
    "        test_labels = [labels[i] for i in test_index]\n",
    "\n",
    "        clf = GaussianNB()\n",
    "        clf.fit(train_fields, train_labels)\n",
    "\n",
    "        res = clf.predict(test_fields).tolist()\n",
    "        \n",
    "        accuracy = []\n",
    "        for i in range(len(res)):\n",
    "            if res[i] == test_labels[i]:\n",
    "                accuracy.append(1)\n",
    "            else:\n",
    "                accuracy.append(0)\n",
    "        #accuracy = [1 if res[i] == test_labels[i] else 0 for i in range(len(res))]\n",
    "        acc = sum(accuracy)/len(accuracy)\n",
    "\n",
    "        if (acc > best_accuracy):\n",
    "            best = train_index, test_index\n",
    "            best_accuracy = acc\n",
    "\n",
    "        print(\"accuracy rate: \", acc)\n",
    "    return best\n",
    "\n",
    "training_w, testing_w = run_kfold(white, income_w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And now for nonwhite people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy rate:  0.6578751857355126\n",
      "accuracy rate:  0.5973254086181278\n",
      "accuracy rate:  0.7035661218424963\n",
      "accuracy rate:  0.5995542347696879\n",
      "accuracy rate:  0.6927934621099554\n",
      "accuracy rate:  0.7069093610698366\n",
      "accuracy rate:  0.6266716196136701\n",
      "accuracy rate:  0.6849925705794948\n",
      "accuracy rate:  0.7094017094017094\n",
      "accuracy rate:  0.6228167967298402\n"
     ]
    }
   ],
   "source": [
    "training_n, testing_n = run_kfold(nonwhite, income_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Differential Accuracy by Race Binary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's examine the false positive and false negative rates respectively. **White people**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results of running the model for white people:\n",
      "false positive rate: 0.190671\n",
      "false negative rate: 0.164060\n",
      "accuracy: 0.645269\n"
     ]
    }
   ],
   "source": [
    "#training and testing index into fields and labels \n",
    "#so max(training) < len(fields) \n",
    "def run_model(training, testing, fields, labels):\n",
    "    train_fields = [fields[i] for i in training]\n",
    "    train_labels = [labels[i] for i in training]\n",
    "    test_fields = [fields[i] for i in testing]\n",
    "    test_labels = [labels[i] for i in testing]\n",
    "\n",
    "    clf = GaussianNB()\n",
    "    clf.fit(train_fields, train_labels)\n",
    "\n",
    "    res = clf.predict(test_fields).tolist()\n",
    "\n",
    "    accuracy = []\n",
    "    for i in range(len(res)):\n",
    "        if res[i] == 1 and test_labels[i] == 0:\n",
    "            accuracy.append(1)\n",
    "        elif res[i] == 0 and test_labels[i] == 1:\n",
    "            accuracy.append(-1)\n",
    "        else:\n",
    "            accuracy.append(0)\n",
    "\n",
    "    fp = sum([1 if accuracy[i] == 1 else 0 for i in range(len(accuracy))])/len(accuracy)\n",
    "    fn = sum([1 if accuracy[i] == -1 else 0 for i in range(len(accuracy))])/len(accuracy)\n",
    "    acc = sum([1 if accuracy[i] == 0 else 0 for i in range(len(accuracy))])/len(accuracy)\n",
    "    print(\"false positive rate: %4f\" % fp)\n",
    "    print(\"false negative rate: %4f\" % fn)\n",
    "    print(\"accuracy: %4f\" % acc)\n",
    "    return res, acc, fp, fn\n",
    "\n",
    "print(\"Results of running the model for white people:\")\n",
    "res_w, acc_w, fp_w, fn_w = run_model(training_w, testing_w, white, income_w)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And now for **nonwhite people:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results of running the model for nonwhite people:\n",
      "false positive rate: 0.144928\n",
      "false negative rate: 0.145671\n",
      "accuracy: 0.709402\n"
     ]
    }
   ],
   "source": [
    "print(\"Results of running the model for nonwhite people:\")\n",
    "res_n, acc_n, fp_n, fn_n = run_model(training_n, testing_n, nonwhite, income_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Overall Accuracy**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "false positive rate: 0.181328\n",
      "false negative rate: 0.160304\n",
      "accuracy: 0.658368\n"
     ]
    }
   ],
   "source": [
    "acc = (acc_n*len(res_n)+acc_w*len(res_w))/(len(res_n)+len(res_w))\n",
    "fp = (fp_n*len(res_n)+fp_w*len(res_w))/(len(res_n)+len(res_w))\n",
    "fn = (fn_n*len(res_n)+fn_w*len(res_w))/(len(res_n)+len(res_w))\n",
    "print(\"false positive rate: %4f\" % fp)\n",
    "print(\"false negative rate: %4f\" % fn)\n",
    "print(\"accuracy: %4f\" % acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Differential Accuracy by Sex"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's examine differential accuracy for different subgroups. Let's initially break it down by sex."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_accuracy(res, labels):\n",
    "    \n",
    "    accuracy = []\n",
    "    for i in range(len(res)):\n",
    "        if res[i] == 1 and labels[i] == 0:\n",
    "            accuracy.append(1)\n",
    "        elif res[i] == 0 and labels[i] == 1:\n",
    "            accuracy.append(-1)\n",
    "        else:\n",
    "            accuracy.append(0)\n",
    "    return accuracy\n",
    "\n",
    "\n",
    "def sex_accuracy(testing, fields, labels, res):\n",
    "    test_fields = [fields[i] for i in testing]\n",
    "    test_labels = [labels[i] for i in testing]\n",
    "        \n",
    "    male = []\n",
    "    female = []\n",
    "    \n",
    "    for i in range(len(testing)):\n",
    "        if fields[testing[i]][8] == 1:\n",
    "            male.append(i)\n",
    "        if fields[testing[i]][8] == 2:\n",
    "            female.append(i)\n",
    "            \n",
    "    accuracy = calc_accuracy([res[i] for i in male], [test_labels[i] for i in male])\n",
    "    fp = sum([1 if accuracy[i] == 1 else 0 for i in range(len(accuracy))])/len(accuracy)\n",
    "    fn = sum([1 if accuracy[i] == -1 else 0 for i in range(len(accuracy))])/len(accuracy)\n",
    "    print(\"Accuracy for male:\")\n",
    "    print(\"    false positive rate: %4f\" % fp)\n",
    "    print(\"    false negative rate: %4f\" % fn)\n",
    "    print(\"    accuracy: %4f\" % (1-(fp + fn)))\n",
    "    \n",
    "    accuracy = calc_accuracy([res[i] for i in female], [test_labels[i] for i in female])\n",
    "    fpf = sum([1 if accuracy[i] == 1 else 0 for i in range(len(accuracy))])/len(accuracy)\n",
    "    fnf = sum([1 if accuracy[i] == -1 else 0 for i in range(len(accuracy))])/len(accuracy)\n",
    "    print(\"Accuracy for female:\")\n",
    "    print(\"    false positive rate: %4f\" % fpf)\n",
    "    print(\"    false negative rate: %4f\" % fnf)\n",
    "    print(\"    accuracy: %4f\" % (1-(fpf + fnf)))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Accuracy broken down by sex for nonwhite people**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for male:\n",
      "    false positive rate: 0.135852\n",
      "    false negative rate: 0.143891\n",
      "    accuracy: 0.720257\n",
      "Accuracy for female:\n",
      "    false positive rate: 0.152730\n",
      "    false negative rate: 0.147201\n",
      "    accuracy: 0.700069\n"
     ]
    }
   ],
   "source": [
    "sex_accuracy(testing_n, nonwhite, income_n, res_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Accuracy broken down by sex for white people**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for male:\n",
      "    false positive rate: 0.192112\n",
      "    false negative rate: 0.161070\n",
      "    accuracy: 0.646818\n",
      "Accuracy for female:\n",
      "    false positive rate: 0.189295\n",
      "    false negative rate: 0.166915\n",
      "    accuracy: 0.643790\n"
     ]
    }
   ],
   "source": [
    "sex_accuracy(testing_w, white, income_w, res_w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Differential Accuracy By Race Categorical"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a little tricker. I first need some way to lookup up the race of each entry. Since I've removed unique identifiers, I'm going to have to go back and collect them. Let's make a list of just the racial categories."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'race_b' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-549ca311e441>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mreader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mrace_b\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m             \u001b[0mrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mrace\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'race_b' is not defined"
     ]
    }
   ],
   "source": [
    "race = []\n",
    "with open(file, 'r') as fi:\n",
    "    reader = csv.reader(fi)\n",
    "    headers = next(reader)\n",
    "    \n",
    "    for line in reader:\n",
    "        if (int(line[race_b]) != 1):\n",
    "            race.append(int(line[race]))\n",
    "print(race)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Accuracy by specific racial group for nonwhite people**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No results for race 0\n",
      "No results for race 1\n",
      "Results for race 2:\n",
      "    1776 ENTRIES TOTAL\n",
      "    false positive rate: 0.128941\n",
      "    false negative rate: 0.114865\n",
      "    accuracy: 0.756194\n",
      "Results for race 3:\n",
      "    183 ENTRIES TOTAL\n",
      "    false positive rate: 0.092896\n",
      "    false negative rate: 0.038251\n",
      "    accuracy: 0.868852\n",
      "Results for race 4:\n",
      "    436 ENTRIES TOTAL\n",
      "    false positive rate: 0.266055\n",
      "    false negative rate: 0.204128\n",
      "    accuracy: 0.529817\n",
      "Results for race 5:\n",
      "    21 ENTRIES TOTAL\n",
      "    false positive rate: 0.380952\n",
      "    false negative rate: 0.190476\n",
      "    accuracy: 0.428571\n",
      "Results for race 6:\n",
      "    112 ENTRIES TOTAL\n",
      "    false positive rate: 0.098214\n",
      "    false negative rate: 0.312500\n",
      "    accuracy: 0.589286\n",
      "Results for race 7:\n",
      "    80 ENTRIES TOTAL\n",
      "    false positive rate: 0.062500\n",
      "    false negative rate: 0.250000\n",
      "    accuracy: 0.687500\n",
      "Results for race 8:\n",
      "    35 ENTRIES TOTAL\n",
      "    false positive rate: 0.028571\n",
      "    false negative rate: 0.600000\n",
      "    accuracy: 0.371429\n",
      "Results for race 9:\n",
      "    11 ENTRIES TOTAL\n",
      "    false positive rate: 0.000000\n",
      "    false negative rate: 0.272727\n",
      "    accuracy: 0.727273\n",
      "Results for race 10:\n",
      "    14 ENTRIES TOTAL\n",
      "    false positive rate: 0.142857\n",
      "    false negative rate: 0.214286\n",
      "    accuracy: 0.642857\n",
      "Results for race 11:\n",
      "    1 ENTRIES TOTAL\n",
      "    false positive rate: 0.000000\n",
      "    false negative rate: 0.000000\n",
      "    accuracy: 1.000000\n",
      "Results for race 12:\n",
      "    2 ENTRIES TOTAL\n",
      "    false positive rate: 0.000000\n",
      "    false negative rate: 0.000000\n",
      "    accuracy: 1.000000\n",
      "No results for race 13\n",
      "No results for race 14\n",
      "Results for race 15:\n",
      "    1 ENTRIES TOTAL\n",
      "    false positive rate: 0.000000\n",
      "    false negative rate: 0.000000\n",
      "    accuracy: 1.000000\n",
      "Results for race 16:\n",
      "    13 ENTRIES TOTAL\n",
      "    false positive rate: 0.076923\n",
      "    false negative rate: 0.307692\n",
      "    accuracy: 0.615385\n",
      "Results for race 17:\n",
      "    2 ENTRIES TOTAL\n",
      "    false positive rate: 0.000000\n",
      "    false negative rate: 0.500000\n",
      "    accuracy: 0.500000\n",
      "No results for race 18\n",
      "Results for race 19:\n",
      "    1 ENTRIES TOTAL\n",
      "    false positive rate: 0.000000\n",
      "    false negative rate: 1.000000\n",
      "    accuracy: 0.000000\n",
      "No results for race 20\n",
      "Results for race 21:\n",
      "    3 ENTRIES TOTAL\n",
      "    false positive rate: 0.000000\n",
      "    false negative rate: 0.000000\n",
      "    accuracy: 1.000000\n",
      "No results for race 22\n",
      "No results for race 23\n",
      "No results for race 24\n",
      "No results for race 25\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "def race_accuracy(testing, labels, res):\n",
    "    test_labels = [labels[i] for i in testing]\n",
    "    data = {\"race\":[], \"false positive\":[], \"false negative\":[], \"accuracy\":[], 'count':[]}\n",
    "    \n",
    "    for r in range(0, max(race)):\n",
    "\n",
    "        # indeces within res for the current group\n",
    "        current = []\n",
    "        for i in range(len(testing)):\n",
    "            if (race[testing[i]] == r):\n",
    "                current.append(i)\n",
    "\n",
    "        # calculate accuracy\n",
    "        accuracy = calc_accuracy([res[i] for i in current], [test_labels[i] for i in current])\n",
    "        try:\n",
    "            fp = sum([1 if accuracy[i] == 1 else 0 for i in range(len(accuracy))])/len(accuracy)\n",
    "            fn = sum([1 if accuracy[i] == -1 else 0 for i in range(len(accuracy))])/len(accuracy)\n",
    "            print(\"Results for race %d:\" % r)\n",
    "            print(\"    %d ENTRIES TOTAL\" % len(current))\n",
    "            print(\"    false positive rate: %4f\" % fp)\n",
    "            print(\"    false negative rate: %4f\" % fn)\n",
    "            print(\"    accuracy: %4f\" % (1-(fp + fn)))\n",
    "            data[\"race\"].append(r)\n",
    "            data[\"false positive\"].append(fp)\n",
    "            data[\"false negative\"].append(fn)\n",
    "            data[\"accuracy\"].append((1-(fp + fn)))\n",
    "            data[\"count\"].append(len(current))\n",
    "        except ZeroDivisionError:\n",
    "            print(\"No results for race %d\" % r)\n",
    "    \n",
    "    return pd.DataFrame(data)\n",
    "\n",
    "df = race_accuracy(testing_n, income_n, res_n)\n",
    "with open(\"2nb_acs.csv\", \"w\") as fo:\n",
    "    writer = csv.writer(fo)\n",
    "    writer.writerow(['race', 'false positive', 'false negative', 'accuracy'])\n",
    "    for index, row in df.iterrows():\n",
    "        writer.writerow([row['race'], row['false positive'], row['false negative'], row['accuracy'], row['count']])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# n-NB Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0\n",
      "14541\n",
      "1944\n",
      "6790\n",
      "791\n",
      "824\n",
      "767\n",
      "540\n",
      "164\n",
      "133\n",
      "45\n",
      "20\n",
      "7\n",
      "2\n",
      "88\n",
      "129\n",
      "8\n",
      "2\n",
      "10\n",
      "4\n",
      "87\n",
      "3\n",
      "2\n",
      "0\n",
      "1\n",
      "16\n"
     ]
    }
   ],
   "source": [
    "# race_fields[i] = 2d array of fields for race i\n",
    "race_fields = [[] for i in range(27)]\n",
    "race_labels = [[] for i in range(27)]\n",
    "\n",
    "# iterate over each row in the data\n",
    "for i in range(len(nonwhite)):\n",
    "    race_fields[race[i]].append(nonwhite[i])\n",
    "    race_labels[race[i]].append(income_n[i])\n",
    "for race_i in race_fields:\n",
    "    print(len(race_i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndentationError",
     "evalue": "unindent does not match any outer indentation level (<tokenize>, line 16)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<tokenize>\"\u001b[0;36m, line \u001b[0;32m16\u001b[0m\n\u001b[0;31m    '''  print()\u001b[0m\n\u001b[0m    ^\u001b[0m\n\u001b[0;31mIndentationError\u001b[0m\u001b[0;31m:\u001b[0m unindent does not match any outer indentation level\n"
     ]
    }
   ],
   "source": [
    "def run_race(race, fields, labels):\n",
    "    \n",
    "    print(\"TRAINING\")\n",
    "    print(\"-----------------------\")\n",
    "    print()\n",
    "    \n",
    "    training, testing = run_kfold(fields, labels)\n",
    "    \n",
    "    print()\n",
    "    print(\"RUNNING THE BEST MODEL\")\n",
    "    print(\"-----------------------\")\n",
    "    print()\n",
    "    \n",
    "    res, acc, fp, fn = run_model(training, testing, fields, labels)\n",
    "    \n",
    "  '''  print()\n",
    "    print(\"ACCURACY BY SEX\")\n",
    "    print(\"-----------------------\")\n",
    "    print()\n",
    "    \n",
    "    sex_accuracy(testing, fields, labels, res) '''\n",
    "    return res, acc, fp, fn "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "RACE 2 \n",
      "\n",
      "TRAINING\n",
      "-----------------------\n",
      "\n",
      "accuracy rate:  0.7353951890034365\n",
      "accuracy rate:  0.7077028885832187\n",
      "accuracy rate:  0.6808803301237965\n",
      "accuracy rate:  0.7049518569463549\n",
      "accuracy rate:  0.6822558459422283\n",
      "accuracy rate:  0.7462173314993122\n",
      "accuracy rate:  0.6306740027510316\n",
      "accuracy rate:  0.734525447042641\n",
      "accuracy rate:  0.7324621733149931\n",
      "accuracy rate:  0.6781292984869326\n",
      "\n",
      "RUNNING THE BEST MODEL\n",
      "-----------------------\n",
      "\n",
      "false positive rate: 0.152682\n",
      "false negative rate: 0.101100\n",
      "accuracy: 0.746217\n",
      "\n",
      "ACCURACY BY SEX\n",
      "-----------------------\n",
      "\n",
      "Accuracy for male:\n",
      "    false positive rate: 0.148036\n",
      "    false negative rate: 0.110272\n",
      "    accuracy: 0.741692\n",
      "Accuracy for female:\n",
      "    false positive rate: 0.156566\n",
      "    false negative rate: 0.093434\n",
      "    accuracy: 0.750000\n",
      "\n",
      "RACE 3 \n",
      "\n",
      "TRAINING\n",
      "-----------------------\n",
      "\n",
      "accuracy rate:  0.36923076923076925\n",
      "accuracy rate:  0.49230769230769234\n",
      "accuracy rate:  0.48717948717948717\n",
      "accuracy rate:  0.3128205128205128\n",
      "accuracy rate:  0.26804123711340205\n",
      "accuracy rate:  0.4175257731958763\n",
      "accuracy rate:  0.30927835051546393\n",
      "accuracy rate:  0.4020618556701031\n",
      "accuracy rate:  0.24742268041237114\n",
      "accuracy rate:  0.4896907216494845\n",
      "\n",
      "RUNNING THE BEST MODEL\n",
      "-----------------------\n",
      "\n",
      "false positive rate: 0.420513\n",
      "false negative rate: 0.087179\n",
      "accuracy: 0.492308\n",
      "\n",
      "ACCURACY BY SEX\n",
      "-----------------------\n",
      "\n",
      "Accuracy for male:\n",
      "    false positive rate: 0.376344\n",
      "    false negative rate: 0.064516\n",
      "    accuracy: 0.559140\n",
      "Accuracy for female:\n",
      "    false positive rate: 0.460784\n",
      "    false negative rate: 0.107843\n",
      "    accuracy: 0.431373\n",
      "\n",
      "RACE 4 \n",
      "\n",
      "TRAINING\n",
      "-----------------------\n",
      "\n",
      "accuracy rate:  0.5905743740795287\n",
      "accuracy rate:  0.5655375552282769\n",
      "accuracy rate:  0.5051546391752577\n",
      "accuracy rate:  0.5773195876288659\n",
      "accuracy rate:  0.5596465390279823\n",
      "accuracy rate:  0.6008836524300442\n",
      "accuracy rate:  0.5434462444771723\n",
      "accuracy rate:  0.5213549337260678\n",
      "accuracy rate:  0.5817378497790869\n",
      "accuracy rate:  0.48306332842415317\n",
      "\n",
      "RUNNING THE BEST MODEL\n",
      "-----------------------\n",
      "\n",
      "false positive rate: 0.175258\n",
      "false negative rate: 0.223859\n",
      "accuracy: 0.600884\n",
      "\n",
      "ACCURACY BY SEX\n",
      "-----------------------\n",
      "\n",
      "Accuracy for male:\n",
      "    false positive rate: 0.176829\n",
      "    false negative rate: 0.219512\n",
      "    accuracy: 0.603659\n",
      "Accuracy for female:\n",
      "    false positive rate: 0.173789\n",
      "    false negative rate: 0.227920\n",
      "    accuracy: 0.598291\n",
      "\n",
      "RACE 5 \n",
      "\n",
      "TRAINING\n",
      "-----------------------\n",
      "\n",
      "accuracy rate:  0.35\n",
      "accuracy rate:  0.35443037974683544\n",
      "accuracy rate:  0.7721518987341772\n",
      "accuracy rate:  0.6075949367088608\n",
      "accuracy rate:  0.4810126582278481\n",
      "accuracy rate:  0.6835443037974683\n",
      "accuracy rate:  0.8227848101265823\n",
      "accuracy rate:  0.7215189873417721\n",
      "accuracy rate:  0.620253164556962\n",
      "accuracy rate:  0.6582278481012658\n",
      "\n",
      "RUNNING THE BEST MODEL\n",
      "-----------------------\n",
      "\n",
      "false positive rate: 0.088608\n",
      "false negative rate: 0.088608\n",
      "accuracy: 0.822785\n",
      "\n",
      "ACCURACY BY SEX\n",
      "-----------------------\n",
      "\n",
      "Accuracy for male:\n",
      "    false positive rate: 0.051282\n",
      "    false negative rate: 0.076923\n",
      "    accuracy: 0.871795\n",
      "Accuracy for female:\n",
      "    false positive rate: 0.125000\n",
      "    false negative rate: 0.100000\n",
      "    accuracy: 0.775000\n",
      "\n",
      "RACE 6 \n",
      "\n",
      "TRAINING\n",
      "-----------------------\n",
      "\n",
      "accuracy rate:  0.6144578313253012\n",
      "accuracy rate:  0.4578313253012048\n",
      "accuracy rate:  0.7590361445783133\n",
      "accuracy rate:  0.6506024096385542\n",
      "accuracy rate:  0.6585365853658537\n",
      "accuracy rate:  0.6707317073170732\n",
      "accuracy rate:  0.7195121951219512\n",
      "accuracy rate:  0.6097560975609756\n",
      "accuracy rate:  0.524390243902439\n",
      "accuracy rate:  0.7804878048780488\n",
      "\n",
      "RUNNING THE BEST MODEL\n",
      "-----------------------\n",
      "\n",
      "false positive rate: 0.012195\n",
      "false negative rate: 0.207317\n",
      "accuracy: 0.780488\n",
      "\n",
      "ACCURACY BY SEX\n",
      "-----------------------\n",
      "\n",
      "Accuracy for male:\n",
      "    false positive rate: 0.000000\n",
      "    false negative rate: 0.200000\n",
      "    accuracy: 0.800000\n",
      "Accuracy for female:\n",
      "    false positive rate: 0.023810\n",
      "    false negative rate: 0.214286\n",
      "    accuracy: 0.761905\n",
      "\n",
      "RACE 7 \n",
      "\n",
      "TRAINING\n",
      "-----------------------\n",
      "\n",
      "accuracy rate:  0.4805194805194805\n",
      "accuracy rate:  0.5064935064935064\n",
      "accuracy rate:  0.36363636363636365\n",
      "accuracy rate:  0.6363636363636364\n",
      "accuracy rate:  0.2987012987012987\n",
      "accuracy rate:  0.4025974025974026\n",
      "accuracy rate:  0.35064935064935066\n",
      "accuracy rate:  0.47368421052631576\n",
      "accuracy rate:  0.47368421052631576\n",
      "accuracy rate:  0.5\n",
      "\n",
      "RUNNING THE BEST MODEL\n",
      "-----------------------\n",
      "\n",
      "false positive rate: 0.324675\n",
      "false negative rate: 0.038961\n",
      "accuracy: 0.636364\n",
      "\n",
      "ACCURACY BY SEX\n",
      "-----------------------\n",
      "\n",
      "Accuracy for male:\n",
      "    false positive rate: 0.344828\n",
      "    false negative rate: 0.068966\n",
      "    accuracy: 0.586207\n",
      "Accuracy for female:\n",
      "    false positive rate: 0.312500\n",
      "    false negative rate: 0.020833\n",
      "    accuracy: 0.666667\n",
      "\n",
      "RACE 8 \n",
      "\n",
      "TRAINING\n",
      "-----------------------\n",
      "\n",
      "accuracy rate:  0.18518518518518517\n",
      "accuracy rate:  0.2777777777777778\n",
      "accuracy rate:  0.3888888888888889\n",
      "accuracy rate:  0.4074074074074074\n",
      "accuracy rate:  0.37037037037037035\n",
      "accuracy rate:  0.25925925925925924\n",
      "accuracy rate:  0.3888888888888889\n",
      "accuracy rate:  0.5\n",
      "accuracy rate:  0.5925925925925926\n",
      "accuracy rate:  0.4074074074074074\n",
      "\n",
      "RUNNING THE BEST MODEL\n",
      "-----------------------\n",
      "\n",
      "false positive rate: 0.240741\n",
      "false negative rate: 0.166667\n",
      "accuracy: 0.592593\n",
      "\n",
      "ACCURACY BY SEX\n",
      "-----------------------\n",
      "\n",
      "Accuracy for male:\n",
      "    false positive rate: 0.227273\n",
      "    false negative rate: 0.136364\n",
      "    accuracy: 0.636364\n",
      "Accuracy for female:\n",
      "    false positive rate: 0.250000\n",
      "    false negative rate: 0.187500\n",
      "    accuracy: 0.562500\n",
      "\n",
      "false positive rate: 0.179008\n",
      "false negative rate: 0.134351\n",
      "accuracy: 0.686641\n"
     ]
    }
   ],
   "source": [
    "fp_overall = 0\n",
    "fn_overall = 0\n",
    "accuracy_overall = 0\n",
    "total = 0\n",
    "with open(\"nNB_acs.csv\", \"w\") as fo:\n",
    "    writer = csv.writer(fo)\n",
    "    writer.writerow(['race', 'false positive', 'false negative', 'accuracy'])\n",
    "    \n",
    "    for i in range(len(race_fields)):\n",
    "        if len(race_fields[i]) > 350:\n",
    "            print()\n",
    "            print(\"RACE %d \" % i)\n",
    "            print()\n",
    "            res, acc, fp, fn = run_race(i, race_fields[i], race_labels[i])\n",
    "            writer.writerow([i, fp, fn, acc])\n",
    "            fp_overall += fp*len(res)\n",
    "            fn_overall += fn*len(res)\n",
    "            accuracy_overall += acc*len(res)\n",
    "            total += len(res)\n",
    "print()\n",
    "print(\"false positive rate: %4f\" % (fp_overall/total))\n",
    "print(\"false negative rate: %4f\" % (fn_overall/total))\n",
    "print(\"accuracy: %4f\" % (accuracy_overall/total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}