data_collection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "     follow_request_sent  has_extended_profile  profile_use_background_image  \\\n0                  False                 False                         False   \n1                  False                  True                         False   \n2                  False                 False                          True   \n3                  False                  True                          True   \n4                  False                 False                          True   \n..                   ...                   ...                           ...   \n514                False                 False                          True   \n515                False                  True                          True   \n516                False                 False                          True   \n517                False                  True                         False   \n518                False                  True                          True   \n\n     default_profile_image                  id  \\\n0                    False           602249341   \n1                    False  797617218511060992   \n2                    False           889925474   \n3                    False            96435556   \n4                    False            16905397   \n..                     ...                 ...   \n514                  False            96766049   \n515                  False            19765236   \n516                  False          1069984033   \n517                  False           986789138   \n518                  False  720338157733068800   \n\n                    profile_background_image_url_https  verified  \\\n0    https://abs.twimg.com/images/themes/theme4/bg.gif     False   \n1    https://abs.twimg.com/images/themes/theme1/bg.png     False   \n2    https://pbs.twimg.com/profile_background_image...     False   \n3    https://abs.twimg.com/images/themes/theme6/bg.gif     False   \n4    https://pbs.twimg.com/profile_background_image...     False   \n..                                                 ...       ...   \n514  https://abs.twimg.com/images/themes/theme1/bg.png     False   \n515  https://pbs.twimg.com/profile_background_image...     False   \n516  https://abs.twimg.com/images/themes/theme1/bg.png     False   \n517  https://abs.twimg.com/images/themes/theme1/bg.png     False   \n518                                               None     False   \n\n    translator_type profile_text_color  \\\n0              none             000000   \n1              none             000000   \n2              none             333333   \n3              none             333333   \n4           regular             666666   \n..              ...                ...   \n514            none             333333   \n515            none             333333   \n516            none             333333   \n517            none             000000   \n518            none             333333   \n\n                               profile_image_url_https  ... notifications  \\\n0    https://pbs.twimg.com/profile_images/923924342...  ...         False   \n1    https://pbs.twimg.com/profile_images/855244571...  ...         False   \n2    https://pbs.twimg.com/profile_images/964079832...  ...         False   \n3    https://pbs.twimg.com/profile_images/311429969...  ...         False   \n4    https://pbs.twimg.com/profile_images/969705141...  ...         False   \n..                                                 ...  ...           ...   \n514  https://pbs.twimg.com/profile_images/783109664...  ...         False   \n515  https://pbs.twimg.com/profile_images/963844227...  ...         False   \n516  https://pbs.twimg.com/profile_images/797121046...  ...         False   \n517  https://pbs.twimg.com/profile_images/932752622...  ...         False   \n518  https://pbs.twimg.com/profile_images/984456758...  ...         False   \n\n                         url                      created_at  \\\n0    https://t.co/e5t6p9w7D8  Thu Jun 07 22:16:27 +0000 2012   \n1                       None  Sun Nov 13 01:48:58 +0000 2016   \n2     http://t.co/7gh2Iu1AT4  Thu Oct 18 23:19:38 +0000 2012   \n3                       None  Sat Dec 12 22:53:04 +0000 2009   \n4    https://t.co/VRgsX8eVR2  Wed Oct 22 13:43:42 +0000 2008   \n..                       ...                             ...   \n514  https://t.co/BgFtjCD2q9  Mon Dec 14 14:01:49 +0000 2009   \n515  https://t.co/80qIwR30Z7  Fri Jan 30 12:31:25 +0000 2009   \n516                     None  Tue Jan 08 04:36:06 +0000 2013   \n517                     None  Mon Dec 03 14:58:40 +0000 2012   \n518                     None  Wed Apr 13 19:49:34 +0000 2016   \n\n    contributors_enabled                   time_zone protected  \\\n0                  False                      London     False   \n1                  False  Pacific Time (US & Canada)     False   \n2                  False                        None     False   \n3                  False                        Rome     False   \n4                  False                        Bern     False   \n..                   ...                         ...       ...   \n514                False                      Dublin     False   \n515                False            America/New_York     False   \n516                False  Eastern Time (US & Canada)     False   \n517                False  Eastern Time (US & Canada)     False   \n518                False                        None     False   \n\n     default_profile  is_translator              collected_at  label  \n0              False          False 2018-04-16 19:28:33+00:00  human  \n1              False          False 2018-04-18 12:55:26+00:00    bot  \n2              False          False 2018-04-17 23:07:08+00:00  human  \n3              False          False 2018-04-18 12:58:28+00:00    bot  \n4              False          False 2018-04-18 14:09:01+00:00  human  \n..               ...            ...                       ...    ...  \n514             True          False 2018-04-18 12:47:09+00:00  human  \n515            False          False 2018-04-18 14:45:18+00:00  human  \n516             True          False 2018-04-18 13:06:00+00:00  human  \n517            False          False 2018-03-31 10:53:34+00:00  human  \n518             True          False 2018-04-18 00:06:49+00:00  human  \n\n[519 rows x 44 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>follow_request_sent</th>\n      <th>has_extended_profile</th>\n      <th>profile_use_background_image</th>\n      <th>default_profile_image</th>\n      <th>id</th>\n      <th>profile_background_image_url_https</th>\n      <th>verified</th>\n      <th>translator_type</th>\n      <th>profile_text_color</th>\n      <th>profile_image_url_https</th>\n      <th>...</th>\n      <th>notifications</th>\n      <th>url</th>\n      <th>created_at</th>\n      <th>contributors_enabled</th>\n      <th>time_zone</th>\n      <th>protected</th>\n      <th>default_profile</th>\n      <th>is_translator</th>\n      <th>collected_at</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>602249341</td>\n      <td>https://abs.twimg.com/images/themes/theme4/bg.gif</td>\n      <td>False</td>\n      <td>none</td>\n      <td>000000</td>\n      <td>https://pbs.twimg.com/profile_images/923924342...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>https://t.co/e5t6p9w7D8</td>\n      <td>Thu Jun 07 22:16:27 +0000 2012</td>\n      <td>False</td>\n      <td>London</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-04-16 19:28:33+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>797617218511060992</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>000000</td>\n      <td>https://pbs.twimg.com/profile_images/855244571...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Sun Nov 13 01:48:58 +0000 2016</td>\n      <td>False</td>\n      <td>Pacific Time (US &amp; Canada)</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-04-18 12:55:26+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>889925474</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/964079832...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>http://t.co/7gh2Iu1AT4</td>\n      <td>Thu Oct 18 23:19:38 +0000 2012</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-04-17 23:07:08+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>False</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>96435556</td>\n      <td>https://abs.twimg.com/images/themes/theme6/bg.gif</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/311429969...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Sat Dec 12 22:53:04 +0000 2009</td>\n      <td>False</td>\n      <td>Rome</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-04-18 12:58:28+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>16905397</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>regular</td>\n      <td>666666</td>\n      <td>https://pbs.twimg.com/profile_images/969705141...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>https://t.co/VRgsX8eVR2</td>\n      <td>Wed Oct 22 13:43:42 +0000 2008</td>\n      <td>False</td>\n      <td>Bern</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-04-18 14:09:01+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>514</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>96766049</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/783109664...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>https://t.co/BgFtjCD2q9</td>\n      <td>Mon Dec 14 14:01:49 +0000 2009</td>\n      <td>False</td>\n      <td>Dublin</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2018-04-18 12:47:09+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>515</th>\n      <td>False</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>19765236</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/963844227...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>https://t.co/80qIwR30Z7</td>\n      <td>Fri Jan 30 12:31:25 +0000 2009</td>\n      <td>False</td>\n      <td>America/New_York</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-04-18 14:45:18+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>516</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>1069984033</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/797121046...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Tue Jan 08 04:36:06 +0000 2013</td>\n      <td>False</td>\n      <td>Eastern Time (US &amp; Canada)</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2018-04-18 13:06:00+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>517</th>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>986789138</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>000000</td>\n      <td>https://pbs.twimg.com/profile_images/932752622...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Mon Dec 03 14:58:40 +0000 2012</td>\n      <td>False</td>\n      <td>Eastern Time (US &amp; Canada)</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-03-31 10:53:34+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>518</th>\n      <td>False</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>720338157733068800</td>\n      <td>None</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/984456758...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Wed Apr 13 19:49:34 +0000 2016</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2018-04-18 00:06:49+00:00</td>\n      <td>human</td>\n    </tr>\n  </tbody>\n</table>\n<p>519 rows × 44 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "# botometer-feedback-2019\n",
    "# Description: Botometer feedback accounts manually labeled by K.C. Yang. Labels and user objects.\n",
    "\n",
    "botometer_feedback_2019 = pd.read_json('botometer-feedback-2019/botometer-feedback-2019_tweets.json', orient='records')\n",
    "botometer_2019_df = pd.DataFrame(list(botometer_feedback_2019['user']))\n",
    "botometer_2019_df['collected_at'] = botometer_feedback_2019['created_at']\n",
    "\n",
    "# data labels\n",
    "botometer_2019_labels = pd.read_csv('botometer-feedback-2019/botometer-feedback-2019.tsv', sep='\\t', header=None)\n",
    "botometer_2019_labels.columns = ['id', 'label']\n",
    "\n",
    "# merge in labels on id\n",
    "botometer_2019_df = botometer_2019_df.merge(botometer_2019_labels, how='left')\n",
    "\n",
    "botometer_2019_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                     id              id_str  \\\n0    969398693249126400  969398693249126400   \n1            2402022649          2402022649   \n2            2976541239          2976541239   \n3            4425828741          4425828741   \n4    817082622824169472  817082622824169472   \n..                  ...                 ...   \n693  925657497238507520  925657497238507520   \n694  772202703212703744  772202703212703744   \n695  709947590268157952  709947590268157952   \n696  993674729830612992  993674729830612992   \n697  971203153269018624  971203153269018624   \n\n                                    name     screen_name           location  \\\n0            Random Goth Lyric Generator     goth_lyrics        City of Dis   \n1                         Transiting Now   transitingnow              Earth   \n2                          TheRiddlerBot   TheRiddlerBot  Coimbra, Portugal   \n3                              🌊🌊🌊🌊🌊🌊🌊🌊🌊       tiny_seas           sui.sexy   \n4                          Wordfindatron   wordfindatron           The Grid   \n..                                   ...             ...                ...   \n693                         Nonogram Bot    nonogram_bot                      \n694                    Clitoris Vulgaris      clitoscope                      \n695                     Tiny Dungeon Bot    TinyDungeons                      \n696                           Connect 4⃣  BotConnectFour                      \n697  fake lyrics from the mountain goats   faketmglyrics        Going to...   \n\n                                           description  \\\n0    Esoteric dark verse from aeons before twitter ...   \n1    Demonstrating the vast number of transiting ex...   \n2    Solve the riddle by replying only the name of ...   \n3    ✨🌊 by @suisea ✨ donate: https://t.co/MS5qM3a8B...   \n4    I make little word-find puzzles. Bot by @joshl...   \n..                                                 ...   \n693  Bot generating nonogram puzzles from 5x5 to15x...   \n694  A compendium of newly discovered species of cl...   \n695  I make tiny dungeons meant to look like someth...   \n696  Challenge a friend to disc-dropping fun with a...   \n697  (fake) nice boys who make the rocking tunes. I...   \n\n                         url  \\\n0    https://t.co/s9UkVMDACm   \n1    https://t.co/L8ARV5JWBz   \n2    https://t.co/1v8BON9QpT   \n3    https://t.co/Q6hC8oOXfQ   \n4    https://t.co/Dn62jWBqAe   \n..                       ...   \n693  https://t.co/bmEXmuW1sn   \n694  https://t.co/lq7rOoR2LL   \n695                     None   \n696  https://t.co/ofTK5VsnFI   \n697  https://t.co/KLRkVFp0Sf   \n\n                                              entities  protected  \\\n0    {'url': {'urls': [{'url': 'https://t.co/s9UkVM...      False   \n1    {'url': {'urls': [{'url': 'https://t.co/L8ARV5...      False   \n2    {'url': {'urls': [{'url': 'https://t.co/1v8BON...      False   \n3    {'url': {'urls': [{'url': 'https://t.co/Q6hC8o...      False   \n4    {'url': {'urls': [{'url': 'https://t.co/Dn62jW...      False   \n..                                                 ...        ...   \n693  {'url': {'urls': [{'url': 'https://t.co/bmEXmu...      False   \n694  {'url': {'urls': [{'url': 'https://t.co/lq7rOo...      False   \n695  {'description': {'urls': [{'url': 'https://t.c...      False   \n696  {'url': {'urls': [{'url': 'https://t.co/ofTK5V...      False   \n697  {'url': {'urls': [{'url': 'https://t.co/KLRkVF...      False   \n\n     followers_count  ...  profile_use_background_image  has_extended_profile  \\\n0                 23  ...                         False                  True   \n1                322  ...                          True                 False   \n2                171  ...                          True                 False   \n3               3374  ...                          True                 False   \n4                  7  ...                         False                 False   \n..               ...  ...                           ...                   ...   \n693               14  ...                          True                 False   \n694              186  ...                         False                 False   \n695              406  ...                          True                 False   \n696               11  ...                          True                 False   \n697               55  ...                          True                 False   \n\n    default_profile  default_profile_image following follow_request_sent  \\\n0             False                  False     False               False   \n1             False                  False     False               False   \n2              True                  False     False               False   \n3              True                  False     False               False   \n4             False                  False     False               False   \n..              ...                    ...       ...                 ...   \n693            True                  False     False               False   \n694           False                  False     False               False   \n695            True                  False     False               False   \n696            True                  False     False               False   \n697            True                  False     False               False   \n\n     notifications  translator_type              collected_at label  \n0            False             none 2019-06-05 21:43:20+00:00   bot  \n1            False             none 2019-04-02 13:26:02+00:00   bot  \n2            False             none 2019-06-05 20:22:03+00:00   bot  \n3            False             none 2019-06-05 18:56:21+00:00   bot  \n4            False             none 2019-06-05 20:54:02+00:00   bot  \n..             ...              ...                       ...   ...  \n693          False             none 2019-06-05 22:38:09+00:00   bot  \n694          False             none 2019-06-05 21:18:54+00:00   bot  \n695          False             none 2019-06-05 20:30:02+00:00   bot  \n696          False             none 2019-05-03 20:17:32+00:00   bot  \n697          False             none 2019-06-05 10:12:31+00:00   bot  \n\n[698 rows x 44 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>id_str</th>\n      <th>name</th>\n      <th>screen_name</th>\n      <th>location</th>\n      <th>description</th>\n      <th>url</th>\n      <th>entities</th>\n      <th>protected</th>\n      <th>followers_count</th>\n      <th>...</th>\n      <th>profile_use_background_image</th>\n      <th>has_extended_profile</th>\n      <th>default_profile</th>\n      <th>default_profile_image</th>\n      <th>following</th>\n      <th>follow_request_sent</th>\n      <th>notifications</th>\n      <th>translator_type</th>\n      <th>collected_at</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>969398693249126400</td>\n      <td>969398693249126400</td>\n      <td>Random Goth Lyric Generator</td>\n      <td>goth_lyrics</td>\n      <td>City of Dis</td>\n      <td>Esoteric dark verse from aeons before twitter ...</td>\n      <td>https://t.co/s9UkVMDACm</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/s9UkVM...</td>\n      <td>False</td>\n      <td>23</td>\n      <td>...</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 21:43:20+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2402022649</td>\n      <td>2402022649</td>\n      <td>Transiting Now</td>\n      <td>transitingnow</td>\n      <td>Earth</td>\n      <td>Demonstrating the vast number of transiting ex...</td>\n      <td>https://t.co/L8ARV5JWBz</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/L8ARV5...</td>\n      <td>False</td>\n      <td>322</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-04-02 13:26:02+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2976541239</td>\n      <td>2976541239</td>\n      <td>TheRiddlerBot</td>\n      <td>TheRiddlerBot</td>\n      <td>Coimbra, Portugal</td>\n      <td>Solve the riddle by replying only the name of ...</td>\n      <td>https://t.co/1v8BON9QpT</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/1v8BON...</td>\n      <td>False</td>\n      <td>171</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 20:22:03+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>4425828741</td>\n      <td>4425828741</td>\n      <td>🌊🌊🌊🌊🌊🌊🌊🌊🌊</td>\n      <td>tiny_seas</td>\n      <td>sui.sexy</td>\n      <td>✨🌊 by @suisea ✨ donate: https://t.co/MS5qM3a8B...</td>\n      <td>https://t.co/Q6hC8oOXfQ</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/Q6hC8o...</td>\n      <td>False</td>\n      <td>3374</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 18:56:21+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>817082622824169472</td>\n      <td>817082622824169472</td>\n      <td>Wordfindatron</td>\n      <td>wordfindatron</td>\n      <td>The Grid</td>\n      <td>I make little word-find puzzles. Bot by @joshl...</td>\n      <td>https://t.co/Dn62jWBqAe</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/Dn62jW...</td>\n      <td>False</td>\n      <td>7</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 20:54:02+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>693</th>\n      <td>925657497238507520</td>\n      <td>925657497238507520</td>\n      <td>Nonogram Bot</td>\n      <td>nonogram_bot</td>\n      <td></td>\n      <td>Bot generating nonogram puzzles from 5x5 to15x...</td>\n      <td>https://t.co/bmEXmuW1sn</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/bmEXmu...</td>\n      <td>False</td>\n      <td>14</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 22:38:09+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>694</th>\n      <td>772202703212703744</td>\n      <td>772202703212703744</td>\n      <td>Clitoris Vulgaris</td>\n      <td>clitoscope</td>\n      <td></td>\n      <td>A compendium of newly discovered species of cl...</td>\n      <td>https://t.co/lq7rOoR2LL</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/lq7rOo...</td>\n      <td>False</td>\n      <td>186</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 21:18:54+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>695</th>\n      <td>709947590268157952</td>\n      <td>709947590268157952</td>\n      <td>Tiny Dungeon Bot</td>\n      <td>TinyDungeons</td>\n      <td></td>\n      <td>I make tiny dungeons meant to look like someth...</td>\n      <td>None</td>\n      <td>{'description': {'urls': [{'url': 'https://t.c...</td>\n      <td>False</td>\n      <td>406</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 20:30:02+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>696</th>\n      <td>993674729830612992</td>\n      <td>993674729830612992</td>\n      <td>Connect 4⃣</td>\n      <td>BotConnectFour</td>\n      <td></td>\n      <td>Challenge a friend to disc-dropping fun with a...</td>\n      <td>https://t.co/ofTK5VsnFI</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/ofTK5V...</td>\n      <td>False</td>\n      <td>11</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-05-03 20:17:32+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>697</th>\n      <td>971203153269018624</td>\n      <td>971203153269018624</td>\n      <td>fake lyrics from the mountain goats</td>\n      <td>faketmglyrics</td>\n      <td>Going to...</td>\n      <td>(fake) nice boys who make the rocking tunes. I...</td>\n      <td>https://t.co/KLRkVFp0Sf</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/KLRkVF...</td>\n      <td>False</td>\n      <td>55</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 10:12:31+00:00</td>\n      <td>bot</td>\n    </tr>\n  </tbody>\n</table>\n<p>698 rows × 44 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "# botwiki-2019\n",
    "# Description: Self-identified bots from https://botwiki.org. Labels and user objects.\n",
    "botwiki_2019_tweets = pd.read_json('botwiki-2019/botwiki-2019_tweets.json')\n",
    "botwiki_2019_df = pd.DataFrame(list(botwiki_2019_tweets['user']))\n",
    "botwiki_2019_df['collected_at'] = botwiki_2019_tweets['created_at']\n",
    "botwiki_2019_df\n",
    "\n",
    "# data labels\n",
    "botwiki_2019_labels = pd.read_csv('botwiki-2019/botwiki-2019.tsv', sep='\\t', header=None)\n",
    "botwiki_2019_labels.columns = ['id', 'label']\n",
    "\n",
    "# merge in labels on id\n",
    "botwiki_2019_df = botwiki_2019_df.merge(botwiki_2019_labels, how='left')\n",
    "\n",
    "botwiki_2019_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "       follow_request_sent  has_extended_profile  \\\n0                    False                 False   \n1                    False                  True   \n2                    False                 False   \n3                    False                 False   \n4                    False                 False   \n...                    ...                   ...   \n20978                False                 False   \n20979                False                 False   \n20980                False                 False   \n20981                False                 False   \n20982                False                 False   \n\n       profile_use_background_image  default_profile_image          id  \\\n0                              True                  False   396086694   \n1                             False                  False    15736267   \n2                             False                  False    47753979   \n3                              True                  False  2330330082   \n4                              True                  False   605640214   \n...                             ...                    ...         ...   \n20978                          True                  False   195720419   \n20979                         False                  False  2469713144   \n20980                          True                  False    15212604   \n20981                          True                  False    50055701   \n20982                          True                  False   124661400   \n\n                      profile_background_image_url_https  verified  \\\n0      https://abs.twimg.com/images/themes/theme9/bg.gif     False   \n1      https://pbs.twimg.com/profile_background_image...      True   \n2      https://abs.twimg.com/images/themes/theme1/bg.png      True   \n3      https://abs.twimg.com/images/themes/theme1/bg.png      True   \n4      https://pbs.twimg.com/profile_background_image...     False   \n...                                                  ...       ...   \n20978  https://abs.twimg.com/images/themes/theme15/bg...     False   \n20979  https://abs.twimg.com/images/themes/theme1/bg.png     False   \n20980  https://pbs.twimg.com/profile_background_image...      True   \n20981  https://pbs.twimg.com/profile_background_image...      True   \n20982  https://abs.twimg.com/images/themes/theme17/bg...     False   \n\n      translator_type profile_text_color  \\\n0             regular             666666   \n1                none             000000   \n2                none             000000   \n3                none             333333   \n4                none             333333   \n...               ...                ...   \n20978            none             333333   \n20979            none             000000   \n20980         regular             0F000F   \n20981            none             000000   \n20982            none             333333   \n\n                                 profile_image_url_https  ...  \\\n0      https://pbs.twimg.com/profile_images/739928180...  ...   \n1      https://pbs.twimg.com/profile_images/937781278...  ...   \n2      https://pbs.twimg.com/profile_images/641944065...  ...   \n3      https://pbs.twimg.com/profile_images/654930765...  ...   \n4      https://pbs.twimg.com/profile_images/827565905...  ...   \n...                                                  ...  ...   \n20978  https://pbs.twimg.com/profile_images/634380552...  ...   \n20979  https://pbs.twimg.com/profile_images/967092763...  ...   \n20980  https://pbs.twimg.com/profile_images/966674205...  ...   \n20981  https://pbs.twimg.com/profile_images/964489716...  ...   \n20982  https://pbs.twimg.com/profile_images/974646547...  ...   \n\n                           url                      created_at  \\\n0      https://t.co/M2XReaeps7  Sat Oct 22 18:34:26 +0000 2011   \n1       http://t.co/v44qPcZZnz  Tue Aug 05 14:06:35 +0000 2008   \n2      https://t.co/QSjNcqaA3I  Tue Jun 16 22:21:50 +0000 2009   \n3      https://t.co/0u1PhNzb4l  Thu Feb 06 13:56:38 +0000 2014   \n4       http://t.co/OsXYRK85qi  Mon Jun 11 18:28:11 +0000 2012   \n...                        ...                             ...   \n20978   http://t.co/mLByVp8LIq  Mon Sep 27 12:27:20 +0000 2010   \n20979   http://t.co/K6L7KXX4T8  Tue Apr 29 18:33:57 +0000 2014   \n20980  https://t.co/sLEZ3lnXXm  Mon Jun 23 22:14:20 +0000 2008   \n20981  https://t.co/DUwhifbm63  Tue Jun 23 18:24:04 +0000 2009   \n20982  https://t.co/3pjGfarW6F  Sat Mar 20 04:19:50 +0000 2010   \n\n       contributors_enabled                   time_zone protected  \\\n0                     False                   New Delhi     False   \n1                     False                       Paris     False   \n2                     False                       Quito     False   \n3                     False                       human     False   \n4                     False  Eastern Time (US & Canada)     False   \n...                     ...                         ...       ...   \n20978                 False                      Muscat     False   \n20979                 False                      London     False   \n20980                 False                 Mexico City     False   \n20981                 False  Eastern Time (US & Canada)     False   \n20982                 False  Central Time (US & Canada)     False   \n\n      default_profile  is_translator  withheld_in_countries  \\\n0               False          False                  human   \n1               False          False                  human   \n2               False          False                  human   \n3                True          False                  human   \n4               False          False                  human   \n...               ...            ...                    ...   \n20978           False          False                  human   \n20979           False          False                  human   \n20980           False          False                  human   \n20981           False          False                  human   \n20982           False          False                  human   \n\n                   collected_at  label  \n0     2018-03-26 02:00:40+00:00  human  \n1     2018-03-26 01:19:57+00:00  human  \n2     2018-03-25 23:46:26+00:00  human  \n3     2018-03-19 11:27:22+00:00  human  \n4     2018-03-23 15:14:40+00:00  human  \n...                         ...    ...  \n20978 2017-01-23 19:09:56+00:00  human  \n20979 2018-03-26 12:56:24+00:00  human  \n20980 2018-03-27 12:05:00+00:00  human  \n20981 2018-03-22 01:12:58+00:00  human  \n20982 2018-03-27 14:27:39+00:00  human  \n\n[20983 rows x 45 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>follow_request_sent</th>\n      <th>has_extended_profile</th>\n      <th>profile_use_background_image</th>\n      <th>default_profile_image</th>\n      <th>id</th>\n      <th>profile_background_image_url_https</th>\n      <th>verified</th>\n      <th>translator_type</th>\n      <th>profile_text_color</th>\n      <th>profile_image_url_https</th>\n      <th>...</th>\n      <th>url</th>\n      <th>created_at</th>\n      <th>contributors_enabled</th>\n      <th>time_zone</th>\n      <th>protected</th>\n      <th>default_profile</th>\n      <th>is_translator</th>\n      <th>withheld_in_countries</th>\n      <th>collected_at</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>396086694</td>\n      <td>https://abs.twimg.com/images/themes/theme9/bg.gif</td>\n      <td>False</td>\n      <td>regular</td>\n      <td>666666</td>\n      <td>https://pbs.twimg.com/profile_images/739928180...</td>\n      <td>...</td>\n      <td>https://t.co/M2XReaeps7</td>\n      <td>Sat Oct 22 18:34:26 +0000 2011</td>\n      <td>False</td>\n      <td>New Delhi</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>human</td>\n      <td>2018-03-26 02:00:40+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>15736267</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>True</td>\n      <td>none</td>\n      <td>000000</td>\n      <td>https://pbs.twimg.com/profile_images/937781278...</td>\n      <td>...</td>\n      <td>http://t.co/v44qPcZZnz</td>\n      <td>Tue Aug 05 14:06:35 +0000 2008</td>\n      <td>False</td>\n      <td>Paris</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>human</td>\n      <td>2018-03-26 01:19:57+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>47753979</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>True</td>\n      <td>none</td>\n      <td>000000</td>\n      <td>https://pbs.twimg.com/profile_images/641944065...</td>\n      <td>...</td>\n      <td>https://t.co/QSjNcqaA3I</td>\n      <td>Tue Jun 16 22:21:50 +0000 2009</td>\n      <td>False</td>\n      <td>Quito</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>human</td>\n      <td>2018-03-25 23:46:26+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2330330082</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>True</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/654930765...</td>\n      <td>...</td>\n      <td>https://t.co/0u1PhNzb4l</td>\n      <td>Thu Feb 06 13:56:38 +0000 2014</td>\n      <td>False</td>\n      <td>human</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>human</td>\n      <td>2018-03-19 11:27:22+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>605640214</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/827565905...</td>\n      <td>...</td>\n      <td>http://t.co/OsXYRK85qi</td>\n      <td>Mon Jun 11 18:28:11 +0000 2012</td>\n      <td>False</td>\n      <td>Eastern Time (US &amp; Canada)</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>human</td>\n      <td>2018-03-23 15:14:40+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>20978</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>195720419</td>\n      <td>https://abs.twimg.com/images/themes/theme15/bg...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/634380552...</td>\n      <td>...</td>\n      <td>http://t.co/mLByVp8LIq</td>\n      <td>Mon Sep 27 12:27:20 +0000 2010</td>\n      <td>False</td>\n      <td>Muscat</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>human</td>\n      <td>2017-01-23 19:09:56+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>20979</th>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2469713144</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>000000</td>\n      <td>https://pbs.twimg.com/profile_images/967092763...</td>\n      <td>...</td>\n      <td>http://t.co/K6L7KXX4T8</td>\n      <td>Tue Apr 29 18:33:57 +0000 2014</td>\n      <td>False</td>\n      <td>London</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>human</td>\n      <td>2018-03-26 12:56:24+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>20980</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>15212604</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>True</td>\n      <td>regular</td>\n      <td>0F000F</td>\n      <td>https://pbs.twimg.com/profile_images/966674205...</td>\n      <td>...</td>\n      <td>https://t.co/sLEZ3lnXXm</td>\n      <td>Mon Jun 23 22:14:20 +0000 2008</td>\n      <td>False</td>\n      <td>Mexico City</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>human</td>\n      <td>2018-03-27 12:05:00+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>20981</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>50055701</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>True</td>\n      <td>none</td>\n      <td>000000</td>\n      <td>https://pbs.twimg.com/profile_images/964489716...</td>\n      <td>...</td>\n      <td>https://t.co/DUwhifbm63</td>\n      <td>Tue Jun 23 18:24:04 +0000 2009</td>\n      <td>False</td>\n      <td>Eastern Time (US &amp; Canada)</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>human</td>\n      <td>2018-03-22 01:12:58+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>20982</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>124661400</td>\n      <td>https://abs.twimg.com/images/themes/theme17/bg...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/974646547...</td>\n      <td>...</td>\n      <td>https://t.co/3pjGfarW6F</td>\n      <td>Sat Mar 20 04:19:50 +0000 2010</td>\n      <td>False</td>\n      <td>Central Time (US &amp; Canada)</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>human</td>\n      <td>2018-03-27 14:27:39+00:00</td>\n      <td>human</td>\n    </tr>\n  </tbody>\n</table>\n<p>20983 rows × 45 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "# celebrity-2019\n",
    "# Description: Celebrity accounts collected as authentic users. Labels and user objects.\n",
    "celebrity_2019_tweets = pd.read_json('celebrity-2019/celebrity-2019_tweets.json', orient='records')\n",
    "celebrity_2019_df = pd.DataFrame(list(celebrity_2019_tweets['user']))\n",
    "celebrity_2019_df['collected_at'] = celebrity_2019_tweets['created_at']\n",
    "\n",
    "# data labels\n",
    "celebrity_2019_labels = pd.read_csv('celebrity-2019/celebrity-2019.tsv', sep='\\t', header=None)\n",
    "celebrity_2019_labels.columns = ['id', 'label']\n",
    "\n",
    "# merge in labels on id\n",
    "celebrity_2019_df = celebrity_2019_df.merge(celebrity_2019_labels, how='left')\n",
    "\n",
    "# they're all human in this df but some show up as NaN after the merge\n",
    "celebrity_2019_df.fillna('human', inplace=True)\n",
    "\n",
    "celebrity_2019_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "             id                  name    screen_name  statuses_count  \\\n0       3610511      Davide Dellacasa         braddd           20370   \n1       5656162        Simone Economo          eKoeS            3131   \n2       5682702                tacone        tacone_            4024   \n3       6067292              alesaura       alesstar           40586   \n4       6015122                Angelo     PerDiletto            2016   \n..          ...                   ...            ...             ...   \n840  1098285654                    K.        k_ksaid               1   \n841  1122936860                Nick.J   Nick_DoubleJ               6   \n842  1129397028                carine  tweetyseventy               1   \n843  1151673038                Dennez  FlorianDennez               5   \n844  1281271908  I Respect the queen!     SearsTammy             207   \n\n     followers_count  friends_count  favourites_count  listed_count  \\\n0               5470           2385               145            52   \n1                506            381                 9            40   \n2                264             87               323            16   \n3                640            622              1118            32   \n4                 62             64                13             0   \n..               ...            ...               ...           ...   \n840                2            325                 0             0   \n841                2            307                 4             0   \n842                2            314                 0             0   \n843                6            482                 0             0   \n844              137            414               106             1   \n\n                         created_at                                url  ...  \\\n0    Fri Apr 06 10:58:22 +0000 2007           http://braddd.tumblr.com  ...   \n1    Mon Apr 30 15:08:42 +0000 2007         http://www.lineheight.net/  ...   \n2    Tue May 01 11:53:40 +0000 2007               http://t.co/LKrl1dZE  ...   \n3    Tue May 15 16:55:16 +0000 2007     http://alesstar.wordpress.com/  ...   \n4    Sun May 13 19:52:00 +0000 2007  http://www.flickr.com/per_diletto  ...   \n..                              ...                                ...  ...   \n840  Thu Jan 17 14:22:31 +0000 2013                                NaN  ...   \n841  Sat Jan 26 19:27:26 +0000 2013                                NaN  ...   \n842  Mon Jan 28 22:38:57 +0000 2013                                NaN  ...   \n843  Tue Feb 05 18:39:07 +0000 2013                                NaN  ...   \n844  Tue Mar 19 19:19:32 +0000 2013                                NaN  ...   \n\n                          profile_background_image_url  \\\n0    http://a0.twimg.com/profile_background_images/...   \n1      http://a0.twimg.com/images/themes/theme1/bg.png   \n2    http://a0.twimg.com/profile_background_images/...   \n3      http://a0.twimg.com/images/themes/theme4/bg.gif   \n4     http://a0.twimg.com/images/themes/theme18/bg.gif   \n..                                                 ...   \n840    http://a0.twimg.com/images/themes/theme1/bg.png   \n841    http://a0.twimg.com/images/themes/theme1/bg.png   \n842    http://a0.twimg.com/images/themes/theme9/bg.gif   \n843    http://a0.twimg.com/images/themes/theme1/bg.png   \n844    http://a0.twimg.com/images/themes/theme1/bg.png   \n\n    profile_background_color profile_link_color  utc_offset  protected  \\\n0                     BADFCD             FF0000      3600.0        NaN   \n1                     C0DEED             0084B4      3600.0        NaN   \n2                     1A1B1F             2FC2EF      3600.0        NaN   \n3                     0099B9             0099B9      3600.0        NaN   \n4                     ACDED6             038543      3600.0        NaN   \n..                       ...                ...         ...        ...   \n840                   C0DEED             0084B4         NaN        NaN   \n841                   C0DEED             0084B4         NaN        NaN   \n842                   1A1B1F             2FC2EF         NaN        NaN   \n843                   C0DEED             0084B4      3600.0        NaN   \n844                   C0DEED             0084B4         NaN        NaN   \n\n     verified                                        description  \\\n0         NaN  Founder of http://www.screenweek.it & http://w...   \n1         NaN  BSc degree (cum laude) in Computer Engineering...   \n2         NaN                             Cogito ergo bestemmio.   \n3         NaN               Se la vita ti dà sarde, scapocciale!   \n4         NaN                                     Je me souviens   \n..        ...                                                ...   \n840       NaN                                                NaN   \n841       NaN                                  세상에서 제일 달콤한것은? :)   \n842       NaN                                                NaN   \n843       NaN  Je suis plusieurs dans ma tête, c'est handicap...   \n844       NaN  I stan for lil Kim i love her music and i a ve...   \n\n            collected_at  dataset  label  \n0    2015-02-14 10:54:49      E13  human  \n1    2015-02-14 10:54:49      E13  human  \n2    2015-02-14 10:54:49      E13  human  \n3    2015-02-14 10:54:49      E13  human  \n4    2015-02-14 10:54:49      E13  human  \n..                   ...      ...    ...  \n840  2015-02-14 09:55:39      TWT    bot  \n841  2015-02-14 09:55:39      TWT    bot  \n842  2015-02-14 09:55:39      TWT    bot  \n843  2015-02-14 09:55:39      TWT    bot  \n844  2015-02-14 09:55:39      TWT    bot  \n\n[5301 rows x 35 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>name</th>\n      <th>screen_name</th>\n      <th>statuses_count</th>\n      <th>followers_count</th>\n      <th>friends_count</th>\n      <th>favourites_count</th>\n      <th>listed_count</th>\n      <th>created_at</th>\n      <th>url</th>\n      <th>...</th>\n      <th>profile_background_image_url</th>\n      <th>profile_background_color</th>\n      <th>profile_link_color</th>\n      <th>utc_offset</th>\n      <th>protected</th>\n      <th>verified</th>\n      <th>description</th>\n      <th>collected_at</th>\n      <th>dataset</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>3610511</td>\n      <td>Davide Dellacasa</td>\n      <td>braddd</td>\n      <td>20370</td>\n      <td>5470</td>\n      <td>2385</td>\n      <td>145</td>\n      <td>52</td>\n      <td>Fri Apr 06 10:58:22 +0000 2007</td>\n      <td>http://braddd.tumblr.com</td>\n      <td>...</td>\n      <td>http://a0.twimg.com/profile_background_images/...</td>\n      <td>BADFCD</td>\n      <td>FF0000</td>\n      <td>3600.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Founder of http://www.screenweek.it &amp; http://w...</td>\n      <td>2015-02-14 10:54:49</td>\n      <td>E13</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>5656162</td>\n      <td>Simone Economo</td>\n      <td>eKoeS</td>\n      <td>3131</td>\n      <td>506</td>\n      <td>381</td>\n      <td>9</td>\n      <td>40</td>\n      <td>Mon Apr 30 15:08:42 +0000 2007</td>\n      <td>http://www.lineheight.net/</td>\n      <td>...</td>\n      <td>http://a0.twimg.com/images/themes/theme1/bg.png</td>\n      <td>C0DEED</td>\n      <td>0084B4</td>\n      <td>3600.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>BSc degree (cum laude) in Computer Engineering...</td>\n      <td>2015-02-14 10:54:49</td>\n      <td>E13</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>5682702</td>\n      <td>tacone</td>\n      <td>tacone_</td>\n      <td>4024</td>\n      <td>264</td>\n      <td>87</td>\n      <td>323</td>\n      <td>16</td>\n      <td>Tue May 01 11:53:40 +0000 2007</td>\n      <td>http://t.co/LKrl1dZE</td>\n      <td>...</td>\n      <td>http://a0.twimg.com/profile_background_images/...</td>\n      <td>1A1B1F</td>\n      <td>2FC2EF</td>\n      <td>3600.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Cogito ergo bestemmio.</td>\n      <td>2015-02-14 10:54:49</td>\n      <td>E13</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>6067292</td>\n      <td>alesaura</td>\n      <td>alesstar</td>\n      <td>40586</td>\n      <td>640</td>\n      <td>622</td>\n      <td>1118</td>\n      <td>32</td>\n      <td>Tue May 15 16:55:16 +0000 2007</td>\n      <td>http://alesstar.wordpress.com/</td>\n      <td>...</td>\n      <td>http://a0.twimg.com/images/themes/theme4/bg.gif</td>\n      <td>0099B9</td>\n      <td>0099B9</td>\n      <td>3600.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Se la vita ti dà sarde, scapocciale!</td>\n      <td>2015-02-14 10:54:49</td>\n      <td>E13</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>6015122</td>\n      <td>Angelo</td>\n      <td>PerDiletto</td>\n      <td>2016</td>\n      <td>62</td>\n      <td>64</td>\n      <td>13</td>\n      <td>0</td>\n      <td>Sun May 13 19:52:00 +0000 2007</td>\n      <td>http://www.flickr.com/per_diletto</td>\n      <td>...</td>\n      <td>http://a0.twimg.com/images/themes/theme18/bg.gif</td>\n      <td>ACDED6</td>\n      <td>038543</td>\n      <td>3600.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Je me souviens</td>\n      <td>2015-02-14 10:54:49</td>\n      <td>E13</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>840</th>\n      <td>1098285654</td>\n      <td>K.</td>\n      <td>k_ksaid</td>\n      <td>1</td>\n      <td>2</td>\n      <td>325</td>\n      <td>0</td>\n      <td>0</td>\n      <td>Thu Jan 17 14:22:31 +0000 2013</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>http://a0.twimg.com/images/themes/theme1/bg.png</td>\n      <td>C0DEED</td>\n      <td>0084B4</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>2015-02-14 09:55:39</td>\n      <td>TWT</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>841</th>\n      <td>1122936860</td>\n      <td>Nick.J</td>\n      <td>Nick_DoubleJ</td>\n      <td>6</td>\n      <td>2</td>\n      <td>307</td>\n      <td>4</td>\n      <td>0</td>\n      <td>Sat Jan 26 19:27:26 +0000 2013</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>http://a0.twimg.com/images/themes/theme1/bg.png</td>\n      <td>C0DEED</td>\n      <td>0084B4</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>세상에서 제일 달콤한것은? :)</td>\n      <td>2015-02-14 09:55:39</td>\n      <td>TWT</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>842</th>\n      <td>1129397028</td>\n      <td>carine</td>\n      <td>tweetyseventy</td>\n      <td>1</td>\n      <td>2</td>\n      <td>314</td>\n      <td>0</td>\n      <td>0</td>\n      <td>Mon Jan 28 22:38:57 +0000 2013</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>http://a0.twimg.com/images/themes/theme9/bg.gif</td>\n      <td>1A1B1F</td>\n      <td>2FC2EF</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>2015-02-14 09:55:39</td>\n      <td>TWT</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>843</th>\n      <td>1151673038</td>\n      <td>Dennez</td>\n      <td>FlorianDennez</td>\n      <td>5</td>\n      <td>6</td>\n      <td>482</td>\n      <td>0</td>\n      <td>0</td>\n      <td>Tue Feb 05 18:39:07 +0000 2013</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>http://a0.twimg.com/images/themes/theme1/bg.png</td>\n      <td>C0DEED</td>\n      <td>0084B4</td>\n      <td>3600.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Je suis plusieurs dans ma tête, c'est handicap...</td>\n      <td>2015-02-14 09:55:39</td>\n      <td>TWT</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>844</th>\n      <td>1281271908</td>\n      <td>I Respect the queen!</td>\n      <td>SearsTammy</td>\n      <td>207</td>\n      <td>137</td>\n      <td>414</td>\n      <td>106</td>\n      <td>1</td>\n      <td>Tue Mar 19 19:19:32 +0000 2013</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>http://a0.twimg.com/images/themes/theme1/bg.png</td>\n      <td>C0DEED</td>\n      <td>0084B4</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>I stan for lil Kim i love her music and i a ve...</td>\n      <td>2015-02-14 09:55:39</td>\n      <td>TWT</td>\n      <td>bot</td>\n    </tr>\n  </tbody>\n</table>\n<p>5301 rows × 35 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "source": [
    "# cresci-2015\n",
    "# Description: A dataset of (i) genuine and (ii) fake Twitter accounts, manually annotated. Released in CSV format.\n",
    "# https://arxiv.org/pdf/1509.04098.pdf\n",
    "\n",
    "# human\n",
    "cresci_2015_E13 = pd.read_csv('cresci-2015/E13/users.csv')\n",
    "cresci_2015_E13['label'] = 'human'\n",
    "\n",
    "# bots\n",
    "cresci_2015_FSF = pd.read_csv('cresci-2015/FSF/users.csv')\n",
    "cresci_2015_FSF['label'] = 'bot'\n",
    "\n",
    "# bots\n",
    "cresci_2015_INT = pd.read_csv('cresci-2015/INT/users.csv')\n",
    "cresci_2015_INT['label'] = 'bot'\n",
    "\n",
    "# human\n",
    "cresci_2015_TFP = pd.read_csv('cresci-2015/TFP/users.csv')\n",
    "cresci_2015_TFP['label'] = 'human'\n",
    "\n",
    "# bots\n",
    "cresci_2015_TWT = pd.read_csv('cresci-2015/TWT/users.csv')\n",
    "cresci_2015_TWT['label'] = 'bot'\n",
    "\n",
    "# combine into one df\n",
    "cresci_2015_df = pd.concat([cresci_2015_E13, cresci_2015_FSF, cresci_2015_INT, cresci_2015_TFP, cresci_2015_TWT])\n",
    "cresci_2015_df.rename(columns={'updated':'collected_at'}, inplace=True)\n",
    "cresci_2015_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "              id                  name      screen_name  statuses_count  \\\n0     1502026416       TASUKU HAYAKAWA         0918Bask            2177   \n1     2492782375                 ro_or         1120Roll            2660   \n2      293212315              bearclaw        14KBBrown            1254   \n3      191839658     pocahontas farida      wadespeters          202968   \n4     3020965143              Ms Kathy  191a5bd05da04dc              82   \n...          ...                   ...              ...             ...   \n1123   818079408  Columbia Facil. Mgmt  tmj_cae_facmgmt              41   \n1124   846108068      NJ Business/Mgmt      tmj_nj_mgmt             218   \n1125   857076271      PR Customer Ser.   tmj_ptr_cstsrv              17   \n1126   872126906      PR Soft Dev Java   tmj_ptr_itjava               2   \n1127   979406250  Harrisburg Cust. Ser   tmj_hpa_cstsrv              47   \n\n      followers_count  friends_count  favourites_count  listed_count  \\\n0                 208            332               265             1   \n1                 330            485              3972             5   \n2                 166            177              1185             0   \n3                2248            981             60304           101   \n4                  21             79                 5             0   \n...               ...            ...               ...           ...   \n1123              169            151                 0            12   \n1124              332            245                 0            31   \n1125              530            300                 0             8   \n1126              247            202                 0             1   \n1127              267            234                 0             7   \n\n                          url lang  ...  \\\n0                         NaN   ja  ...   \n1                         NaN   ja  ...   \n2                         NaN   en  ...   \n3      http://t.co/rGV0HIJGsu   en  ...   \n4                         NaN   en  ...   \n...                       ...  ...  ...   \n1123   http://t.co/TKb82K4tFJ   en  ...   \n1124  https://t.co/DByWt4njnT   en  ...   \n1125   http://t.co/paWgrkSWfT   en  ...   \n1126   http://t.co/OnFSpFHC7G   en  ...   \n1127     http://t.co/Kz252i4r   en  ...   \n\n                                            description contributors_enabled  \\\n0                                 15years ago X.Lines24                  NaN   \n1      保守見習い地元大好き人間。 経済学、電工、仏教を勉強中、ちなDeではいかんのか？ (*^◯^*)                  NaN   \n2                    Let me see what your best move is!                  NaN   \n3     20. menna: #farida #nyc and the 80s actually y...                  NaN   \n4                                         Cosmetologist                  NaN   \n...                                                 ...                  ...   \n1123  Follow this account for geo-targeted Facilitie...                  NaN   \n1124  Follow this account for geo-targeted Business/...                  NaN   \n1125  Follow this account for geo-targeted Customer ...                  NaN   \n1126  Follow this account for geo-targeted Software ...                  NaN   \n1127  Follow this account for geo-targeted Customer ...                  NaN   \n\n      following                      created_at            timestamp  \\\n0           NaN  Tue Jun 11 11:20:35 +0000 2013  2013-06-11 13:20:35   \n1           NaN  Tue May 13 10:37:57 +0000 2014  2014-05-13 12:37:57   \n2           NaN  Wed May 04 23:30:37 +0000 2011  2011-05-05 01:30:37   \n3           NaN  Fri Sep 17 14:02:10 +0000 2010  2010-09-17 16:02:10   \n4           NaN  Fri Feb 06 04:10:49 +0000 2015  2015-02-06 05:10:49   \n...         ...                             ...                  ...   \n1123        NaN  Tue Sep 11 20:00:43 +0000 2012  2012-09-11 22:00:43   \n1124        NaN  Tue Sep 25 19:35:53 +0000 2012  2012-09-25 21:35:53   \n1125        NaN  Mon Oct 01 21:47:23 +0000 2012  2012-10-01 23:47:23   \n1126        NaN  Wed Oct 10 16:44:09 +0000 2012  2012-10-10 18:44:09   \n1127        NaN  Fri Nov 30 00:08:26 +0000 2012  2012-11-30 01:08:26   \n\n               crawled_at         collected_at  test_set_1 test_set_2  label  \n0     2015-05-02 06:41:46  2016-03-15 15:53:47         0.0        0.0  human  \n1     2015-05-01 17:20:27  2016-03-15 15:53:48         0.0        0.0  human  \n2     2015-05-01 18:48:28  2016-03-15 15:53:48         0.0        0.0  human  \n3     2015-05-01 13:55:16  2016-03-15 15:53:48         0.0        0.0  human  \n4     2015-05-02 01:17:32  2016-03-15 15:53:48         0.0        0.0  human  \n...                   ...                  ...         ...        ...    ...  \n1123  2016-03-15 13:49:15  2016-03-15 13:49:15         NaN        NaN    bot  \n1124  2016-03-15 13:49:15  2016-03-15 13:49:15         NaN        NaN    bot  \n1125  2016-03-15 13:49:15  2016-03-15 13:49:15         NaN        NaN    bot  \n1126  2016-03-15 13:49:15  2016-03-15 13:49:15         NaN        NaN    bot  \n1127  2016-03-15 13:49:15  2016-03-15 13:49:15         NaN        NaN    bot  \n\n[11017 rows x 43 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>name</th>\n      <th>screen_name</th>\n      <th>statuses_count</th>\n      <th>followers_count</th>\n      <th>friends_count</th>\n      <th>favourites_count</th>\n      <th>listed_count</th>\n      <th>url</th>\n      <th>lang</th>\n      <th>...</th>\n      <th>description</th>\n      <th>contributors_enabled</th>\n      <th>following</th>\n      <th>created_at</th>\n      <th>timestamp</th>\n      <th>crawled_at</th>\n      <th>collected_at</th>\n      <th>test_set_1</th>\n      <th>test_set_2</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1502026416</td>\n      <td>TASUKU HAYAKAWA</td>\n      <td>0918Bask</td>\n      <td>2177</td>\n      <td>208</td>\n      <td>332</td>\n      <td>265</td>\n      <td>1</td>\n      <td>NaN</td>\n      <td>ja</td>\n      <td>...</td>\n      <td>15years ago X.Lines24</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Tue Jun 11 11:20:35 +0000 2013</td>\n      <td>2013-06-11 13:20:35</td>\n      <td>2015-05-02 06:41:46</td>\n      <td>2016-03-15 15:53:47</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2492782375</td>\n      <td>ro_or</td>\n      <td>1120Roll</td>\n      <td>2660</td>\n      <td>330</td>\n      <td>485</td>\n      <td>3972</td>\n      <td>5</td>\n      <td>NaN</td>\n      <td>ja</td>\n      <td>...</td>\n      <td>保守見習い地元大好き人間。 経済学、電工、仏教を勉強中、ちなDeではいかんのか？ (*^◯^*)</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Tue May 13 10:37:57 +0000 2014</td>\n      <td>2014-05-13 12:37:57</td>\n      <td>2015-05-01 17:20:27</td>\n      <td>2016-03-15 15:53:48</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>293212315</td>\n      <td>bearclaw</td>\n      <td>14KBBrown</td>\n      <td>1254</td>\n      <td>166</td>\n      <td>177</td>\n      <td>1185</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>en</td>\n      <td>...</td>\n      <td>Let me see what your best move is!</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Wed May 04 23:30:37 +0000 2011</td>\n      <td>2011-05-05 01:30:37</td>\n      <td>2015-05-01 18:48:28</td>\n      <td>2016-03-15 15:53:48</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>191839658</td>\n      <td>pocahontas farida</td>\n      <td>wadespeters</td>\n      <td>202968</td>\n      <td>2248</td>\n      <td>981</td>\n      <td>60304</td>\n      <td>101</td>\n      <td>http://t.co/rGV0HIJGsu</td>\n      <td>en</td>\n      <td>...</td>\n      <td>20. menna: #farida #nyc and the 80s actually y...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Fri Sep 17 14:02:10 +0000 2010</td>\n      <td>2010-09-17 16:02:10</td>\n      <td>2015-05-01 13:55:16</td>\n      <td>2016-03-15 15:53:48</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>3020965143</td>\n      <td>Ms Kathy</td>\n      <td>191a5bd05da04dc</td>\n      <td>82</td>\n      <td>21</td>\n      <td>79</td>\n      <td>5</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>en</td>\n      <td>...</td>\n      <td>Cosmetologist</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Fri Feb 06 04:10:49 +0000 2015</td>\n      <td>2015-02-06 05:10:49</td>\n      <td>2015-05-02 01:17:32</td>\n      <td>2016-03-15 15:53:48</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1123</th>\n      <td>818079408</td>\n      <td>Columbia Facil. Mgmt</td>\n      <td>tmj_cae_facmgmt</td>\n      <td>41</td>\n      <td>169</td>\n      <td>151</td>\n      <td>0</td>\n      <td>12</td>\n      <td>http://t.co/TKb82K4tFJ</td>\n      <td>en</td>\n      <td>...</td>\n      <td>Follow this account for geo-targeted Facilitie...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Tue Sep 11 20:00:43 +0000 2012</td>\n      <td>2012-09-11 22:00:43</td>\n      <td>2016-03-15 13:49:15</td>\n      <td>2016-03-15 13:49:15</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1124</th>\n      <td>846108068</td>\n      <td>NJ Business/Mgmt</td>\n      <td>tmj_nj_mgmt</td>\n      <td>218</td>\n      <td>332</td>\n      <td>245</td>\n      <td>0</td>\n      <td>31</td>\n      <td>https://t.co/DByWt4njnT</td>\n      <td>en</td>\n      <td>...</td>\n      <td>Follow this account for geo-targeted Business/...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Tue Sep 25 19:35:53 +0000 2012</td>\n      <td>2012-09-25 21:35:53</td>\n      <td>2016-03-15 13:49:15</td>\n      <td>2016-03-15 13:49:15</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1125</th>\n      <td>857076271</td>\n      <td>PR Customer Ser.</td>\n      <td>tmj_ptr_cstsrv</td>\n      <td>17</td>\n      <td>530</td>\n      <td>300</td>\n      <td>0</td>\n      <td>8</td>\n      <td>http://t.co/paWgrkSWfT</td>\n      <td>en</td>\n      <td>...</td>\n      <td>Follow this account for geo-targeted Customer ...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Mon Oct 01 21:47:23 +0000 2012</td>\n      <td>2012-10-01 23:47:23</td>\n      <td>2016-03-15 13:49:15</td>\n      <td>2016-03-15 13:49:15</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1126</th>\n      <td>872126906</td>\n      <td>PR Soft Dev Java</td>\n      <td>tmj_ptr_itjava</td>\n      <td>2</td>\n      <td>247</td>\n      <td>202</td>\n      <td>0</td>\n      <td>1</td>\n      <td>http://t.co/OnFSpFHC7G</td>\n      <td>en</td>\n      <td>...</td>\n      <td>Follow this account for geo-targeted Software ...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Wed Oct 10 16:44:09 +0000 2012</td>\n      <td>2012-10-10 18:44:09</td>\n      <td>2016-03-15 13:49:15</td>\n      <td>2016-03-15 13:49:15</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1127</th>\n      <td>979406250</td>\n      <td>Harrisburg Cust. Ser</td>\n      <td>tmj_hpa_cstsrv</td>\n      <td>47</td>\n      <td>267</td>\n      <td>234</td>\n      <td>0</td>\n      <td>7</td>\n      <td>http://t.co/Kz252i4r</td>\n      <td>en</td>\n      <td>...</td>\n      <td>Follow this account for geo-targeted Customer ...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Fri Nov 30 00:08:26 +0000 2012</td>\n      <td>2012-11-30 01:08:26</td>\n      <td>2016-03-15 13:49:15</td>\n      <td>2016-03-15 13:49:15</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>bot</td>\n    </tr>\n  </tbody>\n</table>\n<p>11017 rows × 43 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "# cresci-2017\n",
    "# Description: A dataset of (i) genuine, (ii) traditional, and (iii) social spambot Twitter accounts, annotated by CrowdFlower contributors. Released in CSV format.\n",
    "    \n",
    "social_spambots_1 = pd.read_csv('cresci-2017/datasets_full/social_spambots_1/users.csv')\n",
    "social_spambots_2 = pd.read_csv('cresci-2017/datasets_full/social_spambots_2/users.csv')\n",
    "social_spambots_3 = pd.read_csv('cresci-2017/datasets_full/social_spambots_3/users.csv')\n",
    "traditional_spambots_1 = pd.read_csv('cresci-2017/datasets_full/traditional_spambots_1/users.csv')\n",
    "traditional_spambots_2 = pd.read_csv('cresci-2017/datasets_full/traditional_spambots_2/users.csv')\n",
    "traditional_spambots_3 = pd.read_csv('cresci-2017/datasets_full/traditional_spambots_3/users.csv')\n",
    "traditional_spambots_4 = pd.read_csv('cresci-2017/datasets_full/traditional_spambots_4/users.csv')\n",
    "\n",
    "#traditional_spambots_1 has errored created_at data, using 'timestamp' instead\n",
    "traditional_spambots_1['created_at'] = traditional_spambots_1['timestamp']\n",
    "\n",
    "# label genuine dataset\n",
    "cresci_2017_genuine = pd.read_csv('cresci-2017/datasets_full/genuine_accounts/users.csv')\n",
    "cresci_2017_genuine['label'] = 'human'\n",
    "\n",
    "# combine and label bot datasets\n",
    "cresci_2017_bots = pd.concat([social_spambots_1, social_spambots_2, social_spambots_3, traditional_spambots_1, traditional_spambots_2, traditional_spambots_3, traditional_spambots_4])\n",
    "cresci_2017_bots['label'] = 'bot'\n",
    "\n",
    "# combine into one dataset\n",
    "cresci_2017_df = pd.concat([cresci_2017_genuine, cresci_2017_bots])\n",
    "# using 'updated' instead of 'crawled_at'\n",
    "cresci_2017_df.rename(columns={'updated':'collected_at'}, inplace=True)\n",
    "cresci_2017_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Index(['id', 'name', 'screen_name', 'statuses_count', 'followers_count',\n       'friends_count', 'favourites_count', 'listed_count', 'url', 'lang',\n       'time_zone', 'location', 'default_profile', 'default_profile_image',\n       'geo_enabled', 'profile_image_url', 'profile_banner_url',\n       'profile_use_background_image', 'profile_background_image_url_https',\n       'profile_text_color', 'profile_image_url_https',\n       'profile_sidebar_border_color', 'profile_background_tile',\n       'profile_sidebar_fill_color', 'profile_background_image_url',\n       'profile_background_color', 'profile_link_color', 'utc_offset',\n       'is_translator', 'follow_request_sent', 'protected', 'verified',\n       'notifications', 'description', 'contributors_enabled', 'following',\n       'created_at', 'timestamp', 'crawled_at', 'collected_at', 'test_set_1',\n       'test_set_2', 'label'],\n      dtype='object')"
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "cresci_2017_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                     id              id_str  \\\n0            3022357312          3022357312   \n1    753659579582541824  753659579582541824   \n2    901802279623417856  901802279623417856   \n3            2982392825          2982392825   \n4    825436422609969152  825436422609969152   \n..                  ...                 ...   \n688          2982573165          2982573165   \n689  825698658695905280  825698658695905280   \n690  751371698327416832  751371698327416832   \n691  726755736236253184  726755736236253184   \n692  835202996585787392  835202996585787392   \n\n                                        name      screen_name  \\\n0                                  patatavis         pattavis   \n1    #1DMITAMProject ♡ #OneDirectionBestFans  PromotingLouisT   \n2                              Anna 05600885       Anna10mila   \n3                                      Hanna        AlleHanna   \n4                                Zaffiro Blu      ZaffiroBlu3   \n..                                       ...              ...   \n688                           Paola Cacciari          nichilu   \n689                               @FiloRosso       FiloRosso2   \n690                          mario_anselmi72  mario_anselmi72   \n691                                    anna.      papersouls_   \n692                            Camilla Fiore      _CamillaFio   \n\n              location                                        description  \\\n0                                                                           \n1                Italy  ➡ here we promote all of the boys' music 🐼💖\\n\\...   \n2    Milano, Lombardia                                 Sii il cambiamento   \n3                                                                           \n4                                           Se puoi sognarlo puoi farlo 💙   \n..                 ...                                                ...   \n688                                                                         \n689                     { É rosso e forte il filo che tiene due anime ...   \n690                                                                         \n691             Italia  le mie chiavi di casa. @michele_bravi @GreysAB...   \n692                                                                         \n\n                         url  \\\n0                       None   \n1    https://t.co/8HvscWygQd   \n2                       None   \n3                       None   \n4                       None   \n..                       ...   \n688                     None   \n689                     None   \n690                     None   \n691  https://t.co/oJSzdcXfnP   \n692                     None   \n\n                                              entities  protected  \\\n0                        {'description': {'urls': []}}      False   \n1    {'url': {'urls': [{'url': 'https://t.co/8HvscW...      False   \n2                        {'description': {'urls': []}}      False   \n3                        {'description': {'urls': []}}      False   \n4                        {'description': {'urls': []}}      False   \n..                                                 ...        ...   \n688                      {'description': {'urls': []}}      False   \n689                      {'description': {'urls': []}}      False   \n690                      {'description': {'urls': []}}      False   \n691  {'url': {'urls': [{'url': 'https://t.co/oJSzdc...      False   \n692                      {'description': {'urls': []}}      False   \n\n     followers_count  ...  profile_use_background_image  has_extended_profile  \\\n0                218  ...                          True                 False   \n1                256  ...                          True                 False   \n2                287  ...                          True                 False   \n3                266  ...                          True                 False   \n4                286  ...                          True                 False   \n..               ...  ...                           ...                   ...   \n688              152  ...                          True                 False   \n689              301  ...                          True                  True   \n690               35  ...                          True                  True   \n691              134  ...                          True                  True   \n692               79  ...                          True                 False   \n\n    default_profile  default_profile_image following follow_request_sent  \\\n0              True                  False     False               False   \n1              True                  False     False               False   \n2              True                  False     False               False   \n3              True                  False     False               False   \n4              True                  False     False               False   \n..              ...                    ...       ...                 ...   \n688            True                  False     False               False   \n689            True                  False     False               False   \n690            True                  False     False               False   \n691            True                  False     False               False   \n692            True                  False     False               False   \n\n     notifications  translator_type              collected_at  label  \n0            False             none 2019-05-15 16:00:19+00:00    bot  \n1            False             none 2019-05-16 20:07:35+00:00    bot  \n2            False             none 2019-05-17 17:22:28+00:00  human  \n3            False             none 2018-09-24 13:01:10+00:00    bot  \n4            False             none 2019-05-15 16:41:07+00:00    bot  \n..             ...              ...                       ...    ...  \n688          False             none 2019-05-02 12:09:25+00:00    bot  \n689          False             none 2019-05-14 20:28:27+00:00    bot  \n690          False             none 2019-05-17 15:07:18+00:00    bot  \n691          False             none 2019-05-15 14:51:25+00:00  human  \n692          False             none 2019-05-15 17:28:10+00:00    bot  \n\n[693 rows x 44 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>id_str</th>\n      <th>name</th>\n      <th>screen_name</th>\n      <th>location</th>\n      <th>description</th>\n      <th>url</th>\n      <th>entities</th>\n      <th>protected</th>\n      <th>followers_count</th>\n      <th>...</th>\n      <th>profile_use_background_image</th>\n      <th>has_extended_profile</th>\n      <th>default_profile</th>\n      <th>default_profile_image</th>\n      <th>following</th>\n      <th>follow_request_sent</th>\n      <th>notifications</th>\n      <th>translator_type</th>\n      <th>collected_at</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>3022357312</td>\n      <td>3022357312</td>\n      <td>patatavis</td>\n      <td>pattavis</td>\n      <td></td>\n      <td></td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>218</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-05-15 16:00:19+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>753659579582541824</td>\n      <td>753659579582541824</td>\n      <td>#1DMITAMProject ♡ #OneDirectionBestFans</td>\n      <td>PromotingLouisT</td>\n      <td>Italy</td>\n      <td>➡ here we promote all of the boys' music 🐼💖\\n\\...</td>\n      <td>https://t.co/8HvscWygQd</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/8HvscW...</td>\n      <td>False</td>\n      <td>256</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-05-16 20:07:35+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>901802279623417856</td>\n      <td>901802279623417856</td>\n      <td>Anna 05600885</td>\n      <td>Anna10mila</td>\n      <td>Milano, Lombardia</td>\n      <td>Sii il cambiamento</td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>287</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-05-17 17:22:28+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>2982392825</td>\n      <td>2982392825</td>\n      <td>Hanna</td>\n      <td>AlleHanna</td>\n      <td></td>\n      <td></td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>266</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2018-09-24 13:01:10+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>825436422609969152</td>\n      <td>825436422609969152</td>\n      <td>Zaffiro Blu</td>\n      <td>ZaffiroBlu3</td>\n      <td></td>\n      <td>Se puoi sognarlo puoi farlo 💙</td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>286</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-05-15 16:41:07+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>688</th>\n      <td>2982573165</td>\n      <td>2982573165</td>\n      <td>Paola Cacciari</td>\n      <td>nichilu</td>\n      <td></td>\n      <td></td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>152</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-05-02 12:09:25+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>689</th>\n      <td>825698658695905280</td>\n      <td>825698658695905280</td>\n      <td>@FiloRosso</td>\n      <td>FiloRosso2</td>\n      <td></td>\n      <td>{ É rosso e forte il filo che tiene due anime ...</td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>301</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-05-14 20:28:27+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>690</th>\n      <td>751371698327416832</td>\n      <td>751371698327416832</td>\n      <td>mario_anselmi72</td>\n      <td>mario_anselmi72</td>\n      <td></td>\n      <td></td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>35</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-05-17 15:07:18+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>691</th>\n      <td>726755736236253184</td>\n      <td>726755736236253184</td>\n      <td>anna.</td>\n      <td>papersouls_</td>\n      <td>Italia</td>\n      <td>le mie chiavi di casa. @michele_bravi @GreysAB...</td>\n      <td>https://t.co/oJSzdcXfnP</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/oJSzdc...</td>\n      <td>False</td>\n      <td>134</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-05-15 14:51:25+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>692</th>\n      <td>835202996585787392</td>\n      <td>835202996585787392</td>\n      <td>Camilla Fiore</td>\n      <td>_CamillaFio</td>\n      <td></td>\n      <td></td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>79</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-05-15 17:28:10+00:00</td>\n      <td>bot</td>\n    </tr>\n  </tbody>\n</table>\n<p>693 rows × 44 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 8
    }
   ],
   "source": [
    "# cresci-rtbust-2019\n",
    "# Description: Manually annotated bot and human accounts. Labels and user objects.\n",
    "cresci_rtbust_2019 = pd.read_json('cresci-rtbust-2019/cresci-rtbust-2019_tweets.json', orient='records')\n",
    "cresci_rtbust_2019_df = pd.DataFrame(list(cresci_rtbust_2019['user']))\n",
    "cresci_rtbust_2019_df['collected_at'] = cresci_rtbust_2019['created_at']\n",
    "\n",
    "# data labels\n",
    "cresci_rtbust_2019_labels = pd.read_csv('cresci-rtbust-2019/cresci-rtbust-2019.tsv', sep='\\t', header=None)\n",
    "cresci_rtbust_2019_labels.columns = ['id', 'label']\n",
    "\n",
    "# merge in labels on id\n",
    "cresci_rtbust_2019_df = cresci_rtbust_2019_df.merge(cresci_rtbust_2019_labels, how='left')\n",
    "\n",
    "cresci_rtbust_2019_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                       id              id_str                 name  \\\n0               270861589           270861589           Pat Miller   \n1              1831466064          1831466064  Gwen C. Persichetti   \n2              1506843600          1506843600              versace   \n3               621780088           621780088          SamaR Galil   \n4      703798067586338816  703798067586338816               Pixey9   \n...                   ...                 ...                  ...   \n13271           357395130           357395130   Harry Charles Bird   \n13272  772123406091358209  772123406091358209       Tamala Stanley   \n13273           247808509           247808509        angelyca melo   \n13274            78648922            78648922   PCandMACrepair.com   \n13275           940418736           940418736    elizabeth gaffney   \n\n           screen_name         location  \\\n0          millerkills          Midwest   \n1      gwenpersichetti  Atlanta Georgia   \n2               5KSOSS                    \n3           SamaRGalil            EgypT   \n4         PattyRodman1     Florida, USA   \n...                ...              ...   \n13271    HarryCharlesB                    \n13272  PancakesStanley                    \n13273    angelycamelos                    \n13274   pcandmacrepair              USA   \n13275           eagaff                    \n\n                                             description  \\\n0                       working, running, fishing, music   \n1      Catholic, Mom, Wife, Born NY Live ATL ❤ Flower...   \n2      ooOooOohhh she thinks I’m cute she wanna have sex   \n3                                                          \n4          Love People, Love Christ, Love my Pomeranian.   \n...                                                  ...   \n13271                       You aint playn' me about....   \n13272  TeamDarkskin , TeamTakeYourMan, Teamsexy, Team...   \n13273                                                      \n13274  Your one-stop-shop for keeping your computer r...   \n13275                                                      \n\n                          url  \\\n0                        None   \n1                        None   \n2                        None   \n3                        None   \n4                        None   \n...                       ...   \n13271  http://t.co/LVnfPqsfdK   \n13272                    None   \n13273                    None   \n13274  http://t.co/HZSOaFSg5a   \n13275                    None   \n\n                                                entities  protected  \\\n0                          {'description': {'urls': []}}      False   \n1                          {'description': {'urls': []}}      False   \n2                          {'description': {'urls': []}}      False   \n3                          {'description': {'urls': []}}      False   \n4                          {'description': {'urls': []}}      False   \n...                                                  ...        ...   \n13271  {'url': {'urls': [{'url': 'http://t.co/LVnfPqs...      False   \n13272                      {'description': {'urls': []}}      False   \n13273                      {'description': {'urls': []}}      False   \n13274  {'url': {'urls': [{'url': 'http://t.co/HZSOaFS...      False   \n13275                      {'description': {'urls': []}}      False   \n\n       followers_count  ...  has_extended_profile  default_profile  \\\n0                    6  ...                 False             True   \n1                 1716  ...                 False             True   \n2                 2010  ...                  True            False   \n3                  220  ...                 False            False   \n4                  404  ...                  True             True   \n...                ...  ...                   ...              ...   \n13271               44  ...                 False            False   \n13272                2  ...                 False             True   \n13273               19  ...                 False            False   \n13274               36  ...                 False            False   \n13275               21  ...                 False             True   \n\n      default_profile_image  following follow_request_sent notifications  \\\n0                     False      False               False         False   \n1                     False      False               False         False   \n2                     False      False               False         False   \n3                     False      False               False         False   \n4                     False      False               False         False   \n...                     ...        ...                 ...           ...   \n13271                 False      False               False         False   \n13272                 False      False               False         False   \n13273                 False      False               False         False   \n13274                 False      False               False         False   \n13275                 False      False               False         False   \n\n       translator_type                                 profile_banner_url  \\\n0                 none                                                NaN   \n1                 none                                                NaN   \n2              regular  https://pbs.twimg.com/profile_banners/15068436...   \n3                 none                                                NaN   \n4                 none                                                NaN   \n...                ...                                                ...   \n13271             none                                                NaN   \n13272             none                                                NaN   \n13273             none                                                NaN   \n13274             none                                                NaN   \n13275             none  https://pbs.twimg.com/profile_banners/94041873...   \n\n                   collected_at  label  \n0     2017-08-25 00:47:17+00:00    bot  \n1     2019-05-17 13:00:28+00:00  human  \n2     2019-05-17 21:12:18+00:00  human  \n3     2017-08-25 00:52:04+00:00    bot  \n4     2019-05-18 02:21:41+00:00  human  \n...                         ...    ...  \n13271 2017-06-30 05:43:24+00:00  human  \n13272 2017-06-30 10:44:39+00:00  human  \n13273 2017-11-03 00:34:05+00:00    bot  \n13274 2017-09-13 17:39:54+00:00    bot  \n13275 2018-04-07 05:27:52+00:00    bot  \n\n[13276 rows x 44 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>id_str</th>\n      <th>name</th>\n      <th>screen_name</th>\n      <th>location</th>\n      <th>description</th>\n      <th>url</th>\n      <th>entities</th>\n      <th>protected</th>\n      <th>followers_count</th>\n      <th>...</th>\n      <th>has_extended_profile</th>\n      <th>default_profile</th>\n      <th>default_profile_image</th>\n      <th>following</th>\n      <th>follow_request_sent</th>\n      <th>notifications</th>\n      <th>translator_type</th>\n      <th>profile_banner_url</th>\n      <th>collected_at</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>270861589</td>\n      <td>270861589</td>\n      <td>Pat Miller</td>\n      <td>millerkills</td>\n      <td>Midwest</td>\n      <td>working, running, fishing, music</td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>6</td>\n      <td>...</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>NaN</td>\n      <td>2017-08-25 00:47:17+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1831466064</td>\n      <td>1831466064</td>\n      <td>Gwen C. Persichetti</td>\n      <td>gwenpersichetti</td>\n      <td>Atlanta Georgia</td>\n      <td>Catholic, Mom, Wife, Born NY Live ATL ❤ Flower...</td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>1716</td>\n      <td>...</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>NaN</td>\n      <td>2019-05-17 13:00:28+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1506843600</td>\n      <td>1506843600</td>\n      <td>versace</td>\n      <td>5KSOSS</td>\n      <td></td>\n      <td>ooOooOohhh she thinks I’m cute she wanna have sex</td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>2010</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>regular</td>\n      <td>https://pbs.twimg.com/profile_banners/15068436...</td>\n      <td>2019-05-17 21:12:18+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>621780088</td>\n      <td>621780088</td>\n      <td>SamaR Galil</td>\n      <td>SamaRGalil</td>\n      <td>EgypT</td>\n      <td></td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>220</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>NaN</td>\n      <td>2017-08-25 00:52:04+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>703798067586338816</td>\n      <td>703798067586338816</td>\n      <td>Pixey9</td>\n      <td>PattyRodman1</td>\n      <td>Florida, USA</td>\n      <td>Love People, Love Christ, Love my Pomeranian.</td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>404</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>NaN</td>\n      <td>2019-05-18 02:21:41+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>13271</th>\n      <td>357395130</td>\n      <td>357395130</td>\n      <td>Harry Charles Bird</td>\n      <td>HarryCharlesB</td>\n      <td></td>\n      <td>You aint playn' me about....</td>\n      <td>http://t.co/LVnfPqsfdK</td>\n      <td>{'url': {'urls': [{'url': 'http://t.co/LVnfPqs...</td>\n      <td>False</td>\n      <td>44</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>NaN</td>\n      <td>2017-06-30 05:43:24+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>13272</th>\n      <td>772123406091358209</td>\n      <td>772123406091358209</td>\n      <td>Tamala Stanley</td>\n      <td>PancakesStanley</td>\n      <td></td>\n      <td>TeamDarkskin , TeamTakeYourMan, Teamsexy, Team...</td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>2</td>\n      <td>...</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>NaN</td>\n      <td>2017-06-30 10:44:39+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>13273</th>\n      <td>247808509</td>\n      <td>247808509</td>\n      <td>angelyca melo</td>\n      <td>angelycamelos</td>\n      <td></td>\n      <td></td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>19</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>NaN</td>\n      <td>2017-11-03 00:34:05+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>13274</th>\n      <td>78648922</td>\n      <td>78648922</td>\n      <td>PCandMACrepair.com</td>\n      <td>pcandmacrepair</td>\n      <td>USA</td>\n      <td>Your one-stop-shop for keeping your computer r...</td>\n      <td>http://t.co/HZSOaFSg5a</td>\n      <td>{'url': {'urls': [{'url': 'http://t.co/HZSOaFS...</td>\n      <td>False</td>\n      <td>36</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>NaN</td>\n      <td>2017-09-13 17:39:54+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>13275</th>\n      <td>940418736</td>\n      <td>940418736</td>\n      <td>elizabeth gaffney</td>\n      <td>eagaff</td>\n      <td></td>\n      <td></td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>21</td>\n      <td>...</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>https://pbs.twimg.com/profile_banners/94041873...</td>\n      <td>2018-04-07 05:27:52+00:00</td>\n      <td>bot</td>\n    </tr>\n  </tbody>\n</table>\n<p>13276 rows × 44 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "source": [
    "# cresci-stock-2018\n",
    "# Description: Automated accounts that act in coordinate fashion. Labels and user objects.\n",
    "cresci_stock_2018 = pd.read_json('cresci-stock-2018/cresci-stock-2018_tweets.json')\n",
    "cresci_stock_2018_df = pd.DataFrame(list(cresci_stock_2018['user']))\n",
    "cresci_stock_2018_df['collected_at'] = cresci_stock_2018['created_at']\n",
    "\n",
    "# data labels\n",
    "cresci_stock_2018_labels = pd.read_csv('cresci-stock-2018/cresci-stock-2018.tsv', sep='\\t', header=None)\n",
    "cresci_stock_2018_labels.columns = ['id', 'label']\n",
    "\n",
    "# merge in labels on id\n",
    "cresci_stock_2018_df = cresci_stock_2018_df.merge(cresci_stock_2018_labels, how='left')\n",
    "\n",
    "cresci_stock_2018_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "              id      id_str                 name      screen_name  \\\n0      613512037   613512037      Baseball Quotes  BaseballQuotes1   \n1       19263978    19263978  Cleveland Cavaliers             cavs   \n2     1624329463  1624329463          ‏المهاجر 1M      muohajer_12   \n3      550610050   550610050                BC.20            Bc20_   \n4      149714640   149714640          mia 🧜🏾‍♀️🖤💋      _Blkahontas   \n...          ...         ...                  ...              ...   \n2508  3236431915  3236431915        Adam McArthur     BigMacFlashy   \n2509   388843527   388843527               Badrul     BadruulAminn   \n2510  2566083968  2566083968                 Sav🌸       anxietyhes   \n2511   250121556   250121556         遠@原付ベトナム縦断準備           EN_owl   \n2512   498378398   498378398                 Cess     cessdomingo_   \n\n                      location  \\\n0                  The Diamond   \n1        The Q | Cleveland, OH   \n2     المملكة العربية السعودية   \n3                                \n4                Hollywood, FL   \n...                        ...   \n2508               Phoenix, AZ   \n2509        Petaling, Selangor   \n2510                             \n2511                    なごやちほー   \n2512                             \n\n                                            description  \\\n0     Quoting America's Pastime in 280 characters or...   \n1     Official Twitter of the 2016 NBA Champion Clev...   \n2                                                         \n3      R.I.P JRL21//R.I.P Monica💙 IG//b.20c #GLOHIOBOYS   \n4                                      to die for 🔮🤞🏾👸🏾   \n...                                                 ...   \n2508  CEO of @thebambox and @comiconauction. Love De...   \n2509  20 / YouTuber + Streamer / Married /\\n\\nCome s...   \n2510  woke up the girl who looked just like you I al...   \n2511  旧型クロスカブ改(プラウダ戦車色)に乗るガルパンおじさん。ノンナと愛里寿推し。日本一周201...   \n2512                                                      \n\n                          url  \\\n0                        None   \n1     https://t.co/XDJzEZarB0   \n2                        None   \n3                        None   \n4     https://t.co/Y7Md3366Ky   \n...                       ...   \n2508  https://t.co/4MHS29rjze   \n2509  https://t.co/2k86dljV8d   \n2510  https://t.co/ZK3i8HNjoD   \n2511  https://t.co/H4tJIPH6Dk   \n2512                     None   \n\n                                               entities  protected  \\\n0                         {'description': {'urls': []}}      False   \n1     {'url': {'urls': [{'url': 'https://t.co/XDJzEZ...      False   \n2                         {'description': {'urls': []}}      False   \n3                         {'description': {'urls': []}}      False   \n4     {'url': {'urls': [{'url': 'https://t.co/Y7Md33...      False   \n...                                                 ...        ...   \n2508  {'url': {'urls': [{'url': 'https://t.co/4MHS29...      False   \n2509  {'url': {'urls': [{'url': 'https://t.co/2k86dl...      False   \n2510  {'url': {'urls': [{'url': 'https://t.co/ZK3i8H...      False   \n2511  {'url': {'urls': [{'url': 'https://t.co/H4tJIP...      False   \n2512                      {'description': {'urls': []}}      False   \n\n      followers_count  ...  profile_use_background_image  \\\n0              121500  ...                         False   \n1             3227215  ...                          True   \n2              864968  ...                          True   \n3                 951  ...                         False   \n4                1412  ...                          True   \n...               ...  ...                           ...   \n2508              331  ...                         False   \n2509            71105  ...                          True   \n2510            99769  ...                          True   \n2511             1222  ...                          True   \n2512             1308  ...                          True   \n\n      has_extended_profile default_profile  default_profile_image following  \\\n0                    False           False                  False     False   \n1                    False           False                  False     False   \n2                    False            True                  False     False   \n3                     True           False                  False     False   \n4                    False           False                  False     False   \n...                    ...             ...                    ...       ...   \n2508                 False           False                  False     False   \n2509                  True           False                  False     False   \n2510                  True           False                  False     False   \n2511                 False           False                  False     False   \n2512                 False           False                  False     False   \n\n     follow_request_sent  notifications  translator_type  \\\n0                  False          False             none   \n1                  False          False             none   \n2                  False          False             none   \n3                  False          False             none   \n4                  False          False             none   \n...                  ...            ...              ...   \n2508               False          False             none   \n2509               False          False          regular   \n2510               False          False             none   \n2511               False          False             none   \n2512               False          False             none   \n\n                  collected_at  label  \n0    2019-01-21 19:37:21+00:00  human  \n1    2019-01-21 20:42:18+00:00  human  \n2    2019-01-21 19:18:27+00:00    bot  \n3    2019-01-21 14:45:13+00:00  human  \n4    2019-01-21 20:28:43+00:00  human  \n...                        ...    ...  \n2508 2017-06-05 23:53:56+00:00  human  \n2509 2019-01-22 00:06:58+00:00  human  \n2510 2019-01-18 16:45:06+00:00  human  \n2511 2019-01-21 15:26:25+00:00    bot  \n2512 2019-01-21 15:32:03+00:00  human  \n\n[2513 rows x 44 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>id_str</th>\n      <th>name</th>\n      <th>screen_name</th>\n      <th>location</th>\n      <th>description</th>\n      <th>url</th>\n      <th>entities</th>\n      <th>protected</th>\n      <th>followers_count</th>\n      <th>...</th>\n      <th>profile_use_background_image</th>\n      <th>has_extended_profile</th>\n      <th>default_profile</th>\n      <th>default_profile_image</th>\n      <th>following</th>\n      <th>follow_request_sent</th>\n      <th>notifications</th>\n      <th>translator_type</th>\n      <th>collected_at</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>613512037</td>\n      <td>613512037</td>\n      <td>Baseball Quotes</td>\n      <td>BaseballQuotes1</td>\n      <td>The Diamond</td>\n      <td>Quoting America's Pastime in 280 characters or...</td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>121500</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-01-21 19:37:21+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>19263978</td>\n      <td>19263978</td>\n      <td>Cleveland Cavaliers</td>\n      <td>cavs</td>\n      <td>The Q | Cleveland, OH</td>\n      <td>Official Twitter of the 2016 NBA Champion Clev...</td>\n      <td>https://t.co/XDJzEZarB0</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/XDJzEZ...</td>\n      <td>False</td>\n      <td>3227215</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-01-21 20:42:18+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1624329463</td>\n      <td>1624329463</td>\n      <td>‏المهاجر 1M</td>\n      <td>muohajer_12</td>\n      <td>المملكة العربية السعودية</td>\n      <td></td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>864968</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-01-21 19:18:27+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>550610050</td>\n      <td>550610050</td>\n      <td>BC.20</td>\n      <td>Bc20_</td>\n      <td></td>\n      <td>R.I.P JRL21//R.I.P Monica💙 IG//b.20c #GLOHIOBOYS</td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>951</td>\n      <td>...</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-01-21 14:45:13+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>149714640</td>\n      <td>149714640</td>\n      <td>mia 🧜🏾‍♀️🖤💋</td>\n      <td>_Blkahontas</td>\n      <td>Hollywood, FL</td>\n      <td>to die for 🔮🤞🏾👸🏾</td>\n      <td>https://t.co/Y7Md3366Ky</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/Y7Md33...</td>\n      <td>False</td>\n      <td>1412</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-01-21 20:28:43+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2508</th>\n      <td>3236431915</td>\n      <td>3236431915</td>\n      <td>Adam McArthur</td>\n      <td>BigMacFlashy</td>\n      <td>Phoenix, AZ</td>\n      <td>CEO of @thebambox and @comiconauction. Love De...</td>\n      <td>https://t.co/4MHS29rjze</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/4MHS29...</td>\n      <td>False</td>\n      <td>331</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2017-06-05 23:53:56+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>2509</th>\n      <td>388843527</td>\n      <td>388843527</td>\n      <td>Badrul</td>\n      <td>BadruulAminn</td>\n      <td>Petaling, Selangor</td>\n      <td>20 / YouTuber + Streamer / Married /\\n\\nCome s...</td>\n      <td>https://t.co/2k86dljV8d</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/2k86dl...</td>\n      <td>False</td>\n      <td>71105</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>regular</td>\n      <td>2019-01-22 00:06:58+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>2510</th>\n      <td>2566083968</td>\n      <td>2566083968</td>\n      <td>Sav🌸</td>\n      <td>anxietyhes</td>\n      <td></td>\n      <td>woke up the girl who looked just like you I al...</td>\n      <td>https://t.co/ZK3i8HNjoD</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/ZK3i8H...</td>\n      <td>False</td>\n      <td>99769</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-01-18 16:45:06+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>2511</th>\n      <td>250121556</td>\n      <td>250121556</td>\n      <td>遠@原付ベトナム縦断準備</td>\n      <td>EN_owl</td>\n      <td>なごやちほー</td>\n      <td>旧型クロスカブ改(プラウダ戦車色)に乗るガルパンおじさん。ノンナと愛里寿推し。日本一周201...</td>\n      <td>https://t.co/H4tJIPH6Dk</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/H4tJIP...</td>\n      <td>False</td>\n      <td>1222</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-01-21 15:26:25+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>2512</th>\n      <td>498378398</td>\n      <td>498378398</td>\n      <td>Cess</td>\n      <td>cessdomingo_</td>\n      <td></td>\n      <td></td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>1308</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-01-21 15:32:03+00:00</td>\n      <td>human</td>\n    </tr>\n  </tbody>\n</table>\n<p>2513 rows × 44 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "# gilani-2017\n",
    "# Description: Manually annotated human and bot accounts. Labels and user objects.\n",
    "gilani_2017 = pd.read_json('gilani-2017/gilani-2017_tweets.json')\n",
    "gilani_2017_df = pd.DataFrame(list(gilani_2017['user']))\n",
    "gilani_2017_df['collected_at'] = gilani_2017['created_at']\n",
    "\n",
    "# data labels\n",
    "gilani_2017_labels = pd.read_csv('gilani-2017/gilani-2017.tsv', sep='\\t', header=None)\n",
    "gilani_2017_labels.columns = ['id', 'label']\n",
    "\n",
    "# merge in labels on id\n",
    "gilani_2017_df = gilani_2017_df.merge(gilani_2017_labels, how='left')\n",
    "\n",
    "gilani_2017_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Index(['id', 'id_str', 'name', 'screen_name', 'location', 'description', 'url',\n       'entities', 'protected', 'followers_count', 'friends_count',\n       'listed_count', 'created_at', 'favourites_count', 'utc_offset',\n       'time_zone', 'geo_enabled', 'verified', 'statuses_count', 'lang',\n       'contributors_enabled', 'is_translator', 'is_translation_enabled',\n       'profile_background_color', 'profile_background_image_url',\n       'profile_background_image_url_https', 'profile_background_tile',\n       'profile_image_url', 'profile_image_url_https', 'profile_banner_url',\n       'profile_link_color', 'profile_sidebar_border_color',\n       'profile_sidebar_fill_color', 'profile_text_color',\n       'profile_use_background_image', 'has_extended_profile',\n       'default_profile', 'default_profile_image', 'following',\n       'follow_request_sent', 'notifications', 'translator_type',\n       'collected_at', 'label'],\n      dtype='object')"
     },
     "metadata": {},
     "execution_count": 11
    }
   ],
   "source": [
    "gilani_2017_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                   collected_at                   id      screen_name  \\\n0      Tue Nov 06 20:35:08 2018           4107317134    danitheduck21   \n1      Tue Nov 06 17:57:51 2018           4858296837     ncaraballoPR   \n2      Tue Nov 06 20:35:23 2018            232631847       drmendezmd   \n3      Tue Nov 06 19:23:19 2018             16700555      ScottNevins   \n4      Tue Nov 06 20:35:24 2018            334443152         lild1206   \n...                         ...                  ...              ...   \n50533  Sat Dec 29 07:59:42 2018  1078921538739544064    Wilfredsteve1   \n50534  Sat Dec 29 08:54:38 2018  1078923723019874304  VinceGi92042414   \n50535  Sat Dec 29 09:02:18 2018  1078937076878503936  KenFish39349255   \n50536  Sat Dec 29 10:48:30 2018  1078963374015291392        aniston_a   \n50537  Sat Dec 29 11:04:56 2018  1078964200234536960    Joshuashores2   \n\n                    name                                        description  \\\n0               Dani🏳️‍🌈  Dani 💜 She/Her 💜 Randomness all over. Expect l...   \n1      Natalie Caraballo  Things I don’t get tired of: Politics, Amy Win...   \n2                 Wilson  Latin american100%! Let fight for our country,...   \n3           Scott Nevins  TV Personality & Host | Political/News Contrib...   \n4                      D                                               None   \n...                  ...                                                ...   \n50533       Wilfredsteve  I'm a Widow..God bless America!\\nWe rise by he...   \n50534         Vince Gill         Download Vince’s New Album -umgn.us/d2mlbh   \n50535         Ken Fisher  Self-Made Muilt-Billionaire, Global investor, ...   \n50536   Stella.A.Aniston  lady aniston's twitter.\\ncryptosystem investor...   \n50537      Joshua shores                                               None   \n\n               created_at                                url lang  protected  \\\n0     2015-11-03 21:16:13                               None   en        0.0   \n1     2016-01-28 20:03:51                               None   en        0.0   \n2     2010-12-31 18:55:05                               None   en        0.0   \n3     2008-10-11 21:39:34         http://www.ScottNevins.com   en        0.0   \n4     2011-07-13 03:13:52                               None   en        0.0   \n...                   ...                                ...  ...        ...   \n50533 2018-12-29 07:51:55                               None   en        0.0   \n50534 2018-12-29 08:00:35               http://VinceGill.com   en        0.0   \n50535 2018-12-29 08:53:39  http://ken-fisher-investments.com   en        0.0   \n50536 2018-12-29 10:38:09                               None   en        0.0   \n50537 2018-12-29 10:41:26                               None   en        0.0   \n\n       verified  ...  profile_use_background_image  default_profile  \\\n0         False  ...                         False            False   \n1         False  ...                         False            False   \n2         False  ...                          True             True   \n3          True  ...                          True            False   \n4         False  ...                          True             True   \n...         ...  ...                           ...              ...   \n50533     False  ...                          True             True   \n50534     False  ...                          True             True   \n50535     False  ...                          True             True   \n50536     False  ...                          True             True   \n50537     False  ...                          True             True   \n\n       followers_count  friends_count  listed_count  favourites_count  \\\n0                  481            870            26              6542   \n1                  202            712             5              1515   \n2                  278            342             4              4780   \n3                29546            384           402            143163   \n4                   95            668             1              1178   \n...                ...            ...           ...               ...   \n50533                0              4             0                 3   \n50534                0              0             0                 0   \n50535                0              0             0                 0   \n50536                1              9             0                 2   \n50537                0             34             0                 0   \n\n       statuses_count                  tid  label default_profile_image  \n0               67025  1059907055421509632  human                  None  \n1                 158  1059867472810180609  human                  None  \n2                4029  1059907117094711296  human                  None  \n3               53427  1059888980957650944  human                  None  \n4                1315  1059907122408898562  human                  None  \n...               ...                  ...    ...                   ...  \n50533               5  1078923499413098496    bot                  None  \n50534             280  1078937323868434432    bot                  None  \n50535               1  1078939252958875648    bot                  None  \n50536               4  1078965978162216961    bot                  None  \n50537              16  1078970115717242880    bot                  None  \n\n[50538 rows x 21 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>collected_at</th>\n      <th>id</th>\n      <th>screen_name</th>\n      <th>name</th>\n      <th>description</th>\n      <th>created_at</th>\n      <th>url</th>\n      <th>lang</th>\n      <th>protected</th>\n      <th>verified</th>\n      <th>...</th>\n      <th>profile_use_background_image</th>\n      <th>default_profile</th>\n      <th>followers_count</th>\n      <th>friends_count</th>\n      <th>listed_count</th>\n      <th>favourites_count</th>\n      <th>statuses_count</th>\n      <th>tid</th>\n      <th>label</th>\n      <th>default_profile_image</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Tue Nov 06 20:35:08 2018</td>\n      <td>4107317134</td>\n      <td>danitheduck21</td>\n      <td>Dani🏳️‍🌈</td>\n      <td>Dani 💜 She/Her 💜 Randomness all over. Expect l...</td>\n      <td>2015-11-03 21:16:13</td>\n      <td>None</td>\n      <td>en</td>\n      <td>0.0</td>\n      <td>False</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>481</td>\n      <td>870</td>\n      <td>26</td>\n      <td>6542</td>\n      <td>67025</td>\n      <td>1059907055421509632</td>\n      <td>human</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Tue Nov 06 17:57:51 2018</td>\n      <td>4858296837</td>\n      <td>ncaraballoPR</td>\n      <td>Natalie Caraballo</td>\n      <td>Things I don’t get tired of: Politics, Amy Win...</td>\n      <td>2016-01-28 20:03:51</td>\n      <td>None</td>\n      <td>en</td>\n      <td>0.0</td>\n      <td>False</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>202</td>\n      <td>712</td>\n      <td>5</td>\n      <td>1515</td>\n      <td>158</td>\n      <td>1059867472810180609</td>\n      <td>human</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Tue Nov 06 20:35:23 2018</td>\n      <td>232631847</td>\n      <td>drmendezmd</td>\n      <td>Wilson</td>\n      <td>Latin american100%! Let fight for our country,...</td>\n      <td>2010-12-31 18:55:05</td>\n      <td>None</td>\n      <td>en</td>\n      <td>0.0</td>\n      <td>False</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>278</td>\n      <td>342</td>\n      <td>4</td>\n      <td>4780</td>\n      <td>4029</td>\n      <td>1059907117094711296</td>\n      <td>human</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Tue Nov 06 19:23:19 2018</td>\n      <td>16700555</td>\n      <td>ScottNevins</td>\n      <td>Scott Nevins</td>\n      <td>TV Personality &amp; Host | Political/News Contrib...</td>\n      <td>2008-10-11 21:39:34</td>\n      <td>http://www.ScottNevins.com</td>\n      <td>en</td>\n      <td>0.0</td>\n      <td>True</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>29546</td>\n      <td>384</td>\n      <td>402</td>\n      <td>143163</td>\n      <td>53427</td>\n      <td>1059888980957650944</td>\n      <td>human</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Tue Nov 06 20:35:24 2018</td>\n      <td>334443152</td>\n      <td>lild1206</td>\n      <td>D</td>\n      <td>None</td>\n      <td>2011-07-13 03:13:52</td>\n      <td>None</td>\n      <td>en</td>\n      <td>0.0</td>\n      <td>False</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>95</td>\n      <td>668</td>\n      <td>1</td>\n      <td>1178</td>\n      <td>1315</td>\n      <td>1059907122408898562</td>\n      <td>human</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>50533</th>\n      <td>Sat Dec 29 07:59:42 2018</td>\n      <td>1078921538739544064</td>\n      <td>Wilfredsteve1</td>\n      <td>Wilfredsteve</td>\n      <td>I'm a Widow..God bless America!\\nWe rise by he...</td>\n      <td>2018-12-29 07:51:55</td>\n      <td>None</td>\n      <td>en</td>\n      <td>0.0</td>\n      <td>False</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>0</td>\n      <td>4</td>\n      <td>0</td>\n      <td>3</td>\n      <td>5</td>\n      <td>1078923499413098496</td>\n      <td>bot</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>50534</th>\n      <td>Sat Dec 29 08:54:38 2018</td>\n      <td>1078923723019874304</td>\n      <td>VinceGi92042414</td>\n      <td>Vince Gill</td>\n      <td>Download Vince’s New Album -umgn.us/d2mlbh</td>\n      <td>2018-12-29 08:00:35</td>\n      <td>http://VinceGill.com</td>\n      <td>en</td>\n      <td>0.0</td>\n      <td>False</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>280</td>\n      <td>1078937323868434432</td>\n      <td>bot</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>50535</th>\n      <td>Sat Dec 29 09:02:18 2018</td>\n      <td>1078937076878503936</td>\n      <td>KenFish39349255</td>\n      <td>Ken Fisher</td>\n      <td>Self-Made Muilt-Billionaire, Global investor, ...</td>\n      <td>2018-12-29 08:53:39</td>\n      <td>http://ken-fisher-investments.com</td>\n      <td>en</td>\n      <td>0.0</td>\n      <td>False</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1078939252958875648</td>\n      <td>bot</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>50536</th>\n      <td>Sat Dec 29 10:48:30 2018</td>\n      <td>1078963374015291392</td>\n      <td>aniston_a</td>\n      <td>Stella.A.Aniston</td>\n      <td>lady aniston's twitter.\\ncryptosystem investor...</td>\n      <td>2018-12-29 10:38:09</td>\n      <td>None</td>\n      <td>en</td>\n      <td>0.0</td>\n      <td>False</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>1</td>\n      <td>9</td>\n      <td>0</td>\n      <td>2</td>\n      <td>4</td>\n      <td>1078965978162216961</td>\n      <td>bot</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>50537</th>\n      <td>Sat Dec 29 11:04:56 2018</td>\n      <td>1078964200234536960</td>\n      <td>Joshuashores2</td>\n      <td>Joshua shores</td>\n      <td>None</td>\n      <td>2018-12-29 10:41:26</td>\n      <td>None</td>\n      <td>en</td>\n      <td>0.0</td>\n      <td>False</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>0</td>\n      <td>34</td>\n      <td>0</td>\n      <td>0</td>\n      <td>16</td>\n      <td>1078970115717242880</td>\n      <td>bot</td>\n      <td>None</td>\n    </tr>\n  </tbody>\n</table>\n<p>50538 rows × 21 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "# midterm-2018\n",
    "# Description: Manually labeled human and bot accounts from 2018 US midterm elections. Labels and processed user objects.\n",
    "midterm_2018 = pd.read_json('midterm-2018/midterm-2018_processed_user_objects.json')\n",
    "\n",
    "# data labels\n",
    "midterm_2018_labels = pd.read_csv('midterm-2018/midterm-2018.tsv', sep='\\t', header=None)\n",
    "midterm_2018_labels.columns = ['user_id', 'label']\n",
    "\n",
    "# merge in labels on id\n",
    "midterm_2018_df = midterm_2018.merge(midterm_2018_labels, how='left')\n",
    "\n",
    "midterm_2018_df.rename(columns={'probe_timestamp':'collected_at', 'user_created_at':'created_at', 'user_id':'id'}, inplace=True)\n",
    "\n",
    "# add default_profile_image as null and we'll figure out how to handle it during EDA\n",
    "midterm_2018_df['default_profile_image'] = None\n",
    "\n",
    "# important decision: \"profile_use_background_image\" seems to be reversed from standard, uses deprecated format. now 'default_profile'==True implies no changes to background. \n",
    "# will change profile_use_background_image==True to default_profile==False, so changing name and reversing\n",
    "\n",
    "midterm_2018_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Index(['collected_at', 'id', 'screen_name', 'name', 'description',\n       'created_at', 'url', 'lang', 'protected', 'verified', 'geo_enabled',\n       'profile_use_background_image', 'default_profile', 'followers_count',\n       'friends_count', 'listed_count', 'favourites_count', 'statuses_count',\n       'tid', 'label', 'default_profile_image'],\n      dtype='object')"
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "midterm_2018_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    follow_request_sent  has_extended_profile  profile_use_background_image  \\\n0                 False                 False                          True   \n1                 False                 False                          True   \n2                 False                 False                          True   \n3                 False                 False                          True   \n4                 False                 False                          True   \n..                  ...                   ...                           ...   \n57                False                 False                          True   \n58                False                 False                          True   \n59                False                 False                          True   \n60                False                 False                          True   \n61                False                 False                          True   \n\n    default_profile_image          id  \\\n0                   False  2312443350   \n1                   False  1865011970   \n2                   False   701329338   \n3                   False  2318983652   \n4                   False  1551216937   \n..                    ...         ...   \n57                  False   628813356   \n58                  False  2272261705   \n59                  False   737418163   \n60                  False  2384134034   \n61                  False  2308520706   \n\n                   profile_background_image_url_https  verified  \\\n0   https://pbs.twimg.com/profile_background_image...     False   \n1   https://pbs.twimg.com/profile_background_image...     False   \n2   https://pbs.twimg.com/profile_background_image...     False   \n3   https://pbs.twimg.com/profile_background_image...     False   \n4   https://pbs.twimg.com/profile_background_image...     False   \n..                                                ...       ...   \n57  https://pbs.twimg.com/profile_background_image...     False   \n58  https://pbs.twimg.com/profile_background_image...     False   \n59  https://pbs.twimg.com/profile_background_image...     False   \n60  https://abs.twimg.com/images/themes/theme1/bg.png     False   \n61  https://pbs.twimg.com/profile_background_image...     False   \n\n   translator_type profile_text_color  \\\n0             none             333333   \n1             none             333333   \n2             none             333333   \n3             none             333333   \n4             none             333333   \n..             ...                ...   \n57            none             333333   \n58            none             333333   \n59            none             333333   \n60            none             333333   \n61            none             333333   \n\n                              profile_image_url_https  ... notifications  \\\n0   https://pbs.twimg.com/profile_images/859466387...  ...         False   \n1   https://pbs.twimg.com/profile_images/864910995...  ...         False   \n2   https://pbs.twimg.com/profile_images/852604219...  ...         False   \n3   https://pbs.twimg.com/profile_images/889609194...  ...         False   \n4   https://pbs.twimg.com/profile_images/883402465...  ...         False   \n..                                                ...  ...           ...   \n57  https://pbs.twimg.com/profile_images/860669066...  ...         False   \n58  https://pbs.twimg.com/profile_images/852593334...  ...         False   \n59  https://pbs.twimg.com/profile_images/972132970...  ...         False   \n60  https://pbs.twimg.com/profile_images/889253129...  ...         False   \n61  https://pbs.twimg.com/profile_images/863211824...  ...         False   \n\n                       url                      created_at  \\\n0   http://t.co/jYvqmtPizF  Sun Jan 26 20:42:09 +0000 2014   \n1   http://t.co/UvH3kIFnqg  Sat Sep 14 21:09:01 +0000 2013   \n2   http://t.co/dynJvZ70Il  Tue Jul 17 16:18:48 +0000 2012   \n3   http://t.co/ap5l34TcZV  Thu Jan 30 14:20:07 +0000 2014   \n4   http://t.co/BkObE6YBst  Thu Jun 27 18:09:27 +0000 2013   \n..                     ...                             ...   \n57  http://t.co/ap5l34TcZV  Fri Jul 06 21:21:17 +0000 2012   \n58  http://t.co/ap5l34TcZV  Thu Jan 02 00:23:57 +0000 2014   \n59  http://t.co/df2HJ34JZe  Sat Aug 04 21:08:35 +0000 2012   \n60  http://t.co/0jostSd78P  Tue Mar 11 19:51:07 +0000 2014   \n61  http://t.co/jYvqmtPizF  Fri Jan 24 15:20:24 +0000 2014   \n\n   contributors_enabled               time_zone protected  default_profile  \\\n0                 False                    None     False            False   \n1                 False                    None     False            False   \n2                 False  Atlantic Time (Canada)     False            False   \n3                 False                    None     False            False   \n4                 False  Atlantic Time (Canada)     False            False   \n..                  ...                     ...       ...              ...   \n57                False  Atlantic Time (Canada)     False            False   \n58                False                    None     False            False   \n59                False  Atlantic Time (Canada)     False            False   \n60                False                    None     False             True   \n61                False                    None     False            False   \n\n    is_translator              collected_at  label  \n0           False 2017-07-08 01:17:23+00:00    bot  \n1           False 2018-04-18 16:07:19+00:00    bot  \n2           False 2018-04-07 05:01:16+00:00    bot  \n3           False 2018-01-25 20:02:58+00:00    bot  \n4           False 2017-12-01 00:00:46+00:00    bot  \n..            ...                       ...    ...  \n57          False 2017-12-08 19:20:39+00:00    bot  \n58          False 2017-07-11 16:29:17+00:00    bot  \n59          False 2018-03-09 16:12:39+00:00    bot  \n60          False 2017-10-27 21:19:21+00:00    bot  \n61          False 2017-10-25 15:22:08+00:00    bot  \n\n[62 rows x 44 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>follow_request_sent</th>\n      <th>has_extended_profile</th>\n      <th>profile_use_background_image</th>\n      <th>default_profile_image</th>\n      <th>id</th>\n      <th>profile_background_image_url_https</th>\n      <th>verified</th>\n      <th>translator_type</th>\n      <th>profile_text_color</th>\n      <th>profile_image_url_https</th>\n      <th>...</th>\n      <th>notifications</th>\n      <th>url</th>\n      <th>created_at</th>\n      <th>contributors_enabled</th>\n      <th>time_zone</th>\n      <th>protected</th>\n      <th>default_profile</th>\n      <th>is_translator</th>\n      <th>collected_at</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2312443350</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/859466387...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>http://t.co/jYvqmtPizF</td>\n      <td>Sun Jan 26 20:42:09 +0000 2014</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2017-07-08 01:17:23+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>1865011970</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/864910995...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>http://t.co/UvH3kIFnqg</td>\n      <td>Sat Sep 14 21:09:01 +0000 2013</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-04-18 16:07:19+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>701329338</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/852604219...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>http://t.co/dynJvZ70Il</td>\n      <td>Tue Jul 17 16:18:48 +0000 2012</td>\n      <td>False</td>\n      <td>Atlantic Time (Canada)</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-04-07 05:01:16+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2318983652</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/889609194...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>http://t.co/ap5l34TcZV</td>\n      <td>Thu Jan 30 14:20:07 +0000 2014</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-01-25 20:02:58+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>1551216937</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/883402465...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>http://t.co/BkObE6YBst</td>\n      <td>Thu Jun 27 18:09:27 +0000 2013</td>\n      <td>False</td>\n      <td>Atlantic Time (Canada)</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2017-12-01 00:00:46+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>57</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>628813356</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/860669066...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>http://t.co/ap5l34TcZV</td>\n      <td>Fri Jul 06 21:21:17 +0000 2012</td>\n      <td>False</td>\n      <td>Atlantic Time (Canada)</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2017-12-08 19:20:39+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>58</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2272261705</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/852593334...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>http://t.co/ap5l34TcZV</td>\n      <td>Thu Jan 02 00:23:57 +0000 2014</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2017-07-11 16:29:17+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>59</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>737418163</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/972132970...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>http://t.co/df2HJ34JZe</td>\n      <td>Sat Aug 04 21:08:35 +0000 2012</td>\n      <td>False</td>\n      <td>Atlantic Time (Canada)</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-03-09 16:12:39+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>60</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2384134034</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/889253129...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>http://t.co/0jostSd78P</td>\n      <td>Tue Mar 11 19:51:07 +0000 2014</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2017-10-27 21:19:21+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>61</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2308520706</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/863211824...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>http://t.co/jYvqmtPizF</td>\n      <td>Fri Jan 24 15:20:24 +0000 2014</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2017-10-25 15:22:08+00:00</td>\n      <td>bot</td>\n    </tr>\n  </tbody>\n</table>\n<p>62 rows × 44 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 14
    }
   ],
   "source": [
    "# political-bots-2019\n",
    "# Description: Automated political accounts run by @rzazula (now suspended), shared by @josh_emerson on Twitter. Labels and user objects.\n",
    "political_bots_2019 = pd.read_json('political-bots-2019/political-bots-2019_tweets.json')\n",
    "political_2019_df = pd.DataFrame(list(political_bots_2019['user']))\n",
    "political_2019_df['collected_at'] = political_bots_2019['created_at']\n",
    "\n",
    "# all bots\n",
    "political_2019_df['label'] = 'bot'\n",
    "\n",
    "political_2019_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "       follow_request_sent  has_extended_profile  \\\n0                    False                 False   \n1                    False                 False   \n2                    False                 False   \n3                    False                 False   \n4                    False                 False   \n...                    ...                   ...   \n17877                False                 False   \n17878                False                 False   \n17879                False                 False   \n17880                False                 False   \n17881                False                 False   \n\n       profile_use_background_image  default_profile_image  \\\n0                              True                  False   \n1                              True                  False   \n2                              True                  False   \n3                              True                  False   \n4                              True                  False   \n...                             ...                    ...   \n17877                          True                  False   \n17878                          True                  False   \n17879                          True                  False   \n17880                          True                  False   \n17881                          True                  False   \n\n                       id                 profile_background_image_url_https  \\\n0                86166567  https://abs.twimg.com/images/themes/theme5/bg.gif   \n1               108704199  https://abs.twimg.com/images/themes/theme1/bg.png   \n2              2165940152  https://abs.twimg.com/images/themes/theme1/bg.png   \n3               196846880  https://abs.twimg.com/images/themes/theme1/bg.png   \n4               884237364  https://abs.twimg.com/images/themes/theme1/bg.png   \n...                   ...                                                ...   \n17877           188645930  https://abs.twimg.com/images/themes/theme1/bg.png   \n17878           552236544  https://abs.twimg.com/images/themes/theme1/bg.png   \n17879          1933105668  https://abs.twimg.com/images/themes/theme5/bg.gif   \n17880          3222769956  https://abs.twimg.com/images/themes/theme1/bg.png   \n17881  766645102148059138                                               None   \n\n       verified translator_type profile_text_color  \\\n0         False            none             3E4415   \n1         False            none             333333   \n2         False            none             333333   \n3         False            none             333333   \n4         False            none             333333   \n...         ...             ...                ...   \n17877     False            none             333333   \n17878     False            none             333333   \n17879     False            none             3E4415   \n17880     False            none             333333   \n17881     False            none             333333   \n\n                                 profile_image_url_https  ... notifications  \\\n0      https://pbs.twimg.com/profile_images/967640193...  ...         False   \n1      https://pbs.twimg.com/profile_images/971197066...  ...         False   \n2      https://pbs.twimg.com/profile_images/971723310...  ...         False   \n3      https://pbs.twimg.com/profile_images/971647812...  ...         False   \n4      https://pbs.twimg.com/profile_images/971628685...  ...         False   \n...                                                  ...  ...           ...   \n17877  https://pbs.twimg.com/profile_images/969003065...  ...         False   \n17878  https://pbs.twimg.com/profile_images/971837126...  ...         False   \n17879  https://pbs.twimg.com/profile_images/971290681...  ...         False   \n17880  https://pbs.twimg.com/profile_images/972321759...  ...         False   \n17881  https://pbs.twimg.com/profile_images/971846378...  ...         False   \n\n        url                      created_at contributors_enabled time_zone  \\\n0      None  Thu Oct 29 21:29:14 +0000 2009                False      None   \n1      None  Tue Jan 26 20:12:22 +0000 2010                False      None   \n2      None  Thu Oct 31 03:48:18 +0000 2013                False      None   \n3      None  Thu Sep 30 01:33:02 +0000 2010                False      None   \n4      None  Tue Oct 16 11:24:54 +0000 2012                False      None   \n...     ...                             ...                  ...       ...   \n17877  None  Thu Sep 09 06:54:00 +0000 2010                False      None   \n17878  None  Thu Apr 12 20:41:15 +0000 2012                False      None   \n17879  None  Fri Oct 04 05:18:59 +0000 2013                False      None   \n17880  None  Fri May 22 01:21:33 +0000 2015                False      None   \n17881  None  Fri Aug 19 14:36:51 +0000 2016                False      None   \n\n      protected  default_profile  is_translator              collected_at  \\\n0         False            False          False 2018-03-03 10:38:12+00:00   \n1         False             True          False 2018-03-16 09:25:13+00:00   \n2         False             True          False 2018-03-10 16:25:03+00:00   \n3         False             True          False 2018-03-10 12:09:03+00:00   \n4         False             True          False 2018-03-10 09:24:54+00:00   \n...         ...              ...            ...                       ...   \n17877     False             True          False 2018-03-02 22:13:07+00:00   \n17878     False             True          False 2018-03-10 04:52:16+00:00   \n17879     False            False          False 2018-03-16 19:28:34+00:00   \n17880     False             True          False 2018-03-10 04:42:55+00:00   \n17881     False             True          False 2018-03-10 05:55:45+00:00   \n\n       label  \n0        bot  \n1        bot  \n2        bot  \n3        bot  \n4        bot  \n...      ...  \n17877    bot  \n17878    bot  \n17879    bot  \n17880    bot  \n17881    bot  \n\n[17882 rows x 44 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>follow_request_sent</th>\n      <th>has_extended_profile</th>\n      <th>profile_use_background_image</th>\n      <th>default_profile_image</th>\n      <th>id</th>\n      <th>profile_background_image_url_https</th>\n      <th>verified</th>\n      <th>translator_type</th>\n      <th>profile_text_color</th>\n      <th>profile_image_url_https</th>\n      <th>...</th>\n      <th>notifications</th>\n      <th>url</th>\n      <th>created_at</th>\n      <th>contributors_enabled</th>\n      <th>time_zone</th>\n      <th>protected</th>\n      <th>default_profile</th>\n      <th>is_translator</th>\n      <th>collected_at</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>86166567</td>\n      <td>https://abs.twimg.com/images/themes/theme5/bg.gif</td>\n      <td>False</td>\n      <td>none</td>\n      <td>3E4415</td>\n      <td>https://pbs.twimg.com/profile_images/967640193...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Thu Oct 29 21:29:14 +0000 2009</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-03-03 10:38:12+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>108704199</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/971197066...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Tue Jan 26 20:12:22 +0000 2010</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2018-03-16 09:25:13+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2165940152</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/971723310...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Thu Oct 31 03:48:18 +0000 2013</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2018-03-10 16:25:03+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>196846880</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/971647812...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Thu Sep 30 01:33:02 +0000 2010</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2018-03-10 12:09:03+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>884237364</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/971628685...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Tue Oct 16 11:24:54 +0000 2012</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2018-03-10 09:24:54+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>17877</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>188645930</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/969003065...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Thu Sep 09 06:54:00 +0000 2010</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2018-03-02 22:13:07+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>17878</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>552236544</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/971837126...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Thu Apr 12 20:41:15 +0000 2012</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2018-03-10 04:52:16+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>17879</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>1933105668</td>\n      <td>https://abs.twimg.com/images/themes/theme5/bg.gif</td>\n      <td>False</td>\n      <td>none</td>\n      <td>3E4415</td>\n      <td>https://pbs.twimg.com/profile_images/971290681...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Fri Oct 04 05:18:59 +0000 2013</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>2018-03-16 19:28:34+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>17880</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>3222769956</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/972321759...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Fri May 22 01:21:33 +0000 2015</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2018-03-10 04:42:55+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>17881</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>766645102148059138</td>\n      <td>None</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/971846378...</td>\n      <td>...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>Fri Aug 19 14:36:51 +0000 2016</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>2018-03-10 05:55:45+00:00</td>\n      <td>bot</td>\n    </tr>\n  </tbody>\n</table>\n<p>17882 rows × 44 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "source": [
    "# pronbots-2019\n",
    "# Description: Pronbots shared by Andy Patel (github.com/r0zetta/pronbot2). Labels and user objects.\n",
    "pronbots = pd.read_json('pronbots-2019/pronbots-2019_tweets.json')\n",
    "pronbots_df = pd.DataFrame(list(pronbots['user']))\n",
    "pronbots_df['collected_at'] = pronbots['created_at']\n",
    "\n",
    "# all bots\n",
    "pronbots_df['label'] = 'bot'\n",
    "\n",
    "pronbots_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "             id           created_at         collected_at  friends_count  \\\n0          6301  2006-09-18 01:07:50  2010-01-17 20:38:25           3269   \n1         10836  2006-10-27 14:38:04  2010-06-18 03:35:34           1949   \n2         10997  2006-10-29 09:50:38  2010-04-24 01:12:40           1119   \n3        633293  2007-01-14 12:40:10  2010-01-24 11:59:38           2174   \n4        717883  2007-01-27 22:14:18  2010-02-06 06:25:58           7731   \n...         ...                  ...                  ...            ...   \n19271  93390990  2009-11-29 06:34:30  2009-11-29 07:50:54              5   \n19272  93402679  2009-11-29 07:47:50  2009-11-29 07:56:28             20   \n19273  93419256  2009-11-29 09:23:30  2009-11-29 09:30:48              0   \n19274  93426370  2009-11-29 10:04:26  2009-11-29 10:13:17             20   \n19275  93442002  2009-11-29 11:27:25  2009-11-29 11:36:52              1   \n\n       followers_count  statuses_count  screen_name_length  \\\n0                 3071             861                   8   \n1                  793             226                   9   \n2                 9644           38674                  12   \n3                 6029           12718                  11   \n4                 7029             873                   6   \n...                ...             ...                 ...   \n19271                0               5                  11   \n19272                1               1                  12   \n19273                0               1                   8   \n19274                1               1                  10   \n19275                0               1                   7   \n\n       profile_desc_length  label  \n0                      132    bot  \n1                      134    bot  \n2                      158    bot  \n3                      121    bot  \n4                       70    bot  \n...                    ...    ...  \n19271                    0  human  \n19272                    0  human  \n19273                    0  human  \n19274                    0  human  \n19275                    0  human  \n\n[41499 rows x 9 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>created_at</th>\n      <th>collected_at</th>\n      <th>friends_count</th>\n      <th>followers_count</th>\n      <th>statuses_count</th>\n      <th>screen_name_length</th>\n      <th>profile_desc_length</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>6301</td>\n      <td>2006-09-18 01:07:50</td>\n      <td>2010-01-17 20:38:25</td>\n      <td>3269</td>\n      <td>3071</td>\n      <td>861</td>\n      <td>8</td>\n      <td>132</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>10836</td>\n      <td>2006-10-27 14:38:04</td>\n      <td>2010-06-18 03:35:34</td>\n      <td>1949</td>\n      <td>793</td>\n      <td>226</td>\n      <td>9</td>\n      <td>134</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>10997</td>\n      <td>2006-10-29 09:50:38</td>\n      <td>2010-04-24 01:12:40</td>\n      <td>1119</td>\n      <td>9644</td>\n      <td>38674</td>\n      <td>12</td>\n      <td>158</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>633293</td>\n      <td>2007-01-14 12:40:10</td>\n      <td>2010-01-24 11:59:38</td>\n      <td>2174</td>\n      <td>6029</td>\n      <td>12718</td>\n      <td>11</td>\n      <td>121</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>717883</td>\n      <td>2007-01-27 22:14:18</td>\n      <td>2010-02-06 06:25:58</td>\n      <td>7731</td>\n      <td>7029</td>\n      <td>873</td>\n      <td>6</td>\n      <td>70</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>19271</th>\n      <td>93390990</td>\n      <td>2009-11-29 06:34:30</td>\n      <td>2009-11-29 07:50:54</td>\n      <td>5</td>\n      <td>0</td>\n      <td>5</td>\n      <td>11</td>\n      <td>0</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>19272</th>\n      <td>93402679</td>\n      <td>2009-11-29 07:47:50</td>\n      <td>2009-11-29 07:56:28</td>\n      <td>20</td>\n      <td>1</td>\n      <td>1</td>\n      <td>12</td>\n      <td>0</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>19273</th>\n      <td>93419256</td>\n      <td>2009-11-29 09:23:30</td>\n      <td>2009-11-29 09:30:48</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>8</td>\n      <td>0</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>19274</th>\n      <td>93426370</td>\n      <td>2009-11-29 10:04:26</td>\n      <td>2009-11-29 10:13:17</td>\n      <td>20</td>\n      <td>1</td>\n      <td>1</td>\n      <td>10</td>\n      <td>0</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>19275</th>\n      <td>93442002</td>\n      <td>2009-11-29 11:27:25</td>\n      <td>2009-11-29 11:36:52</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>7</td>\n      <td>0</td>\n      <td>human</td>\n    </tr>\n  </tbody>\n</table>\n<p>41499 rows × 9 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 16
    }
   ],
   "source": [
    "# caverlee-2011\n",
    "# Description: This social honeypot dataset collected from December 30, 2009 to August 2, 2010 on Twitter. \n",
    "# The dataset contains 22,223 content polluters, their number of followings over time 2,353,473 tweets, \n",
    "# and 19,276 legitimate users, their number of followings over time and 3,259,693 tweets.\n",
    "# file:///Users/Kelso/Downloads/Seven_Months_with_the_Devils_A_Long-Term_Study_of_.pdf\n",
    "# Note: missing most metadata features, likely not usable\n",
    "honeypot_2011_bots_df = pd.read_csv('social_honeypot_icwsm_2011/content_polluters.txt', sep='\\t', header=None)\n",
    "honeypot_2011_bots_df.columns=['UserID', 'created_at', 'collected_at', 'following_count', 'followers_count', 'tweets_count', 'screen_name_length', 'profile_desc_length']\n",
    "\n",
    "honeypot_2011_legitimate_df = pd.read_csv('social_honeypot_icwsm_2011/legitimate_users.txt', sep='\\t', header=None)\n",
    "honeypot_2011_legitimate_df.columns=['UserID', 'created_at', 'collected_at', 'following_count', 'followers_count', 'tweets_count', 'screen_name_length', 'profile_desc_length']\n",
    "\n",
    "# labels\n",
    "honeypot_2011_bots_df['label'] = 'bot'\n",
    "honeypot_2011_legitimate_df['label'] = 'human'\n",
    "\n",
    "honeypot_2011_df = pd.concat([honeypot_2011_bots_df, honeypot_2011_legitimate_df])\n",
    "\n",
    "honeypot_2011_df.rename(columns={'UserID':'id', 'following_count':'friends_count', 'tweets_count':'statuses_count'}, inplace=True)\n",
    "\n",
    "honeypot_2011_df\n",
    "# follower counts over time\n",
    "# pd.read_csv('social_honeypot_icwsm_2011/content_polluters_followings.txt', sep='\\t', header=None)\n",
    "# pd.read_csv('social_honeypot_icwsm_2011/legitimate_users_followings.txt', sep='\\t', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "      follow_request_sent  has_extended_profile  profile_use_background_image  \\\n0                   False                 False                          True   \n1                   False                 False                          True   \n2                   False                 False                          True   \n3                   False                 False                          True   \n4                   False                 False                          True   \n...                   ...                   ...                           ...   \n1083                False                 False                         False   \n1084                False                 False                         False   \n1085                False                 False                          True   \n1086                False                  True                          True   \n1087                False                 False                          True   \n\n                      id                 profile_background_image_url_https  \\\n0               53805021  https://abs.twimg.com/images/themes/theme1/bg.png   \n1               66317516  https://abs.twimg.com/images/themes/theme1/bg.png   \n2              483428005  https://pbs.twimg.com/profile_background_image...   \n3              512754110  https://abs.twimg.com/images/themes/theme1/bg.png   \n4             2583514062  https://abs.twimg.com/images/themes/theme1/bg.png   \n...                  ...                                                ...   \n1083          1409637493  https://pbs.twimg.com/profile_background_image...   \n1084           623584562  https://abs.twimg.com/images/themes/theme1/bg.png   \n1085          2906301805  https://abs.twimg.com/images/themes/theme1/bg.png   \n1086  948314138886987777                                               None   \n1087          3434325791  https://abs.twimg.com/images/themes/theme1/bg.png   \n\n      verified translator_type profile_text_color  \\\n0        False            none             333333   \n1        False            none             333333   \n2        False            none             333333   \n3        False            none             333333   \n4        False            none             333333   \n...        ...             ...                ...   \n1083     False            none             827972   \n1084     False         regular             333333   \n1085     False            none             333333   \n1086     False            none             333333   \n1087     False            none             333333   \n\n                                profile_image_url_https  \\\n0     https://abs.twimg.com/sticky/default_profile_i...   \n1     https://pbs.twimg.com/profile_images/946806849...   \n2     https://pbs.twimg.com/profile_images/180475720...   \n3     https://pbs.twimg.com/profile_images/186863532...   \n4     https://pbs.twimg.com/profile_images/480973233...   \n...                                                 ...   \n1083  https://pbs.twimg.com/profile_images/878063076...   \n1084  https://pbs.twimg.com/profile_images/538768842...   \n1085  https://pbs.twimg.com/profile_images/909393909...   \n1086  https://pbs.twimg.com/profile_images/950794598...   \n1087  https://pbs.twimg.com/profile_images/634755126...   \n\n     profile_sidebar_fill_color  ...                      url  \\\n0                        DDEEF6  ...                     None   \n1                        DDEEF6  ...                     None   \n2                        DDEEF6  ...                     None   \n3                        DDEEF6  ...                     None   \n4                        DDEEF6  ...   http://t.co/2oTcpHdIA4   \n...                         ...  ...                      ...   \n1083                     061127  ...  https://t.co/UwIM5mT9TB   \n1084                     DDEEF6  ...  https://t.co/RkOBDnTANL   \n1085                     DDEEF6  ...                     None   \n1086                     DDEEF6  ...  https://t.co/RuRHCm4qJE   \n1087                     DDEEF6  ...  https://t.co/M6vM20XX9K   \n\n                          created_at  contributors_enabled  \\\n0     Sun Jul 05 01:14:40 +0000 2009                 False   \n1     Mon Aug 17 09:21:25 +0000 2009                 False   \n2     Sun Feb 05 00:49:59 +0000 2012                 False   \n3     Sat Mar 03 03:53:35 +0000 2012                 False   \n4     Mon Jun 23 07:15:59 +0000 2014                 False   \n...                              ...                   ...   \n1083  Tue May 07 07:25:47 +0000 2013                 False   \n1084  Sun Jul 01 09:29:35 +0000 2012                 False   \n1085  Fri Nov 21 17:00:28 +0000 2014                 False   \n1086  Tue Jan 02 22:04:44 +0000 2018                 False   \n1087  Fri Aug 21 15:47:34 +0000 2015                 False   \n\n                       time_zone protected  default_profile  following  \\\n0                        Beijing     False             True      False   \n1                          Paris     False             True      False   \n2     Eastern Time (US & Canada)     False            False      False   \n3                           None     False             True      False   \n4                           None     False             True      False   \n...                          ...       ...              ...        ...   \n1083                    Brisbane     False            False      False   \n1084  Central Time (US & Canada)     False            False      False   \n1085  Pacific Time (US & Canada)     False             True      False   \n1086                        None     False             True      False   \n1087                        None     False             True      False   \n\n                                     profile_banner_url  \\\n0                                                   NaN   \n1     https://pbs.twimg.com/profile_banners/66317516...   \n2                                                   NaN   \n3                                                   NaN   \n4     https://pbs.twimg.com/profile_banners/25835140...   \n...                                                 ...   \n1083  https://pbs.twimg.com/profile_banners/14096374...   \n1084  https://pbs.twimg.com/profile_banners/62358456...   \n1085  https://pbs.twimg.com/profile_banners/29063018...   \n1086  https://pbs.twimg.com/profile_banners/94831413...   \n1087  https://pbs.twimg.com/profile_banners/34343257...   \n\n                  collected_at  label  \n0    2018-01-09 16:38:26+00:00    bot  \n1    2018-01-12 21:55:46+00:00    bot  \n2    2018-01-13 20:16:44+00:00    bot  \n3    2018-01-11 06:31:06+00:00    bot  \n4    2018-01-09 02:16:16+00:00    bot  \n...                        ...    ...  \n1083 2018-02-01 00:07:44+00:00    bot  \n1084 2018-01-31 00:48:16+00:00    bot  \n1085 2018-01-31 00:44:44+00:00    bot  \n1086 2018-02-01 07:32:20+00:00    bot  \n1087 2018-01-31 17:31:00+00:00    bot  \n\n[1088 rows x 44 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>follow_request_sent</th>\n      <th>has_extended_profile</th>\n      <th>profile_use_background_image</th>\n      <th>id</th>\n      <th>profile_background_image_url_https</th>\n      <th>verified</th>\n      <th>translator_type</th>\n      <th>profile_text_color</th>\n      <th>profile_image_url_https</th>\n      <th>profile_sidebar_fill_color</th>\n      <th>...</th>\n      <th>url</th>\n      <th>created_at</th>\n      <th>contributors_enabled</th>\n      <th>time_zone</th>\n      <th>protected</th>\n      <th>default_profile</th>\n      <th>following</th>\n      <th>profile_banner_url</th>\n      <th>collected_at</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>53805021</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://abs.twimg.com/sticky/default_profile_i...</td>\n      <td>DDEEF6</td>\n      <td>...</td>\n      <td>None</td>\n      <td>Sun Jul 05 01:14:40 +0000 2009</td>\n      <td>False</td>\n      <td>Beijing</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>NaN</td>\n      <td>2018-01-09 16:38:26+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>66317516</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/946806849...</td>\n      <td>DDEEF6</td>\n      <td>...</td>\n      <td>None</td>\n      <td>Mon Aug 17 09:21:25 +0000 2009</td>\n      <td>False</td>\n      <td>Paris</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>https://pbs.twimg.com/profile_banners/66317516...</td>\n      <td>2018-01-12 21:55:46+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>483428005</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/180475720...</td>\n      <td>DDEEF6</td>\n      <td>...</td>\n      <td>None</td>\n      <td>Sun Feb 05 00:49:59 +0000 2012</td>\n      <td>False</td>\n      <td>Eastern Time (US &amp; Canada)</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>NaN</td>\n      <td>2018-01-13 20:16:44+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>512754110</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/186863532...</td>\n      <td>DDEEF6</td>\n      <td>...</td>\n      <td>None</td>\n      <td>Sat Mar 03 03:53:35 +0000 2012</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>NaN</td>\n      <td>2018-01-11 06:31:06+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>2583514062</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/480973233...</td>\n      <td>DDEEF6</td>\n      <td>...</td>\n      <td>http://t.co/2oTcpHdIA4</td>\n      <td>Mon Jun 23 07:15:59 +0000 2014</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>https://pbs.twimg.com/profile_banners/25835140...</td>\n      <td>2018-01-09 02:16:16+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1083</th>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>1409637493</td>\n      <td>https://pbs.twimg.com/profile_background_image...</td>\n      <td>False</td>\n      <td>none</td>\n      <td>827972</td>\n      <td>https://pbs.twimg.com/profile_images/878063076...</td>\n      <td>061127</td>\n      <td>...</td>\n      <td>https://t.co/UwIM5mT9TB</td>\n      <td>Tue May 07 07:25:47 +0000 2013</td>\n      <td>False</td>\n      <td>Brisbane</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>https://pbs.twimg.com/profile_banners/14096374...</td>\n      <td>2018-02-01 00:07:44+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1084</th>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>623584562</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>regular</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/538768842...</td>\n      <td>DDEEF6</td>\n      <td>...</td>\n      <td>https://t.co/RkOBDnTANL</td>\n      <td>Sun Jul 01 09:29:35 +0000 2012</td>\n      <td>False</td>\n      <td>Central Time (US &amp; Canada)</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>https://pbs.twimg.com/profile_banners/62358456...</td>\n      <td>2018-01-31 00:48:16+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1085</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>2906301805</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/909393909...</td>\n      <td>DDEEF6</td>\n      <td>...</td>\n      <td>None</td>\n      <td>Fri Nov 21 17:00:28 +0000 2014</td>\n      <td>False</td>\n      <td>Pacific Time (US &amp; Canada)</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>https://pbs.twimg.com/profile_banners/29063018...</td>\n      <td>2018-01-31 00:44:44+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1086</th>\n      <td>False</td>\n      <td>True</td>\n      <td>True</td>\n      <td>948314138886987777</td>\n      <td>None</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/950794598...</td>\n      <td>DDEEF6</td>\n      <td>...</td>\n      <td>https://t.co/RuRHCm4qJE</td>\n      <td>Tue Jan 02 22:04:44 +0000 2018</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>https://pbs.twimg.com/profile_banners/94831413...</td>\n      <td>2018-02-01 07:32:20+00:00</td>\n      <td>bot</td>\n    </tr>\n    <tr>\n      <th>1087</th>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>3434325791</td>\n      <td>https://abs.twimg.com/images/themes/theme1/bg.png</td>\n      <td>False</td>\n      <td>none</td>\n      <td>333333</td>\n      <td>https://pbs.twimg.com/profile_images/634755126...</td>\n      <td>DDEEF6</td>\n      <td>...</td>\n      <td>https://t.co/M6vM20XX9K</td>\n      <td>Fri Aug 21 15:47:34 +0000 2015</td>\n      <td>False</td>\n      <td>None</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>https://pbs.twimg.com/profile_banners/34343257...</td>\n      <td>2018-01-31 17:31:00+00:00</td>\n      <td>bot</td>\n    </tr>\n  </tbody>\n</table>\n<p>1088 rows × 44 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "# vendor-purchased-2019\n",
    "# Description: Fake follower accounts purchased from several companies. Labels and user objects.\n",
    "vendor_purchased_2019 = pd.read_json('vendor-purchased-2019/vendor-purchased-2019_tweets.json')\n",
    "vendor_2019_df = pd.DataFrame(list(vendor_purchased_2019['user']))\n",
    "vendor_2019_df['collected_at'] = vendor_purchased_2019['created_at']\n",
    "\n",
    "# all bots\n",
    "vendor_2019_df['label'] = 'bot'\n",
    "\n",
    "vendor_2019_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "              id      id_str              name      screen_name  \\\n0       31098240    31098240         Netwerk24        Netwerk24   \n1        6892002     6892002   WBIR Channel 10             wbir   \n2       20083388    20083388   Patricia Mazzei   PatriciaMazzei   \n3      385587235   385587235      Jeff Skowron      Jeffskowron   \n4       51240839    51240839  Heather Crawford       HeatherFCN   \n...          ...         ...               ...              ...   \n1982    76235237    76235237  Tire Discounters  TireDiscounters   \n1983  2159329092  2159329092  eillie anzilotti       eillieanzi   \n1984    25527618    25527618    Debra Birnbaum    debrabirnbaum   \n1985    43654274    43654274    艾未未  Ai Weiwei             aiww   \n1986    19084210    19084210        LexisNexis       LexisNexis   \n\n               location                                        description  \\\n0          Johannesburg  Al die nuus, politiek, sport, geldsake & verma...   \n1         Knoxville, TN  News, Sports, Weather & Live At Five at 4:00 -...   \n2                 Miami  @nytimes Miami bureau chief. Former @MiamiHera...   \n3           los angeles  Actor, writer, co-creator of @greganddonny and...   \n4      Jacksonville, FL  Wife, Mom, @FCN2Go Anchor @TSIChildren Mentor,...   \n...                 ...                                                ...   \n1982  Higher Standards.  Family owned & operated business for 40 years!...   \n1983        Seattle, WA  (it's eye-lee.) sustainability, cities, and eq...   \n1984        Los Angeles  Head of awards at Amazon. Former editor at Var...   \n1985                 北京  Instagram: https://t.co/dASr1tt9PE https://t.c...   \n1986       New York, NY  The official feed of LexisNexis®, a leading gl...   \n\n                          url  \\\n0     https://t.co/Y0QuSqwwOL   \n1                        None   \n2     https://t.co/zGJ4T9rCnp   \n3     https://t.co/sLdAcQfZXV   \n4     https://t.co/eOw8ZfXBWc   \n...                       ...   \n1982  https://t.co/XWFZ4aqcZ4   \n1983  https://t.co/OJ69l0pxyk   \n1984  https://t.co/IiGhu084D5   \n1985  https://t.co/wBpiJ5kRRf   \n1986                     None   \n\n                                               entities  protected  \\\n0     {'url': {'urls': [{'url': 'https://t.co/Y0QuSq...      False   \n1                         {'description': {'urls': []}}      False   \n2     {'url': {'urls': [{'url': 'https://t.co/zGJ4T9...      False   \n3     {'url': {'urls': [{'url': 'https://t.co/sLdAcQ...      False   \n4     {'url': {'urls': [{'url': 'https://t.co/eOw8Zf...      False   \n...                                                 ...        ...   \n1982  {'url': {'urls': [{'url': 'https://t.co/XWFZ4a...      False   \n1983  {'url': {'urls': [{'url': 'https://t.co/OJ69l0...      False   \n1984  {'url': {'urls': [{'url': 'https://t.co/IiGhu0...      False   \n1985  {'url': {'urls': [{'url': 'https://t.co/wBpiJ5...      False   \n1986                      {'description': {'urls': []}}      False   \n\n      followers_count  ...  profile_use_background_image  \\\n0              188419  ...                         False   \n1              180759  ...                          True   \n2               19311  ...                          True   \n3                 398  ...                         False   \n4                7125  ...                          True   \n...               ...  ...                           ...   \n1982             3020  ...                         False   \n1983             4074  ...                         False   \n1984            17014  ...                          True   \n1985           367516  ...                          True   \n1986            55512  ...                          True   \n\n      has_extended_profile default_profile  default_profile_image following  \\\n0                    False           False                  False     False   \n1                    False           False                  False     False   \n2                    False           False                  False     False   \n3                    False           False                  False     False   \n4                    False           False                  False     False   \n...                    ...             ...                    ...       ...   \n1982                  True           False                  False     False   \n1983                 False           False                  False     False   \n1984                  True           False                  False     False   \n1985                 False           False                  False     False   \n1986                 False           False                  False     False   \n\n     follow_request_sent  notifications  translator_type  \\\n0                  False          False             none   \n1                  False          False             none   \n2                  False          False             none   \n3                  False          False             none   \n4                  False          False          regular   \n...                  ...            ...              ...   \n1982               False          False             none   \n1983               False          False             none   \n1984               False          False             none   \n1985               False          False             none   \n1986               False          False             none   \n\n                  collected_at  label  \n0    2019-06-05 20:09:15+00:00  human  \n1    2019-06-05 22:40:04+00:00  human  \n2    2019-06-04 22:04:21+00:00  human  \n3    2019-06-05 10:58:25+00:00  human  \n4    2019-06-05 18:21:32+00:00  human  \n...                        ...    ...  \n1982 2019-06-05 12:26:47+00:00  human  \n1983 2019-06-05 22:07:56+00:00  human  \n1984 2019-06-05 14:16:01+00:00  human  \n1985 2019-06-05 16:33:33+00:00  human  \n1986 2019-06-06 01:00:12+00:00  human  \n\n[1987 rows x 44 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>id_str</th>\n      <th>name</th>\n      <th>screen_name</th>\n      <th>location</th>\n      <th>description</th>\n      <th>url</th>\n      <th>entities</th>\n      <th>protected</th>\n      <th>followers_count</th>\n      <th>...</th>\n      <th>profile_use_background_image</th>\n      <th>has_extended_profile</th>\n      <th>default_profile</th>\n      <th>default_profile_image</th>\n      <th>following</th>\n      <th>follow_request_sent</th>\n      <th>notifications</th>\n      <th>translator_type</th>\n      <th>collected_at</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>31098240</td>\n      <td>31098240</td>\n      <td>Netwerk24</td>\n      <td>Netwerk24</td>\n      <td>Johannesburg</td>\n      <td>Al die nuus, politiek, sport, geldsake &amp; verma...</td>\n      <td>https://t.co/Y0QuSqwwOL</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/Y0QuSq...</td>\n      <td>False</td>\n      <td>188419</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 20:09:15+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>6892002</td>\n      <td>6892002</td>\n      <td>WBIR Channel 10</td>\n      <td>wbir</td>\n      <td>Knoxville, TN</td>\n      <td>News, Sports, Weather &amp; Live At Five at 4:00 -...</td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>180759</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 22:40:04+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>20083388</td>\n      <td>20083388</td>\n      <td>Patricia Mazzei</td>\n      <td>PatriciaMazzei</td>\n      <td>Miami</td>\n      <td>@nytimes Miami bureau chief. Former @MiamiHera...</td>\n      <td>https://t.co/zGJ4T9rCnp</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/zGJ4T9...</td>\n      <td>False</td>\n      <td>19311</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-04 22:04:21+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>385587235</td>\n      <td>385587235</td>\n      <td>Jeff Skowron</td>\n      <td>Jeffskowron</td>\n      <td>los angeles</td>\n      <td>Actor, writer, co-creator of @greganddonny and...</td>\n      <td>https://t.co/sLdAcQfZXV</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/sLdAcQ...</td>\n      <td>False</td>\n      <td>398</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 10:58:25+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>51240839</td>\n      <td>51240839</td>\n      <td>Heather Crawford</td>\n      <td>HeatherFCN</td>\n      <td>Jacksonville, FL</td>\n      <td>Wife, Mom, @FCN2Go Anchor @TSIChildren Mentor,...</td>\n      <td>https://t.co/eOw8ZfXBWc</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/eOw8Zf...</td>\n      <td>False</td>\n      <td>7125</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>regular</td>\n      <td>2019-06-05 18:21:32+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1982</th>\n      <td>76235237</td>\n      <td>76235237</td>\n      <td>Tire Discounters</td>\n      <td>TireDiscounters</td>\n      <td>Higher Standards.</td>\n      <td>Family owned &amp; operated business for 40 years!...</td>\n      <td>https://t.co/XWFZ4aqcZ4</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/XWFZ4a...</td>\n      <td>False</td>\n      <td>3020</td>\n      <td>...</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 12:26:47+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>1983</th>\n      <td>2159329092</td>\n      <td>2159329092</td>\n      <td>eillie anzilotti</td>\n      <td>eillieanzi</td>\n      <td>Seattle, WA</td>\n      <td>(it's eye-lee.) sustainability, cities, and eq...</td>\n      <td>https://t.co/OJ69l0pxyk</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/OJ69l0...</td>\n      <td>False</td>\n      <td>4074</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 22:07:56+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>1984</th>\n      <td>25527618</td>\n      <td>25527618</td>\n      <td>Debra Birnbaum</td>\n      <td>debrabirnbaum</td>\n      <td>Los Angeles</td>\n      <td>Head of awards at Amazon. Former editor at Var...</td>\n      <td>https://t.co/IiGhu084D5</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/IiGhu0...</td>\n      <td>False</td>\n      <td>17014</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 14:16:01+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>1985</th>\n      <td>43654274</td>\n      <td>43654274</td>\n      <td>艾未未  Ai Weiwei</td>\n      <td>aiww</td>\n      <td>北京</td>\n      <td>Instagram: https://t.co/dASr1tt9PE https://t.c...</td>\n      <td>https://t.co/wBpiJ5kRRf</td>\n      <td>{'url': {'urls': [{'url': 'https://t.co/wBpiJ5...</td>\n      <td>False</td>\n      <td>367516</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-05 16:33:33+00:00</td>\n      <td>human</td>\n    </tr>\n    <tr>\n      <th>1986</th>\n      <td>19084210</td>\n      <td>19084210</td>\n      <td>LexisNexis</td>\n      <td>LexisNexis</td>\n      <td>New York, NY</td>\n      <td>The official feed of LexisNexis®, a leading gl...</td>\n      <td>None</td>\n      <td>{'description': {'urls': []}}</td>\n      <td>False</td>\n      <td>55512</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>none</td>\n      <td>2019-06-06 01:00:12+00:00</td>\n      <td>human</td>\n    </tr>\n  </tbody>\n</table>\n<p>1987 rows × 44 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 18
    }
   ],
   "source": [
    "# verified-2019\n",
    "# Description: Verified human accounts. Labels and user objects.\n",
    "verified_2019 = pd.read_json('verified-2019/verified-2019_tweets.json')\n",
    "verified_2019_df = pd.DataFrame(list(verified_2019['user']))\n",
    "verified_2019_df['collected_at'] = verified_2019['created_at']\n",
    "\n",
    "# all human\n",
    "verified_2019_df['label'] = 'human'\n",
    "\n",
    "verified_2019_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# botometer_2019_df\n",
    "# botwiki_2019_df\n",
    "# celebrity_2019_df\n",
    "# cresci_2015_df\n",
    "# cresci_2017_df\n",
    "# cresci_rtbust_2019_df\n",
    "# cresci_stock_2018_df\n",
    "# gilani_2017_df\n",
    "# midterm_2018_df\n",
    "# political_2019_df\n",
    "# pronbots_df\n",
    "# honeypot_2011_df\n",
    "# vendor_2019_df\n",
    "# verified_2019_df\n",
    "\n",
    "# Note, much of this data is dated and many of these user-object features have been deprecated. Initial model building will include only modern features for better futureproofing. \n",
    "\n",
    "# potential later inclusions, need additional work to make numerical: \n",
    "# name\n",
    "# screen_name\n",
    "# location\n",
    "# description\n",
    "\n",
    "# drop_list = [ 'description', 'id', 'id_str', 'location', 'name', 'profile_banner_url', 'profile_image_url_https', 'screen_name', 'url', 'utc_offset''witheld_in_countries']\n",
    "keep_cols = ['created_at', 'collected_at', 'default_profile', 'default_profile_image', 'favourites_count', 'followers_count', 'friends_count', 'listed_count', 'protected', 'statuses_count', 'verified', 'label']\n",
    "\n",
    "df_list = [botometer_2019_df, botwiki_2019_df, celebrity_2019_df, cresci_2015_df, cresci_2017_df, cresci_rtbust_2019_df, cresci_stock_2018_df, gilani_2017_df, midterm_2018_df, political_2019_df, pronbots_df, honeypot_2011_df, vendor_2019_df, verified_2019_df]\n",
    "# for df in df_list: \n",
    "#     df = df[keep_cols]\n",
    "#     df = df.reindex(sorted(df.columns), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Missing Features: \n",
    "#### cresci_2015_df: \n",
    "follow_request_sent\n",
    "has_extended_profile\n",
    "translator_type\n",
    "entities\n",
    "id_str\n",
    "is_translation_enabled\n",
    "following\n",
    "notifications\n",
    "contributors_enabled\n",
    "is_translator\n",
    "withheld_in_countries\n",
    "\n",
    "#### cresci_2017_df\n",
    "has_extended_profile\n",
    "translator_type\n",
    "entities\n",
    "id_str\n",
    "is_translation_enabled\n",
    "withheld_in_countries\n",
    "\n",
    "#### midterm_2018_df \n",
    "follow_request_sent\n",
    "has_extended_profile\n",
    "default_profile_image\n",
    "id\n",
    "profile_background_image_url_https\n",
    "translator_type\n",
    "profile_text_color\n",
    "profile_image_url_https\n",
    "profile_sidebar_fill_color\n",
    "entities\n",
    "profile_sidebar_border_color\n",
    "id_str\n",
    "profile_background_color\n",
    "is_translation_enabled\n",
    "utc_offset\n",
    "location\n",
    "profile_link_color\n",
    "profile_image_url\n",
    "following\n",
    "profile_banner_url\n",
    "profile_background_image_url\n",
    "profile_background_tile\n",
    "notifications\n",
    "created_at\n",
    "contributors_enabled\n",
    "time_zone\n",
    "is_translator\n",
    "withheld_in_countries\n",
    "\n",
    "#### honeypot_2011_df\n",
    "follow_request_sent\n",
    "has_extended_profile\n",
    "profile_use_background_image\n",
    "default_profile_image\n",
    "id\n",
    "profile_background_image_url_https\n",
    "verified\n",
    "translator_type\n",
    "profile_text_color\n",
    "profile_image_url_https\n",
    "profile_sidebar_fill_color\n",
    "entities\n",
    "profile_sidebar_border_color\n",
    "id_str\n",
    "profile_background_color\n",
    "listed_count\n",
    "is_translation_enabled\n",
    "utc_offset\n",
    "statuses_count\n",
    "description\n",
    "friends_count\n",
    "location\n",
    "profile_link_color\n",
    "profile_image_url\n",
    "following\n",
    "geo_enabled\n",
    "profile_banner_url\n",
    "profile_background_image_url\n",
    "screen_name\n",
    "lang\n",
    "profile_background_tile\n",
    "favourites_count\n",
    "name\n",
    "notifications\n",
    "url\n",
    "contributors_enabled\n",
    "time_zone\n",
    "protected\n",
    "default_profile\n",
    "is_translator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "------\n------\n------\n------\n------\n------\n------\n------\n------\n------\n------\n------\ndefault_profile\ndefault_profile_image\nfavourites_count\nlisted_count\nprotected\nverified\n------\n------\n"
    }
   ],
   "source": [
    "# for df in df_list: \n",
    "#     print(len(df.columns))\n",
    "\n",
    "for df in df_list:\n",
    "    print('------')\n",
    "    for column in keep_cols:\n",
    "        if column not in df.columns:\n",
    "            print(column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "(518, 519)\n(698, 698)\n(20983, 20983)\n(5301, 5301)\n(11017, 11017)\n(693, 693)\n(13276, 13276)\n(2494, 2513)\n(50538, 50538)\n(62, 62)\n(17882, 17882)\n(41455, 41499)\n(1088, 1088)\n(1987, 1987)\n"
    }
   ],
   "source": [
    "# check for duplicate id values \n",
    "for df in df_list: \n",
    "    if 'id' in df.columns:\n",
    "        print((df['id'].nunique(), len(df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove honeypot\n",
    "df_list_v2 = [botometer_2019_df, botwiki_2019_df, celebrity_2019_df, cresci_2015_df, cresci_2017_df, cresci_rtbust_2019_df, cresci_stock_2018_df, gilani_2017_df, midterm_2018_df, political_2019_df, pronbots_df, vendor_2019_df, verified_2019_df]\n",
    "\n",
    "# reduce to initial features\n",
    "key_features_df_list = []\n",
    "for df in df_list_v2:\n",
    "    key_features_df_list.append(df[keep_cols])\n",
    "    \n",
    "starter_df = pd.concat(key_features_df_list)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "starter_df.to_pickle('starter.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "       screen_name  followers_count\n0       EmmaDingle              790\n1        Wolv_2018            16039\n2  RobPalmWhistler             7618\n3       Marianocrt              388\n4       herrlarbig            19677\n5        bartvalou               27\n6       DadaistBot              260\n7          Fando_C              239\n8      Todd_Scheve            23025\n9    edward_spratt               50",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>screen_name</th>\n      <th>followers_count</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>EmmaDingle</td>\n      <td>790</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Wolv_2018</td>\n      <td>16039</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>RobPalmWhistler</td>\n      <td>7618</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Marianocrt</td>\n      <td>388</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>herrlarbig</td>\n      <td>19677</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>bartvalou</td>\n      <td>27</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>DadaistBot</td>\n      <td>260</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>Fando_C</td>\n      <td>239</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>Todd_Scheve</td>\n      <td>23025</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>edward_spratt</td>\n      <td>50</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 40
    }
   ],
   "source": [
    "features_screen_name_df.iloc[0:10][['screen_name', 'followers_count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# leave in id and profile\n",
    "features_screen_name = []\n",
    "for df in df_list_v2: \n",
    "    features_screen_name.append(df[keep_cols_v2])\n",
    "features_screen_name_df = pd.concat(features_screen_name)\n",
    "features_screen_name_df.to_pickle('screen_name_starter.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "['created_at',\n 'collected_at',\n 'default_profile',\n 'default_profile_image',\n 'favourites_count',\n 'followers_count',\n 'friends_count',\n 'listed_count',\n 'protected',\n 'statuses_count',\n 'verified',\n 'label',\n 'screen_name']"
     },
     "metadata": {},
     "execution_count": 32
    }
   ],
   "source": [
    "keep_cols_v2 = keep_cols + ['screen_name']\n",
    "keep_cols_v2"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}