generations.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python_defaultSpec_1599952335559",
   "display_name": "Python 3.7.7 64-bit ('anaconda3': virtualenv)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pull in generated text files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "['data/generated_text/gpt2_gentext_20200912_202521.txt',\n 'data/generated_text/gpt2_gentext_20200912_201909.txt',\n 'data/generated_text/gpt2_gentext_20200912_202807.txt',\n 'data/generated_text/gpt2_gentext_20200912_203040.txt',\n 'data/generated_text/gpt2_gentext_20200912_201420.txt']"
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "file_names = glob.glob('data/generated_text/*.txt')\n",
    "file_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "open_files = [open(file_name) for file_name in file_names]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean and split generator output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_generated_file(text):\n",
    "    \"\"\"\n",
    "    take in a generated text file (single string containing multiple samples in mutliple batches)\n",
    "    split into individual strings and remove generator artifact tokens\n",
    "\n",
    "    input format (single string):\n",
    "    \n",
    "    'texttexttext<|endoftext|>\n",
    "    <|startoftext|>textexttext<|endoftext|>\n",
    "    ...\n",
    "    \\n====================\\n\n",
    "    texttexttext<|endoftext|>\n",
    "    <|startoftext|>textexttext<|endoftext|>\n",
    "    ...'\n",
    "    \n",
    "    input: string (file.read())\n",
    "    returns: list of strings (separated and cleaned)\n",
    "    \"\"\"\n",
    " \n",
    "    # there's some messiness in the endoftext/startoftext tokens that makes splitting on them inconsistent\n",
    "    # so after some experimenting this approach seems to work better\n",
    "    \n",
    "    # split on < and > from '<|startoftext|>' and '<|endoftext|>' tokens\n",
    "    text = text.split('<') # str -> ['str', 'str',...]\n",
    "    text = [x.split('>') for x in text] # [['str'], ['str', 'str'], ...]\n",
    "    text = list(chain(*text)) # flatten list ['str', 'str',...]\n",
    "    # split elements on batch separator token\n",
    "    text = [x.split('\\n====================\\n') for x in text] \n",
    "    text = list(chain(*text)) # flatten list\n",
    "    # remove generator token artifacts\n",
    "    rem_tokens = ['|startoftext|', '|endoftext|', '\\n', '']\n",
    "    text = [x for x in text if x not in rem_tokens]\n",
    "    text = [x for x in text if '|' not in x] # catch tokens that somehow got split\n",
    "    text = [x[1:] if x[0]=='\\n' else x for x in text] # remove initial newlines\n",
    "\n",
    "    # return list of cleaned texts\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = open_files[3].read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_test = clean_generated_file(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import scoring model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import NMF\n",
    "from score_text import score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('models/rfr_final.pkl', 'rb') as f:\n",
    "    model = pkl.load(f)\n",
    "\n",
    "with open('transformers/final_nmf.pkl', 'rb') as f:\n",
    "    fac = pkl.load(f)\n",
    "\n",
    "with open('transformers/final_vec.pkl', 'rb') as f:\n",
    "    vec = pkl.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "'High school is about to get a little scary when a group of fireflies is attacked by a giant blob of black specks, breaking her nose. At school, a few girls get their hands on a special toy, and they join forces with the blob warriors to fight...'"
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "source": [
    "clean_test[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "6.39"
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "score(clean_test[0], vec, fac, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combined clean and scoring function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def clean_and_score_file(gen_file_string):\n",
    "    # clean and split file into list of synopses\n",
    "    synopsis_list = clean_generated_file(gen_file_string)\n",
    "    \n",
    "    scores = []\n",
    "    for synopsis in synopsis_list:\n",
    "        scores.append(score(synopsis, vec, fac, model))\n",
    "\n",
    "    # zip scores and synopses, sort by score and reverse to have highest score first\n",
    "    zipped = [list(zip(scores, synopsis_list))[idx] for idx in np.argsort(scores)]\n",
    "    zipped.reverse()\n",
    "    \n",
    "    return zipped\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_scores = clean_and_score_file(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "[(7.46,\n  'A god is born in the Wind World, where man and machine are made. Miko Sakamaki is the sole survivor of all of the creations, and as such, she does not have any friends. While attending freshman year, she meets Sayaka Hayase, an aspiring manga editor who lives in the Wind World. Hayase is particularly interested in the subject of money, and encourages her to start working at a \"recycled jacket shop\" to learn more about the topic. As she gradually grows interested in the daily grind of going from job to job, she begins to find that the path she has chosen is one that will prove to be a challenge. Thus, Miko attempts to support other people living within the wind world as well as reach out to her own kind. In the end, Miko finds herself being pursued by a strange entity who has her enslaved by the evil goddess, as she must not let her captors sustain her own life as she makes her way toward a new existence.'),\n (7.39,\n  \"While having trouble fitting in, Yasumi is asked by her older sister-in-law to participate in a contest with a mysterious prize, but they refuse. Instead, he proposes that they play the game of life through tea time. At the same time, a mysterious transfer student named Saori is visiting his family. They have no idea what is going on, but when they find out that Saori is a rabbit, the first thing they do is rotate their television. They can't do that until the final reward arrives. Even though they start to get nervous, they play, wait, and wait, but they learn that Saori is a real rabbit. After that, they determine each other as if nothing happened, wound up with an idea, and decided to meet each other, after which Saori joins them. No one knows what will happen to Saori and Yasumi, but after that, they begin to brother and sister together with the mysterious yuria.\"),\n (7.29,\n  \"Shinobu Yamaguchi is a decision-making genius who believes in his own dream and is willing to solve problems through thought. He has all the skills and talent of a genius, but is actually a mere soccer without a passion, and won't even complete psychological tests, because of his lack of any knowledge of soccer. He will be a great soccer player in the world of chess, with his last name, Satoshi. Shinobu's solution to a puzzle is to continue to solve the puzzle to God. One day, Shinobu notices that a client requests him to become the client. After helping out his client, Satoshi, Satoshi, and Shinobu set out to meet to try to solve the client's problem. Satoshi, who is very interested in playing chess, ends up getting bored and decides to help Shinobu solve the client's problem. The problem is the mother of all problems! Shinobu's solution becomes Satoshi's dream, and that is just the start of his journey.\"),\n (7.29,\n  \"In the Ozarks, Les Misérables and Elie Bancan often watch over their great-grandfather's castle in an age where they cannot bring themselves to call the Peeping Life—the only known intelligences—an evil, pitiful joke. But the evil is not limited to the Peeping Life: In the Underwoods neighborhood, a loveless bureaucracy known as the Ark Administration exists to manage all sorts of secret projects throughout the years. The mysterious Andalite Queens of Heart, the Queen's Guard, and a sinister organization known as the Underworld exist only to serve the Ark Administration. When an army of mysterious disappearances strike the town of Wellpoint, the faces of the employees begin to change, and the walls of Wellpoint begin to crumble. A young girl named Leila is sent to investigate; however, with help from a strange Webbing, she is able to escape. Meanwhile, on an island far away from the bustling city, supernatural forces seek out the Hand, the magical guardian of the Hmeimimiya Castle. As they make their way through a series of mysterious buildings, the Precure begin to take notice, seeking to wield an important power. In order to restore Wellpoint to its former glory, Leila is given the task of protecting Beretta, the Precure's only remaining remaining spiritual partner. However, she soon learns the hand of fate is even more powerful than she thought...\"),\n (7.26,\n  \"Mitsuomi's friend Shizuku visits a rural sports academy. Shizuku thinks she can do some good in the classes, but the girls realize that with training and hard work they really don't have much. Suddenly, a group of students, including two of Mitsuomi's friends, picks up Andoei Kachan, a transfer student from Japan. Andoei's father is a soccer great, but he is killed in an accident 50 years ago. And to make things worse, he is the one who saved her mother's life, and turned her into an idol. Now, new years have come and there are problems with a mysterious bullet in her left eye. As the students and teachers go about their troubles, they use the desperate condition that their hearts have become even more hopeless to find answers. And it seems as though every time the girls come to see the academy, they think that somebody has entered the past life...\")]"
     },
     "metadata": {},
     "execution_count": 29
    }
   ],
   "source": [
    "test_scores[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Clean, score and store all of our generated files\n",
    "* Combine all of them\n",
    "* Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reset read cursor\n",
    "open_files = [open(file_name) for file_name in file_names]\n",
    "\n",
    "scored_files = []\n",
    "\n",
    "# clean and score each file and store the lists of scored synopses\n",
    "for gen_file in open_files:\n",
    "    text = gen_file.read()\n",
    "    cleaned_file = clean_and_score_file(text)\n",
    "    scored_files.append(cleaned_file)\n",
    "\n",
    "# flatten the lists and re-sort\n",
    "scored_files = list(chain(*scored_files))\n",
    "scored_files.sort(reverse=True, key=lambda x: x[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "[(7.46,\n  \"As a year progresses, the galaxy has begun to build up more and more the beautiful galaxy, and the 26th year of the Fourth Ledger is just around the corner. The leaders of the galaxy declare war on each other, and it seems they have something that their enemies doesn't... After years of hard work they finally defeat the galactic clothed ones and establish a peaceful galaxy together. That's right, friends Mikazuchi and Reiber are now fighting each other. Will they be able to settle a long standing dispute between the two? As the war goes on Mikazuchi, Reiber and Haibara discover the truth behind the war and their true motives. They even find yet another way to merge. Who is this mysterious entity that is attacking Mikazuchi? Is it really a new game or just the latest step of the battle royales for Mikazuchi and Reiber?\"),\n (7.46,\n  'A god is born in the Wind World, where man and machine are made. Miko Sakamaki is the sole survivor of all of the creations, and as such, she does not have any friends. While attending freshman year, she meets Sayaka Hayase, an aspiring manga editor who lives in the Wind World. Hayase is particularly interested in the subject of money, and encourages her to start working at a \"recycled jacket shop\" to learn more about the topic. As she gradually grows interested in the daily grind of going from job to job, she begins to find that the path she has chosen is one that will prove to be a challenge. Thus, Miko attempts to support other people living within the wind world as well as reach out to her own kind. In the end, Miko finds herself being pursued by a strange entity who has her enslaved by the evil goddess, as she must not let her captors sustain her own life as she makes her way toward a new existence.'),\n (7.39,\n  \"While having trouble fitting in, Yasumi is asked by her older sister-in-law to participate in a contest with a mysterious prize, but they refuse. Instead, he proposes that they play the game of life through tea time. At the same time, a mysterious transfer student named Saori is visiting his family. They have no idea what is going on, but when they find out that Saori is a rabbit, the first thing they do is rotate their television. They can't do that until the final reward arrives. Even though they start to get nervous, they play, wait, and wait, but they learn that Saori is a real rabbit. After that, they determine each other as if nothing happened, wound up with an idea, and decided to meet each other, after which Saori joins them. No one knows what will happen to Saori and Yasumi, but after that, they begin to brother and sister together with the mysterious yuria.\"),\n (7.37,\n  \"In order to repay the debt he owes the Santa Claus family, he makes a vow to find an answer to his last request: if he ever meets the priestess whom he abandoned, he'll grant him his freedom. But if he fails, he's not alone. Tasked with becoming a full-fledged priestess, Rumiya's only legal requirement is to repay all the debt, which leaves the rest to their fate. As they approach the end of their third year of elementary school tradition begins to change, Rumiya arrives down at the building where he hopes to meet up with the priestess. Not knowing where he's going, he runs into Takeshi, a ruthless and powerful man with a burning desire to beat the crap out of Rumiya. And thus begins the tale of the two boys' friendship in this fast paced tale of vengeance and battle.\"),\n (7.37,\n  'A small side-effect occurs when the user\\'s Candy Card is consumed while in sleep. This phenomenon is known as \"Sneak Attack.\" A boy named Mihoshi finds himself in a strange situation on his way home from school where he hears a voice that reminds him of a familiar voice. He immediately rushes over and discovers that his Candy Card has been consumed and the sound of a bell is coming from the school\\'s new school building. After he takes a closer look at the building, he spot a girl with a bright red Candy Card. She turns out to be the voice of the headmaster of the school, Shuuji Akamatsu, who seemed to be sleeping in his dreams. Akamatsu was trying to prevent the creation of a magical girl named Creamy. So, they began a battle with each other, with the goal of getting more Candy Cards. But, they soon discovered that they would have to fight with each other to get Creamy\\'s Candy Card. The girls of the school are going to help each other out if they want to have a normal school life.')]"
     },
     "metadata": {},
     "execution_count": 43
    }
   ],
   "source": [
    "scored_files[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "with open('output/scored_gens_v1.csv', 'w') as f:\n",
    "    writer = csv.writer(f)\n",
    "    writer.writerow(['', 'score', 'text'])\n",
    "    for idx, syn in enumerate(scored_files):\n",
    "        writer.writerow([idx, syn[0], syn[1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}