deep_learning_model.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"deep_learning_model.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"j9oQwdYbpJCe","executionInfo":{"status":"ok","timestamp":1638469971937,"user_tz":300,"elapsed":17850,"user":{"displayName":"Zhaozheng Shen","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15945483903558522134"}},"outputId":"cd3064c2-3fb6-4f93-c7ff-0d6a16569b14"},"source":["from google.colab import drive\n","drive.mount('/content/gdrive')\n","# !wget https://github.com/explosion/spacy-models/releases/download/en_vectors_web_lg-2.1.0/en_vectors_web_lg-2.1.0.tar.gz -O en_vectors_web_lg-2.1.0.tar.gz\n","# !pip install en_vectors_web_lg-2.1.0.tar.gz\n","data_dir = '/content/gdrive/Shareddrives/520_Project'\n","# !pip install /content/gdrive/Shareddrives/520_Project/en_vectors_web_lg-2.1.0.tar.gz"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/gdrive\n"]}]},{"cell_type":"code","metadata":{"id":"NLpErMAE2VO7"},"source":["import pandas as pd\n","import numpy as np\n","import os\n","import torch\n","import re\n","import time\n","import copy\n","import math\n","import pickle\n","from sklearn.model_selection import train_test_split\n","from torch.utils.data import DataLoader, Dataset\n","import torch.optim as optimizer\n","import torch.nn.functional as F\n","from torch import nn\n","from sklearn.metrics import accuracy_score\n","# import en_vectors_web_lg\n","\n","import seaborn as sns\n","import matplotlib as mpl\n","import matplotlib.pyplot as plt\n","plt.rcParams['font.family'] = \"sans-serif\"\n","plt.rcParams['font.sans-serif'] = ['Times New Roman']\n","sns.set_style(\"whitegrid\")\n","sns.set_style({'font.family':'serif', 'font.serif':'Times New Roman'})\n","sns.set(font_scale=1.2)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"ALRfWp_JpLss","executionInfo":{"status":"ok","timestamp":1638469982148,"user_tz":300,"elapsed":1027,"user":{"displayName":"Zhaozheng Shen","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15945483903558522134"}},"outputId":"7fdd3e3c-dc96-4211-db72-91ebda078765"},"source":["data = pd.read_csv(os.path.join(data_dir, \"Womens_Clothing_E-Commerce_Reviews.csv\"))\n","data.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Unnamed: 0</th>\n","      <th>Clothing ID</th>\n","      <th>Age</th>\n","      <th>Title</th>\n","      <th>Review Text</th>\n","      <th>Rating</th>\n","      <th>Recommended IND</th>\n","      <th>Positive Feedback Count</th>\n","      <th>Division Name</th>\n","      <th>Department Name</th>\n","      <th>Class Name</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>767</td>\n","      <td>33</td>\n","      <td>NaN</td>\n","      <td>Absolutely wonderful - silky and sexy and comf...</td>\n","      <td>4</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Initmates</td>\n","      <td>Intimate</td>\n","      <td>Intimates</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1</td>\n","      <td>1080</td>\n","      <td>34</td>\n","      <td>NaN</td>\n","      <td>Love this dress!  it's sooo pretty.  i happene...</td>\n","      <td>5</td>\n","      <td>1</td>\n","      <td>4</td>\n","      <td>General</td>\n","      <td>Dresses</td>\n","      <td>Dresses</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2</td>\n","      <td>1077</td>\n","      <td>60</td>\n","      <td>Some major design flaws</td>\n","      <td>I had such high hopes for this dress and reall...</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>General</td>\n","      <td>Dresses</td>\n","      <td>Dresses</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>3</td>\n","      <td>1049</td>\n","      <td>50</td>\n","      <td>My favorite buy!</td>\n","      <td>I love, love, love this jumpsuit. it's fun, fl...</td>\n","      <td>5</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>General Petite</td>\n","      <td>Bottoms</td>\n","      <td>Pants</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>4</td>\n","      <td>847</td>\n","      <td>47</td>\n","      <td>Flattering shirt</td>\n","      <td>This shirt is very flattering to all due to th...</td>\n","      <td>5</td>\n","      <td>1</td>\n","      <td>6</td>\n","      <td>General</td>\n","      <td>Tops</td>\n","      <td>Blouses</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   Unnamed: 0  Clothing ID  Age  ...   Division Name Department Name  Class Name\n","0           0          767   33  ...       Initmates        Intimate   Intimates\n","1           1         1080   34  ...         General         Dresses     Dresses\n","2           2         1077   60  ...         General         Dresses     Dresses\n","3           3         1049   50  ...  General Petite         Bottoms       Pants\n","4           4          847   47  ...         General            Tops     Blouses\n","\n","[5 rows x 11 columns]"]},"metadata":{},"execution_count":3}]},{"cell_type":"code","metadata":{"id":"I1MZosu64FTu"},"source":["def clean_text(w):\n","    return re.sub(\n","            r\"([.,'!?\\\"()*#:;])\",\n","            '',\n","            w.lower()\n","            ).replace('-', ' ').replace('/', ' ')\n","\n","def get_glove_embedding(reviews, data_dir):\n","  token_file = os.path.join(data_dir,'token_to_ix.pkl')\n","  glove_file = os.path.join(data_dir,'train_glove.npy')\n","  if os.path.exists(glove_file) and os.path.exists(token_file):\n","        print(\"Loading saved embedding\")\n","        return pickle.load(open(token_file, \"rb\")), np.load(glove_file)\n","  all_reviews = {}\n","  for idx, s in enumerate(reviews):\n","    all_reviews[idx] = clean_text(s).split()\n","\n","  from collections import defaultdict\n","  token_to_ix = defaultdict(int)\n","  token_to_ix['UNK'] = 1\n","\n","  spacy_tool = en_vectors_web_lg.load()\n","  pretrained_emb = []\n","  pretrained_emb.append(spacy_tool('UNK').vector)\n","  \n","  for k, v in all_reviews.items():\n","      for word in v:\n","          if word not in token_to_ix:\n","              token_to_ix[word] = len(token_to_ix)\n","              pretrained_emb.append(spacy_tool(word).vector)\n","\n","  pretrained_emb = np.array(pretrained_emb)\n","  np.save(glove_file, pretrained_emb)\n","  pickle.dump(token_to_ix, open(token_file, \"wb\"))\n","  return token_to_ix, pretrained_emb\n","\n","def embed_text(x, max_len, token2ix):\n","  ques_ix = np.zeros(max_len, np.int64)\n","  x = clean_text(x).split()\n","  for ix, word in enumerate(x):\n","    if word in token2ix:\n","      ques_ix[ix] = token2ix[word]\n","    else:\n","      ques_ix[ix] = 1\n","    if ix + 1 == max_len:\n","      break\n","  return ques_ix\n","def tokenize(reviews):\n","  token2ix = {'PAD': 0, 'UNK': 1, 'SS' : 2,}\n","  for r in reviews:\n","    r = clean_text(r).split()\n","    for word in r:\n","      if word not in token2ix:\n","        token2ix[word] = len(token2ix)\n","  return token2ix\n","def category_from_output(output):\n","  top_n, top_i = output[0].topk(1)\n","  category_i = top_i[0].item()\n","  return category_i"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"pvGQuDRL4Vp-"},"source":["class CNN_model(nn.Module):\n","    def __init__(self, token_size, pretrained_emb):\n","        super(CNN_model, self).__init__()\n","        dropout_rate = 0.5\n","        self.embedding = nn.Embedding(\n","            num_embeddings=token_size,\n","            embedding_dim=300\n","        )\n","        self.embedding.weight.data.copy_(torch.from_numpy(pretrained_emb))\n","\n","        hidden_size = 32\n","        print('hidden_size', hidden_size)\n","        self.conv1d = torch.nn.Conv1d(in_channels=300, out_channels=hidden_size, kernel_size=3)\n","        self.conv1d1 = torch.nn.Conv1d(in_channels=hidden_size, out_channels=hidden_size, kernel_size=3)\n","        self.conv_unit = nn.Sequential(self.conv1d, nn.ReLU(),nn.Dropout(0.5),\n","                                       self.conv1d1, nn.ReLU(),\n","                                      #  nn.Dropout(0.5),\n","                                      #  self.conv1d1, nn.ReLU(),\n","                                       )\n","        self.dropout = nn.Dropout(dropout_rate)\n","        self.fc = nn.Linear(hidden_size, 1)\n","\n","    def forward(self, x):\n","      x = self.embedding(x) # print('0',x.shape)  # [bsz, len, 300]\n","      x = torch.transpose(x,1,2) # print('1',x.shape) # [bsz, 300, len]\n","      x = self.conv_unit(x) # print('2',x.shape) # [bsz, 128, len]\n","      out = torch.transpose(x, 1, 2) \n","      length = out.shape[1]\n","      embedding_v = out[:,-1,:] # print('3',embedding_v.shape) # [bsz, 128]\n","      logit = self.fc(embedding_v)\n","      # logit = self.fc_unit(embedding_v) # print('logit.shape',logit.shape) # [bsz, 1]\n","      return logit\n","    \n","# net = CNN_model(len(token2ix), pretrained_emb).to(device)\n","# criteon = nn.BCEWithLogitsLoss().to(device)\n","# for idx,(x,y) in enumerate(zip(X_train,y_train)):\n","#   embed_vec, label = torch.LongTensor([x]).to(device), torch.Tensor([y]).to(device)\n","#   output = net(embed_vec)\n","#   loss = criteon(output[0],label)\n","#   print(output.squeeze(1))\n","#   result = torch.gt(torch.sigmoid(output[0]),0.5).int() \n","#   print(result.squeeze(0))\n","#   break"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"O2ddpTph3SEx"},"source":["class cloth_dataset(Dataset):\n","  def __init__(self, encodings, labels):\n","    self.embedded = encodings\n","    self.label = labels\n","  def __getitem__(self, index):\n","    return self.embedded[index],\\\n","          self.label[index],\n","  def __len__(self):\n","    return len(self.label)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5hFnBIal4fh1","executionInfo":{"status":"ok","timestamp":1637894357290,"user_tz":300,"elapsed":1005,"user":{"displayName":"Yixuan Meng","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"00008408025985696509"}},"outputId":"0c502f33-d327-4c4f-c94f-67d958bb8bca"},"source":["device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n","print(device)\n","EMBEDDING_DIM = 300\n","BATCH_SIZE = 256\n","LEARNING_RATE = 0.005\n","EPOCH = 11\n","data['Review Text'] = data['Review Text'].fillna(' ')\n","token2ix, pretrained_emb = get_glove_embedding(data['Review Text'],data_dir)\n","print(pretrained_emb.shape) # (len(vocab), embedding_dim)\n","\n","lengths = [len(x.split()) for x in data['Review Text']]\n","max_len = int(np.percentile(lengths,90))\n","data['embedded'] = data['Review Text'].apply(lambda x : embed_text(x,max_len,token2ix))\n","\n","X_train, X_dev, y_train, y_dev = train_test_split(np.array(data['embedded']), np.array(data['Recommended IND']), test_size=0.2, random_state=42)\n","X_dev, X_test, y_dev, y_test = train_test_split(X_dev, y_dev, test_size=0.5, random_state=42)\n","\n","train_dataset = cloth_dataset(X_train, y_train)\n","train_data_iter = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=False)\n","dev_dataset = cloth_dataset(X_dev, y_dev)\n","dev_data_iter = DataLoader(dev_dataset, batch_size=BATCH_SIZE, shuffle=False)\n","test_dataset = cloth_dataset(X_test, y_test)\n","test_data_iter = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["cuda:0\n","Loading saved embedding\n","(16335, 300)\n"]}]},{"cell_type":"code","metadata":{"id":"pODOmaDB5mka"},"source":["def train(epoch, train_data_iter, dev_data_iter ,opt,criteon,net,device):\n","  def timeSince(since):\n","      now = time.time()\n","      s = now - since\n","      m = math.floor(s / 60)\n","      s -= m * 60\n","      return '%dm %ds' % (m, s)\n","  train_losses, dev_losses, dev_acc_list = [], [], []\n","  best_model, best_val_acc = None, float('-inf')\n","  cnt_step = 0\n","  current_loss = 0\n","  plot_every = 2\n","  dev_every = 2\n","  print('train len:',len(train_data_iter),'dev len:',len(dev_data_iter))\n","  print('learning_rate',LEARNING_RATE,'n_iters',EPOCH, 'optim','Adam','batch size ',BATCH_SIZE, 'lr_scheduler',None, 'device',device)\n","  start = time.time()\n","  for e in range(epoch): \n","    print('Epoch', e)\n","    net.train()\n","    for batch_idx, (text, label) in enumerate(train_data_iter):\n","      text, label = text.to(device), label.to(device)\n","      net.zero_grad()\n","      opt.zero_grad()\n","      output = net(text)\n","      # loss = criteon(output[0],label)\n","      output = output.squeeze(-1)\n","      loss = criteon(output,label.float())\n","      current_loss += loss\n","      cnt_step += 1\n","      loss.backward()\n","      opt.step()\n","    if e==0:\n","      print(time.time()-start)\n","    if e % plot_every == 0:\n","      tmp_loss = current_loss.item() / cnt_step\n","      train_losses.append(tmp_loss)\n","      current_loss, cnt_step = 0, 0\n","      print('%d %d%% (%s) loss: %.4f ' % (e, e / EPOCH * 100, timeSince(start), tmp_loss))\n","    if e % dev_every ==0:\n","      net.eval()\n","      eval_loss = 0\n","      y_pred, y_true = [], []\n","      cnt_eval_step = 0\n","      for batch_idx, (text, label) in enumerate(dev_data_iter):\n","        text, label = text.to(device), label.to(device)\n","        output = net(text)\n","        output = output.squeeze(-1)\n","        loss = criteon(output,label.float())\n","        # loss = criteon(output[0],label)\n","        result = torch.gt(torch.sigmoid(output),0.5).int() \n","        eval_loss += loss\n","        cnt_eval_step += 1\n","        y_pred += result.tolist()\n","        y_true += label.tolist()\n","      # print(cnt_eval_step, eval_loss, len(dev_data_iter))\n","      dev_losses.append(eval_loss.item()/cnt_eval_step)\n","      acc = accuracy_score(y_pred,y_true)\n","      dev_acc_list.append(acc)\n","      if acc>best_val_acc:\n","        best_val_acc = acc\n","        best_model = copy.deepcopy(net)\n","      print('%d %d%% (%s) loss:%.4f %s %s acc:%.4f' % (e, e / EPOCH * 100, timeSince(start), eval_loss.item()/cnt_eval_step, result.tolist()[:4], label.tolist()[:4], acc))\n","  print('best acc', best_val_acc)\n","  return train_losses, dev_losses, dev_acc_list, best_model # best_model"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"bhzT-IbB5sHF"},"source":["LEARNING_RATE = 0.002"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VKtd29tj76xe","executionInfo":{"status":"ok","timestamp":1637895482345,"user_tz":300,"elapsed":633942,"user":{"displayName":"Yixuan Meng","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"00008408025985696509"}},"outputId":"b214506f-fde4-4872-c23c-2f74c8e7085f"},"source":["# device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n","net = CNN_model(len(token2ix), pretrained_emb).to(device)\n","# opt = optimizer.Adam(net.parameters(), lr=LEARNING_RATE)\n","opt = optimizer.SGD(net.parameters(), lr=LEARNING_RATE)\n","# criteon = nn.CrossEntropyLoss().to(device)\n","criteon = nn.BCEWithLogitsLoss().to(device)\n","train_losses, dev_losses, dev_acc_list, best_model = train(500,train_data_iter, dev_data_iter ,opt,criteon,net,device)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["hidden_size 32\n","train len: 74 dev len: 10\n","learning_rate 0.002 n_iters 11 optim Adam batch size  256 lr_scheduler None device cuda:0\n","Epoch 0\n","1.3985779285430908\n","0 0% (0m 1s) loss: 0.7213 \n","0 0% (0m 1s) loss:0.6872 [1, 0, 1, 0] [0, 0, 0, 1] acc:0.7063\n","Epoch 1\n","Epoch 2\n","2 18% (0m 3s) loss: 0.6320 \n","2 18% (0m 4s) loss:0.5610 [1, 0, 1, 1] [0, 0, 0, 1] acc:0.7701\n","Epoch 3\n","Epoch 4\n","4 36% (0m 6s) loss: 0.5395 \n","4 36% (0m 6s) loss:0.5062 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 5\n","Epoch 6\n","6 54% (0m 8s) loss: 0.5141 \n","6 54% (0m 9s) loss:0.4980 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 7\n","Epoch 8\n","8 72% (0m 11s) loss: 0.5100 \n","8 72% (0m 11s) loss:0.4955 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 9\n","Epoch 10\n","10 90% (0m 14s) loss: 0.5064 \n","10 90% (0m 14s) loss:0.4932 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 11\n","Epoch 12\n","12 109% (0m 16s) loss: 0.5027 \n","12 109% (0m 16s) loss:0.4909 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 13\n","Epoch 14\n","14 127% (0m 19s) loss: 0.5005 \n","14 127% (0m 19s) loss:0.4886 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 15\n","Epoch 16\n","16 145% (0m 21s) loss: 0.4961 \n","16 145% (0m 21s) loss:0.4863 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 17\n","Epoch 18\n","18 163% (0m 24s) loss: 0.4944 \n","18 163% (0m 24s) loss:0.4841 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 19\n","Epoch 20\n","20 181% (0m 26s) loss: 0.4921 \n","20 181% (0m 26s) loss:0.4821 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 21\n","Epoch 22\n","22 200% (0m 29s) loss: 0.4893 \n","22 200% (0m 29s) loss:0.4802 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 23\n","Epoch 24\n","24 218% (0m 31s) loss: 0.4870 \n","24 218% (0m 31s) loss:0.4784 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 25\n","Epoch 26\n","26 236% (0m 34s) loss: 0.4848 \n","26 236% (0m 34s) loss:0.4768 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 27\n","Epoch 28\n","28 254% (0m 36s) loss: 0.4837 \n","28 254% (0m 36s) loss:0.4754 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 29\n","Epoch 30\n","30 272% (0m 39s) loss: 0.4821 \n","30 272% (0m 39s) loss:0.4740 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 31\n","Epoch 32\n","32 290% (0m 41s) loss: 0.4814 \n","32 290% (0m 41s) loss:0.4729 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 33\n","Epoch 34\n","34 309% (0m 44s) loss: 0.4787 \n","34 309% (0m 44s) loss:0.4718 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 35\n","Epoch 36\n","36 327% (0m 46s) loss: 0.4774 \n","36 327% (0m 47s) loss:0.4709 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 37\n","Epoch 38\n","38 345% (0m 49s) loss: 0.4771 \n","38 345% (0m 49s) loss:0.4702 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 39\n","Epoch 40\n","40 363% (0m 52s) loss: 0.4768 \n","40 363% (0m 52s) loss:0.4695 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 41\n","Epoch 42\n","42 381% (0m 54s) loss: 0.4754 \n","42 381% (0m 54s) loss:0.4689 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 43\n","Epoch 44\n","44 400% (0m 57s) loss: 0.4750 \n","44 400% (0m 57s) loss:0.4684 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 45\n","Epoch 46\n","46 418% (0m 59s) loss: 0.4743 \n","46 418% (0m 59s) loss:0.4680 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 47\n","Epoch 48\n","48 436% (1m 2s) loss: 0.4750 \n","48 436% (1m 2s) loss:0.4677 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 49\n","Epoch 50\n","50 454% (1m 4s) loss: 0.4743 \n","50 454% (1m 4s) loss:0.4674 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 51\n","Epoch 52\n","52 472% (1m 7s) loss: 0.4739 \n","52 472% (1m 7s) loss:0.4672 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 53\n","Epoch 54\n","54 490% (1m 9s) loss: 0.4742 \n","54 490% (1m 9s) loss:0.4670 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 55\n","Epoch 56\n","56 509% (1m 12s) loss: 0.4729 \n","56 509% (1m 12s) loss:0.4668 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 57\n","Epoch 58\n","58 527% (1m 14s) loss: 0.4730 \n","58 527% (1m 14s) loss:0.4667 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 59\n","Epoch 60\n","60 545% (1m 17s) loss: 0.4719 \n","60 545% (1m 17s) loss:0.4666 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 61\n","Epoch 62\n","62 563% (1m 19s) loss: 0.4722 \n","62 563% (1m 19s) loss:0.4665 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 63\n","Epoch 64\n","64 581% (1m 22s) loss: 0.4718 \n","64 581% (1m 22s) loss:0.4664 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 65\n","Epoch 66\n","66 600% (1m 24s) loss: 0.4718 \n","66 600% (1m 25s) loss:0.4663 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 67\n","Epoch 68\n","68 618% (1m 27s) loss: 0.4716 \n","68 618% (1m 27s) loss:0.4663 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 69\n","Epoch 70\n","70 636% (1m 30s) loss: 0.4717 \n","70 636% (1m 30s) loss:0.4662 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 71\n","Epoch 72\n","72 654% (1m 32s) loss: 0.4714 \n","72 654% (1m 32s) loss:0.4662 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 73\n","Epoch 74\n","74 672% (1m 35s) loss: 0.4715 \n","74 672% (1m 35s) loss:0.4661 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 75\n","Epoch 76\n","76 690% (1m 37s) loss: 0.4723 \n","76 690% (1m 37s) loss:0.4661 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 77\n","Epoch 78\n","78 709% (1m 40s) loss: 0.4723 \n","78 709% (1m 40s) loss:0.4661 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 79\n","Epoch 80\n","80 727% (1m 42s) loss: 0.4717 \n","80 727% (1m 42s) loss:0.4661 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 81\n","Epoch 82\n","82 745% (1m 45s) loss: 0.4721 \n","82 745% (1m 45s) loss:0.4660 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 83\n","Epoch 84\n","84 763% (1m 47s) loss: 0.4717 \n","84 763% (1m 47s) loss:0.4660 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 85\n","Epoch 86\n","86 781% (1m 50s) loss: 0.4712 \n","86 781% (1m 50s) loss:0.4660 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 87\n","Epoch 88\n","88 800% (1m 52s) loss: 0.4720 \n","88 800% (1m 52s) loss:0.4660 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 89\n","Epoch 90\n","90 818% (1m 55s) loss: 0.4711 \n","90 818% (1m 55s) loss:0.4660 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 91\n","Epoch 92\n","92 836% (1m 57s) loss: 0.4718 \n","92 836% (1m 57s) loss:0.4659 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 93\n","Epoch 94\n","94 854% (2m 0s) loss: 0.4709 \n","94 854% (2m 0s) loss:0.4659 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 95\n","Epoch 96\n","96 872% (2m 2s) loss: 0.4708 \n","96 872% (2m 3s) loss:0.4659 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 97\n","Epoch 98\n","98 890% (2m 5s) loss: 0.4720 \n","98 890% (2m 5s) loss:0.4659 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 99\n","Epoch 100\n","100 909% (2m 8s) loss: 0.4715 \n","100 909% (2m 8s) loss:0.4659 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 101\n","Epoch 102\n","102 927% (2m 10s) loss: 0.4711 \n","102 927% (2m 10s) loss:0.4659 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 103\n","Epoch 104\n","104 945% (2m 13s) loss: 0.4709 \n","104 945% (2m 13s) loss:0.4659 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 105\n","Epoch 106\n","106 963% (2m 15s) loss: 0.4710 \n","106 963% (2m 15s) loss:0.4659 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 107\n","Epoch 108\n","108 981% (2m 18s) loss: 0.4700 \n","108 981% (2m 18s) loss:0.4658 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 109\n","Epoch 110\n","110 1000% (2m 20s) loss: 0.4706 \n","110 1000% (2m 20s) loss:0.4658 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 111\n","Epoch 112\n","112 1018% (2m 23s) loss: 0.4706 \n","112 1018% (2m 23s) loss:0.4658 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 113\n","Epoch 114\n","114 1036% (2m 25s) loss: 0.4705 \n","114 1036% (2m 25s) loss:0.4658 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 115\n","Epoch 116\n","116 1054% (2m 28s) loss: 0.4705 \n","116 1054% (2m 28s) loss:0.4658 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 117\n","Epoch 118\n","118 1072% (2m 30s) loss: 0.4711 \n","118 1072% (2m 30s) loss:0.4658 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 119\n","Epoch 120\n","120 1090% (2m 33s) loss: 0.4704 \n","120 1090% (2m 33s) loss:0.4658 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 121\n","Epoch 122\n","122 1109% (2m 35s) loss: 0.4702 \n","122 1109% (2m 35s) loss:0.4658 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 123\n","Epoch 124\n","124 1127% (2m 38s) loss: 0.4710 \n","124 1127% (2m 38s) loss:0.4658 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 125\n","Epoch 126\n","126 1145% (2m 40s) loss: 0.4705 \n","126 1145% (2m 41s) loss:0.4658 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 127\n","Epoch 128\n","128 1163% (2m 43s) loss: 0.4710 \n","128 1163% (2m 43s) loss:0.4658 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 129\n","Epoch 130\n","130 1181% (2m 46s) loss: 0.4702 \n","130 1181% (2m 46s) loss:0.4658 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 131\n","Epoch 132\n","132 1200% (2m 48s) loss: 0.4698 \n","132 1200% (2m 48s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 133\n","Epoch 134\n","134 1218% (2m 51s) loss: 0.4703 \n","134 1218% (2m 51s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 135\n","Epoch 136\n","136 1236% (2m 53s) loss: 0.4702 \n","136 1236% (2m 53s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 137\n","Epoch 138\n","138 1254% (2m 56s) loss: 0.4702 \n","138 1254% (2m 56s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 139\n","Epoch 140\n","140 1272% (2m 58s) loss: 0.4700 \n","140 1272% (2m 58s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 141\n","Epoch 142\n","142 1290% (3m 1s) loss: 0.4695 \n","142 1290% (3m 1s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 143\n","Epoch 144\n","144 1309% (3m 3s) loss: 0.4704 \n","144 1309% (3m 3s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 145\n","Epoch 146\n","146 1327% (3m 6s) loss: 0.4705 \n","146 1327% (3m 6s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 147\n","Epoch 148\n","148 1345% (3m 8s) loss: 0.4697 \n","148 1345% (3m 8s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 149\n","Epoch 150\n","150 1363% (3m 11s) loss: 0.4699 \n","150 1363% (3m 11s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 151\n","Epoch 152\n","152 1381% (3m 13s) loss: 0.4699 \n","152 1381% (3m 14s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 153\n","Epoch 154\n","154 1400% (3m 16s) loss: 0.4698 \n","154 1400% (3m 16s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 155\n","Epoch 156\n","156 1418% (3m 19s) loss: 0.4707 \n","156 1418% (3m 19s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 157\n","Epoch 158\n","158 1436% (3m 21s) loss: 0.4694 \n","158 1436% (3m 21s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 159\n","Epoch 160\n","160 1454% (3m 24s) loss: 0.4706 \n","160 1454% (3m 24s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 161\n","Epoch 162\n","162 1472% (3m 26s) loss: 0.4702 \n","162 1472% (3m 26s) loss:0.4657 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 163\n","Epoch 164\n","164 1490% (3m 29s) loss: 0.4691 \n","164 1490% (3m 29s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 165\n","Epoch 166\n","166 1509% (3m 31s) loss: 0.4696 \n","166 1509% (3m 31s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 167\n","Epoch 168\n","168 1527% (3m 34s) loss: 0.4699 \n","168 1527% (3m 34s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 169\n","Epoch 170\n","170 1545% (3m 36s) loss: 0.4694 \n","170 1545% (3m 36s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 171\n","Epoch 172\n","172 1563% (3m 39s) loss: 0.4697 \n","172 1563% (3m 39s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 173\n","Epoch 174\n","174 1581% (3m 41s) loss: 0.4703 \n","174 1581% (3m 41s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 175\n","Epoch 176\n","176 1600% (3m 44s) loss: 0.4702 \n","176 1600% (3m 44s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 177\n","Epoch 178\n","178 1618% (3m 46s) loss: 0.4692 \n","178 1618% (3m 47s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 179\n","Epoch 180\n","180 1636% (3m 49s) loss: 0.4698 \n","180 1636% (3m 49s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 181\n","Epoch 182\n","182 1654% (3m 52s) loss: 0.4691 \n","182 1654% (3m 52s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 183\n","Epoch 184\n","184 1672% (3m 54s) loss: 0.4698 \n","184 1672% (3m 54s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 185\n","Epoch 186\n","186 1690% (3m 57s) loss: 0.4692 \n","186 1690% (3m 57s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 187\n","Epoch 188\n","188 1709% (3m 59s) loss: 0.4697 \n","188 1709% (3m 59s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 189\n","Epoch 190\n","190 1727% (4m 2s) loss: 0.4701 \n","190 1727% (4m 2s) loss:0.4656 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 191\n","Epoch 192\n","192 1745% (4m 4s) loss: 0.4691 \n","192 1745% (4m 4s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 193\n","Epoch 194\n","194 1763% (4m 7s) loss: 0.4699 \n","194 1763% (4m 7s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 195\n","Epoch 196\n","196 1781% (4m 9s) loss: 0.4693 \n","196 1781% (4m 9s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 197\n","Epoch 198\n","198 1800% (4m 12s) loss: 0.4699 \n","198 1800% (4m 12s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 199\n","Epoch 200\n","200 1818% (4m 14s) loss: 0.4697 \n","200 1818% (4m 14s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 201\n","Epoch 202\n","202 1836% (4m 17s) loss: 0.4692 \n","202 1836% (4m 17s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 203\n","Epoch 204\n","204 1854% (4m 19s) loss: 0.4691 \n","204 1854% (4m 19s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 205\n","Epoch 206\n","206 1872% (4m 22s) loss: 0.4688 \n","206 1872% (4m 22s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 207\n","Epoch 208\n","208 1890% (4m 24s) loss: 0.4682 \n","208 1890% (4m 25s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 209\n","Epoch 210\n","210 1909% (4m 27s) loss: 0.4689 \n","210 1909% (4m 27s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 211\n","Epoch 212\n","212 1927% (4m 30s) loss: 0.4688 \n","212 1927% (4m 30s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 213\n","Epoch 214\n","214 1945% (4m 32s) loss: 0.4693 \n","214 1945% (4m 32s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 215\n","Epoch 216\n","216 1963% (4m 35s) loss: 0.4694 \n","216 1963% (4m 35s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 217\n","Epoch 218\n","218 1981% (4m 37s) loss: 0.4689 \n","218 1981% (4m 37s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 219\n","Epoch 220\n","220 2000% (4m 40s) loss: 0.4685 \n","220 2000% (4m 40s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 221\n","Epoch 222\n","222 2018% (4m 42s) loss: 0.4698 \n","222 2018% (4m 42s) loss:0.4655 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 223\n","Epoch 224\n","224 2036% (4m 45s) loss: 0.4694 \n","224 2036% (4m 45s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 225\n","Epoch 226\n","226 2054% (4m 47s) loss: 0.4690 \n","226 2054% (4m 47s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 227\n","Epoch 228\n","228 2072% (4m 50s) loss: 0.4693 \n","228 2072% (4m 50s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 229\n","Epoch 230\n","230 2090% (4m 52s) loss: 0.4688 \n","230 2090% (4m 52s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 231\n","Epoch 232\n","232 2109% (4m 55s) loss: 0.4687 \n","232 2109% (4m 55s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 233\n","Epoch 234\n","234 2127% (4m 57s) loss: 0.4686 \n","234 2127% (4m 58s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 235\n","Epoch 236\n","236 2145% (5m 0s) loss: 0.4689 \n","236 2145% (5m 0s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 237\n","Epoch 238\n","238 2163% (5m 3s) loss: 0.4691 \n","238 2163% (5m 3s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 239\n","Epoch 240\n","240 2181% (5m 5s) loss: 0.4691 \n","240 2181% (5m 5s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 241\n","Epoch 242\n","242 2200% (5m 8s) loss: 0.4689 \n","242 2200% (5m 8s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 243\n","Epoch 244\n","244 2218% (5m 10s) loss: 0.4688 \n","244 2218% (5m 10s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 245\n","Epoch 246\n","246 2236% (5m 13s) loss: 0.4684 \n","246 2236% (5m 13s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 247\n","Epoch 248\n","248 2254% (5m 15s) loss: 0.4684 \n","248 2254% (5m 15s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 249\n","Epoch 250\n","250 2272% (5m 18s) loss: 0.4688 \n","250 2272% (5m 18s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 251\n","Epoch 252\n","252 2290% (5m 20s) loss: 0.4687 \n","252 2290% (5m 20s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 253\n","Epoch 254\n","254 2309% (5m 23s) loss: 0.4682 \n","254 2309% (5m 23s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 255\n","Epoch 256\n","256 2327% (5m 25s) loss: 0.4691 \n","256 2327% (5m 25s) loss:0.4654 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 257\n","Epoch 258\n","258 2345% (5m 28s) loss: 0.4683 \n","258 2345% (5m 28s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 259\n","Epoch 260\n","260 2363% (5m 30s) loss: 0.4691 \n","260 2363% (5m 30s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 261\n","Epoch 262\n","262 2381% (5m 33s) loss: 0.4683 \n","262 2381% (5m 33s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 263\n","Epoch 264\n","264 2400% (5m 36s) loss: 0.4687 \n","264 2400% (5m 36s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 265\n","Epoch 266\n","266 2418% (5m 38s) loss: 0.4694 \n","266 2418% (5m 38s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 267\n","Epoch 268\n","268 2436% (5m 41s) loss: 0.4684 \n","268 2436% (5m 41s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 269\n","Epoch 270\n","270 2454% (5m 43s) loss: 0.4691 \n","270 2454% (5m 43s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 271\n","Epoch 272\n","272 2472% (5m 46s) loss: 0.4684 \n","272 2472% (5m 46s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 273\n","Epoch 274\n","274 2490% (5m 48s) loss: 0.4688 \n","274 2490% (5m 48s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 275\n","Epoch 276\n","276 2509% (5m 51s) loss: 0.4685 \n","276 2509% (5m 51s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 277\n","Epoch 278\n","278 2527% (5m 53s) loss: 0.4685 \n","278 2527% (5m 53s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 279\n","Epoch 280\n","280 2545% (5m 56s) loss: 0.4684 \n","280 2545% (5m 56s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 281\n","Epoch 282\n","282 2563% (5m 58s) loss: 0.4690 \n","282 2563% (5m 58s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 283\n","Epoch 284\n","284 2581% (6m 1s) loss: 0.4687 \n","284 2581% (6m 1s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 285\n","Epoch 286\n","286 2600% (6m 3s) loss: 0.4685 \n","286 2600% (6m 3s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 287\n","Epoch 288\n","288 2618% (6m 6s) loss: 0.4686 \n","288 2618% (6m 6s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 289\n","Epoch 290\n","290 2636% (6m 8s) loss: 0.4684 \n","290 2636% (6m 8s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 291\n","Epoch 292\n","292 2654% (6m 11s) loss: 0.4686 \n","292 2654% (6m 11s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 293\n","Epoch 294\n","294 2672% (6m 13s) loss: 0.4683 \n","294 2672% (6m 14s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 295\n","Epoch 296\n","296 2690% (6m 16s) loss: 0.4681 \n","296 2690% (6m 16s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 297\n","Epoch 298\n","298 2709% (6m 19s) loss: 0.4679 \n","298 2709% (6m 19s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 299\n","Epoch 300\n","300 2727% (6m 21s) loss: 0.4692 \n","300 2727% (6m 21s) loss:0.4653 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 301\n","Epoch 302\n","302 2745% (6m 24s) loss: 0.4683 \n","302 2745% (6m 24s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 303\n","Epoch 304\n","304 2763% (6m 26s) loss: 0.4679 \n","304 2763% (6m 26s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 305\n","Epoch 306\n","306 2781% (6m 29s) loss: 0.4687 \n","306 2781% (6m 29s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 307\n","Epoch 308\n","308 2800% (6m 31s) loss: 0.4690 \n","308 2800% (6m 31s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 309\n","Epoch 310\n","310 2818% (6m 34s) loss: 0.4681 \n","310 2818% (6m 34s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 311\n","Epoch 312\n","312 2836% (6m 36s) loss: 0.4677 \n","312 2836% (6m 36s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 313\n","Epoch 314\n","314 2854% (6m 39s) loss: 0.4677 \n","314 2854% (6m 39s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 315\n","Epoch 316\n","316 2872% (6m 41s) loss: 0.4680 \n","316 2872% (6m 41s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 317\n","Epoch 318\n","318 2890% (6m 44s) loss: 0.4682 \n","318 2890% (6m 44s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 319\n","Epoch 320\n","320 2909% (6m 46s) loss: 0.4680 \n","320 2909% (6m 46s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 321\n","Epoch 322\n","322 2927% (6m 49s) loss: 0.4681 \n","322 2927% (6m 49s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 323\n","Epoch 324\n","324 2945% (6m 51s) loss: 0.4686 \n","324 2945% (6m 52s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 325\n","Epoch 326\n","326 2963% (6m 54s) loss: 0.4682 \n","326 2963% (6m 54s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 327\n","Epoch 328\n","328 2981% (6m 57s) loss: 0.4685 \n","328 2981% (6m 57s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 329\n","Epoch 330\n","330 3000% (6m 59s) loss: 0.4677 \n","330 3000% (6m 59s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 331\n","Epoch 332\n","332 3018% (7m 2s) loss: 0.4682 \n","332 3018% (7m 2s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 333\n","Epoch 334\n","334 3036% (7m 4s) loss: 0.4679 \n","334 3036% (7m 4s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 335\n","Epoch 336\n","336 3054% (7m 7s) loss: 0.4680 \n","336 3054% (7m 7s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 337\n","Epoch 338\n","338 3072% (7m 9s) loss: 0.4678 \n","338 3072% (7m 9s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 339\n","Epoch 340\n","340 3090% (7m 12s) loss: 0.4683 \n","340 3090% (7m 12s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 341\n","Epoch 342\n","342 3109% (7m 14s) loss: 0.4678 \n","342 3109% (7m 14s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 343\n","Epoch 344\n","344 3127% (7m 17s) loss: 0.4680 \n","344 3127% (7m 17s) loss:0.4652 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 345\n","Epoch 346\n","346 3145% (7m 19s) loss: 0.4680 \n","346 3145% (7m 19s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 347\n","Epoch 348\n","348 3163% (7m 22s) loss: 0.4677 \n","348 3163% (7m 22s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 349\n","Epoch 350\n","350 3181% (7m 24s) loss: 0.4681 \n","350 3181% (7m 24s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 351\n","Epoch 352\n","352 3200% (7m 27s) loss: 0.4678 \n","352 3200% (7m 27s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 353\n","Epoch 354\n","354 3218% (7m 29s) loss: 0.4679 \n","354 3218% (7m 29s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 355\n","Epoch 356\n","356 3236% (7m 32s) loss: 0.4677 \n","356 3236% (7m 32s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 357\n","Epoch 358\n","358 3254% (7m 34s) loss: 0.4682 \n","358 3254% (7m 35s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 359\n","Epoch 360\n","360 3272% (7m 37s) loss: 0.4684 \n","360 3272% (7m 37s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 361\n","Epoch 362\n","362 3290% (7m 40s) loss: 0.4681 \n","362 3290% (7m 40s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 363\n","Epoch 364\n","364 3309% (7m 42s) loss: 0.4683 \n","364 3309% (7m 42s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 365\n","Epoch 366\n","366 3327% (7m 45s) loss: 0.4676 \n","366 3327% (7m 45s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 367\n","Epoch 368\n","368 3345% (7m 47s) loss: 0.4679 \n","368 3345% (7m 47s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 369\n","Epoch 370\n","370 3363% (7m 50s) loss: 0.4679 \n","370 3363% (7m 50s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 371\n","Epoch 372\n","372 3381% (7m 52s) loss: 0.4672 \n","372 3381% (7m 52s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 373\n","Epoch 374\n","374 3400% (7m 55s) loss: 0.4676 \n","374 3400% (7m 55s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 375\n","Epoch 376\n","376 3418% (7m 57s) loss: 0.4679 \n","376 3418% (7m 57s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 377\n","Epoch 378\n","378 3436% (8m 0s) loss: 0.4677 \n","378 3436% (8m 0s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 379\n","Epoch 380\n","380 3454% (8m 2s) loss: 0.4672 \n","380 3454% (8m 2s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 381\n","Epoch 382\n","382 3472% (8m 5s) loss: 0.4683 \n","382 3472% (8m 5s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 383\n","Epoch 384\n","384 3490% (8m 7s) loss: 0.4677 \n","384 3490% (8m 7s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 385\n","Epoch 386\n","386 3509% (8m 10s) loss: 0.4675 \n","386 3509% (8m 10s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 387\n","Epoch 388\n","388 3527% (8m 12s) loss: 0.4683 \n","388 3527% (8m 13s) loss:0.4651 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 389\n","Epoch 390\n","390 3545% (8m 15s) loss: 0.4674 \n","390 3545% (8m 15s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 391\n","Epoch 392\n","392 3563% (8m 18s) loss: 0.4677 \n","392 3563% (8m 18s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 393\n","Epoch 394\n","394 3581% (8m 20s) loss: 0.4670 \n","394 3581% (8m 20s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 395\n","Epoch 396\n","396 3600% (8m 23s) loss: 0.4673 \n","396 3600% (8m 23s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 397\n","Epoch 398\n","398 3618% (8m 25s) loss: 0.4675 \n","398 3618% (8m 25s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 399\n","Epoch 400\n","400 3636% (8m 28s) loss: 0.4676 \n","400 3636% (8m 28s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 401\n","Epoch 402\n","402 3654% (8m 30s) loss: 0.4674 \n","402 3654% (8m 30s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 403\n","Epoch 404\n","404 3672% (8m 33s) loss: 0.4676 \n","404 3672% (8m 33s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 405\n","Epoch 406\n","406 3690% (8m 35s) loss: 0.4671 \n","406 3690% (8m 35s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 407\n","Epoch 408\n","408 3709% (8m 38s) loss: 0.4675 \n","408 3709% (8m 38s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 409\n","Epoch 410\n","410 3727% (8m 40s) loss: 0.4675 \n","410 3727% (8m 40s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 411\n","Epoch 412\n","412 3745% (8m 43s) loss: 0.4675 \n","412 3745% (8m 43s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 413\n","Epoch 414\n","414 3763% (8m 45s) loss: 0.4679 \n","414 3763% (8m 46s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 415\n","Epoch 416\n","416 3781% (8m 48s) loss: 0.4679 \n","416 3781% (8m 48s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 417\n","Epoch 418\n","418 3800% (8m 51s) loss: 0.4676 \n","418 3800% (8m 51s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 419\n","Epoch 420\n","420 3818% (8m 53s) loss: 0.4671 \n","420 3818% (8m 53s) loss:0.4650 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 421\n","Epoch 422\n","422 3836% (8m 56s) loss: 0.4671 \n","422 3836% (8m 56s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 423\n","Epoch 424\n","424 3854% (8m 58s) loss: 0.4670 \n","424 3854% (8m 58s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 425\n","Epoch 426\n","426 3872% (9m 1s) loss: 0.4674 \n","426 3872% (9m 1s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 427\n","Epoch 428\n","428 3890% (9m 3s) loss: 0.4674 \n","428 3890% (9m 3s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 429\n","Epoch 430\n","430 3909% (9m 6s) loss: 0.4672 \n","430 3909% (9m 6s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 431\n","Epoch 432\n","432 3927% (9m 8s) loss: 0.4672 \n","432 3927% (9m 8s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 433\n","Epoch 434\n","434 3945% (9m 11s) loss: 0.4678 \n","434 3945% (9m 11s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 435\n","Epoch 436\n","436 3963% (9m 13s) loss: 0.4674 \n","436 3963% (9m 13s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 437\n","Epoch 438\n","438 3981% (9m 16s) loss: 0.4674 \n","438 3981% (9m 16s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 439\n","Epoch 440\n","440 4000% (9m 18s) loss: 0.4671 \n","440 4000% (9m 18s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 441\n","Epoch 442\n","442 4018% (9m 21s) loss: 0.4664 \n","442 4018% (9m 21s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 443\n","Epoch 444\n","444 4036% (9m 23s) loss: 0.4670 \n","444 4036% (9m 24s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 445\n","Epoch 446\n","446 4054% (9m 26s) loss: 0.4671 \n","446 4054% (9m 26s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 447\n","Epoch 448\n","448 4072% (9m 29s) loss: 0.4674 \n","448 4072% (9m 29s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 449\n","Epoch 450\n","450 4090% (9m 31s) loss: 0.4670 \n","450 4090% (9m 31s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 451\n","Epoch 452\n","452 4109% (9m 34s) loss: 0.4674 \n","452 4109% (9m 34s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 453\n","Epoch 454\n","454 4127% (9m 36s) loss: 0.4676 \n","454 4127% (9m 36s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 455\n","Epoch 456\n","456 4145% (9m 39s) loss: 0.4670 \n","456 4145% (9m 39s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 457\n","Epoch 458\n","458 4163% (9m 41s) loss: 0.4672 \n","458 4163% (9m 41s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 459\n","Epoch 460\n","460 4181% (9m 44s) loss: 0.4670 \n","460 4181% (9m 44s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 461\n","Epoch 462\n","462 4200% (9m 46s) loss: 0.4676 \n","462 4200% (9m 46s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 463\n","Epoch 464\n","464 4218% (9m 49s) loss: 0.4666 \n","464 4218% (9m 49s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 465\n","Epoch 466\n","466 4236% (9m 51s) loss: 0.4672 \n","466 4236% (9m 51s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 467\n","Epoch 468\n","468 4254% (9m 54s) loss: 0.4669 \n","468 4254% (9m 54s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 469\n","Epoch 470\n","470 4272% (9m 56s) loss: 0.4667 \n","470 4272% (9m 57s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 471\n","Epoch 472\n","472 4290% (9m 59s) loss: 0.4675 \n","472 4290% (9m 59s) loss:0.4649 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 473\n","Epoch 474\n","474 4309% (10m 2s) loss: 0.4671 \n","474 4309% (10m 2s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 475\n","Epoch 476\n","476 4327% (10m 4s) loss: 0.4673 \n","476 4327% (10m 4s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 477\n","Epoch 478\n","478 4345% (10m 7s) loss: 0.4672 \n","478 4345% (10m 7s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 479\n","Epoch 480\n","480 4363% (10m 9s) loss: 0.4676 \n","480 4363% (10m 9s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 481\n","Epoch 482\n","482 4381% (10m 12s) loss: 0.4665 \n","482 4381% (10m 12s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 483\n","Epoch 484\n","484 4400% (10m 14s) loss: 0.4673 \n","484 4400% (10m 14s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 485\n","Epoch 486\n","486 4418% (10m 17s) loss: 0.4663 \n","486 4418% (10m 17s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 487\n","Epoch 488\n","488 4436% (10m 19s) loss: 0.4668 \n","488 4436% (10m 19s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 489\n","Epoch 490\n","490 4454% (10m 22s) loss: 0.4669 \n","490 4454% (10m 22s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 491\n","Epoch 492\n","492 4472% (10m 24s) loss: 0.4669 \n","492 4472% (10m 24s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 493\n","Epoch 494\n","494 4490% (10m 27s) loss: 0.4667 \n","494 4490% (10m 27s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 495\n","Epoch 496\n","496 4509% (10m 29s) loss: 0.4670 \n","496 4509% (10m 29s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 497\n","Epoch 498\n","498 4527% (10m 32s) loss: 0.4672 \n","498 4527% (10m 32s) loss:0.4648 [1, 1, 1, 1] [0, 0, 0, 1] acc:0.8259\n","Epoch 499\n","best acc 0.8258833546189868\n"]}]},{"cell_type":"code","metadata":{"id":"qQHCB3f1Mdsk"},"source":["dev_acc_list = dev_acc_list[:50]\n","train_losses = train_losses[:50]\n","dev_losses = dev_losses[:50]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"uoOBmaj1UA_1","colab":{"base_uri":"https://localhost:8080/","height":360},"executionInfo":{"status":"ok","timestamp":1637895795079,"user_tz":300,"elapsed":1391,"user":{"displayName":"Yixuan Meng","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"00008408025985696509"}},"outputId":"3ed2a209-ba9a-4c19-f13d-1ba41a77a33a"},"source":["import warnings\n","warnings.filterwarnings(\"ignore\")\n","fig, ((ax1, ax2))= plt.subplots(1,2,figsize = (15,5))\n","x_axis = [i*2 for i in range(len(dev_losses))]\n","sns.lineplot(x_axis, dev_acc_list, ax = ax1)\n","ax1.set_ylabel('Accuracy')\n","ax1.set_xlabel(\"Number of Iterations\")\n","sns.lineplot(x_axis, train_losses, ax = ax2, label = 'train loss')\n","sns.lineplot(x_axis, dev_losses, ax = ax2, label = 'dev loss')\n","ax2.set_ylabel(\"Loss\")\n","ax2.set_xlabel(\"Number of Iterations\")\n","ax2.legend()\n","plt.tight_layout()\n","plt.show()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABCUAAAFXCAYAAACV2vT/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde3iT5fkH8O/7pkmavE3SUkpLC2vlpGg99IAoIIITlSIIQ1h/1Q03ikNhP5m11sOQAVPBKjIR6qHoVGD1sAHlMKeiOGBDDtb97MRNsVAMUsqpTXNq0uT3R9pASA8pvG0O/X6ui+uC5z3kzlMvm/fO/dyP4Ha73SAiIiIiIiIi6mZisAMgIiIiIiIiop6JSQkiIiIiIiIiCgomJYiIiIiIiIgoKJiUICIiIiIiIqKgYFKCiIiIiIiIiIKCSQkiIiIiIiIiCgomJYiIiIiIiIgoKKKCHUAoOH3aDJfLLdv94uNjcPJkg2z3I85pV+Ccyo9zKi/Op/w6M6eiKCAuTuriiCILP0+EPs6p/Din8uOcyo9zKq/OzmdHnymYlADgcrll/RDRck+SF+dUfpxT+XFO5cX5lB/ntOvw80R44JzKj3MqP86p/Din8pJzPrl8g4iIiIiIiIiCgkkJIiIiIiIiIgoKJiWIiIiIiIiIKCiYlCAiIiIiIiKioGCjSyIiIiIiIgo5TU1OnD5dC6ez8aLuc/y4CJfLJVNUdP58iqICGk0MYmIMEASh0/djUoKIiIiIiIhCzunTtYiO1kKSki7oYbdFVJQIp5NJCbmcO59utxtNTU6YTGdw+nQtevXq0+n7cfkGERERERERhRynsxGSpL+ohAR1LUEQEBWlRGxsPBobbRd0DyYliIiIiIiIKCQxIREeBEEE4L6ga5mUICIiIiIiIqKgYE+JEPThviP44psTwQ4jpCiVCjgcTcEOI6JwTuXHOZUX57NzFAoBP71pMFJ6S8EOhWTwrbEOH31uxM2ZKcEOhYiIutndd0/Hz3/+C9xyy/gLun7r1k147bVX8N57m2SOrGswKRGCPv3iKOrNjegbrw12KCFDEAU4m9icRk6cU/lxTuXF+ewsEW73hZVNUuip/O4kNv3jEMZe0xcKkYWtREShbu7ce5GRkYWZM3910fdas+YdGSIKH0xKhCCz1YHMIQm4Z/xlwQ4lZCQk6FBbawp2GBGFcyo/zqm8OJ/UkxkkFdxuwGRxIDZGHexwiIhIBg6HA0qlMthhhBwmJUKM2+2G2eaApOGPhoiIqKfSS55ERF1DI5MSREQhrrj4Kfzf/32Bysr/Q1nZGmg0WpSX/w2rV7+Mior9SE+/Cn/96yb06ZOIV199E0uX/h6fffZPmEz1iI/vjWnTcjF16k+997vzzon45S/vRU7ORPzww1FMmzYJ8+cvwrp1b+Lo0aMYOHAQHn30CaSmpgUUn91uwyuvlGD79m2wWq249NLL8L//W4BLLhkAAPj8831YufIP+P77aigUUUhNTcXSpcuh1+uxbduHeP31V3H8eA2UyigMGnQpVq58Sdb545NviGl0uOBsciMmmhk0IiKinsogqQAAdebGIEdCRBQ6dn35A3b+3w+dvk4QgM6ucBx1VV+MvLJvQOcWFj6Gw4cPtbp848sv/4Vrr70O7767CS6XZ1nqFVek495758BgMGDPnt149NEC/OhHqRg27Lo2X+Nvf9uK559fCUmKwcKFv8WyZUvxhz+UBBTfihXL8dVXX2LlylcRF9cLr732Cn7zmzlYt+49aLUSFi2aj1mz7kNOzkQ4nU785z8HoFQqYbPZsHjxfDz33ApkZQ2D3W5HZeX/BfSancFFiiHGbHMAACQNkxJEREQ9lT6mJSlhD3IkRER0MeLje+NnP/sFVCoVoqOjAQC33z4ZcXFxEEUR1103AsOHX4+9e/e0e59f/GIWevWKh1qtxoQJk/DVV/8O6PVdLhe2bt2EWbPuQ1JSX6jVatx77/1wuVz4xz92AgCUSiWMxu9x4kQtlEol0tOvgkajAQBERUXh0KEqnDlzBmq1GllZwy5iNlrHSokQY7Y5AQBSNH80REREPZVB60lK1LNSgojIa+SVgVcvnCsqSoTTGZzm2X37JkMQBO+/3W433nhjNT788H2cOFELQRBgs9mg1xvavU/v3gnev2s0GlitloBev67uDBob7UhO7ucdUygUSErqi5qaYwCAJUuW4a23XsfMmT+DVqvFLbeMx4wZMxEdHY3nnluBsrK1WL36ZSQkJGDSpCn46U//pzNT0CE++YYYs9VTKaHl8g0iIqIeS61SQKNWcPkGEVGYENvYKenchAQAfPjh3/DnP7+DZctexMCBgyCKIh555MEu20HLYIiFSqXG0aNGbw+KpqYm1NT8gMTEJADAwIGD8LvfPQkA+Oab/+DBB3+NxMQkTJgwCVdfnYGrr86A2+1GRcV+PPTQ/2LQoEG4+uos2WLk8o0Q412+wUoJIiKiHi1WF81KCSKiMNGrVzyOHKnu8DyzuQEKhQKxsbFwu9349NNPsHfvZ10WlyiKyMm5HaWlL+HYsWOw2+0oLX0JgIARI0bB4XBgy5ZynD59GgAgSTEQRRGiKOLkyRP4+OOPYDKZIAgCdDodAKHNBMyF4pNviGlZvhHDnhJEREQ9WpxOzaQEEVGYyM29G08/vRC33TYGGo0W69dvbfW8nJzb8cUXn+Puu6dBqVTihhvGYtSoG7s0trlzf4NXXlmJ+++fCavViiFDLsPzz78IrVaCw+HA9u3bUFKyAjabFXq9ATk5E3HrrTk4deoUNm78M4qLn4LT6UCvXvH41a/uR0ZGlqzLYQR3V9WJtMLlcmH58uV47733YLVakZmZiUWLFiElJaXV88vLy/Hqq6/CaDQiJiYGt956KwoLC6FSqdDY2Ijf//732L17N2pra2EwGHDbbbdh3rx53gYigTp5sgEul3zTkJCgQ22t6YKu/etnh/HuJwdR8uCNUKsUssUU7i5mTql1nFP5cU7lxfmUX2fmVBQFxMfHdHFEkUXuzxOlWw+gyliHJ2e13Y2dOof/X5Ef51R+nNOzjh07jKSk1Iu+TzB7SkSituazrZ9XR58punX5RmlpKTZv3ow1a9Zg586dSE5OxuzZs71bo5zr66+/RlFREebMmYP9+/fjT3/6E3bu3IlVq1YBAJxOJ+Li4lBSUoJ9+/bhrbfewu7du1FcXNydb0l2ZqsTClGASsmVNURERD1ZnC4adQ2slCAiosjWrU++ZWVlyM/Px4ABAyBJEgoLC1FVVYX9+/f7nXvkyBFv9YMgCEhJScGYMWPw9ddfAwC0Wi1+85vfYODAgVAoFOjfvz/uvPNO7NnT/lYqoc5sc0DSKP0aohAREVHPEqdTw2J3wsFv94iIKIJ1W1LCZDLBaDQiPT3dO6bX65GamooDBw74nT9q1Cj069cPW7ZsQVNTE6qrq/Hxxx9j3Lhxbb7GP//5T1x22WVdEn93MducbHJJREREiNV5lqOyrwQREUWybnv6bWhoAOBJRJxLp9N5j51Lo9HgzjvvxIIFC1BYWIimpiZMmTIFkydPbvX+paWl+Pzzz/HnP/+507F1xZrZhATdBV3naHIhVhd9wddHMs6J/Din8uOcyovzKT/OafiI06kBAHXmRsQbOtcvi4iIKFx0W1IiJsbz4G8y+TZtMZlM3mPnWr9+PZYtW4aXXnoJmZmZOHHiBObPn4+ioiI8++yzPueuXr0ar7/+Ot544w0kJyd3OrZQanR5ut6GuBg1m9uchw1/5Mc5lR/nVF6cT/mx0WV4iW1OSrBSgoiIIlm3Ld/Q6XRISUlBZWWld8xkMqG6uhpDhw71O7+yshLDhw9HdnY2RFFEnz59MH36dGzbts3nvBUrVuCNN97AW2+9hSFDhnT5++hqZqsTErcDJSIi6vHimpdv1JntQY6EiIio63Rro8vc3FysXr0aVVVVsFgsKC4uRlpaGrKysvzOzcrKwp49e1BRUQG3242TJ0/inXfe8elJsXTpUqxfvx5r167FgAEDuvOtdBmzzQEpmkkJIiKini5WpwLgWb5BREQUqbq1o2J+fj5MJhPy8vJgtVqRlZWFkpISiKKIffv2YdasWdiyZQuSk5ORk5OD2tpaPProo6ipqYFGo8G1116L3/3udwAAo9GI1157DUqlEpMmTfJ5nYqKiu58W7JxNrlga2yCpGGjSyIiop5OGaWAFB3F5RtERBTRuvXpVxRFFBQUoKCgwO9Ydna2XzJhxowZmDFjRqv3SklJwX/+858uiTNYLHYnALBSgoiIqB0ulwvLly/He++9B6vViszMTCxatAgpKSl+55aXl2PBggU+Y3a7HYMGDUJ5eXmn79fd9JKKlRJERBTRunX5BrXPbHUAALcEJSIiakdpaSk2b96MNWvWYOfOnUhOTsbs2bPhcrn8zp00aRIqKiq8f/bs2YO4uDjccccdF3S/7mZgUoKIKGyNGpWNzz/f12X3v/POidi6dVOX3b+7MCkRQsy25koJNrokIiJqU1lZGfLz8zFgwABIkoTCwkJUVVVh//79HV77wQcfoKGhAVOnTpXlfl1NL6lQ38CkBBERRS4mJUKIxdZSKcGkBBERUWtMJhOMRqNP42u9Xo/U1FQcOHCgw+vXrVuHnJwcxMbGynK/rmaQ1KizMClBRESRi+sEQojZ2lIpwR8LERFRaxoaGgB4Egfn0ul03mNt+e9//4t9+/ahqKhIlvudLz4+plPnB6JvnxjYG5sQo9dAo+bnAzkkJOiCHULE4ZzKj3Pqcfy4iKgoeb5Hl+s+bTl16hSWLn0S+/fvhV5vwOzZcwAACsXZ91BZ+SVWrnwBBw9+C61Wg/Hjb8fMmfciKioKjz9eBJ1Oh0ce+a33nvv378VDD83D5s0fQJKkVl9XFAXv/b/4ogIrV/4BVVXfITY2FpMmTcFdd/0cCoUCDocDzz//LLZv/xg2mxV6vQF5eT/D9Om5MJlMWLr0SezduwcOhwPx8fG47765uOmmm9t8v63NpyiKF/TfLn+7hZAGVkoQERG1KybG8+BvMpl8xk0mk/dYW9atW4f09HRcddVVstzvfCdPNsDlcnfqmvYkJOi8H9S+O3wSfeK0st27p0pI0KG21tTxiRQwzqn8OKdnuVwuOJ1n+/s4/rsLjv/8vdP3EQQBbnfn/v+svHQ0lENGBnz+ggWPIyoqCu+842mi/NRTvwMANDV53kN19SH8+tf34dFHn8Do0WNw4kQtHnmkAFFRSsyYMRM5OZPwxBOPYO7cBxEdHQ0A2LhxA8aOvRlqtcZnHs7lcrnhdLpw7NgPeOCB+/HrX/8Gt98+GYcOVeHhh+dBFBXIzb0bmzaV48sv/4U333wbsbGxOHnyBE6cOAGn04U33/wjzGYz3n13IzQaLWpqjsFqtbb5mlFRYqvHXC5Xq//tiqLQbuKeyzdCSEujSy2/CSEiImqVTqdDSkoKKisrvWMmkwnV1dUYOnRom9c1NDSgvLwceXl5styvuxhiVACAerMjyJEQEVFbamuPY+/ezzBnzjzo9Xro9XrMnv1rn3P+8pd3MXr0jbjpppsRFRWFpKS++NnP7vE2qhw2bDhiYnT45JOPAHh+F23f/jFuv/0Ov9drzYcfvo8BAwZi8uQ7ERUVhUGDBuOuu36O8vL1AAClUgmr1YpDh76D0+lEfHxvXHrpZd5jdXV1OHz4ENxuN5KS+uKSSwbINT0d4tNvCLHYnNCqoyCKQrBDISIiClm5ublYvXo1rrvuOiQmJqK4uBhpaWnIyspq85qNGzdCqVRiwoQJstyvu+i1nqREndke5EiIiIJPOWRkp6oXWrT1zb5cjh8/DgDo2zfZO3bu3wHgyJEjqKjYh127dnjHXC433G5PXKIoIidnIrZsKcf48bfjww/fR9++fXHVVdcEFENNTQ2Sk323sk5J6Y+ammMAgFtuGY/Tp09j5co/oLr6EK688mrce+/9GDLkMuTl/RwulwtLlvwex4/XIDv7WsyePRcpKf06PxkXgJUSIcRsc7CfBBERUQfy8/Mxfvx45OXlYcSIETAajSgpKYEoiti3bx8yMjJw9OhRn2vKysowZcoUb0lsoPcLtpZKCW4LSkQUuvr06QMAOHbsB+/YuX8HgPj4eNx66wS8//52758PPvgUH354NkkxYcIkfPnlv/D990ewZUs5JkwIrEoCABITE/1+9xmN3yMxMQkAoFAokJf3M7z66htYv/6vSE1Nw6OPPgQAiI6ORn7+bLzxxp/w9tvrIYoinn56Uecm4SIE/7cteZltTmjZT4KIiKhdoiiioKAA//znP/HFF19g9erV6NfP821OdnY2KioqkJzs+w3Vpk2b8Mgjj3T6fsGm0yohAKhnUoKIKGQlJPRBVta1WLnyD6ivr0d9fT1eeulFn3OmTLkTn3zyIbZv3waHw4GmpiZ8//0R7N79D+85SUl9kZU1DC+88BwOHvwG48f7V/e15eabb8V3332LjRv/AqfTie+++xbr1r2J22+fDMDTNPPrr7+Cw+GASqWCRqOFKCoAADt3forvvjsIp9MJtToaarW6WxPz/Fo+hJitDsRE80dCREREHgpRhE6rZKUEEVGIe+KJRXj22acxbdpE6PUG3Hvv/dixY7v3+NChV+C5515EaWkJnn32aTidTiQlJWPy5Kk+95kw4Q4sWPAobrxxLOLiegX8+n37JuO551agpGQFVq36AwwGz+4bP/2pp5fS6dOnsHx5MY4d+wEKhafnxOLFTwMAjh49ihdfXI6TJ09AqVThiivSUVT02/ZeTlaCu7NtSCNQV3TLvpCOuY++shupiTGYfUd6xyf3MOxCLD/Oqfw4p/LifMqvM3PaUads8tdVnyeeWP0Zehs0+N87r+r4ImoX/78iP86p/DinZx07dhhJSakXfZ+u7inR07Q1n239vLj7Rhix2BzcDpSIiIh8GCQV6i2slCAiosjEpESIcLvdMFud0HL5BhEREZ1DL6lR18CkBBERRSYmJUKErbEJLreblRJERETkwyCpUGduBFfcEhFRJGJSIkSYrQ4A4JagRERE5EMvqeBscsFqdwY7FCIiItkxKREizDbPB40YVkoQERH1eE3Hv8OZf6wHABhiVADAHTiIqEdilVh4uJifE5MSIcJsa6mUYFKCiIiop3MersCp7evgdruhlzxJiXomJYiohxFFBZqaWCUWDhyORigUF1b1z6REiGiplGCjSyIiIhLUWsDtAhw2GCRWShBRz6TRxMBkOgO3m9t5hiq3243GRjvOnKlFTEzsBd2DT8AhwttTgss3iIiISKUFALjtZhgkPQAmJYio54mJMeD06VrU1HwP4CKWB4giXC4mNuRy/nwqFFHQ6eKg0UgXdD8mJUKEd/kGKyWIiIh6PEHt+WDntpshxcdDFAQu3yCiHkcQBPTq1eei75OQoENtrUmGiAiQfz65fCNEmG1OqKJEqJSKYIdCREREQeZNSjRaIAoC9JISdQ1MShARUeRhUiJEmK0O9pMgIiIiAIDgXb5hAQAYJDXqLUxKEBFR5GFSIkSYbU7uvEFEREQAzlZKwG4G4NkWlJUSREQUiZiUCBFmq4NNLomIiAhA8+4bANyNnqSEXqtCndkezJCIiIi6BJMSIcJsc7LJJREREXkoowFBPLt8I0YFk8UBl/vCu88TERGFIiYlQoTZxkoJIiIi8hAEEWK01puU0EsqNLnc3i3EiYiIIgWTEiHCbHNA0rBSgoiIiDzE6Bjv8g2DpAIA1HFbUCIiijBMSoQAh9OFRoeLlRJERETkJaqlc3bf8CQl6pmUICKiCMOkRAiw2DylmNx9g4iIiFooNBLczbtv6FkpQUREEYpJiRDQYHMCABtdEhERkZcYLQGNvpUS3BaUiIgiDZMSIaClaRWXbxAREVELMTrGWymhUUchSiFy+QYREUUcJiVCgNm7fIOVEkREROQhRnt6SrjdbgiCAIOk4vINIiKKOExKhACLd/kGKyWIiIjIQxEtAS4n0ORJROglFerN9iBHRUREJC8mJUIAl28QERHR+cToGADw2YGDlRJERBRpujUp4XK5sGzZMowYMQIZGRmYOXMmjEZjm+eXl5dj4sSJyMzMxOjRo/Hkk0+isfHsL2ObzYYnnngC1157LTIzMzFv3jycOXOmO96KrBpsTggCEK1WBDsUIiIiChFitATgnKREjIo9JYiIKOJ0a1KitLQUmzdvxpo1a7Bz504kJydj9uzZcLlcfud+/fXXKCoqwpw5c7B//3786U9/ws6dO7Fq1SrvOU899RQqKyuxadMmfPLJJ7BYLCgqKurOtyQLs80BKVoJURCCHQoRERGFCG+lRGPztqBaFUwWB5pa+dxEREQUrro1KVFWVob8/HwMGDAAkiShsLAQVVVV2L9/v9+5R44cgcFgwG233QZBEJCSkoIxY8bg66+/BuCpktiwYQMeeOABJCYmwmAwoKioCNu3b8fRo0e7821dNLPVwe1AiYiIyEdLpQSad+AwxKjgBmCyOIIXFBERkcy6LSlhMplgNBqRnp7uHdPr9UhNTcWBAwf8zh81ahT69euHLVu2oKmpCdXV1fj4448xbtw4AMChQ4dgt9tx5ZVXeq8ZOHAgNBpNq/cLZRabE5KG/SSIiIjoLMX5yzckFQBwCQcREUWUbvt6vqGhAYAnEXEunU7nPXYujUaDO++8EwsWLEBhYSGampowZcoUTJ482ed+Op0uoPu1Jz4+plPnByIhQdfxSc3sThdi9dGduqYn4vzIj3MqP86pvDif8uOcho+zyzdakhJqAGCzSyIiiijdlpSIifH8YjWZTD7jJpPJe+xc69evx7Jly/DSSy8hMzMTJ06cwPz581FUVIRnn33W5369evXq8H7tOXmyAS6Xu7NvqU0JCTrU1po6PrFZncmOeJ26U9f0NJ2dU+oY51R+nFN5cT7l15k5FUWhS5L2FDgxWgsAcDcv39BLnqrKugYmJYiIKHJ02/INnU6HlJQUVFZWesdMJhOqq6sxdOhQv/MrKysxfPhwZGdnQxRF9OnTB9OnT8e2bdsAAGlpaVCr1T73O3jwIKxWKy677LKuf0Myaml0SURERNRCEBWAMvqcpETz8g0LkxJERBQ5urXRZW5uLlavXo2qqipYLBYUFxcjLS0NWVlZfudmZWVhz549qKiogNvtxsmTJ/HOO+94e1JER0dj8uTJeOGFF3D8+HHU1dWhuLgYN954I1JSUrrzbV0Ul9vd3FOCjS6JiIjIl6CWvD0lolVRUKsUrJQgIqKI0q1Jifz8fIwfPx55eXkYMWIEjEYjSkpKIIoi9u3bh4yMDO/OGTk5Obj//vvx6KOPIjMzExMnToRGo8Ezzzzjvd9jjz2GoUOHYsKECRg7dizUarXP8XBgtTvhBqBlpQQRERGdR1BrvZUSAGDQqlBntgcxIiIiInl169fzoiiioKAABQUFfseys7NRUVHhMzZjxgzMmDGjzftFR0dj8eLFWLx4seyxdhez1bOtF7cEJSIiovMJKglobnQJAPoYFXffICKiiNKtlRLkz2xzAgC3BCUiIiI/nkqJs0kJg6Ti7htERBRRmJQIMrPNUykRw+UbREREdD6VBHfj2eUbeomVEkREFFmYlAgys7WlUoLLN4iIiMiXX08JSQWzzQmH0xXEqIiIiOTDpESQtVRKsNElERERnU9QS4CzEe4mz5cYhuZtQU3cFpSIiCIEkxJBxkaXRERE1BZBrQUAuJubXeqbkxLsK0FERJGCSYkgM9ucUKsUiFLwR0FERES+BJUnKYHmJRwGSQ0AqGtgUoKIiCIDn4SDzGxzIIZVEkRERNQKQS0BgLevRMvyjXou3yAiogjBpESQma1O9pMgIiLqBJfLhWXLlmHEiBHIyMjAzJkzYTQa2zzfZrNhyZIlGD16NK655hqMGzcOn376qff4I488giuuuAIZGRneP8XFxd3xVjp0NinRsnzD85mhrsEetJiIiIjkxK/og8xsc7CfBBERUSeUlpZi8+bNWLNmDRITE7FkyRLMnj0bGzduhCj6ft/idrsxZ84cAMDatWvRv39/HDt2DE6n0+e8iRMnYsmSJd32HgJ2Xk8JZZQCWnUUe0oQEVHE4NNwkJltTvSN1wY7DCIiorBRVlaG/Px8DBgwAABQWFiIESNGYP/+/Rg2bJjPubt27cLevXuxfft29OrVCwCQlJTU7TFfqJaeEj7bgsaoUM+kBBERRQgmJYLMUynB5RtERESBMJlMMBqNSE9P947p9XqkpqbiwIEDfkmJ3bt3o1+/figpKcHWrVuhVqsxduxYPPjgg5AkyXveRx99hOHDh0On02HEiBGYN2+eN4kRqPj4mIt7c63ok5KEKgDaKCfiEnQAgN6xWlgam5DQ/G/qHM6b/Din8uOcyo9zKi8555NJiSByu90wW51cvkFERBSghoYGAJ5ExLl0Op332LlOnz6NgwcPYuTIkfjoo49w+vRpzJ07F0uXLsWiRYsAAHfffTcKCgrQu3dvHDlyBAsXLsR9992HsrIyCIIQcGwnTzbA5XJfxLvzlZCgw4nTNkChQsPp03DWmgAAGpWIQ8fMqG3+NwUuIUHHeZMZ51R+nFP5cU7l1dn5FEWh3cQ9G10GUaPTBWeTC5KGlRJERESBiInxfKgxmXw/DJlMJu+xc0mSBIVCgYceeggajQbJycmYNWsWPvroI+856enpSEhIgCAI+NGPfoQnn3wSX3zxBQ4dOtSl7yVQgloLNDe6BAC9xOUbREQUOZiUCCKz1QEArJQgIiIKkE6nQ0pKCiorK71jJpMJ1dXVGDp0qN/5l19+OQD4VDx0VP3Qctztlq/q4WIIasm3p4Skgq2xCfbGpiBGRUREJA8mJYLIYvN0/mZPCSIiosDl5uZi9erVqKqqgsViQXFxMdLS0pCVleV37rhx4xAfH4/nn38ejY2NqKmpQWlpKW699VYAgN1ux/vvv++tvDAajZg/fz6uuOIKpKWldefbapOg0np33wA8lRIAUGdhtQQREYU/JiWCyGxjpQQREVFn5efnY/z48cjLy8OIESNgNBpRUlICURSxb98+ZGRk4OjRowA8yzdee+01VFZWYgdbBzEAACAASURBVPjw4Zg2bRoyMzPx8MMPAwBcLhfefPNN/PjHP8Y111yDu+++G0lJSXj55Zf9thcNGrX2vEoJNQBwCQcREUUEPg0HUYO1uVKCPSWIiIgCJooiCgoKUFBQ4HcsOzsbFRUVPmODBw/GW2+91eq9NBoN1q1b1yVxykVQS3Cd+t77b0NLpUQDkxJERBT+QuQrgJ7pbKUEkxJERETUOkEttbp8o95sD1ZIREREsmFSIoi8SQkNC1aIiIiodYJKCzRa4Xa5AAA6rRICgDou3yAiogjApEQQWWxOKEQBaqUi2KEQERFRiBLUWs9fmqslohQiYrRK9pQgIqKIwKREEJmtDkjRUR1uTUZEREQ9l6CWAMBvCQcrJYiIKBIwKRFEDTYnm1wSERFRuwRVc1LCfjYpYWBSgoiIIgSTEkHkqZRgUoKIiIja0bx8w3dbUBWXbxARUURgUiKILDYnpGg2uSQiIqK2tfSUcDeem5RQo87cCLfbHaywiIiIZMGkRBCZbQ5oWSlBRERE7Wht+YZeUsHhdMFqbwpWWERERLJgUiKIzDYHtwMlIiKidnkbXZ63fAMA6i1cwkFEROGNSYkgaXJ5vt2IYaUEERERtSdKBYgK75agAKCP8SQl6hrswYqKiIhIFkxKBInF5gQA7r5BRERE7RIEAYJK61spoW1OSrDZJRERhTkmJYLE3JyU0LLRJREREXVELfn2lIhhUoKIiCIDkxJBYrY6AIBbghIREVGHBLVvpUSMRglRELgtKBERhT0mJYLEbGtOSrDRJREREXVAUEtwn9NTQhQE6LRKJiWIiCjsMSkRJGarZ/kGG10SERFRRzw9JSw+Y3pJBZPFEaSIiIiI5MGkRJCcrZRgUoKIiIjaJ6gl4JzlGwCg1yrZU4KIiMIekxJB4m10qebyDSIiImqfoNLC3WiB2+32juklFZdvEBFR2OvWpITL5cKyZcswYsQIZGRkYObMmTAaja2eW15ejoyMDJ8/l19+OSZNmuQ958SJEygoKMD111+P7Oxs5ObmYu/evd31di6K2eqARh0FURSCHQoRERGFOEEtAW4X4LB5x/SSCvWWRp9EBRERUbjp1qREaWkpNm/ejDVr1mDnzp1ITk7G7Nmz4XK5/M6dNGkSKioqvH/27NmDuLg43HHHHd5zFi5ciOPHj2PLli347LPPcMstt+Dee+9FfX19d76tC2K2OSBxO1AiIiIKhFoLAD7NLvWSCg6nC7bGpmBFRUREdNG6NSlRVlaG/Px8DBgwAJIkobCwEFVVVdi/f3+H137wwQdoaGjA1KlTvWOHDx/Gbbfdhl69ekGhUOCnP/0pLBYLqquru/JtyMJsc7KfBBEREQVEUDUnJc7pK6HXqgCASziIiCisdVtSwmQywWg0Ij093Tum1+uRmpqKAwcOdHj9unXrkJOTg9jYWO/YrFmz8MEHH6C2thYOhwNr165FWloahgwZ0iXvQU6slCAiIqJACWoJgG9SwiB5khJsdklEROEsoKfiW2+9FdOnT8eUKVPQq1evC3qhhoYGAJ5ExLl0Op33WFv++9//Yt++fSgqKvIZz8jIwIYNGzBq1CgoFArExsZi5cqVUKlUnYotPj6mU+cHIiFB1+5xW6MLfXvHdHgencW5kh/nVH6cU3lxPuXHOQ1PZ5MSvss3AFZKEBFReAsoKTFx4kSsXbsWy5cvx0033YTc3Fxcf/31nXqhmBjPg7/JZPIZN5lM3mNtWbduHdLT03HVVVd5x1wuF+655x4MHz4ce/bsgSRJ2L59O2bNmoW1a9fi0ksvDTi2kycb4HLJ1yQqIUGH2lpTu+fUm+2IEoUOzyOPQOaUOodzKj/Oqbw4n/LrzJyKotAlSXu6MEJzTwk0+iclTBYmJYiIKHwFtHxj7ty52LZtG1atWgW3241Zs2bh5ptvxiuvvIITJ04E9EI6nQ4pKSmorKz0jplMJlRXV2Po0KFtXtfQ0IDy8nLk5eX5jNfV1eHIkSP4+c9/DoPBgKioKNx8883o378/du3aFVBMweJ2u2GxObl8g4iIiALSWk+JmObeVFy+QURE4SzgnhKCIOCGG27ACy+8gL///e+YOnUqXnzxRYwZMwa//vWvA2pWmZubi9WrV6OqqgoWiwXFxcVIS0tDVlZWm9ds3LgRSqUSEyZM8BmPi4vDwIEDsXbtWjQ0NMDlcmHbtm345ptvcMUVVwT6toLC1tiEJpcbUjQbXRIREVEAVBoAgk9SIkohIkajRL3FEby4iIiILlKnG11+9913ePXVV/HGG29Aq9XirrvugkKhwIwZM/CHP/yh3Wvz8/Mxfvx45OXlYcSIETAajSgpKYEoiti3bx8yMjJw9OhRn2vKysowZcoUREdH+91v1apVOHHiBMaNG4fs7GwsX74cCxYswPDhwzv7trqV2eb58MBKCSIiIgqEIIiAWuvTUwLwLOFgTwkiIgpnAT0V2+12/PWvf8W7776Lzz//HJmZmXjsscdw2223eZtK7tixA/PmzcMDDzzQ5n1EUURBQQEKCgr8jmVnZ6OiosJvfNOmTW3eLy0tDatWrQrkLYQUs9UJANwSlIiIiAImqLRwN5p9xvRaJZMSREQU1gJKSowaNQqiKGLixIlYuHAhBg0a5HfO1VdfDYPBIHuAkYiVEkRERNRZQhuVEod+YENYIiIKXwE9FT/22GPIycmBWq1u8xy9Xo+PP/5YtsAimcXWXCnBnhJEREQUIEEtwd3on5So4+4bREQUxgLqKXHzzTfDarX6jZ85cwYNDQ2yBxXpGloqJbh8g4iIiAIkqLSA3Xf5hkFSwd7YBLujKUhRERERXZyAkhIFBQWt9nbYsmULHnroIdmDinRmK5dvEBERUecIasln9w0A0Gs9vb1M7CtBRERhKqCkxL/+9a9Wd7S49tpr8cUXX8geVKQz25xQRolQKRXBDoWIiIjChUrrt3xDJ3mSElzCQURE4SqgpITVaoVC4f8ALYoiLBZLK1dQeyw2B6skiIiIqFMEtQQ0OeF2nk1AGJqTEtyBg4iIwlVASYlBgwbhww8/9Bv/29/+hgEDBsgeVKQzW51scklERESdIqi1AOCzhKNl+QaTEkREFK4C+ro+Pz8fDz/8ME6cOIGRI0cCAHbt2oWysjIsWbKkSwOMRGZWShAREVEnCWoJADzbgkpxAAC95PmSg0kJIiIKVwE9Gefk5MBqteLFF1/EmjVrAABJSUlYsGABbr/99i4NMBI1WJ1IiI0OdhhEREQURrxJicazlRLKKAU06ijUmx3BCouIiOiiBPx1/dSpUzF16lScOnUKANCrV68uCyrSWewOSNG6YIdBREREYURQeZZvwO7bz0svqdjokoiIwlan1xAwGXHxzFYntFy+QURERJ3QWk8JADBolVy+QUREYSvgJ+P169dj8+bNMBqNcDh8SwS3bdsme2CRyuF0we5ogqRho0siIiLqBO/yDf9Kie9rza1dQUREFPIC2n3j9ddfx+LFizFgwAAYjUbceOONSE1NRV1dHe64446ujjGiWGyehE4MKyWIiKiH2rVrF/bv3+/99zvvvIOpU6fikUceQUNDQxAjC20tyzfOr5TQSyqYuHyDiIjCVEBJibfffhsLFy7E448/jqioKPziF7/Aa6+9hrvvvht1dXVdHWNEabA5AYCVEkRE1GMVFxfj9OnTAICqqiosWrQI6enpqKysxDPPPBPk6EKXICoAZbRn941z6LUqmG1OOJtcQYqMiIjowgWUlPjhhx+QmZkJAFCr1TCbPRn6yZMnY8uWLV0XXQRqqZRgTwkiIuqpqqurMWTIEADAhx9+iOuvvx4LFy7E4sWLsX379oDu4XK5sGzZMowYMQIZGRmYOXMmjEZjm+fbbDYsWbIEo0ePxjXXXINx48bh008/9Tn+xBNP4Nprr0VmZibmzZuHM2fOXNT77AqCSuuz+wbgqZQAuC0oERGFp4CSEr169YLJZALg2Qr0wIEDAICamho4nc6uiy4CmVsqJaJZKUFERD2XIAgAgL1792LkyJEAgMTExIATAaWlpdi8eTPWrFmDnTt3Ijk5GbNnz4bL5V8t4Ha7MWfOHHzzzTdYu3YtvvjiC7z11lsYOHCg95ynnnoKlZWV2LRpEz755BNYLBYUFRXJ8E7lJailVnffAIB6LuEgIqIwFNDX9dnZ2dixYwcuu+wy5OTk4KmnnsKOHTuwZ88e3HDDDV0dY0SxNicltGpWShARUc906aWXYt26dbjpppuwe/du78P/Dz/8EPAuX2VlZcjPz8eAAQMAAIWFhRgxYgT279+PYcOG+Zy7a9cu7N27F9u3b/fePykpyXvcZrNhw4YNWLFiBRITEwEARUVFyMnJwdGjR5GcnHzR71kuglrbak8JgJUSREQUngJ6Mp4/fz4aGz2/6GbNmgVRFLFv3z5MnDgRc+bM6dIAI43F7klKaLh8g4iIeqiCggLMmTMHf/zjH3HnnXdi0KBBAIBPPvkEV155ZYfXm0wmGI1GpKene8f0ej1SU1Nx4MABv6TE7t270a9fP5SUlGDr1q1Qq9UYO3YsHnzwQUiShEOHDsFut/u89sCBA6HRaHDgwIHQSkqotHDV1/qMtSQl6piUICKiMNThk7HT6cTWrVsxduxYAJ5yy/z8fOTn53d5cJGoJSnBSgkiIuqpsrOz8Y9//ANmsxl6vd47Pn36dGg0mg6vb9mh49xrAUCn07W6e8fp06dx8OBBjBw5Eh999BFOnz6NuXPnYunSpVi0aJH3Gp1OF9D92hIfHxPwuYFKSPCN6bghFtZT1T7jOr1nzpog+J1P/jhH8uOcyo9zKj/OqbzknM8On4yjoqLw9NNPc5mGTKw2J1RRIqIUAbXzICIiikgKhcKbVHC73fj222/Rt29fxMR0/GDfck5Lv6sWJpOp1eslSYJCocBDDz0EtVoNjUaDWbNmYfHixVi0aJHP/c5dPtLW/dpy8mQDXC53wOd3JCFBh9pa3/dod6vQZG3wG1crFfjhuP84+WptTunicE7lxzmVH+dUXp2dT1EU2k3cB/RkfMUVV+Cbb74J+EWpbRa7ExpWSRARUQ+2dOlSvPvuuwA8CYkZM2Zg4sSJuPHGG1FRUdHh9TqdDikpKaisrPSOmUwmVFdXY+jQoX7nX3755QDONtc8/+9paWlQq9U+9zt48CCsVisuu+yyzr/BLiSotYDTDrfLt9G4XlLCxEaXREQUhgJKSvzqV7/CM888g02bNuHQoUOoqanx+UOBs9id3A6UiIh6tPfffx+DBw8GAOzYsQP/+c9/8Pbbb2Py5MlYtmxZQPfIzc3F6tWrUVVVBYvFguLiYqSlpSErK8vv3HHjxiE+Ph7PP/88GhsbUVNTg9LSUtx6660AgOjoaEyePBkvvPACjh8/jrq6OhQXF+PGG29ESkqKfG9cBoJKAgC4z9+BQ6tiTwkiIgpLAT0d/+pXvwLg6Wx97jcLbrcbgiB4twiljllZKUFERD3ciRMnvLtf7NixA7fddhuuvvpqGAwGTJs2LaB75Ofnw2QyIS8vD1arFVlZWSgpKfE24541axa2bNmC5ORkSJKE1157DYsWLcLw4cOh0+lw22234Te/+Y33fo899hiefPJJTJgwAU1NTbjhhhuwcOHCLnn/F0NQaz1/sZsBzdmeGnpJheNnrEGKioiI6MIF9HT85ptvdnUcPYbF5oTESgkiIurBDAYDamtrkZSUhN27d+O+++4D4Pmyo6mpKaB7iKKIgoICFBQU+B3Lzs72WwYyePBgvPXWW23eLzo6GosXL8bixYs78U66X0tSwt14XqWEpMK3xrpghERERHRRAno6vvbaa7s6jh7DaneityE62GEQEREFzQ033ID58+dj6NCh+P777zFq1CgAwLfffhtyyyVCzdnlG2afcb1WhQaLA00uFxQim2kTEVH4CCgpsXfv3naPn78fOLWNPSWIiKine/zxx7F8+XIcPXoUK1as8O7CUVlZifHjxwc5uhCnbqOnhKSCG0CDxQFDjDoIgREREV2YgJ6Of/azn0EQBLjdZ7e5Ore3BHtKBI49JYiIqKeLiYnBb3/7W7/xc3s8UOu8yzfOq5QwSCoAQJ25kUkJIiIKKwE9HX/66ac+/3Y4HPj3v/+NVatW4eGHH+6SwCKRw+mCw+mClkkJIiLq4ZxOJ7Zs2YJvv/0WADBkyBCMHz8eUVH8HdkeQdV2TwkAqOe2oEREFGYC+s2fmJjoN9avXz9oNBqsXLkSI0eOlD2wSGS1e/YUZ6UEERH1ZNXV1cjPz0dNTQ0uueQSAJ6m2i+++CJKS0vRv3//IEcYuoQoFaBQ+veUaE5KmMyOYIRFRER0wS6qE1Jqaiq++uoruWKJeJbmpAR7ShARUU/21FNPISkpCR9//DE2bNiADRs2YNu2bejTpw+eeuqpYIcX8gS1BJzfU0KrBOBZvkFERBROLjgpcerUKbz88svskt0JrJQgIiICPvvsMzzyyCOIj4/3jvXu3RtFRUX47LPPghhZeBBUWr9KCY06ClEKgcs3iIgo7AT0dHzFFVf4NLYEgKamJmi1WixbtqxLAotEFltzpQSTEkRE1MOd/7kCAERuZRkYtdavp4QgCNBLKtSzUoKIiMJMQE/Hixcv9vnwIAgC4uPjcdVVV8FgMHRZcJGmpVKCSQkiIurJhg0bhmeeeQbLly/3fo44c+YMiouLkZ2dHeToQp+gluA2n/Eb12uZlCAiovAT0NPxT37yk66Oo0dgTwkiIiLg0UcfxS9/+UuMGTMGAwcOBAAcPHgQcXFxeO2114IcXegTVFq4Thv9xvWSCmdM9iBEREREdOEC3hJUEASMHj3aZ/zvf/87APiNt8XlcmH58uV47733YLVakZmZiUWLFrXal6K8vBwLFizwGbPb7Rg0aBDKy8u9Y3v27MHy5ctx4MABKJVKZGVloaSkJKB4ulvL8g32lCAiop7skksuwfvvv4/y8nIcPHgQAJCbm4uMjAzcf//92Lp1a5AjDG2CWvLrKQF4khKHa0xBiIiIiOjCBfR0vGzZMjzwwAN+4y1JhkCTEqWlpdi8eTPWrFmDxMRELFmyBLNnz8bGjRv91pFOmjQJkyZN8v7b4XBgzJgxuOOOO7xje/fuxX333YcFCxbglltugSiK+PrrrwOKJRgsdicEAVCrFMEOhYiIKKjUajWmTZvmM/b111+jqqoqSBGFD0GtBRqtcLtcEM75/GSQVDCZHXC53RBb6dlBREQUigLqKHX48GEMHjzYb3zQoEE4fPhwwC9WVlaG/Px8DBgwAJIkobCwEFVVVdi/f3+H137wwQdoaGjA1KlTvWPPPfccpk+fjkmTJiE6OhoqlQpXXXVVwPF0N6vdCa06ih8UiIiI6IIJKsnzF4fVZ1yvVcHldnsrM4mIiMJBQJUSarUaJ06cQP/+/X3Gjx8/jqiowJYimEwmGI1GpKene8f0ej1SU1Nx4MABDBs2rN3r161bh5ycHMTGxgIALBYL/vWvfyEjIwM/+clPYDQakZaWhnnz5uH6668PKKYW8fExnTo/EAkJOr8xFwBJq2r1GHWM8yY/zqn8OKfy4nzKj3Ma/gS1FgDgtpshqCXvuE5SAgDqzI2I0SiDEhsREVFnBZRRGD58OFasWIGSkhKo1WoAgM1mw4svvojrrrsuoBdqaGgA4ElEnEun03mPteW///0v9u3bh6KiIu9YfX09XC4XNm3ahFdeeQWDBw/G+vXrMXv2bGzevNkvgdKekycb4HK5Az6/IwkJOtTW+q/pPF1nQ3SU2Ooxal9bc0oXjnMqP86pvDif8uvMnIqi0CVJe5JBcyLCbffdFtSgVQEA6s2NSOkt+V1GREQUigJKShQWFiI3Nxc//vGPkZmZCQD4/PPP4Xa7sW7duoBeKCbG88HGZPL9MGQymbzH2rJu3Tqkp6f7LM2QJM8v26lTp+Lyyy8HAEyfPh1vvPEGduzYgby8vIDi6k4Wu5NNLomIqMf65S9/2e5xi8XS7nHyEFRnKyXOpZfOJiWIiIjCRUBPyP3798fGjRuxZs0afPXVVwCAadOmIS8vDwkJCQG9kE6nQ0pKCiorK3HllVcC8CQkqqurMXTo0Dava2hoQHl5OR5//HG/+7VWDSGEcL8Gi82JhNjoYIdBREQUFImJiR2ec8kll3RDJOGtZcmGu5FJCSIiCn8Bf23fu3dvzJs376JeLDc3F6tXr8Z1112HxMREFBcXIy0tDVlZWW1es3HjRiiVSkyYMMHv2F133YXVq1cjJycHgwYNwoYNG2A0GgPeDaS7WVkpQUREPdjTTz8d7BAiwtmeEr6VJZJGCVEQUG9hUoKIiMJHQE/If/nLXxAdHY2cnByf8a1bt6KxsRGTJ08O6MXy8/NhMpmQl5cHq9WKrKwslJSUQBRF7Nu3D7NmzcKWLVuQnJzsvaasrAxTpkxBdLR/hcE999wDs9mMmTNnwmKxYPDgwXj55ZfRr1+/gOLpbpbm3TeIiIiILpTQRk8JURCgk5SoY6UEERGFkYCekF999VXMnz/fbzwuLg6LFy8OOCkhiiIKCgpQUFDgdyw7OxsVFRV+45s2bWrzfoIgYO7cuZg7d25Arx9MLrcbNlZKEBER0cWKUgOCCJy3fAPwNLvk8g0iIgonYiAnGY1G/OhHP/Ib79+/P4xGo+xBRSKbvQluANpoJiWIiIjowgmCAEEt+TW6BDx9JUxcvkFERGEkoKSETqfD999/7zd+5MgRaLVa2YOKRFa7EwBYKUFEREQXT631W74BeJISrJQgIqJwElBSYvTo0XjmmWdw/Phx71hNTQ2Ki4tx4403dllwkcTSnJRgTwkiIiK6WIJKgruxlaSEVoU6swNutzsIUREREXVeQE/IhYWFuOuuuzBu3DgMHDgQAPDtt98iOTkZhYWFXRpgpPBWSnD5BhEREV0kQa1tc/mGs8kFq72JS0aJiCgsBPTbqlevXtiwYQPKy8vx1VdfAfBsx3nNNddgzZo1eOCBB7o0yEhgsbFSgoiIiOQhqLRw1df6jeslJQCg3tLIpAQREYWFgJZvAIBarca0adMwf/58jBw5Eu+//z4mTZqEt99+uyvjixhWLt8gIiIimQhqCWijUgIA+0oQEVHYCPgJ+ejRo3j33Xfx5z//GbW1tZgwYQJeeeUVXHfddV0ZX8SwcPkGERERyURQe3pKuN1uCILgHddrmZQgIqLw0m6lhMvlwkcffYRZs2bhlltuQWVlJR5++GGIoojZs2dj5MiRUCgU3RVrWGOjSyIiIpKLoNYCbhfgsPmMG5orJeqYlCAiojDR7hPymDFjoNfrcccdd+DJJ59Enz59AABFRUXdElwksdqcUEWJiFIEvGKGiIiIqHUqz5bs7kYLBJXGOxyjVUIAKyWIiCh8tPuEfOrUKQwYMACDBw9G7969uyumiGSxO6FhlQQRERHJQFBLAOC3A4dCFBGjVcJkYVKCiIjCQ7tPydu2bcN7772HRYsWobGxEZMmTcKUKVN81i5SYCx2J7tgExERkSzOJiUsfsf0WhWXbxARUdhot1IiMTERc+bMwbZt2/D73/8ehw4dwuTJk9HU1IQtW7bg2LFj3RVn2LOyUoKIiIhkIqhblm+0vgNHPSsliIgoTAT0lCwIAsaMGYMxY8agpqYG7777Lv7yl7/g5ZdfxtVXX40//elPXR1n2LPYnJBYKUFEREQyEFSeSgm0VikhqfDd0bpujoiIiOjCdLrrYmJiIubOnYtt27Zh1apViIuL64q4Ig4rJYiIiEguQnTz8g2bye+YXqtCvdnR3SERERFdkAt+Sj63eoI6xp4SREREJBdBpYWgjoGr7rjfMb2khN3RBHtjE9Qqbt1OREShjftTdhNWShAREZGchNgkuOp+8BvXSyoAQB37ShARURhgUqIbOJwuOJwuaJmUICIiIpmIhiS4zvg3HTc0JyXquQMHERGFASYluoHV7gQAVkoQERGRbERDEtzWOrgbrT7jLZUSJiYliIgoDDAp0Q0szUkJ9pQgIiIiuYixSQAAV51vtYRey+UbREQUPpiU6AYtlRJcvkFERERyEQ19AfgnJXRaLt8gIqLwwaREN7DYuHyDiIiI5CXqEwAIfn0llFEitOooJiWIiCgs8Cm5G3D5BhERkXxcLheWL1+O9957D1arFZmZmVi0aBFSUlJaPf/SSy+FWq2GQnF2e8yysjJceumlAIAVK1Zg1apViI6O9h4fO3Ysli1b1rVv5CIJUSoIut5wnWl9Bw4mJYiIKBzwKbkbcPkGERGRfEpLS7F582asWbMGiYmJWLJkCWbPno2NGzdCFFsvAn311VcxfPjwNu+ZnZ2Nt956q6tC7jJibBJcdTV+40xKEBFRuODyjW7A5RtERETyKSsrQ35+PgYMGABJklBYWIiqqirs378/2KF1O9GQBFfdMbjdbp9xvaRCncURpKiIiIgCx6REN7DYnRAEIFql6PhkIiIiapPJZILRaER6erp3TK/XIzU1FQcOHGjzuoKCAgwfPhxTpkzBO++843e8srIS1113HcaOHYuCggIcOXKkS+KXm2hIApx2uC1nfMYNWlZKEBFReOBX993AanNCq46CIAjBDoWIiCisNTQ0APAkIs6l0+m8x873xz/+ERkZGRBFEbt378ZDDz0Ep9OJvLw8AMCtt96Kn/zkJ0hOTsbx48fx3HPP4Re/+AU2btwISZICji0+PuYC31XbEhJ07R63pF6CY7sAPeqgSfiRd7xvnxhYP3ciNk4LZRS/FDlXR3NKncc5lR/nVH6cU3nJOZ9MSnQDi93JpRtEREQyiInxPPibTCafcZPJ5D12vuuvv97799GjR+Oee+5BeXm5NykxZMgQ7/HExEQ8+eSTyM7ORkVFBUaNGhVwbCdPNsDlcnd8YoASEnSorTW1e44LBgDAqcPfQSWleccV8MTx3eFT6KWPbu3SsU9jCgAAIABJREFUHimQOaXO4ZzKj3MqP86pvDo7n6IotJu45/KNbmC1O9nkkoiISAY6nQ4pKSmorKz0jplMJlRXV2Po0KEB3UMURb8eDOcSBAGCILR7TqgQpDggSuW3LaheqwIA1HEJBxERhTgmJboBKyWIiIjkk5ubi9WrV6OqqgoWiwXFxcVIS0tDVlaW37n//ve/8eWXX6KxsRFOpxO7du3C66+/jgkTJnjP2bp1K06dOgUAOHnyJObPn49evXohIyOj297ThRIEEaIhEa6685ISkicpwb4SREQU6vik3A0sNicSYlk6SUREJIf8/HyYTCbk5eXBarUiKysLJSUlEEUR+/btw6xZs7BlyxYkJyejpqYGxcXFOHbsGBQKBZKTkzFv3jz8z//8j/d+5eXlWLRoEaxWK/R6PYYNG4bXX3+9zeUgoUY09EXTiUM+Y0xKEBFRuGBSohtYWSlBREQkG1EUUVBQgIKCAr9jLb0gWtx000246aab2r3fSy+9JHuM3UmMTYKzai/cTQ4ICiWAc5ISFiYliIgotHH5RjewsKcEERERdRHRkAS43XDVH/eOqZUKqFUK9pQgIqKQx6REF3O53bCxUoKIiIi6iGj4//buPDyq6u4D+PfeO3sySSbJhGyQsGUChEDYBCyiUt9WRHwUK0jF2goWtfW14gZ9rbWb1viqtVVc27rx8ghvFRBsebW1irILsoOsCYFACCTMZPa55/3jToYMSSDALAl8P4/z3Jlz7j1z5hAn5/5yllwAaLWuRLrFwOkbRETU6TEoEWdeXwgCgMXEoAQRERHFnpwRDkqcvgNHigFOdyAZVSIiIuqwhAYlVFXFs88+i9GjR6OiogJ33nknampq2jx38eLFqKioiHr0798fEydObPP8e++9Fw6HA6tXr47nRzhnHl8QADhSgoiIiOJCMlggmdMg2tiBgyMliIios0toUOL111/Hhx9+iHfeeQcrVqxAfn4+Zs6cCVVVW507ceJEbNiwIfJYs2YNbDYbbrjhhlbnfvDBB/B6vYn4COfMHQ5KcE0JIiIiihc5I6/VSIn0FAOOO71QVZGkWhEREZ1dQoMS8+fPx/Tp09GrVy+kpKTgoYcewr59+7B+/fqzXrt8+XK4XC5MmjQpKr22thbPP/88fv3rX8er2hckMlKC0zeIiIgoTuT0bq3WlCgtssHjC+Gbgw1JqhUREdHZJexO2el0oqamBmVlZZG0tLQ0FBUVYfv27Rg+fPgZr583bx7Gjx+PjIyMSJoQAnPmzMHdd9+N/Pz8865bVlbs9yG3260AgH1HmwAABblpkTQ6P2y/2GObxh7bNLbYnrHHNr04yel5EN7PILwuSCatXzOwVyb0OhnrdtbB0cOW5BoSERG1LWFBCZfLBUALRLRktVojee3ZtWsX1q1bh0ceeSQqfd68eRBCYPLkyRdUt/p6V0yHNtrtVtTVOQEAtUe1o8/tj6TRuWvZphQbbNPYY5vGFtsz9s6lTWVZikvQnuIjsthlYy0UUx8AgMmgQ1nPTKzfeRS3frsvZElKZhWJiIjalLDpG6mpWsfG6YzuDDmdzkhee+bNm4eysjKUl5dH0qqqqjB37lz85je/iX1lY8jN6RtEREQUZ+1tCzrMkYMGlx97D51MRrWIiIjOKmFBCavVioKCAmzZsiWS5nQ6UVVVhX79+rV7ncvlwuLFizF16tSo9HXr1qGhoQE33XQTLrvsMlx22WUAgHvuuQePP/54fD7EeeBCl0RERBRvUpodkORWi10O6pMNRZawfufRJNWMiIjozBJ6pzxlyhS88cYbGDlyJLp164bKykoUFxdj6NCh7V6zaNEi6PV6XHfddVHp1157LUaPHh2VNnbsWPzmN79plZ5MHm8QBp0MnZLQNUWJiIjoEiLJOkhp9lYjJSwmHQb0zMS6HXW45ao+kDiFg4iIOpmEBiWmT58Op9OJqVOnwuPxYOjQoZg7dy5kWca6deswY8YMLF26NGrRyvnz5+PGG2+EyWSKKstsNsNsNrd6j8zMTKSnp8f9s3SU2xeEmaMkiIiIKM7k9NxWQQkAGFpix6Y99ThwxIni3LQ2riQiIkqehN4ty7KMWbNmYdasWa3yhg0bhg0bNrRKX7JkSYfL37lz5wXVLx7cviAsXE+CiIiI4kzOyEOgZhuEUCFJp0ZoVpTY8ebfd2L9zjoGJYiIqNPhnII48/iCXE+CiIiI4k5OzwVCAQhXfVR6qlmP0qIMrNtxFELEbrcxIiKiWGBQIs7cXk7fICIiovg7tQPHkVZ5Qx05OHLCg5q6pkRXi4iI6IwYlIgzD6dvEBERUQLIGeGgRMPhVnlDSuyQAKzjLhxERNTJMCgRZ1zokoiIiBJBMqcDelObi12mpxjQt3sG1u+qS0LNiIiI2segRJy5vVxTgoiIiOJPkiTIGXlQG1oHJQBgqMOOmromHK7nFA4iIuo8GJSIo0AwhGBI5UgJIiIiSgg5vVubIyUAbWtQAFi/k6MliIio82BQIo7cvhAAcE0JIiIiSgg5PQ/CVQ8R9LfKy0wzoXd+GoMSRETUqTAoEUceXxAAOFKCiIiIEiKy2GUbO3AA2i4cB444cbTBk8hqERERtYtBiThye7WgBNeUICIiokQ4tS1o++tKAMBXHC1BRESdBIMSccSREkRERJRIkaBEG9uCAoA9w4yiblas59agRETUSTAoEUfucFCCa0oQERFRIkh6I6SUzHZHSgDaaIk9h07i+ElvAmtGRETUNgYl4qh5pASnbxAREVGinGkHDuDUFI71uziFg4iIko9BiThqXlOC0zeIiIgoUeSMPKgNtRBCtJmfl5WCAnsKd+EgIqJOgUGJOHL7gpAkwGRQkl0VIiIiukTI6d0AvxvC62z3nKEldnxT3YDGptZbhxIRESUSgxJx5PEGYTHqIElSsqtCRERElwg5PQ9A+ztwAMAwRw4EgK84hYOIiJKMQYk4cvuCnLpBRERECSVnnHkHDgAosKegm83MXTiIiCjpGJSII48vyEUuiYiIKKGk1GxA1kE0Hmn/HEnCsNIc7DjQAJcnkMDaERERRWNQIo44UoKIiIgSTZJlyOk5ZxwpAWi7cKhCYO4HW7C7pjFBtSMiIorGoEQcub1BWEwMShAREVFiyel5UM8wUgIAirpZceu4vqg+6sLv3l6P/56/AbuqGxJUQyIiIg3vmOPIw5ESRERElARyejcEqzZCqCFIctu7gEmShGuGd8eYQXn4dMMh/H31ATz17lco7ZGBiZf3RGmRLcG1JiKiSxHvmOPIzTUliIiIKAnkjDxADUE4j0FK73bGc00GHb57WQ9cNaQA/95Qg49WV+Hp/9mAku4ZmHh5MfoV2biTGBERxQ3vmONEFQJejpQgIiKiJJDSwztwNNZCPktQoplRr+A/RvTAlRUF+OzrQ1i26gCemb8RvfLT8B/Du2NIiR06hTN/iYgotnjHHCdeXwgC4JoSRERElHCntgWtBXoMOqdrDXoF3x7WHWMH5+PzTYexfE01Xl60FTarEVcPKcDYwQVINevjUW0iIroE8Y45Tjy+IABwpAQRERElnGyyAsYUqI21512GXqfg6iGFuHJwATbtqcf/ravG//57L5Z8sR+jynLx7aGFKLCnxrDWRER0KeIdc5y4w0EJrilBREREySCn515QUCJSjixhcN9sDO6bjYNHXfh4fTW+3FKLf288hAHFNnx7WHcM7J0FmetOEBHReeAdc5w0j5Tg9A0iIiJKBjkjD8EDG6C6jkNOzYxJmYU5qbjj2n6YNLY3/r3xEP751UH8YeEm5GVZ8J0RPTBqQDfodW3v9kFERNQWrlYUJ24vp28QERFR8hgGfBtQQ3B/+BRUV31My7ZaDJgwuhhP3z0ad13fH3qdjL9+tAMPzV2JJV/uh8sTiOn7ERHRxYtBiTjhSAkiIiJKJsVeDMv4ByE8TriXxD4wAQA6RcbIAbl4/I7heHDKYPTolor3P9uLB1/6Au/+3y7UNXhi/p5ERHRxYVAiTtxc6JKIiCguVFXFs88+i9GjR6OiogJ33nknampq2j3f4XCgvLwcFRUVkcfOnTvPu7yuROnWB5brHoTwurTAhPNYXN5HkiT0L87EA7cMxq9+NALDHDn4dEMNHn1lJeZ+sAU7q04gGFLj8t5ERNS1MSgRJ26vNmyRC10SERHF1uuvv44PP/wQ77zzDlasWIH8/HzMnDkTqtr+Te9rr72GDRs2RB4Oh+OCyutKlJzesFz3EITPpU3lcNbF9f0Kc1IxfUJ//H7mKHxnRA9s2VeP38/bgJ/+4XM8v+BrLF9TheqjLqhCxLUeRETUNfCOOU48vhAMOhk6hXEfIiKiWJo/fz6mT5+OXr16AQAeeughjB49GuvXr8fw4cOTXl5npOT0guW6h+FeWgn3kqdgmfAo5DR7XN8zM82EW67qg+tHF2PrvuPYduAEtu8/jk17tGkkVose/Yps6FdkQ//iTNgzzHGtDxERdU4MSsSJ2xeAmetJEBERxZTT6URNTQ3KysoiaWlpaSgqKsL27dvbDSLMmjULgUAA+fn5uPXWW3HLLbdcUHldkWLvCcuEcGDiw6dgmfAI5LScuL+v2ajDsNIcDCvV3qu+0YvtB05g+4Hj2Lb/BNZsPwoAGNQ7CzeN7Y3uOalxrxMREXUevGuOE7cvxKkbREREMeZyuQBogYOWrFZrJO90f/3rX1FRUQFZlrFq1So8+OCDCAaDmDp16nmV156srNjfTNvt1hgXOBA+2y9xeN4T8C57Gvm3PQG9LTe273G2KtitKO1jx40AhBCoPuLEl5sP44N/78Ev/7IGVwwuxPe/W4q87JS4vT/FFts09timscc2ja1YtmdC75pVVcXzzz+PhQsXwuPxYMiQIfjVr36FgoKCVucuXrwYjz/+eFSaz+dDnz59sHjxYgBAZWUlPv30Uxw+fBgWiwVXXHEFHnroIdhstoR8njPxeAMMShAREcVYaqp24+90OqPSnU5nJO90o0aNijy/4oorcMcdd2Dx4sWYOnXqeZXXnvp6F1Q1dusk2O1W1NU5z37iuVLsMI1/GJ4Pn8bBNx+DceRk6LoPhGSwxP69OsCsSBg3OB8jS+34aFUVPl5XjRVf12DMoHxcP7oYNqsxZu8Vtza9hLFNY49tGnts09g61/aUZemMgfuELnhwLgtJTZw4MWpBqjVr1sBms+GGG26InKMoCiorK7F69WosWrQItbW1mD17diI/UrvcvhB33iAiIooxq9WKgoICbNmyJZLmdDpRVVWFfv36dagMWZYhwossxqK8rkjJ6gHzhEcAAN5P5sL15k/hXvo0/Fv+D+rJ+C6E2Z4Ukx43X9kbT80chbGD8/H514cw+5WVWPCv3XB5AkmpExERxV9C75ovZCGp5cuXw+VyYdKkSZG0Bx54IPI8KysL06ZNw6xZs+JT+XPk9gVhzzAluxpEREQXnSlTpuCNN97AyJEj0a1bN1RWVqK4uBhDhw5tde7WrVuhqiocDgdkWcbq1avxl7/8Bffee+95lXcxUbK6I+XWZxA6uhuhAxsRPLARvi/fhe/LdyHbCqErGgxd0WDI9l6Q5MT9HSsj1Yjb/sOB/xjRA4s+34e/r67CpxtrMKY8H30K0tErPw02qxGSJCWsTkREFD8JC0pc6EJS8+bNw/jx45GRkdHuOStXrkRpaWnM6nwhPL4gR0oQERHFwfTp0+F0OjF16lR4PB4MHToUc+fOhSzLWLduHWbMmIGlS5ciPz8fR44cQWVlJWpra6EoCvLz83H//ffj1ltv7VB5FztJlqHLLYEutwTGy26B2ngEwQMbETywAf6vl8G/8UNIJiuUwjLoug+EUlgG2Zx29oJjICfDjBnX98e1I3vg/c/24pP1B7F8bTUAID3FgJ55aeiZn4ZeeWnomWeFxaRPSL2IiCi2JCESs0n04cOHceWVV+If//gHiouLI+lTpkzBFVdcgXvuuafda3ft2oXrr78eCxYsQHl5eZvnLFu2DD//+c/xzjvvYMCAAbGu/jm76ZElmDimF+6YkPy6EBERUfx1mTUlOkj4mhCs3oRg1SaEDm6B8Gp1kbOLtQBF93IoOb0gyUpC6hMIqqg+6sK+wyex99BJ7Dt8ErXH3ZH8bpkW9C+2YWCvLPTrYYPR0LpeyW7TixHbNPbYprHHNo2tWK8pkbA/5V/IQlLz5s1DWVlZuwGJpUuX4pe//CXmzp17XgGJWHci0jMsCARViJDKH/4Y4RdJ7LFNY49tGltsz9g7lzY9WweCLn6SMQX6PqOg7zMKQqhQjx1AsHozQtWb4d/4IbBhCWAwQ1cwAEr3gdAVDoScmhm3+uh1Mnrlp6FXfhrGhWfWuL0B7Kt1Yt+hk9hd04gvNh/Gv76qgU6RUNI9A2U9szCwdxbysyyc7kFE1EklLCjRciGpgQMHAujYQlIulwuLFy/Gz3/+8zbzFyxYgMrKSrz88sudZu5nk1dbjMli4vQNIiIi6vokSYZi7wnF3hMYMlEbRVGzDaHqzQge3IzgvnXwAZBthVqAovtAKLklkJT49oUsJj0GFGdiQLEWDAkEVXxzsAGb99Zjy97jeO9fu/Hev3YjK82Isl5ZuGJIdxRmmqHXXfxTc4iIuoqE3jWfz0JSixYtgl6vx3XXXdcq76233sKLL76IN954IxLo6Azc3iAAcE0JIiIiuihJxhToew2HvtdwCCGgnjioBSiqNyOwZTkCmz4CdEboCvqfGkWRZo97vfQ6Gf2LM9G/OBOTrwbqG73Ysq8em/cex+ptR/DvjYdgNiqo6GvH8NIcDOiZCZ3CAAURUTIl9K75XBamajZ//nzceOONMJla72Tx29/+FjqdDrfffntU+ullJFpTeNsqC4MSREREdJGTJAlKZncomd1hGDQeIuBF6NB2BKs3a2tSHNgAHwApvRt0hWXQFZZBye8HSR//Xcqy0k0YO7gAYwcXIBhScajBi49XHcBXu+rw5ZZaWIw6DCmxY3i/HPQrsjFAQUSUBAlb6LIzi/WaEgePe/CLV1fi0e8PQUn39ncLoY7j3PLYY5vGHts0ttiescc1JeLrYlvoMhaEEBCNtQge3ILgwS0IHdoOBP2ArEDp1je8q0cZ5KwekKT4BwSa2zQYUrF133Gs3XEUG76pg8cXQopJC1CU985G/2IbR7x20MXwc9rZsE1jj20aW112octLCdeUICIiItJGUUgZeTBk5MFQdg1EKIBQ7TcIhYMU/rUL4V+7MLzt6ADoCgdCKRwA2RLfP+roFBmD+mRjUJ9sBIJagGLNjiNYu+MoPt90GLIkoU9BGgb0ykJZz0wU5Vohc6FMIqK44F1zHDR5tDUlOH2DiIiI6BRJ0UNX0B+6gv4wXnYLVHcDQjXbtF09arYiuHsVAEDO6g6loCy8YGZfSIo+bnXS62QM7puNwX2zEQyp2FPTiC37jmPL3uN4/7O9eP+zvUg16zGgZybKemZiQM9MZKQa41YfIqJLDe+a46B5TQkO+yMiIiJqn2zJgNx3NPR9R2vbjtZXI3hwM0LVW1osmGmAkleqrUVROAByRn7ctvfUKTIcPWxw9LBh0tjeONnkx9b9WoBi635tsUwAyM9OQb8iG/oXaedydCwR0fnjN2gcuL0BSBJgMijJrgoRERFRlyBJMpTsIijZRcDgCRB+D0KHdyBYrU318FVv0s6zZEDJ7wdd4QAo+f0hp2bGrU5pKQaMGpCLUQNyoQqBg0dd2Lr/OLbvP4HPNx3CJ+sPQpKA4tw09C+2oV+RDX0L06HXsQ9IRNRRDErEQZMnAItRF7coPhEREdHFTjKYoSuqgK6oAgCgOusQrNmGUM02bU2K3SsBAHJGHpSC/lAK+kOX64Bkis8CrbIkoUc3K3p0s+Lay4oQCKrYe6gR2/afwPYDJ/DRqiosXXkAiizBnmFGjs0Me7oZdpsZORna0Z5ugkHPgAURUUsMSsRBkzfAqRtEREREMSRb7TCUjgVKx2pTPY4f1NahqNmOwM7PEdj6CQAJclYhlLxSKHkOKHkOyCZrXOqj152a6nEjAI8viJ3VDfjmYAOOnvCg7oQHu6ob4PWHoq7LSDUgLysFPbqlakGOnFTkZlmgyNyOlIguTbxzjoMmT5CLXBIRERHFiSTJULJ6QMnqAUP5tRChIEJH9yB0eAdCh3cisP3fCGz5PwCAbCuIBCiCpqGIV/fXbNRhcJ9sDO6THUkTQsDpCaDuhAdHGzyoa/Dg6AkPao414ZP1BxEMaVvI6nUyCu0pkSBFYU4qbKlGWC0GGDkdmIgucrxzjgOOlCAiIiJKHEnRQZfngC7PAQAQoSDUun0IHt6J0OEdCHzzJQLb/omqTwDJaoeS2xdKbgmUbn0h2/IgSfEZpSBJEtIsBqRZDOhdkB6VFwypqK13o+qoE1VHXKg64sTa7Ufx742Hos4z6GWkWQywWgxIs+hhtRhgTdGje04qBvXOZp+TiLo8fovFQZMnAFuqIdnVICIiIrokSYouHHjoC1RMgFBDUI8dgMl5ACf3bEGoejOC33ypnWxMgdKtD5Ru2vmKvRiSLv5bfuoUGYXhURGjy7Q0IQTqG72oOdaEk24/nO4ATjb54XT7cdIdwAmXD1VHXTjZ5EdIFdApEvoXZ2Kow46KvnakmuO3dSoRUbwwKBEHbm8ABVmWZFeDiIiIiABIsgIlpxcyBgxCoPdVEEJAnDyCUO032uPIN/BXfd18MuTMAij23pBzekLJ6QU5owBSAtZ8kCQJ2RlmZGeYz3ieKgT21DRi/c46rN9Zh0176vGmtBOlRRkY6sjBkL7ZSE+Nf2CFiCgWGJSIgyYPp28QERERdVaSJEFKz4Wcngu9YwwAQPU6oR7ZjdDRvQjV7UNg7xpgx6faBTojFHsxZHtPKPZeULKLIKXZ4zbt42xkSULfwgz0LczA5Kv74MARJ9bvrMO6nXV4+x878c4/dqJ3QTqKulmRn21BXlYK8rNTYLXouTscEXU6vHOOMVUIuH1BWExsWiIiIqKuQjZZIbfYglQIFaLxKEJ1e7VAxdG9CGz5GAE1qF2gN0PJ7gE5qwhKdhHk7CLIGXmQ5MQuTClJEopz01Ccm4abruiFmmNNWL+zDlv21uPLrYfh8Z3a/SPVrEd+lgX52SnIy05BRqoRep0Mg06GQa/AoJPDrxUY9NpRr5chM5BBRHHEO+cY8/pCEAIcKUFERETUhUmSDCkjF3JGLvR9RwMARCgA9UQNQscOQD12AKFjBxDY/ikCIb92kaKHnNkdSlZ3yJmF2vPMQkim1ATVWUKhPRWF9lTc8K2eEEKgweXHoWNN2qNeO67dcRRN3mCHy9UpEvQ6JRLAaA5c6PUy8rJTkZVqiAQ6utnM0Cnc3pSIOo53zjHm9gUAgFuCEhEREV1kJEUPJbsYSnZxJE2oIagNtVDrD0SCFYF964Ad/z51XYoNcmYhlMxwsMJWoI2q0MV3YXRJkmCzGmGzGjGgZ+apOguBk+4AnG4/AkEV/kAIgaAKX0BFIBiCv0WaP6jCHwwhENCeB4It0gMhfFN9Aivq3RDhshVZQo7NjPzsFORnpaBbphlmow4mgw4mgxJ+6GDUa89lmaMwiC51vHOOseYhchwpQURERHTxk2QFSmYBlMyCUyMqhIBwN0A9Xg31+EGE6quhnjgIf802QG2eTiFBsmZDzsiDbMvXjhn5UDLy4j6yQpIkpKcYkJ5y4UERu92Kg4caUFvvjozEOHSsCQfrmvDVrjoIcebrDToZKWY90sL1SbMYkJbS/NAj3WKAyaiDy9O8E0kgvDNJy91JAtApElIteljNBu3Y/NysPU8162Ex6WA2ag+DTu7w+hohVYXPr0KS2Mcnigf+XxVjbm94pATXlCAiIiK6JEmSpI2OSLEB3csj6UINaqMqThyC2nAofDyMwKFtQOjUdArJnAYpLQdyWjfIaTmQ03O0Y1o3wJjS6RarNOoVFOVaUZRrjUoPBFXUn/TC4wvC5w/B6w/BGwhqR18IvkAIXn8QLncAjW4/Glw+VB1xwukOIKS2H83QKTLSUvSwWrQgRl5WCkKqCqc7gPqTXuyvPXnWMhRZgtmog8XYHKhQoArAFwjBH9Dq5vOH4AuoCIbUyHVpFj1ys1KQl2VBbqZFO2alIDvNxFEfROeJd84xxpESRERERNQWSdZBySyEklkYlS5UFcJ1LByoOAy14TDUk0cROrQdwW++iC7EYA4HK+yQUrMhp9khp2Zroy6s2XGfEnIu9DoZuZmWc75OFQJubxAnm/w42eSH1x/SRjyk6JFmMcBkUM4amBFCwOsPwekJwOUOwOXxw+0LwuMLwe0NwOMLweMLwuMLwh1+yJKENIsBRr0Mo16BwaDApFe053oFqhCoPe5G7XE31p22LodOkdHNZobRoEAIAVXV6qAKQEBAVQWE0NbnyE43w55hhj3DhByb9jw73QS97syLpKpCIBBQoQrRoTbo6nyBEBqb/BBCaIE+AJKk7T4jSRIkCZCg/ZyZjDouyNqF8c45xiJrSnCkBBERERF1gCTL4ZEROUCPwVF5IuiH6qyDaDwK9eRRqCePaAGL+iqI/RsANXrBSsmcpgUoUrMhpWZCTrFBSmk+2iBZMhK+Q8i5kiUJqWZtykV+dsp5lSFJUmSqRk6GOcY11DjdftQed+NwvRaoqK13IxhSIUkSZAmRG2dZkiDJWpo/oKKu0YNtB47DH1CjyrNZjchON8Fg0KHJ7Q+P2lC1YzAUdb4kAWbDqekoFqMCi0kPs1GB2ag766gNWZIgyxKU8OPUcznyXAgtkKJGHU8FWCQJ0OsUGPXa7i368C4uRp0MfXg3l+br1PB1qqoFarSjFjhqcPrQ4PKhweUPH7XnHl/HF2OVJG1NvxSTNk0nxaxHikl7bTbqoDfo0HDSEx4Fo7WpLxCC3x+CL6gCQsDYHIQy6LTAlEGBSa9RV/2jAAAZ6UlEQVSD0aBE8kwGBSZj9PooJoPW5qGQisYmPxrDwbSWx0aXD25vENYUAzKtRmSmmSJHm9WIzDQjTAbt/jEYUrWAnNuPRleLMpr8cHuDSEvRw5ZqhC18rS3ViPRUw3ktMBsMqahr8ODoCQ+OnPDgyAk36ho8GDsoH0MdOedc3vninXOMcaQEEREREcWKpDNAsRUAtoJWeUKoEO5GqM5jEM668PEYVNcxhI7thzjwVdS0EK1ACZI5PRyoyNCem9MgWdIhWdIhm7WjZE7vVKMuOiOrxQCrxYC+hRnnfG3zYqN1Jzyoa2jxaPRCkoC0FAMMOjkySkM7aq8lSQqP+jj1cHuDOH7SG0k/w8wVAOEAgyoQCj+STadIyEg1IiPViPzsFPQvzkRGqra2iE6WI4ERgfBRnDr6gyqavEG4vQE0eYNo8gbg9gZxrMGDJq/WHga9DL1OC5QYDVp7GvUKUqxa0AFAeLpOCC6PH8caW07hCSEYOr82Mujl8PotRmRYjTjZ5Ef1URdONvlbnWsx6iBJaHdnnObgk9MdgD8YHdCSAKSlagEPi1EHRZG1IJMiQydLUBQt4KQoEoRAOBDhxrFGb9S6L2ajDt1s5oRPReKdc4xF1pRgUIKIiIiI4kiS5FNrV+T2bZUvhAB8TVCbjkM0HYfqOqEdmxq0Y+NRiNpvILzOtt/AYIZkSoNktkI2WSGZrJDM4WP4uTfQDapHgmSwAAYLJJnbgXZEy8VG+xSmR+XZ7VbU1bXzbxIHosVIhlCLhwRAliVIkCDLCI8ACY/+kCWoqojs3uJv86hGztVGZgBKeIRGc5pRryDDakSKSRfX6SgX2qbBkKqtheIPwusLnXruD8ETTlMU7d80smhriiEy+qGt8k44fTh+0ovjLY4Q0K5NNSDdEj6GF4A16LXgiRACTd4gTjh94Ye3xXMfPP4gQt4ggiGBkKoiFD4GVYFQOLiSlW5Cz7w0jOyfixybGd0yLehmMyPVrE/KtCDeOceYxxeCQa9wf2YiIiIiSipJkgBTKhRTKpDVo93zhBqE8DghPI0Qbu2hehohPCe1dO9JbRRG3T4IjxMQoci1nuh31AIZxhRIplRIBov23GDR0g1mLc1ghmQMBzEMZkh6M6A3QtKbACU5N0WXMkmSoEgSFBnQn8N1siJBp8iXxAhxnSIj1Swj1XwuLXTm8rR1Rc59apHUYnpT95z47tSTKBf/T1CCuX0BpJrZrERERETUNUiyDlKKDUixnfVcIQTgd0N4nRAeJ6wmFY1Hj0H4XBA+d4tjE4SvCaqrXjvf7wFCgQ5URokEKCSDCdCZIOmNgM6oTSeJHA2Q9C2eKwZAp9eCGoo+nKa9hk4feR5JU3SQJP4Rkagz4N1zjLl9IVhMsYmgERERERF1JpIkaduSGlOA9Fyk2K1wZ3RsWLwIBSH87kiQQnu4gYAXwu+FCHi15wEvRMAHBDzhNJ8W6Aj6IIJ+iKAPCPo7FuQ4E1mnBSrCwQzIOm0RUEXR8mRFe938XNGOp/J0kXO163RaUEVWtGkskgLIsnaNJGvnSOHX4fy2zvN6UxFq9ITPlbWjJGllSNGvTz9Kp6dBCr+Wol9D4ogU6jQYlIgxjzeAlBgN6yEiIiIiulhIig6SOQ0wp8WkPCFUIBjQghShABD0Q4QCQCgAEQwAofDrYDgt1OLYVloopO1mooYg1BCgaq+18oMQLfNCQS2v+bzw61jwnP2UGGknYCHJUXlSqwCH3GaQQ/vvtPIgRd6q+ZxTCafntQyStJ8ntXzd5jUt0sLPDxt08PtDp9Wt5bmnBWlapLdKOz1POv2ztNRG4Kfl9ae1R3Sc6PQyT3/f0887/b1OK/9sQagWK17q+46GktPrzOfHEIMSMdYzPw2pKaZkV4OIiIiI6KImSXJ4qocx2VUBEJ7aIoS25oaqRo5CDQFCDQc8VECoEC3ytbyQFmRRQ0hPM6GxwRVOF1p6y4eqAtDeS8sTLfLFafki8hyiOR0tnp/Ka30uTpXZ/IB6qtzmPO3Dn7q2ZbmnWifqtYg8b3k92k5rr5zTy4i8L6LyBQRUhCACwTaubz5fQETet+V7tazTaa9F8xWnX3fa5zj1odv+HK3yWpbZXhlnKhvhurVVfvgztBX8ACKBC9mWz6BEV3bTFb0TvmIuEREREREllxT5K7wMKC3Sz7Eci92KJt5LxBTvzzo3ru5CREREREREREnBoAQRERERERERJQWDEkRERERERESUFAxKEBEREREREVFSMChBREREREREREnBoAQRERERERERJQWDEkRERERERESUFAxKEBEREREREVFSMChBREREREREREnBoAQRERERERERJYUu2RXoDGRZ6hJlXurYprHHNo09tmlssT1jr6NtyrY/d+xPdA1s09hjm8Ye2zT22KaxdS7tebZzJSGEuNAKERERERERERGdK07fICIiIiIiIqKkYFCCiIiIiIiIiJKCQQkiIiIiIiIiSgoGJYiIiIiIiIgoKRiUICIiIiIiIqKkYFCCiIiIiIiIiJKCQQkiIiIiIiIiSgoGJYiIiIiIiIgoKRiUICIiIiIiIqKkYFCCiIiIiIiIiJKCQYkYUlUVzz77LEaPHo2KigrceeedqKmpSXa1uoTKykpcd911GDJkCL71rW9hzpw5OHHiRNQ527Ztw5QpUzBo0CBceeWVeOutt5JU267p3nvvhcPhwOrVqyNpX375JSZOnIhBgwbhO9/5DpYtW5bEGnYda9aswdSpU1FRUYERI0bg7rvvjuSxTc/dsWPHMGvWLIwaNQrDhg3DlClTsHbt2kg+2/TMli5diqlTp2LIkCFwOByt8s/23en1evGLX/wCI0aMwJAhQ3D//fejoaEhUdWnNrA/cWHYp4gv9idih/2J2GJ/4sIktT8hKGZeeeUVcdVVV4k9e/YIl8sl/uu//ktMmDBBhEKhZFet0/vv//5vsXXrVuH3+8WxY8fED3/4Q/HjH/84ku90OsWoUaPEH//4R+H1esWGDRvE8OHDxUcffZTEWncd77//vvjRj34kSkpKxKpVq4QQQlRXV4vy8nLx3nvvCZ/PJ/75z3+K8vJysXHjxiTXtnNbs2aNGDJkiFi0aJHweDzC5/OJr7/+WgjBNj1fP/nJT8Rtt90m6uvrRTAYFG+88YYYPHiwaGxsZJt2wGeffSaWLFkiFixYIEpKSqLyOvLd+dhjj4kbb7xR1NbWioaGBjFjxgxx1113JfpjUAvsT1wY9inih/2J2GF/IvbYn7gwyexPMCgRQ1dddZV49913I68bGxvFgAEDxJo1a5JYq67pn//8p6ioqIi8/t///V9x+eWXR3XInn76aTFt2rRkVK9LOXz4sBg7dqyoqamJ6kS88MIL4uabb4469z//8z/Fo48+moxqdhmTJ08WTz31VJt5bNPzc/3114t33nkn8trlcomSkhKxefNmtuk5WLVqVatOxNm+Oz0ejxg4cKD49NNPI/m7d+8WJSUloqamJjEVp1bYn4gt9ilig/2J2GJ/IvbYn4iNZPQnOH0jRpxOJ2pqalBWVhZJS0tLQ1FREbZv357EmnVNK1euRGlpaeT1jh070L9/f8jyqR/ZsrIy7NixIxnV6zKEEJgzZw7uvvtu5OfnR+Xt2LEj6ucVYJuejdvtxtdffw0AuOmmm3DZZZdh8uTJWLlyJQC26fmaMWMGli9fjrq6OgQCAbz77rsoLi5GSUkJ2/QCne27c//+/fD5fBg4cGAkv3fv3jCbzfzdlSTsT8Qe+xQXjv2J2GJ/Ij7Yn4ifePcndLGv8qXJ5XIB0DoOLVmt1kgedcyyZcuwYMECvPPOO5E0l8sFq9UadV5aWhrb9izmzZsHIQQmT57cKs/lcqFPnz5RaWzTMzt58iRUVcWSJUvw6quvom/fvnj//fcxc+ZMfPjhh2zT81RRUYEPPvgA3/rWt6AoCjIyMvDiiy/CYDCwTS/Q2b47m4+nn8PfXcnD/kRssU8RG+xPxBb7E/HB/kT8xLs/wZESMZKamgpA+wtHS06nM5JHZ7d06VI8/vjjmDt3LgYMGBBJT01NbfUDffLkSbbtGVRVVWHu3Ln4zW9+02Z+ampqq59XtumZpaSkAAAmTZqE/v37Q6/X45ZbbkFhYSE+//xztul5UFUVd9xxB3Jzc7FmzRps2rQJv/rVrzBjxgzs3LmTbXqBzvbdyd9dnQ//TWKHfYrYYH8i9tifiD32J+Ir3v0JBiVixGq1oqCgAFu2bImkOZ1OVFVVoV+/fkmsWdexYMECPPHEE3j55ZcxcuTIqLzS0lJs27YNqqpG0rZu3Ro1HJOirVu3Dg0NDZFhgZdddhkA4J577sHjjz+O0tLSqJ9XgG16NlarFd27d2+VLkkSALBNz0NjYyOqq6tx++23Iz09HTqdDt/+9rfRvXt3fPHFF2zTC3S2787i4mIYjcaoNt6zZw88Hg/bOEnYn4gN9ilih/2J2GN/IvbYn4ivuPcnYrAWBoW98sorYty4cWLv3r2iqalJPPbYY1wtu4PefPNNMWLECLFp06Y285tXfP3Tn/4UWZ14xIgRYtmyZQmuadfhdrvF4cOHox4lJSVi2bJloqGhQVRVVYny8nKxcOFC4ff7xaeffioGDRrEVYjP4s9//rO4/PLLxY4dO0QwGBQLFy4UgwcPFtXV1WzT83TttdeKxx57TDidThEKhcTHH38sBgwYIFatWsU27YBgMCi8Xq/4/PPPRUlJifB6vcLr9YpQKNSh787HHntMTJo0SRw5ckQ0NDSIH//4x2LGjBlJ/ETE/sSFYZ8ittifiA/2J2KP/YkLk8z+hCSEEDEPpVyiVFXFc889h4ULF8Lj8WDo0KF44oknUFhYmOyqdXoOhwM6nQ4GgyEqfenSpZEFlbZt24YnnngC27dvh81mw5133onbb789GdXtshwOB956663IXzm+/PJLPPnkkzhw4AByc3Nx//33Y/z48UmuZecmhMCLL76I+fPnw+12o2/fvpg1axZGjBgBgG16Pvbv34+nn34aGzZsgM/nQ0FBAW6//XZ873vfA8A2PZu//e1vmD17dqv05v/Xz/bd6fV68dvf/hZ///vfEQqFMGbMGDzxxBPIyMhI5MegFtifuDDsU8Qf+xMXjv2J2GN/4sIksz/BoAQRERERERERJQXXlCAiIiIiIiKipGBQgoiIiIiIiIiSgkEJIiIiIiIiIkoKBiWIiIiIiIiIKCkYlCAiIiIiIiKipGBQgoiIiIiIiIiSgkEJokvIwYMH4XA4sG7dumRXJYrL5cK9996LoUOHwuFw4ODBg8mu0jlZvXo1HA4Hamtrk10VIiKihGCfIj7Yp6BLEYMSRAny6KOPwuFw4Omnn45Kr62thcPhwOrVq5NUs+SbN28eNm7ciHnz5mHFihXIy8trdc7pnZ9ktVv//v3xt7/9LSqtoqICK1asQE5OTkLrQkRElyb2KdrHPgVR18OgBFECGY1GvP3226ipqUl2VWIuEAic97UHDhxAnz594HA4YLfboShKDGt2dhdSdwAwGAyw2+2QZX6lEhFRYrBP0Tb2KYi6Hv60EyVQRUUFSktL8eyzz7Z7TnvDIa+55hr88Y9/jLx2OBx4++23cf/992Pw4MG48sor8fe//x1OpxOzZs1CRUUFxo0bh3/84x+t3qOmpgY/+MEPUF5ejnHjxmHp0qVR+ceOHcOjjz6KkSNHoqKiAlOmTMHatWsj+c1DCz/99FPceuutGDhwIBYsWNDm5wkEAnjmmWcwZswYlJWVYfz48ViyZEkk/+qrr8bChQuxatUqOBwOTJs27cyNGDZ27FgAwO233w6Hw4Grr746kvfFF19gypQpKC8vx5gxYzB79mycOHEikv/oo4/ijjvuwNtvv42rr74aAwcOhNfrxRdffIFp06ZhxIgRGDp0KG677TZs2rQpqq6hUAizZ8+Gw+GAw+GIao+WQy03btyI73//+ygvL8fw4cMxa9Ys1NfXR/L/+Mc/4pprrsHHH3+M7373uxg8eDCmTZuG/fv3R85xuVyYPXs2Lr/8cpSVlWHs2LF48sknO9Q+RER0cWOfgn2KZuxTUFfHoARRAkmShEceeQRLly7F5s2bL7i8l19+GWPHjsWiRYtw5ZVX4uGHH8bPfvYzXH755fjggw9w5ZVX4pFHHon65QkAlZWVmDRpEj744ANMmDABDz74ILZt2wYA8Hq9uP3229HU1ITXXnsNH3zwAcaOHYsf/vCH2LNnT1Q5Tz31FGbMmIFly5ZF/QJv6dlnn8WCBQswZ84cLFmyBBMnTsRDDz2ElStXAgAWLlyIa6+9FsOGDcOKFSuiOkln8v777wPQfhGvWLECCxcuBACsXLkS99xzD6677josXrwYL774Ig4ePIif/vSnEEJErt+0aRNWrVqFl156CYsWLYJer4fb7catt96K+fPnY/78+SgqKsL06dMj7bdw4UIoioI5c+ZgxYoVWLFiRZt1q6urw49+9CPk5uZiwYIFmDt3Lnbt2oX77ruv1Xn/8z//g2eeeQbz589HU1MT5syZE8l//vnnsXXrVrz00ktYvnw5nnvuOfTu3btD7UNERBc39inYpzj9PPYpqKtiUIIowYYNG4Zx48a1mgd6PsaPH48bb7wRRUVFuO++++Dz+VBUVISbbropkubxeLBx48ao626++WZMnDgRvXr1ws9+9jMMGjQIf/nLXwAAy5Ytg8vlwnPPPYeBAweiqKgId999N4YMGYL58+dHlTNz5kxcffXV6N69O3Jzc1vVz+Px4O2338Z9992Ha6+9Fj179sTMmTMxbtw4zJ07FwCQmZkJk8kEvV4Pu92OjIyMDn32zMxMAEB6ejrsdnvk9UsvvYRp06Zh2rRpKC4uRnl5OX7/+99j7dq12LFjR+R6WZbx9NNPo7S0FA6HA4qi4JprrsH48ePRq1cv9O3bF7/+9a8hhMDnn38e9Z5WqxV2ux12u73Nur377rtITU3Fk08+CYfDgWHDhqGyshLr1q2L+uuQ3+9HZWUlysrKUFpaiunTp+Orr76Cz+cDoP31qX///hg0aBDy8/MxZMgQ3HLLLR1qHyIiuvixT8E+RTP2Kagr0yW7AkSXogcffBATJkzAJ598ggEDBpx3OaWlpZHnmZmZUBQlMvwP0H656vX6qCF+gDbk8/TXq1atAgBs3rwZx44dw/Dhw6PO8fv9MJlMUWnl5eVnrN+BAwcQCARalTV8+HC8+uqrZ/l052fz5s3YuHEj3n333VZ5+/fvR79+/QAAvXv3RkpKSlR+dXU1XnjhBWzcuBH19fUQQsDj8eDQoUPnVIfdu3dj8ODBMBgMkbTS0lJYrVZ88803kfbIycmJdEqaXwshUF9fj/z8fEydOhX33XcftmzZgpEjR2LMmDEYM2YM55kSEVEE+xTsUwDsU1DXxqAEURL07NkTkydPxjPPPIPXXnstKq+9Xw7BYLBVmk7X+n/h09MkSYoaYng2qqqid+/e+NOf/tQq7/QOhMVi6XC5iaKqKmbMmIEbbrihVV52dnbkudlsbpU/c+ZM2Gw2/OIXv0BeXh70ej2mTp16wYtWtUev17eZrqoqAGDMmDH417/+hRUrVmDNmjV4+OGHUVJSgr/+9a8JX7iLiIg6J/Yp4od9CqLEYFCCKEl+8pOfYNGiRXjvvfei0puj3EePHo2k1dfX48iRIzF7740bN0YWdQKADRs2ROYVlpWVYdGiRUhNTUVWVtYFvU9RUREMBgPWrl2LkpKSSPratWvRt2/fCyq7+Zdv8y/bZmVlZdi9ezeKiorOqbwTJ05g9+7dePXVVzFmzBgA2hZhp/9FSK/XIxQKnbGsPn364G9/+xv8fn/kLxs7duyA0+mMaoeOyMjIwIQJEzBhwgTcdNNNmDx5Mnbv3h311ysiIrq0sU/BPkVHsE9BnRXH6xAlSWZmJu666y68+eabUekmkwlDhgzB66+/jh07dmDLli14+OGHo4btXaiFCxdiyZIl2LdvH/7whz9g48aN+OEPfwgAmDhxIgoLC3HXXXdhxYoVOHjwIL7++mu88sor+Pjjj8/pfcxmM6ZNm4YXXngBH330Efbt24eXX34Zn3zyCWbOnHlBn8Fms8FisWDFihWoq6tDY2MjAOC+++7DJ598gieffBLbt29HVVUVPvvsM8yZMwder7fd8tLT05GZmYkFCxZg37592LBhAx544IFWf8kpLCzE6tWrceTIERw/frzNsm677bbIKte7du3CunXr8NBDD2HYsGEYNmxYhz/jc889h+XLl2Pv3r3Yv38/lixZAovFgvz8/A6XQUREFz/2KdinOBv2KagzY1CCKInuuOMO2Gy2Vum/+93vYLFYMGXKFDzwwAOYPHlyuwsgnY9Zs2bhvffew8SJE7Fo0SJUVlZG5qE273teVlaG2bNn47vf/S5+8pOfYNOmTef1i+tnP/sZvve97+F3v/sdrr/+eixevBiVlZUYNWrUBX0GWZbx+OOP46OPPsLYsWNx4403AgBGjhyJN998Ezt37sTUqVMxceJEPPnkk0hJSWlzaGrL8v7whz+gqqoKEydOxKOPPoof/OAHrdr9kUcewdatWzFu3Lh2P0N2djb+/Oc/o7a2FjfffDNmzpyJkpISvPDCC+f0GQ0GA1544QVMmjQJkyZNws6dO/Haa6/BarWeUzlERHTxY5/i/LFPQZRckjiXiWFERERERERERDHCkRJERERERERElBQMShARERERERFRUjAoQURERERERERJwaAEERERERERESUFgxJERERERERElBQMShARERERERFRUjAoQURERERERERJwaAEERERERERESUFgxJERERERERElBT/D40unQqhagvhAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 1080x360 with 2 Axes>"]},"metadata":{}}]},{"cell_type":"code","metadata":{"id":"GKC7NvHq6joJ"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"EpN-mR4j9a-v"},"source":["class BiLSTM_Attention(nn.Module):\n","\n","    def __init__(self, embedding_dim, pretrained_emb, hidden_dim, n_layers, token_size, USE_GLOVE=True):\n","\n","        super(BiLSTM_Attention, self).__init__()\n","\n","        self.embedding = nn.Embedding(\n","            num_embeddings=token_size,\n","            embedding_dim=embedding_dim\n","        )\n","        if USE_GLOVE:\n","            self.embedding.weight.data.copy_(torch.from_numpy(pretrained_emb).type(torch.float))\n","\n","        # self.token_size = token_to_ix.__len__()\n","        self.hidden_dim = hidden_dim\n","        self.n_layers = n_layers\n","        self.rnn = nn.LSTM(embedding_dim, hidden_dim, num_layers=n_layers, bidirectional=True, dropout=0.5)\n","        self.fc = nn.Linear(hidden_dim * 2, 1)\n","        self.dropout = nn.Dropout(0.5)\n","\n","        # 初始时间步和最终时间步的隐藏状态作为全连接层输入\n","        self.w_omega = nn.Parameter(torch.Tensor(hidden_dim * 2, hidden_dim * 2))\n","        self.u_omega = nn.Parameter(torch.Tensor(hidden_dim * 2, 1))\n","\n","        nn.init.uniform_(self.w_omega, -0.1, 0.1)\n","        nn.init.uniform_(self.u_omega, -0.1, 0.1)\n","\n","\n","    def attention_net(self, x):       #x:[batch, seq_len, hidden_dim*2]\n","\n","        u = torch.tanh(torch.matmul(x, self.w_omega))         #[batch, seq_len, hidden_dim*2]\n","        att = torch.matmul(u, self.u_omega)                   #[batch, seq_len, 1]\n","        att_score = F.softmax(att, dim=1)\n","\n","        scored_x = x * att_score                              #[batch, seq_len, hidden_dim*2]\n","\n","        context = torch.sum(scored_x, dim=1)                  #[batch, hidden_dim*2]\n","        return context\n","\n","    def forward(self, x):\n","        # print('lan x.shape',x.shape)\n","        embedding = self.dropout(self.embedding(x))       #[seq_len, batch, embedding_dim]\n","        # print(embedding.shape)\n","        embedding = torch.transpose(embedding,0,1)\n","        # print('embedding',embedding.shape)\n","        # output: [seq_len, batch, hidden_dim*2]     hidden/cell: [n_layers*2, batch, hidden_dim]\n","        output, (final_hidden_state, final_cell_state) = self.rnn(embedding)\n","        output = output.permute(1, 0, 2)                  #[batch, seq_len, hidden_dim*2]\n","        attn_output = self.attention_net(output)\n","        # print(attn_output.shape)\n","        logit = self.fc(attn_output)\n","        # print('logit',logit.shape)\n","        return logit\n","        # return attn_output"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"5IV4L7ImBJU_"},"source":["LEARNING_RATE = 0.001"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"lfFScNkP9dqj"},"source":["# device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n","net = BiLSTM_Attention(pretrained_emb.shape[1], pretrained_emb, hidden_dim=32, n_layers=2, token_size=len(token2ix)).to(device)\n","# opt = optimizer.Adam(net.parameters(), lr=LEARNING_RATE)\n","opt = optimizer.SGD(net.parameters(), lr=LEARNING_RATE)\n","# criteon = nn.CrossEntropyLoss().to(device)\n","criteon = nn.BCEWithLogitsLoss().to(device)\n","train_losses, dev_losses, dev_acc_list, best_model = train(11,train_data_iter, dev_data_iter ,opt,criteon,net,device)"],"execution_count":null,"outputs":[]}]}