haven-jeon/naver_review_classifications_gluon_attention.ipynb

## naver_review_classifications_gluon_attention.ipynb
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from konlpy.tag import Mecab\n",
    "from mxnet.gluon import nn, rnn\n",
    "from mxnet import gluon, autograd\n",
    "import gluonnlp as nlp\n",
    "from mxnet import nd \n",
    "import mxnet as mx\n",
    "import multiprocessing as mp\n",
    "import time\n",
    "import itertools\n",
    "from tqdm import tqdm\n",
    "from matplotlib import font_manager, rc\n",
    "from  matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "rc('font', family='NanumGothic')\n",
    "mecab = Mecab()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "rating = pd.read_csv(\"../../../part1/examples/naver_review/ratings.txt\",sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>document</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>199970</th>\n",
       "      <td>10258554</td>\n",
       "      <td>이게 왜 이렇게 높은지 모르겠네요..</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199971</th>\n",
       "      <td>306234</td>\n",
       "      <td>필름 낭비하고자 기를 쓰는 영화.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199972</th>\n",
       "      <td>5906178</td>\n",
       "      <td>전개가 일단 원작에 비해 생략도 많고 .. 권투씬도 졸려서 잠오고..</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199973</th>\n",
       "      <td>10188249</td>\n",
       "      <td>진짜 재미없네요. 볼까 말까 하다가 본 제가 다 멍청한듯</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199974</th>\n",
       "      <td>8449365</td>\n",
       "      <td>전윤수 미인도에서 9살 어린여자아이 옷벗기는 장면 명백한 아동학대범죄이다. 18금영...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199975</th>\n",
       "      <td>307000</td>\n",
       "      <td>삐질삐질;;</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199976</th>\n",
       "      <td>5502875</td>\n",
       "      <td>업로드 좀 빨리해라 어제부터 기다리는데...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199977</th>\n",
       "      <td>8682884</td>\n",
       "      <td>가지고 있는 이야기에 비해 정말로 수준떨어지는 연출, 편집...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199978</th>\n",
       "      <td>10211978</td>\n",
       "      <td>그린스크린 수정도 안하고 홍콩에서 시어스 타워가보인단다 ㄷㄷ 마이클 베이 가 영화를...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199979</th>\n",
       "      <td>3781237</td>\n",
       "      <td>2.9점이면 적당하네요</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199980</th>\n",
       "      <td>5535592</td>\n",
       "      <td>0점은 없나? 짜증나--</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199981</th>\n",
       "      <td>4645862</td>\n",
       "      <td>영상은 좋으나 반전까지 보기엔 산만하고 지루하고 개연성이떨어진다.. 평점에낚였네</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199982</th>\n",
       "      <td>7432883</td>\n",
       "      <td>헐 미쳣다 이영화가이평점이라닠ㅋㅋㅋㅋ</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199983</th>\n",
       "      <td>9986958</td>\n",
       "      <td>아 진짜 너무 오글거린다... 못봐줄정도네....</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199984</th>\n",
       "      <td>6624437</td>\n",
       "      <td>미국인이나,일본인이 아니면,제대로 만든 상업용 애니메이션을 해올수 있는,인력이 없다...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199985</th>\n",
       "      <td>8597016</td>\n",
       "      <td>내평점을 지운듯하다.. 그 당시 봤을때, 너무 밋밋하고 임팩트 없는 내용이 대부분이...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199986</th>\n",
       "      <td>9191516</td>\n",
       "      <td>지브리라는 이름이 아까운...스토리가 빈약하고 뜬금없는 요소도 많고. 어린 아이들이...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199987</th>\n",
       "      <td>9177798</td>\n",
       "      <td>정극에는 도무지 어울리지 않는 조니뎁, 혁신적이나 클래식하진않은 촬영기법, 거울을 ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199988</th>\n",
       "      <td>8312249</td>\n",
       "      <td>설리에 관한 모든것 인줄 아랐네 ㅡㅡ</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199989</th>\n",
       "      <td>5711373</td>\n",
       "      <td>재미없고 억지스럽다</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199990</th>\n",
       "      <td>5465496</td>\n",
       "      <td>장르는 무협인데 내가 보기엔 코믹이던데 막장 평점 2점도 아깝다</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199991</th>\n",
       "      <td>8965828</td>\n",
       "      <td>나치입장에서 본 영화가 갑자기 연속으로 나오네? 뭔일 있었나...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199992</th>\n",
       "      <td>2228930</td>\n",
       "      <td>태권도???</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199993</th>\n",
       "      <td>417815</td>\n",
       "      <td>음 왜 봤을까? 예고편이 다 -</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199994</th>\n",
       "      <td>4834376</td>\n",
       "      <td>개연성이 없어요.. 별루다...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199995</th>\n",
       "      <td>8963373</td>\n",
       "      <td>포켓 몬스터 짜가 ㅡㅡ;;</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199996</th>\n",
       "      <td>3302770</td>\n",
       "      <td>쓰.레.기</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199997</th>\n",
       "      <td>5458175</td>\n",
       "      <td>완전 사이코영화. 마지막은 더욱더 이 영화의질을 떨어트린다.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199998</th>\n",
       "      <td>6908648</td>\n",
       "      <td>왜난 재미없었지 ㅠㅠ 라따뚜이 보고나서 스머프 봐서 그런가 ㅋㅋ</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199999</th>\n",
       "      <td>8548411</td>\n",
       "      <td>포풍저그가나가신다영차영차영차</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              id                                           document  label\n",
       "199970  10258554                               이게 왜 이렇게 높은지 모르겠네요..      0\n",
       "199971    306234                                 필름 낭비하고자 기를 쓰는 영화.      0\n",
       "199972   5906178             전개가 일단 원작에 비해 생략도 많고 .. 권투씬도 졸려서 잠오고..      0\n",
       "199973  10188249                    진짜 재미없네요. 볼까 말까 하다가 본 제가 다 멍청한듯      0\n",
       "199974   8449365  전윤수 미인도에서 9살 어린여자아이 옷벗기는 장면 명백한 아동학대범죄이다. 18금영...      0\n",
       "199975    307000                                             삐질삐질;;      0\n",
       "199976   5502875                           업로드 좀 빨리해라 어제부터 기다리는데...      0\n",
       "199977   8682884                가지고 있는 이야기에 비해 정말로 수준떨어지는 연출, 편집...      0\n",
       "199978  10211978  그린스크린 수정도 안하고 홍콩에서 시어스 타워가보인단다 ㄷㄷ 마이클 베이 가 영화를...      0\n",
       "199979   3781237                                       2.9점이면 적당하네요      0\n",
       "199980   5535592                                      0점은 없나? 짜증나--      0\n",
       "199981   4645862       영상은 좋으나 반전까지 보기엔 산만하고 지루하고 개연성이떨어진다.. 평점에낚였네      0\n",
       "199982   7432883                               헐 미쳣다 이영화가이평점이라닠ㅋㅋㅋㅋ      0\n",
       "199983   9986958                        아 진짜 너무 오글거린다... 못봐줄정도네....      0\n",
       "199984   6624437  미국인이나,일본인이 아니면,제대로 만든 상업용 애니메이션을 해올수 있는,인력이 없다...      0\n",
       "199985   8597016  내평점을 지운듯하다.. 그 당시 봤을때, 너무 밋밋하고 임팩트 없는 내용이 대부분이...      0\n",
       "199986   9191516  지브리라는 이름이 아까운...스토리가 빈약하고 뜬금없는 요소도 많고. 어린 아이들이...      0\n",
       "199987   9177798  정극에는 도무지 어울리지 않는 조니뎁, 혁신적이나 클래식하진않은 촬영기법, 거울을 ...      0\n",
       "199988   8312249                               설리에 관한 모든것 인줄 아랐네 ㅡㅡ      0\n",
       "199989   5711373                                         재미없고 억지스럽다      0\n",
       "199990   5465496                장르는 무협인데 내가 보기엔 코믹이던데 막장 평점 2점도 아깝다      0\n",
       "199991   8965828               나치입장에서 본 영화가 갑자기 연속으로 나오네? 뭔일 있었나...      0\n",
       "199992   2228930                                             태권도???      0\n",
       "199993    417815                                  음 왜 봤을까? 예고편이 다 -      0\n",
       "199994   4834376                                  개연성이 없어요.. 별루다...      0\n",
       "199995   8963373                                     포켓 몬스터 짜가 ㅡㅡ;;      0\n",
       "199996   3302770                                              쓰.레.기      0\n",
       "199997   5458175                  완전 사이코영화. 마지막은 더욱더 이 영화의질을 떨어트린다.      0\n",
       "199998   6908648                왜난 재미없었지 ㅠㅠ 라따뚜이 보고나서 스머프 봐서 그런가 ㅋㅋ      0\n",
       "199999   8548411                                    포풍저그가나가신다영차영차영차      0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating.tail(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = [(d, l) for d,l in zip(rating['document'], rating['label'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_len = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "length_clip = nlp.data.PadSequence(seq_len, pad_val=\"<pad>\")\n",
    "\n",
    "def preprocess(data):\n",
    "    comment, label = data\n",
    "    morphs = mecab.morphs(str(comment).strip())\n",
    "    return(length_clip(morphs), label)\n",
    "\n",
    "def preprocess_dataset(dataset):\n",
    "    start = time.time()\n",
    "    with mp.Pool() as pool:\n",
    "        dataset = gluon.data.SimpleDataset(pool.map(preprocess, dataset))\n",
    "    end = time.time()\n",
    "    print('Done! Tokenizing Time={:.2f}s, #Sentences={}'\n",
    "          .format(end - start, len(dataset)))\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done! Tokenizing Time=9.77s, #Sentences=200000\n"
     ]
    }
   ],
   "source": [
    "preprocessed  = preprocess_dataset(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['어릴',\n",
       "  '때',\n",
       "  '보',\n",
       "  '고',\n",
       "  '지금',\n",
       "  '다시',\n",
       "  '봐도',\n",
       "  '재밌',\n",
       "  '어요',\n",
       "  'ㅋㅋ',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>',\n",
       "  '<pad>'],\n",
       " 1)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessed[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = nlp.data.count_tokens(itertools.chain.from_iterable([c for c, _ in preprocessed]))\n",
    "\n",
    "vocab = nlp.Vocab(counter,bos_token=None, eos_token=None, min_freq=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed_encoded  = [(vocab[data], label)  for data, label in preprocessed ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = nlp.data.train_valid_split(preprocessed_encoded, valid_ratio=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "batchify_fn = nlp.data.batchify.Tuple(nlp.data.batchify.Stack(),\n",
    "                                      nlp.data.batchify.Stack('float32'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataloader  = gluon.data.DataLoader(train, batch_size=100, batchify_fn=batchify_fn, shuffle=True, last_batch='discard')\n",
    "test_dataloader  = gluon.data.DataLoader(test, batch_size=100, batchify_fn=batchify_fn, shuffle=True, last_batch='discard')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100, 30)\n"
     ]
    }
   ],
   "source": [
    "for data, label in train_dataloader:\n",
    "    print(data.shape)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SentClassificationModelAtt(gluon.HybridBlock):\n",
    "    def __init__(self, vocab_size, num_embed, seq_len, hidden_size, **kwargs):\n",
    "        super(SentClassificationModelAtt, self).__init__(**kwargs)\n",
    "        self.seq_len = seq_len\n",
    "        self.hidden_size = hidden_size \n",
    "        with self.name_scope():\n",
    "            self.embed = nn.Embedding(input_dim=vocab_size, output_dim=num_embed)\n",
    "            self.drop = nn.Dropout(0.3)\n",
    "            self.bigru = rnn.GRU(self.hidden_size,dropout=0.2, bidirectional=True)\n",
    "            self.attention = nlp.model.MLPAttentionCell(30, dropout=0.2)\n",
    "            self.dense = nn.Dense(2)  \n",
    "    def hybrid_forward(self, F ,inputs):\n",
    "        em_out = self.drop(self.embed(inputs))\n",
    "        bigruout = self.bigru(em_out).transpose((1,0,2))\n",
    "        ctx_vector, weigth_vector = self.attention(bigruout, bigruout)\n",
    "        outs = self.dense(ctx_vector) \n",
    "        return(outs, weigth_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "ctx = mx.gpu()\n",
    "\n",
    "#모형 인스턴스 생성 및 트래이너, loss 정의 \n",
    "model = SentClassificationModelAtt(vocab_size = len(vocab.idx_to_token), num_embed=50, seq_len=seq_len, hidden_size=30)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.initialize(mx.init.Xavier(),ctx=ctx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = gluon.Trainer(model.collect_params(), 'adam')\n",
    "loss = gluon.loss.SoftmaxCrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SentClassificationModelAtt(\n",
      "  (dense): Dense(None -> 2, linear)\n",
      "  (bigru): GRU(None -> 30, TNC, dropout=0.2, bidirectional)\n",
      "  (drop): Dropout(p = 0.3, axes=())\n",
      "  (embed): Embedding(8303 -> 50, float32)\n",
      "  (attention): MLPAttentionCell(\n",
      "    (_dropout_layer): Dropout(p = 0.2, axes=())\n",
      "    (_act): Activation(tanh)\n",
      "    (_key_mid_layer): Dense(None -> 30, linear)\n",
      "    (_query_mid_layer): Dense(None -> 30, linear)\n",
      "    (_attention_score): Dense(30 -> 1, linear)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-12-17T08:09:11.112480Z",
     "start_time": "2017-12-17T08:09:11.107263Z"
    }
   },
   "outputs": [],
   "source": [
    "model.hybridize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 학습 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_accuracy(model, data_iter, ctx=ctx):\n",
    "    acc = mx.metric.Accuracy()\n",
    "    for i, (data, label) in enumerate(data_iter):\n",
    "        data = data.as_in_context(ctx)\n",
    "        label = label.as_in_context(ctx)\n",
    "        output, _ = model(data.T)\n",
    "        predictions = nd.argmax(output, axis=1)\n",
    "        acc.update(preds=predictions, labels=label)\n",
    "    return(acc.get()[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-12-17T08:09:18.821490Z",
     "start_time": "2017-12-17T08:09:18.813080Z"
    }
   },
   "outputs": [],
   "source": [
    "def calculate_loss(model, data_iter, loss_obj, ctx=ctx):\n",
    "    test_loss = []\n",
    "    for i, (te_data, te_label) in enumerate(data_iter):\n",
    "        te_data = te_data.as_in_context(ctx)\n",
    "        te_label = te_label.as_in_context(ctx)\n",
    "        with autograd.predict_mode():\n",
    "            te_output, _ = model(te_data.T)\n",
    "            loss_te = loss_obj(te_output, te_label)\n",
    "        curr_loss = nd.mean(loss_te).asscalar()\n",
    "        test_loss.append(curr_loss)\n",
    "    return(np.mean(test_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-12-17T08:12:53.863045Z",
     "start_time": "2017-12-17T08:09:24.910904Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1800/1800 [01:06<00:00, 26.96it/s]\n",
      "  0%|          | 3/1800 [00:00<01:07, 26.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0. Train Loss: 0.36828366, Test Loss : 0.3297405, Test Accuracy : 0.85735\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1800/1800 [01:06<00:00, 27.04it/s]\n",
      "  0%|          | 3/1800 [00:00<01:06, 26.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Train Loss: 0.3086045, Test Loss : 0.3247741, Test Accuracy : 0.85945\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1800/1800 [01:06<00:00, 27.03it/s]\n",
      "  0%|          | 3/1800 [00:00<01:08, 26.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2. Train Loss: 0.2874814, Test Loss : 0.32170543, Test Accuracy : 0.8627\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1800/1800 [01:06<00:00, 27.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3. Train Loss: 0.2719606, Test Loss : 0.31289443, Test Accuracy : 0.86415\n"
     ]
    }
   ],
   "source": [
    "epochs = 4\n",
    "\n",
    "\n",
    "tot_test_loss = []\n",
    "tot_test_accu = []\n",
    "tot_train_loss = []\n",
    "for e in range(epochs):\n",
    "    train_loss = []\n",
    "    #batch training \n",
    "    for i, (data, label) in enumerate(tqdm(train_dataloader)):\n",
    "        data = data.as_in_context(ctx)\n",
    "        label = label.as_in_context(ctx)\n",
    "        with autograd.record():\n",
    "            output, _ = model(data.T)\n",
    "            loss_ = loss(output, label)\n",
    "            loss_.backward()\n",
    "        trainer.step(data.shape[0])\n",
    "\n",
    "        curr_loss = nd.mean(loss_).asscalar()\n",
    "        train_loss.append(curr_loss)\n",
    "\n",
    "    #caculate test loss\n",
    "    test_loss = calculate_loss(model, test_dataloader, loss_obj = loss, ctx=ctx) \n",
    "    test_accu = evaluate_accuracy(model, test_dataloader,  ctx=ctx)\n",
    "\n",
    "    print(\"Epoch %s. Train Loss: %s, Test Loss : %s, Test Accuracy : %s\" % (e, np.mean(train_loss), test_loss, test_accu))    \n",
    "    tot_test_loss.append(test_loss)\n",
    "    tot_train_loss.append(np.mean(train_loss))\n",
    "    tot_test_accu.append(test_accu)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent = '가지고 있는 이야기에 비해 정말로 수준떨어지는 연출, 편집'\n",
    "\n",
    "morphs = mecab.morphs(str(sent).strip())\n",
    "len_raw = len(morphs)\n",
    "seqs = length_clip(morphs)\n",
    "onehot_seq = vocab(seqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "ret, weight = model(nd.array([onehot_seq],ctx=ctx).T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[[ 2.3831623 -2.5675967]]\n",
       "<NDArray 1x2 @gpu(0)>"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['가지',\n",
       " '고',\n",
       " '있',\n",
       " '는',\n",
       " '이야기',\n",
       " '에',\n",
       " '비해',\n",
       " '정말로',\n",
       " '수준',\n",
       " '떨어지',\n",
       " '는',\n",
       " '연출',\n",
       " ',',\n",
       " '편집',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>']"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seqs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "mat = np.corrcoef(weight.asnumpy()[0,][:len_raw,:len_raw])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEOCAYAAAD8Nx6LAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcHVWZ//HPt7MT1pAICAgKCEhQEUYRRwQB2RREBkY2Ydg0MsrizxEVXAZkGETFFYwgAuICqIDI5gIqoAg6KooCgopsAmEJZE/39/fHqZtUOn2XvlX39q3u5/163Vdyq+rUqb7dffrUOaeeR7YJIYSxrG+kLyCEEEZaNIQhhDEvGsIQwpgXDWEIYcyLhjCEMOZFQxhCGPOiIQwhjHnREIYQxrxoCMOIk3TxSF9DGNvGj/QFhNZJuoz63zMBi2y/vYuXNCySTgZeANxm+wpJn7L9PmDDDtc71Oem3P97+nMLnRcNYYXYPnCkr6GgtwKHAl8GrgC26Ualo+BzCx0Wt8YVI2mKpIkjfR1tmm/7L6zYGwPo+APvkiYPev8aSSd1ut5QDdEjrJ7fAb+T1AdcYPvakb6gAraW9F1g3S7Udb+kW4ALgUWknukxXag3VED0CKvnYdsHAAcBL5X0FUlV/YP2B9tvA/7ZhbruJTV+M4HPAu+0PdCFekMFRENYPQawvdj2OcBFwOyRvaRhGydpEl24Jc6x7SW2zwbeA5zXxbpDj4uGsHpWGF+zfQtwm6TDR+h6huNH2b+/Bb4KfD17340GcVnvz/ZPgSskfaQL9YYKUARmrRZJ02w/NcT2l9q+dySuqShJV9j+tw7XMcH2kkHb9rR9XSfrDdUQPcKKsf2UpP+svZe0f7a9Eo1g/tprhtMISjq4nXoHN4LZtq42gu1ee+i8yg2yZ79IjRrwfttf7Nb1dJOkVYFxwIGSLiJ9DrtJ+mF2yIDt50fsAhsYdO0XA0ttz5e0GmlZTb+kV9r+bYNzHA/sC3wjt21v2z9oUvchWd319Nu+dDhfz3BJ2gs4GvhG9lkMAFva/nWH6hv8Nc8F+oG1svdP2/5+J+quoso1hKTxpYY/1N26kBFwDqnxux/4XG77Z7N/lwDHdqLirPEaz/IxyunAk7lDFttuNE55Dun7dj/peh+VtBh4KenJktcDnwbeOETdnwdOBDYb4rzvAxo2hKRGoPbH8zTg1EH7O/YzI+ko4FrgNdn7KcCNwDTgEYb4ekvyLCv+njxP+jr7gNNJDXE0hJnKNYTZ5MAKJE0F3mT7eyNwSV1j++gRrPsd+feSbrJ90DDKr3Ttkj5NWst3TW1TneI7A2cCtwMvG3yaFupe9gsv6XjbV7VyzUVlt8JvB7YEnsg2TyE13Lt2sm7b1+TfSzrU9tez/59Ad2fse17lGkIASRcCPyXdIk0gLR/56IheVJdkt5IfAbYgNQJ/AE7v9C1xdis+YdC2b+TeLrZ9RJNznE/qCQH8Irer2S/lE8BUUgNypKT/zrZ/toWytbpfDHwBmCbpCtK6wk/YntdK+TYdk13fPOBPwB7AOztY3wqyz+kiYFNgvW7VW0WVbAiBzYHvAZcBE4Hjs0e3xoKLgC/bfj8sG3u6GHhbJyut3fZKOsH2OdmTLZsDj9h+tsXTbGb7Ddl5bgZ+M4xLuIrU+Bv4drZt7jDKnwt83PYvsvpfR3rKpBvPIf+Y1BM08GdW7tV2yt6kIYy1SQvwQx1VnTVeaPtq4ADSX/aZI3w93TTD9g21N9kjdjO6UbGko4F9smedrwD2By6VtGeLp8iPxeV7cuMlrUn93p2Bm8gae9t/zF4rzQQ3sEqtEczOcSud7yWdlf17B/Av2f9/2uE6854FjiON5U6S9JZsvem6wDpdvI6eV9WGUJCWRNg+AdhZ0g4jfE3d8mzWmwEg+7pb7ZEVdUj272uBG2yfTnpsrdUJmqmSNpS0Eel7+CtgMXAl8ClSD20o99teBKwKIGmSpO9K+h7Lb7WbeVrSLrU3knYD5rRYti3Z8pzbbS8AVs8295Nule8HJnWyftI6YZMa5ONJEybPACeTGsiQqeSCakl71YINSNoU+BvwSdsnFjzvkIuVh1G+9oPXsTolrUWa+dyENPP3APAR208P62LbIOkmUu/scOATwAnAW4CX2v5wC+VPZXnD9X+2hxWQVdJOWb1vGe73SdLqwCmkiQuAu0ljhMO5vc6fb7jft82B82zv3E597ZD0ltpEkaSP2x4T4+jtqGRDmCfpJ7ZLWYIg6ce2d2mwfwIrP+K2OLf/K8CJw5m4yNcpabztpcO/8pbqOZTm6y8brqXLln5ca3tnSa8lja/dTxqzbHibmq2dG1z/fNtLJU3Mf44NzrE9sBT430bfpyZ1i7SGcd5w1jAOcc6m37chvubNgPtIQzuLJb0he9yvdHU+b3J1v9H2TzpRdxVVbrJkULRhsTyUU+39Ytv/3uQcVwELSL0bkcZQ/pfmQwUXk9Zm1RpDZ0tANgB2B77RaiMo6VLbhwyq83oaLKsoGKH6MVZsDAb/BWy4li73i3VC1rv6I8tn6qdImtTka6+tI9yd9HUCXCVpLvAxSQYOtv1InfpXB2bZPjybqBmOwXU/IgngJTRZwzjEdQzn+3YOsFe2X9kxPwR+QnrO+qOt1Nmmz5B+VnYnrb2dCDwI3ApcQOodR0OYqVxDWFK04dVt71t7I2k6aQayYfd4qHVzkl4C7EgaAxrOWN1QA/WNFooX/dp3yp2/9gcgbylpdrOeWmOywiVl/4omi7lr6wiz9YdHSlrd9lxJNwB7Aq8E3ksavxrKKSxfRD6s2xjbR2fRbq7PzvNqUgPUyhrGwVr+vmX13mT7SABJ19X+P8w6h832MVmdN5H+yL8I+EVudUXH6q6iyjWEJVnhF8n2kwBZL6GhbMb03aQfpC/ZfgA4S6nwxZJmFVjT1/QXPOsNvQPYLdt0A3BJC2OT57H8F/abpIW++S+4YY+wjMXc2R+N70p6IWmB9DuA8dmjdncAHxqizATS5z0n9ziaJK2y/NK8oIXqf0Dq0c8j3dLXnoopY2yo0Tkk6bPAfFa+4+jouJSkL5PGcf9K+kP1IaDWEFd7TKxklWwIJf0R+AfwNPAZ279q8zyrAN+qvaW15RTnkn6p+oDzgcOUAqPuDzzUhWd9P0eaIKk1xsdk297TqJDthyStD/w36Vb+TNKyjnNtL2ylYkn/y8o9iT/b/mqL134TcAmpd/lfg/ZNBoa6jm+QejODbyHPy66lHziihbqnkcYztyOtq3uy8eGl2YQ0G/4z4P2SdrP9wyZlyvI20tIqkRbDd3qWurIq2RACj9veI5tB/YCkf3fKhjYstucD+9TeZ7cRzWxg+7vZ8e/MBvDPID35sNNwr6EN29nOR3D5pKRftlj2EuBTto+CNKtIWqDdcEw154ukX6paj7IvO2erDeEDpPGpD9j+W7btn5K2BbYnNZQrsH2ApH8Fvi1pv2xSxoMf+WvBz0kzxiLFRaxlzhufDY10qof0d9sXAbUsfl+T9CdS73yVhiWLq0UAr93J3JItXRpP+nkNmao2hLUozU8DJ0t6s6TP227YK8qpdw/cyi/DJEj3O8AE278E3ihpa1IvpW7ggewXoY/0S1Dr/YyTdES2rZXcHQOSXmD78eyc67V43QATnYvUYvv7kj7QYllsP5jVuSj3/+HMctv2X7M1gLNsn0saFzydFIDgtDqFbpF0LvBBUo+2nfGtrW0fn13zj0iNd20N41nA1+oVLPh9W6TlsRBfTPo6T8zOV+iZZ0lTmgwLvETSGbXDSY3frOyaryhS92hT1YZw8BKWayRtlvUYWgm8cFad7a38gp2bDfAPkG6Ta9dwl6RLm6wvu570Q2hWHPh/LNt+aAv1Hw1cJOl5lv9wH9VCOYCfSvoiKerIeFJ6zVZ6wYO5zv+b+TOA7RslfVXSN7Px2Xc1rdD+gaQiuYfXl/RV0uf8qO3ac9KfbqFske/b6cB1kh4kDb0cYbusHC03S/pDrYc/hMdIDbyy1762601GjWmVXEc4VGOT9dC2tX1ngfOekj0t0ey4yQCtjq2VUecQ5SZm19B0/d2gcq8kjZMB3GH7d8Mo+21ST2Zr4PdkY0+23zqca8jOtQ7wjNMTI8MtO+zPLJtkmkDqlQ7rMyvjGpr8gSxyHevbfrjOvl1s/zj3/uDcH4CQU8mGMIQQylTVZ41DCKE00RCGEMa8UdMQSioUor5I+ag76h4LdY9mo6YhpHiujiLlo+6oeyzU3XWSxkn6hKTr6+zfVdIPJF2m9Nx/w+31jKaGMIQw+rwZuJohlvplK0U+CLwtew5/vqTd6m1vVEllZo2nTxvnjTecUHf/E3P6mbF2/ZgFcwcaLxF89qmlrDFt6GWVC1y/XoB5Ty1h6rT6x9iN65739GKmrjVxyH1T+hqv9Hj2qX7WmFb/616tb6Bh+Sfn9DO9zuf2fNPPrHHdDz7dOHB2/7x5jJta/wGHiQ/XTyeyhEVMaPDEWP+0xg9OLF04j/GT6x/jafXXiS99dj7j16j/UMj0SY3ToDz/9GJWrfP9Bpg2rv6qrDlzBlh77fr9l4Emv89znhpg7Wn1y99119InbReKeL77zlM956nmiQF//ftFf2TFxypn25491LGSfmR710HbNgf+s/YghaTtSI8VXjTUdtsrPcteU5kF1RtvOIFf3bBh8wPruHF+48askbsXrd92WYBFA+3XPXPKPwrVveuU59oue8vCyYXqnnV5sbuwF5/8i+YH1TF3j+0L1b3ooPbj3B616W2F6j54tXvaLvucG//ha+bFGz7290InAJ58qp/bb9ig6XET1rt/oe3tmh5Y39pAfm3mU9m2etvrqkxDGEKoCtNfsEFu0RyWJ6yHFFhjToPtdUVDGEIolYGB7kT5+gswUyko8CJgX1JyrHrb6+p4QyjpctsHNNj/euCVtj/f6WsJIXTHAKX3CFdKBZGlWDiNlEnxeVL+6xtte6jtjU5eSkMoaW3gO7lN6wAvy4KFTsiO2Rt4f7Z/EnCvU67ccTSJzBxCqA5jlpR8a2x7WcrYLODsR2z/0/ZNDB2+bcjt9ZTSENqeQy4Wn6QbB0dMzsI//SDbvy9NBi9DCNVkoL+Dt8a231n2OUu/NZY0jRQ5ut7+CaRw4a0GAw0hVEyXxghL04kF1R8gZdCq+VdJN0t6afb+bNKtcdNGWNKxku6UdOcTc5qvSwohjDwD/XbTVy8ptSHMQr9PzKI219xieyfgr5I+T8rdcDzwFaW8snXZnm17O9vbNVosHULoLQMtvHpJWZMlfaRsWZtQP4nQJ4Fv274lK/NR4E00Wd8TQqgW446OEXZCWWOE25KS1NR9uNn2CYPe3wvcK2mnkq4hhNADbFhSrXawtFnjO0ipIYey0vqfQfppklM3hFAlor9i+eM7vqC60WLqbP/PSakWQwijgIGBsdgjDCGEvOgRdsjcARWKIPOmVZrdodf3hin3tV0WKLRUYL7bv26Af/a3X/fmE4adYG4F++3efvQYgOse36Htss9vUmy05fCN7mq77FaThkwq17Ib5rcf7eiRJWs1P6ihxwqWry2ojoYwhDCGGVjiasV8joYwhFAqI/orFvw+GsIQQukGmkRl7zXREIYQShVjhE1I2gv4ryF2PWf7Ld28lhBCp4j+GCOsz/a1wLWDt0u6rJvXEULonBShOhrClkiaaruW7qtiyy9DCPXYYrGrFSRlJMcIvwU0vB2WdCxZQuoXvDCGM0OoioGSxgglHUKKXdoP/ML2Wbl9Aj4BrAssAP5m+5PZvoNIuUqeIyVymmX7iXr1jGT/VQCSZlAnAk0+DFe9nMMhhN6SJkv6mr6aycL0HQbsa3s/YGtJm+UO2Q1YYPtI28cBz0h6ebbvOOAg28cAlwMHN6qrF27k5wJnjvRFhBDKkiZLmr2A6bXAy9lrcCLsHYAf5tJ+XAXsnNs/nxVTfswAXpv9/1fAFpLGkaJjrTQ3kde1bpakU4FdcpvWk3Rzbr+BA2w/2a1rCiGUbxiTJU82SfA+VKL2ZT1C27dIepmkC0i3wP8EVsl2XwAcBfwJeAh4oNGFdK0htH0acFq36gshjJz+chZUzwG2yr1fKVG77dnAbABJxwGPSloHeJ/tI7PtrwI+DpxSr6JeuDUOIYwiRizx+KavFtwO7JpNigDsQ0r1sRJJawAHAjcAa7K8ZwhpImXjRhXFDEQIoVS1yZLC57GfkXQJ8E1JS4Hf2v5zbX/WQH6elAJlOnB8tiTvHkm/lPRN0hzENFJSuboq0xAu8ATuXtR+eKIiobQmqf3wXwBFVhIMDBRLc3PPkoltl91o/PxCdW879a+Fyl+22avbLrvaes8VqnvLKe2H0lqlr1j4sn8smdZ22ccWrVGo7jIYlXVrjO1vAt/Mb5P0HeBA2/3Af9Ypd85w6qlMQxhCqI5OPllie/+yzxkNYQihVDbxrHEIYWxLkyXxiF0IYYyrWmDWnrhaSd+SNGmkryOEUJwRA27+6iXdjkf4dmAWsJi02nuW7YXZdfTWJxNCaFv0COuQNAU4EtjZ9m7ANcC7u1V/CKE7Ul7jvqavXtLtMcJ83ME+ohcYwiikCNVfj+0Fki4EbpK0hPSA9NGNyuTjEa613uTOX2QIobCUzjNmjeuy/S3gW5J2sH0bgKRNgE9nY4WDj1/2QPWGM1ePKNYhVICtnrv1bWakrvYTkmrPfv0b0P6zcyGEntNiPMKe0fWrkbQ1sCqwv6Q+UtDEPXIRJkIIFZbiEarpq5d0tSGUtDspnedOwCuAS4CzgEuB87NosiGESms5QnXP6Pas8a+BG7PQ2ydLWt32XABJd2bRJEIIFZaWz/RWj6+Zbk+WPDno/dyh/h9CqK541riDbLFooP24gP0uMOlcrT9uK2gxEvCQikVChIkq2MHva/97Nk7FFhmMG8FU20sG2v+e9UoDFAneQwhjWgrDNeJ5jWeQciRNJj3S+wXbv69XTzSEIYTSlTFGmMtrvKdtS7pE0ma2a+Hml+U1zo4/RtLLswbvbOBU2w+2Ulc0hCGEUqXoMy3dGk+XdGfu/ezsIYqaenmNaw3hkHmNJf0ze3+SpGnAn2z/T6ML6Xb0mW2APZpdVAihutIjdiOa13gjYBtgxywB1MmSDrN9Sb2Kuj2iOS57hRBGLZUVfWYOsFbu/ZB5jW0fZfsEUsa6v5N6ij+3/Ux22FWkBzfq6nZD6BGoM4TQZSU9WdJuXuP7gE1zD2i8Bqg7UQLdHyO8H9hB0s2Dtj9q+6AuX0sIoQPKmjUukNcYSZ8Hvi1pDqmH+P8a1dXtBdXPAHu0enw+DNeaEYYrhMooK/pMgbzG15CCP7ekaw2hpFNJ090irQkazMAB+adP8mG4NthqjQjDFUIF1HKWdOz8Vc5rbPs0Sb8G1rX91W7VG0LoLgNLeyyoQjPdHiPsIyZLQhj1qhaYtdsN4aPAJyUdOmj7AGl94eIuX08IoWw9mK6zmW5Plvwa2LKbdYYQuqsWmLVK4hG7EELpokfYIVP6FjNzyj/aLj/fS9ouOzBQNCBV+1btK7ZsaKPx7Yd5nN43sflBDdyzcL1C5bW4/XGmqZOKjbLMKPC5bTy+WN2vmfqXtsvOmbxqobq/WKh0EoFZQwhjnhFLB2KyJIQwxsUYYQhhbHPcGocQxrgqjhEWupGXdHmT/RdIWr3oeUII1TKQrSVs9OolLfUIJW0FnE+KJXgn8F7bS4EJ2f5ppBzFawJPAIfbfjY7vi87Zm3gO7nTbgC8IosW0X5WphBCTzGiv2KTJa1e7aeAt9l+NfAA8I5B+08DzrT9OuDLwKm5fddIeq/tObZ3qr2A3zF08IUQQsWVFI+wa5o2hJImAfNsP5ptOp8URSZvE9s/B7B9HSs+PfJm258b4px9the2feUhhJ5kj9Jb40GG+goGrzhu9lW+G7igaUW5eIQzXhh3zyFUhXusoWumaY/Q9iJgVUkvzDYdQwqHnfdXSa8DkLQX8IfcvrMkHVB7I+n1wMwscGKzumfb3s72dmtMi1QnIVRD895gVXuEJwHfldQH3EEaM8w7FbhEUm2yJD+G+F+1JCqSjgFeR9bLCyGMTmX1CNtN8J475izSpOzujeppqSG0/Udg+wb7nwL2bnQOSZsDS20f0UqdIYRqsqF/YMQTvCPp3cDVwKua1VV0QXWzp8uXko0f2r4HuKfN84QQKqTFWeGOJHgHfi9pJ1LH65blSfDqK9QQ2j6wyf6jyzhPCKE6TMu3xh1J8C5pQ+BNtj/U6jXHI3YhhJKVNhkyB9gq937IBO9kCd4kHUeKgr8/sK6k87LDtpB0qu3T6lVUmYZwtb4Bdp3yXNvl/9nffhK8e5YUi8u3xO1/zEXiCQK8eEL78ekeXPp8obq/9ue6w8ot2eSy9kdMHt5x3UJ1X7jm69sue+iM2wrVvfXE9r/nzwwU+3kpi8vJOXk7cLykz2S3x/sAZwx1YC7B+1613Ma5fT9q1AhChRrCEEJ1lDFrXCTB+yCLmtUVDWEIoVRp1nhkE7wPOkfDFS0QDWEIoQNKujWuc+4KJ3ivJ5vheRdwMyn5+yUje0UhhKKq9ohd1xtCSWcAbwCWkBZLjhv0CiFUmFHlGsKuBg2TtA2wVhau62zg492sP4TQHW7h1Uu6HT1xO+D7AFnQhd2Ab3X5GkIInWTwgJq+eslIhJEdD8umvm8F3l7vQEnHSrpT0p1PzokYriFUha2mr17S7YbwDmDf7P9vyd7XlQ/DNX3tGD4MoSrs5q9e0tXJEtu/lfSopFuBR4DDgRd08xpCCJ01jGeNe0bXZ41tn5J/30pkiBBChRiIhnDY+ge9QggV12u3vs2MeENo+x/Ah0f6OkIIZem9WeFmRrwhDCGMQtEj7IznB8QtCye3XX7zCU0DUNS10fj5bZeFlVP8Dcf0vmIhwIqE0nrR+PZDeAEcsnnDRQFNffXAN7RddqMtHi5U96x1ftJ22ZkTirUC9y5p/9fymYEphepO4fwKckyWhBBC9AhDCKF5avPeEg1hCKF8RcaDRkA0hCGEcsU6wpVlGaZOtL1SMgVJF9t+h6RLbB/W6WsJIXRHWesI203wLul0UrKnqcBdts9uVE9pzxpLuiz3/42yDPOQYgz2SZoi6SZJN0s6JttXmxKdUNZ1hBB6QAlxuHIJ3ve1vR+wtaTNcocsS/Bu+zjgGUkvh/QEm+132z4c2E3S1EZ1ldkjzDeqGvR+Y+Cc3PtDJBVbmxFC6F2t3Rp3LMF7bUPWaxwg9RjrKrMh3FrSzdn/JwO35Pb9zfZOkl4EbELq4i6UFLEIQxiF1NqtcUcSvA86x/HAhbYbTt+UGYbrLts72d6JIWIMStqBdD+/OfAtSeNISZhvJrXkK8nHI3z2qXgMOYRKsGCghVdzc4C1cu+HTPBu+yjbJwBzgb/X9kk6EJho+zKa6GY8wgNJkybnAb8hNYiPZQ3nE0MVyMcjXGNaxCMMoTLKidV/O7Crloeo2gf42VAH5hK835C93xd4WX5ypZEyb40n5G6NJwAXDtr/APBmSVcB25MSM4cQRqMSZo3bTfAuaSNgNvA9Sedlh5+TLztYaQ2h7X2bHHIe8DHga8CXbD8dsQhDGKVKWj7TToJ3238H1hlOPd1YUL0UGLC9GPjQoH2Ls3+XdOE6Qgjd0OEF1ZVM8G776Ab73pH9G4upQxhFWpw17hnxiF0IoXzREHbGg0/PYNblx7Zdfr/df9F22W2n/rXtsgAT1f7Sn3sWrleo7q/9efu2yxaNJ3jK9Lpj0y2ZvstzbZd9aPG0QnU/tnSNtste/ewmher+xs92aLvs+OeLLgR5X8HySfQIQwghgi6EEMa01tcJ9oxoCEMI5YuGMIQw1qligVkLjaxKukDS6u3uDyGMUuU8Ytc1LfUIJZ0LbJm93QTY2fZfWB5r8CPAm0lhcfqAv9g+srY/d54Pk2KI5a0CXGn7jCJfSAihN8ijdNbY9iwASS8Bjgb2lrQfsEV2yDTSIy9/y467Mlf8GkmX2f6c7U+QItAskwVaPK7QVxFC6C2jddZY0u7A20gxwr5k+7OSvtZC0TfbfiY7x+eAbUhht2sMnFunzmOBYwHGr7nWUIeEEHrRaOsRSloX+B/gTtvvzHqFZ0rKP+z8f8BFkkyKTn1ttv0x0rPGNTOAN9luGC22JotWOxtg8gYbVuyjDWHsGo23xs8C77P9FIDtB4BDAbKQWvNtXyRpCfDqrMx6ks4hBUqclzvXj4Frs/A5M4E/ZNv/z/aJhb+aEMLIc/VmjZs2hFnvbYGkGcDnSBmjTLq9PTOLKgNwBXD1oOJfAVYnNabYPh84H9I4ou23lvFFhBB6zCjsEdacDnza9h0AWVao6yTdansh8B7graw4/jeRNJNciyGWT7RSC9Nf85DtQ4f/JYQQes4obggfBbaXdA+wEHhFtn1R9u9MUtq9p4Yq3IkYYiGE3lTWGGGBvMa7AieShuYesn1So3qG0xCeBswiheCfQhrfOziXau8m4Du5/AI1J9n+zTDqCSGEfF7jPW1b0iWSNrNdS+e5LK9xdvwxWV7ju4APAnvZXiTpdEm72f5hvbpabgizsNhfyF5D7b8YuLjV84UQRrFyeoTt5jVeBNxtu3a3eiVp6V/xhnCkTXx4Hi8+uf2Ygtc93n6Mt8s2e3Xzgxrpa/+nQouLxZfb5LLFzQ+q46sHvqFQ3UXiCQK8a82H2y7720UPFKr7qrnbtF3223dvW6jul174fNtl++Yvan5QA/cXKp1pfda4WYL3dvMaD1Uu32CupDINYQihQspJ8D4H2Cr3fsi8xmRrjSUdR5rLaJoPebBu5jUOIYwBYvnzxo1eLWg3r/FfgJmSJmW79wV+2qii6BGGEMo3gnmNs32nAZdKeh54ArixUV0daQglXQEckhusDCGMFSVGn2knr3FW7ibSSpaWlN4QSvpXYCPg3cBnBu07H9g0t2kGcLPtiD4TwmjSwUfsOrEmudQxQklvJiVxfyMwSdLZ2b07sCzH8Z7AWcA9pPv5eMY4hFGmpDHCrimtRyjp66RByv2yW+IzJf0L8IUssOsk4O3A88B3SQ3m3sBnJN1n+5yyriWEMMJ6rKFrpsxz+OhEAAAMN0lEQVRb48NyCx8ByJ5LPixrEM8GaovaBi+02krSk7a/nt+Yj0c4mVVKvNQQQsf0YCj+ZkppCCWdCuyy8tN1y5g09T3P9gJJM4G32z6l0Xnza4RW17SKfbQhjF29duvbTCkNoe3TSM8iA8tuk99le4Ul8pK+B+xXRp0hhB42FhvCYah1Ge8FYkwwhFFq1AVmbdMAQ/9NeHE+BmHuVvpTtr/foWsJIXTTWB0jHMz2O+psf8VQ20MIo4dYfutXFfGIXQihfNEj7Iz+aVOZu8f2bZd/fpP+5gfVsdp6xcJJjSswhTZ1UvthtAAe3nHdtstutEX7YbAAHlo8rVD5IqG0XjlpUvODGuhbo/1YwvO3nFio7iv3f23bZQuPzZ1asHxmTM4ahxDCCqIhDCGMaaMxnWcIIQxb9AhDCGNd1cYIC0WfkXSBpNUb7N9H0mEtnOfyItcRQugxbuHVQ5r2CCWNA/4EPAL8Hng98DSwFzAO6MtCbV1CyhOwAJhl+35SQzsuO89E4Lrae+ClwKa25wMTSvyaQggjrGo9wlZujScAt9k+QtK7SCG0NgGuB7bIjjkF+LztH0ragRQ+e6/8SWwvBnaBZY3ipVkjGEIYTUxpgVkbJXjP9p9Iima1mNTJmmV7vqSDSLlKniN10GbZfqJePa3cGjt3nEiN53eAQ0iNIcCWteTJtm8D1s8epTu9zjmPJyWKb0jSsZLulHTn0oXzWrjUEMJIKyt5Uy7B+7629wO2lrRZbv+awK62D82SvN9NSvoOcBxwkO1jgMuBgxvV1bQhzIKs9kn6MbAa8F/AucCrSbfLS1n5iZr7be9E6ikO/uLeABxOSsLcrO7Ztrezvd34yVObHR5C6BWtjRFOr3V0stexg85SL8F7zbPAo5LWkzSFlCLklmzfr4AtsqG9bYFrG11uS7PGtg+VdCiwPSlyDKTb3LnAPOBuSW+yfaOk1+WOWYGkPYEDgFcB50mab7v9rO0hhJ4ktzRI2CyvcbME75Z0ISk/0hzgVtu1/MUXAEeR5jceAho+pjSc5TOXAVcO2vYVYHXSLfDFkj5KSrD8H4MLS9qL1Is82vZA1vqfAERDGMJoUt6scMME75JeDrzZ9gez92+TdDTwfeB92e0ykl4FfJwh7lBrhtMQvgd4K2nQsmYiMN/2EtLAZF22ryXXPbW9lBS+P4QwypQ0a3w7cLykz2S3x/sAZ+T2r8eKw3ILgI2BNWGF3B617XUNpyGcCexv+/FhlOlnxYaznmKRBUIIPaWMR+yaJXgnJW3fUdLFpDmHVYD32p4j6ZeSvkkavpsGfKBRXcNpCG8Gvq2VE5OcZHvIUB2tBlu1feAwriOE0Ou6l+D9w3XKDSsCfssNoe2LgIuGc/IQwhjU4bzFnUjwXplnjT1tKYsOerrt8odvdFfbZbecUiwu37gCfx5njJ9bqO4L13x922VnrfOTQnU/tnSNQuWvmrtN22WLxBMEePnEyW2XnbH2Lc0PauDxXVdru+yC/mIPad1XUjzCXnuErpnKNIQhhGqoLaiukmgIQwil00C1WsJoCEMI5erB6DLNREMYQihd1SJUF4pH2A5J20j6YLfrDSF00WiLR9gB41gekzCEMArFZElz+bBeIYTRxkBrQRd6xkg0hPcDO2TxCvMetX1QfkMWmOFYgAkz6mYECCH0mKqNEXa9IbT9DLBHi8fOBmYDrLLZetX6ExPCGBXrCJuQdCopgqwYOhiDgQNsP9nN6wohlMiOW+NGbJ8m6dfAura/2s26QwjdEz3C5vqIyZIQRrdoCJt6FPhkFvo/bwDYI8t2F0KosOgRNmH718CW3a43hNAlBvqr1RJW5hG76ZPmcdSmt7VdfqtJ7YfSWqWvacK9jtl4fLEO8qEz2v/MZk4o9sN89bObFCr/7bu3bbvs/C0nFqq7SCit9cavWqjud77g5rbLznOxr/vyQqWXix5hCCGUNGtcIMH7DOA0YHK27wu2f1+vnmgIQwilK6NHmEvwvmeWuvMSSZvZvi/bX0vwvnf2/gOk5XlXkRLDnWr7wVbqitnbEEK5Wgm40MEE75LWyfafJOniVoK8dLxHKOkC4ETbc7P3e7I8o9RXbF/a6WsIIXSPALU2WdKRBO+SXg1sA+yYZcI7WdJhti+pV1FpPUJJb5T0E0m3SnpI0k+z54n3rtUj6UxWTKt3pKRjyrqGEEJvkN301YI5wFq59/USvJ+aZa1bkCV4nw/8PHucF1JPsuHMWykNYZbi88PAXrZfB5wK/N72TsD1teNsn0xqGD8G/Bz4Hbmk7yGEUaD1W+Nmbgd2zaUQ3gf4WW5/vQTv9wGbSqqF+3sNUHeiBMq7NZ4G/MP2wuz9T4HN8wdI2gI4ipRw+Q7gTtJ6wndKut32D0q6lhDCiCrnWeMCCd4XSfo8KQ/7HFIP8f81qquUhjC7L19P0jbAA8D7SS3y9cDWpPv6T+aK7DLoFDtKWsv21/Mb82G41nrhpDIuNYTQBWWtIyyQ4P0a4JpW6ylzsuQI4BRgfeBc2zcASPoacJ/tnSStRVrrsyOpN/hlYIHteUOdMB+G60UzV6/YEs0QxrAORp/p6QTvth+VtIjUNc2v3fkJULtlPgKYmtv3LmBvSbvXZpVDCBXnlmeNe0bZy2dWiixj++Lc25nA4Oeu1iYFXAghjBbVagdLbwgfJw1QLhi0/eO2byLdDg9+ePaRbPsdJV9LCGGEtLg8pmeU2hDaPgM4o8H+HcqsL4TQo8ZyQxhCCJjKDXZFQxhCKJVo+cmRnlGZhnDauIUcvNo9bZe/Yf76bZf9x5JpbZcFWDLQ/sf8mql/KVT31hPbn4y/d0mxH49v/KzYSMhLL3y+7bJX7v/aQnU/vutqbZctEk8QYPvJ45ofVNdQOdFGwEC1uoSVaQhDCBURt8YhhDDGZ41DCAGIWeMQwlgXCd5DCGNdZLELIYQYIwwhhLg1LlM+HuEG6xdZWxVC6BoDA9VqCHs6i53t2ba3s73d2mv39KWGEJbJJkuavXpIT/cIQwgVNcIJ3nP7zwJeYXv3RvVENyuEUC4D/QPNX03kErzva3s/YGtJm+X21xK8H2r7SOBuUoL32v53A1eTGsiGeqYhlPQrSVObHxlC6G0GDzR/dSjBO4CknYCltm9p5Yp74tZY0gbAH+vlLgkhVExrt8adSvC+IfAm2x9q9XJ7oiG0/RDwHyN9HSGEEpQ3azwH2Cr3vl6C9w9m79+WJXhfFVhX0nnZoVtIOtX2afUq6omGsBUDNs+5/ZAWjyxZq+2yjy1ao+2yAEvc/tKfOZNXLVT3MwPth+F6ZmBKobrHP19s5KVv/qK2y6pg9JMF/RPaLjvPE4tV3iuhtIooZ7LkduB4SZ/Jbo/3YcUI+EMmeLd9Sv4kkn7UqBGECjWEIYQKGcEE70Ocqulf1GgIQwjlsqG/nF5tuwneB51j72bHREMYQihfxRK8d3z5jKTzJa3eYP9bJB3W6esIIXRRxZ4sKaUhlLRrtg7wQUk3S3pS0ley3eOBPkn7ZPtulvQbSadn+8fRwoLHEEJVOM0aN3v1kFJujW3/SNJDwBG2T5Z0JfAlSTcDW2THXA1cLakP+CJwURl1hxB6jMEFVniMhE7eGv8V+Bjw29oGSRsDXwCmA5t2sO4Qwkgq4RG7bip9skTSJGACaSHki4CJ2fb/BhYC7ydNZ58kaQbwCD30qF8IoSC7cuk8yxojPAT4HrAv6Vm/ycAxwC+BBwFsfyR7v6vtpbbPsn0xafXokHPtko6tPYc456lqfbAhjGkVmywpa4zwUuDSLOTNebYfqO2TdCupJwiwOrDmoOJrDLGtdt7ZwGyAl798Qm99ciGEulyxHmHZt8Z9DOpl2v7KoGNOlnRE7v104OySryOEMGJ6r8fXTNkN4ePANyTNH7T9dNs/IoXNGfwJGXi65OsIIYyUCobqL7UhzKLHntVg/03AlmXWGULoLQZc0iN23RKP2IUQymXXAq9WRjSEIYTSuWK3xnJFBjUlPQH8vcEh04EnC1RRpHzUHXWPlro3sj2jwPmRdH1WTzNP2t6jSF1lqUxD2IykO5uE/e5Y+ag76h4LdY9m8URHCGHMi4YwhDDmjaaGcPYIlo+6o+6xUPeoNWrGCEMIoV2jqUcYQghtiYYwhDDmRUMYQhjzoiEMIYx50RCGEMa8/w9kZYe0spUifQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "alpha = seqs[:len_raw]\n",
    "\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(1,1,1)\n",
    "cax = ax.matshow(mat)\n",
    "fig.colorbar(cax)\n",
    "\n",
    "plt.xticks(np.arange(len_raw))\n",
    "plt.yticks(np.arange(len_raw))\n",
    "ax.set_xticklabels(alpha, rotation='vertical')\n",
    "ax.set_yticklabels(alpha)\n",
    "\n",
    "#plt.show()\n",
    "plt.savefig('attention_weight.png', format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-12-17T08:17:24.140031Z",
     "start_time": "2017-12-17T08:17:24.051574Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAETCAYAAADDIPqYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VPW9//HXJzuBBJIQCCEkrIqCbCK4Ky6tWhVbWxdob9t7f+5b9Xa7S2+xi62211v3lvZeaxHQWhTcWuu+i0UIqIgSthDCEhKWELJOPr8/ZoBJzJAAmUyW9/PxmEfmbHM+OYTznu9ZvsfcHRERkZbExboAERHpvBQSIiISkUJCREQiUkiIiEhECgkREYlIISEiIhEpJETagZn9p5l9vQ3zfRph/Etmltf+lYkcGYWESPtICL1ak3yEy4t0KIWEiIhEpG8u0qOY2TvAy8DpBL/V3wRcCxwF7AX+yd23huadCdwA1IUW/7m7vxiadh7wM6AG2AWUAetD0zKBe4FBQCIwx91/fwg1Hg/cRfD/ZxywALjH3d3MfgqcC1QBr7j7z1sad+hbRiQCd9dLrx7zIrgjvyj0fgywA7gwNHwB8IvQ+9OAt4H00PBA4CNgOJAKrANGhKYNAkqBb4WG/whMDb2PA54HRu9bf4S6XgOGAv2Az4CjQuOTgL8AM4E04JNmy31unF56tedLh5ukJ3oewN0/BgLu/mxo/EqCO3yA6cD/uPvu0LxbgTnA+QRbHcvcfU1o2mbgibDP/wJwp5m9BrwC5AIj21jbqcDf3f2z0GfXAb8CLnH3SuBhM7vLzEaEpn9unEh70uEm6XHcPRA2uCfSbBHGN4Ze1mx8Ytj7ADDN3Q+n98yDrRd3v8vM+gKzzGydu9/b0rjDWK9Ii9SSEGnZQuC20M4XM8sBvgH8FVgFHLXvm7uZDQMuDVv2feCWfQNmlnII630LONfMRoeWTQK+DzxpZgbg7ruAe4BvtjTu0H9VkcjUkpCepvYgw4HQC3d/28zuBZ4zs3qCLYdb3H09gJl9G3gktJMuBx4DGkKfcyNwv5ldAVQTPF8xM8L692kAGtx9V2i5+8wsEYgHnnD3x83saDP7C8ET5SnA7QTDqvk4kXZjh9ciFhGRnkCHm0REJCKFhIiIRKSQEBGRiBQSIiISUZe/uql///4+dOjQWJchItKlfPDBB9vdPbu1+bp8SAwdOpQlS5bEugwRkS7FzDa0ZT4dbhIRkYgUEiIiEpFCQkREIury5yRaUl9fT0lJCTU1NbEuJWpSUlLIy8sjMTGx9ZlFRA5TtwyJkpIS0tLSGDp0KKH+z7oVd6e8vJySkhKGDRsW63JEpBvrloebampqyMrK6pYBAWBmZGVldeuWkoh0Dt0yJIBuGxD7dPffT0Q6h24bEq1pdGfzrmrqGgKtzywi0kP12JBoCDRSsaeO4opqGqPUXfpVV13Fli1b2jTvs88+y2OPPRaVOkREDle3PHHdFkkJ8QzO6EVxxV627q5hUN9e7b6O+vp6GhoaWp8RuPDCC9t9/SIiR6rbh8Ttz3zMytLdEafXNjTSEGgkJTGe+Li2Hec/NjedH1805qDz/PGPf+T9999n1qxZDBkyhNraWlatWsWdd97Je++9R2FhIQAFBQXcfPPNzJkzh4SEBK688kpOO+00jjnmGJKTkykvL+ePf/wjSUlJbf+lRUTaSY893LRPckIccXFGbUNjxCfQH45vfetbTJkyhVmzZuHupKWl8eSTTzJq1CiGDh1KdXU1cXFxzJ49G4BAIEAgEDw/UlpaykMPPcR9993HuHHjePHFF9uxMhGRtuv2LYnWvvED1NQHKNq2h15J8Qzv3zsqVw6ddNJJAGzdupUf/ehHPPPMM6SlpfHmm29+bt5hw4YRHx8PQE5ODjt27Gj3ekRE2qLHtyQAUhLjye3Xi6raBrZVRnpO/aGLj4/ff04iISGYx8XFxUyZMoW0tDQ2b97MypUr2219IiLtTSERktk7iYzUJLburmFPTX27fOYZZ5zBd77zHeLj4/e3DCZNmkR5eTk333wzP/7xjznzzDMBmswTfv4hfLyISEczj9Llnx1l8uTJ3vx5Ep988gnHHHPMIX9WoNEp2raHRndGDuhDYnznztDD/T1FRMzsA3ef3Np8nXsv2MHi44z8zFQaGp2SHdV09QAVETlSColmeiXFk9s3hcqaesr2tN/5CRGRrkgh0YLM3kn07ZXI1l21VNW27WY4EZHuSCHRAjMjL6MXiQnGxoq9NAQaY12SiEhMRO0+CTObCVwOBIB33f2uZtMfBBKB3sBn7j4rND4b+CmQAtQB97v7imjVGUl8XBz5mamsKauiZEc1BVmp6nlVRHqcqISEmaUB3wDOd3c3szlmNsrdV++bx92vD5v/ETM72t0/BX4N/Mjdi6NR26FITUogJz2FzbuqKa+qo3+f5FiXJCLSoaJ1uOlk4EU/cHnQImBaSzOaWQaQDWw1s4Gh0beZ2Z/M7N8iLHO1mS0xsyVlZWXtXXsT/fskkZ6SyOZdNeyta//zE+effz4QuRfY+fPnM2fOnHZfr4hIW0QrJLKAirDhitC4/cxspJnNBZYCs919J1AATARmufs/AW5m32j+4e4+290nu/vk7OzsKP0K++skL6MXCXFGccVeAo3te36ivj54496FF17IFVdc8bnp4X06iYh0tGidkygHwjtNygyN28/di4CZZpYAzDez94C9wJuhwIBgC+Qa4PC/Sv/1h7Dlw8NeHIIb6Sh3qusC1McbcXnjsfPvPOgyV111Fbfffju5ubmsXr2aO++8k4EDB1JdXU1NTQ1XX301EyZM2D9/eC+wv/zlL/noo48YMGAA69atY/r06UdUv4jI4YpWS2IxcI4dONN7MfBGSzO6ewMQDyQBq4GRZravH4qpQIeftG5JvBlJCXE0BJzahtZbE1deeeX+w0dz5szhuuuuIzs7m9raWhobG3n44YebzL+vxbB582Y++OADHn30Ue6++24GDhzY0seLiHSIqLQk3H2nmc0h2EJoAArdfdW+6WY2CbgN2AOkAwv2nag2s/uAx82snGDL4rtHVMz5vzyixcMlulOyvYq9dQFG1gdISYzcp9K0adN44IEHuPXWW1m1ahWZmZnU19fzwAMP8NFHH/Gb3/ymxeWKi4s59thj9w8ff/zx7Va/iMihitolsO4+H5gfPs7MFgCXuftS4OsRlnsWeDZadR0JM2NIZiqrt+5hQ/leRg7oE/FBRWbGsccey8MPP8xZZ53Fhx9+yA033ADAyy+/HHEdw4cP59NPP90//O6773L66ae37y8iItJGHfo8CXe/tCPXFw2J8XHkZ/Zi7fYqSndWMyQzNeK83/zmNznjjDNYuXIlJ598MrNmzWLgwIEMGjRof8+uiYmJwIHeXrOzsznzzDP55je/SWZmJikpKeoFVkRiRr3AHqYtu2rYVlnDkMxUMlJj82hR9QIrIodLvcBG2cD0ZHonJbBpRzW19bpEVUS6J4XEYdp3fiLOYEPFXhobu3aLTESkJd02JDriMFpSQhx5GanU1AfYvKsm6usL19UPE4pI19AtQyIlJYXy8vIO2ZGm90oku08y5VW17NpbF/X1QTAgysvLSUlJ6ZD1iUjP1aFXN3WUvLw8SkpKiHa/Tvu4Ozv21FG2sZEB6ckkxEU/e1NSUsjLy4v6ekSkZ+uWIZGYmMiwYcM6dJ0bK/Zywb1vMjy7D09ccxJJCd2ykSYiPYz2ZO1kSGYqd106juUbd/KrF1a1voCISBegkGhH5x83iH86qYDfv7mOlz/ZGutyRESOmEKinf37Bcdw7KB0/vWJ5WzeVR3rckREjohCop2lJMZz/4yJ1DU0csv8Qj0fW0S6NIVEFAzP7sMdXz6O99dXcM/Lq1tfQESkk1JIRMklEwdz2eQ87n+1iLdWb491OSIih0UhEUWzLh7DiOw+fOfxQrZVduwd2SIi7UEhEUWpSQk8MGMSlTX13Pb4cvXvJCJdjkIiyo7OSeP2i8fwVtF2Hnp9TazLERE5JAqJDnD5CUO4eHwu//33T3l/XUWsyxERaTOFRAcwM37+5bHkZ6Zy8/xlVFR1TEeAIiJHSiHRQdJSErl/xiQqqur43hPL1dW3iHQJCokONHZwX/79gtG8vGob//vWuliXIyLSKoVEB/vmyUP5wrEDufNvqyjcuDPW5YiIHJRCooOZGb/66ngGpKVw0/yl7Kquj3VJIiIRKSRioG9qIvdeOZHSnTX825MrdH5CRDothUSMHF+Qwfe+eDTPf7iFRxcXx7ocEZEWKSRi6OrThnPGUdn89NmVrCzdHetyREQ+RyERQ3Fxxt2XjScjNZEb5y2lqrYh1iWJiDQRtZAws5lm9rSZPWVm329h+oNm9nszm2dms1qYfpeZvRCt+jqLrD7J3HPFRNaXV/GfCz/S+QkR6VSiEhJmlgZ8A5ju7l8GjjOzUeHzuPv17n6Vu88AhpnZ0WHLXw88DcRH+PyrzWyJmS0pKyuLxq/QoU4cnsXNZ4/iqWWb+MsHJbEuR0Rkv2i1JE4GXvQDX4sXAdNamtHMMoBsYGto+Eygwd3fivTh7j7b3Se7++Ts7Ox2LTxWbjprFCcOz+S/Fn1M0bbKWJcjIgJELySygPCe7CpC4/Yzs5FmNhdYCsx2951mNgT4grvPjlJdnVZ8nHHPFRNJTYrnhrnLqKkPxLokEZGohUQ5kBE2nBkat5+7F7n7TGAUMNPMcoBLgRwz+62Z/RYYbWY/ilKNnc7A9BTuvnwCn26t5PZnVsa6HBGRqIXEYuAcM7PQ8MXAGy3N6O4NBM89JLn7b9z9n939Wne/Fljl7j+NUo2d0hlHZXPdmSOY/34xTy8vjXU5ItLDJUTjQ0OHjuYA882sASh091X7ppvZJOA2YA+QDixw95buKKuNRn2d3W3nHsXiteX8+5MfMm5wX4b27x3rkkSkh7KOvOTSzBYAl7l7ux1wnzx5si9ZsqS9Pq7TKNmxly/d+xb5man85bqTSE5o8UIvEZHDYmYfuPvk1ubr0Jvp3P3S9gyI7iwvI5VffXUcH27axS//uqr1BUREokB3XHdiXxiTw7dPGcrDb6/nhY+3xLocEemBFBKd3A/PH83Ywel874nllOzYG+tyRKSHUUh0cskJ8dx/5SQaHW6ev4z6QGOsSxKRHkQh0QUM7d+bO75yHEuLd3L3i5/FuhwR6UGicglsl1C3Fx48ETKHQ9YIyBxx4Ge/fEhIinWFTVw8Ppd315Tz0GtrOHF4Fmcc1T26IxGRVrjDnm2wcwPs2AA71sPO9cH36YPhK7+L6up7bkjU74W8E6BiDax4Amp3HZhm8dBvSNPgyBoRDJR++RCfGJOSf3zRsSzdsIPbHi/k+VtOY2B6SkzqEJF2VlsZFgAbmr7fWRzcX4XrMxD6FUDv/lEvrUPvk4iGdrlPwh32VgQDo3xNs59roS6sw724hGBQhAdI5nDIGg598yE+urlbtK2Si+57mwlD+vHo/5tKfJy1vpCIxFagHnZtDO74d2wIBUHY+73lTedPSoOMoZBREAyD8Pf98iEp9YhLaut9Ej23JRHODHpnBV9DpjSd5g5V21sOkOJ3oW7PgXnjEoP/kOEtj32Hs/oOgbgjvyFu5IA0fjJ9DN/7ywruf6WIW84Z1fpCIhJd+w4JtdQS2LEBdpeAh110su/LZr8COOaiYAjsD4Oh0CsjuF/qBBQSrTGDPtnBV/6JTaft+8MID46KtcHWx/o3mzYR45OC//jhLY99YZKeB3Ftv4bgq8fn8e6acu55+TOmDMvkpBFZrS8kIkemZveBnX54S2DH+uAhoYbqpvP3yQl+aSw4KRQABQfCID23Xb40dgQdbooWd6jcEhYgaw8cvqpY2/QPKj4ZMoc1bXnsC5C03BYDpKq2gYvue4s9tQ389ZbTyOqT3IG/nEg31FAXPCT0uZbA+uBwdUXT+ZPTP7/z339YKB8Se3X873AI2nq4SSERC42NULk5rOUR9nPHOmioOTBvQgpkDDtw+CosQFZW9uaSh97hpOFZPPytE4jT+QmRyNxhz9bIJ4h3b2p2SCgxeAHL/gBoFgad6JDQ4VBIdFWNjcE/1v0tj2YBEqg7MG9CL3ak5PHern7kDBvDxAnHHziclZbTpf+ARQ7L/kNC61s4QdzCIaG0QS0EQOh92qAuc0jocCgkuqPGQDBAwq688ooitq5bSWZdKUkW1ndiYu/Pn/vIDL3vM0ABIl3TvkNCkU4Qt3RIqMkVQkMPvO83pNMfEoomXd3UHcXFh66IyIcRwUeGG9Crup5z732N7MB2HpmeRe+qDQeCZOvHsOo5aGw48DlJacFzIE0u4Q29791fASKxs/+Q0PqWTxBXljY9JBSfFLxyMKMAcid+/nLRLn5IqDNQSHQDfXslcu+ME/jqb9/h1g8y+d03zsXC/2MEGmBXceikedghrM3LYeXTEN57e3J6WGgMb3o/SGqm/sNJ2zUGoGYXVO8IvvZWHHhfHf4+bFrl5qbn5CB42CdjKAw99fOHhbr5IaHOQCHRTYwf0o8fnDeanz33CY+8s55vnTLswMT4hANXTnFO0wUD9cFjtU0u4V0Dmz6Aj59q+q0tpS/0zg4GSUp62M++kJzWbFzYtH3DibpDvEs6nJ199Y7gMkQ6nG3Bv6deGcFXambwy0ifgU0PC/XL199NjCkkupF/OXUY764p547nVzF5aCZjB/dtfaH4xOB/zqwRn5/WUBds7u+/hHdt8M7Q2t3BE4SVW4I/a3c3vakw4rqSmoZIclpwR/G5YAn/GT49DRJT1Zo5XB25s9833Cuz6bR971P6qgXQRejEdTezo6qOC+59k+SEOJ656VTSUjqon6nGwIHwqK0Me787uJNpMhzhZ20lkXdGIXEJwbBoqaUS6WfzcUl9unbQdNTOfv+OXjv77kgnrnuojN5J3HvlRK6Y/R7//tRH3HvFhKbnJ6IlLv7AjuNwNTYGWyQtBklLQRMKo50bgx007psWfoisJRYXCpoWAuZzh80ihFBS2iHdJd/y76tv9tL5KSS6oROGZnLrOaP49d8/45QRWVwxJT/WJbVNXFxwB5ySDm04UtYid6iraiFQdjVtsTQPod2lULvqwHD41WAtsgMtmojnY9KD97VoZy9dmEKim7ruzJG8t7aCHz/9MRPzMzg6Jy3WJXUMM0juE3yl5x7eZ7hDfXWzQ2EHO1wWauVUlQVP/u8bH6jjczv7XhmRd/bhO3zt7KWT0DmJbmxbZQ0X3PMWGamJLLrxFFKT9J2gQ9XXBC8M0M5eOqG2npPQ40u7sQFpKfzm8gkUle1h1tMfx7qcnicxRQEhXZ5Cops7dVR/bpw2kj8vKeGpZSWxLkdEuhiFRA9wy9mjmDI0k/946iPWlrXhfgYRkRCFRA+QEB/HPVdOIDkhjhvnLaOmPtD6QiIiRDEkzGymmT1tZk+Z2fdbmP6gmf3ezOaZ2ayw8T8LTXvEzL4brfp6mkF9e/Hfl41n5ebd3PH8J7EuR0S6iKhc7mJmacA3gPPd3c1sjpmNcvfV++Zx9+vD5n/EzI5290/d/T/Dxr9gZg+5e1Wzz78auBogP7+L3APQCZw1eiBXnTaM37+5jpOGZ3H+cYNiXZKIdHLRakmcDLzoB66vXQRMa2lGM8sAsoGtzcYb0AhUN1/G3We7+2R3n5ydnd2uhXd33/viaMYP6cf3F6xgY8Xe1hcQkR4tWiGRBYQ//aMiNG4/MxtpZnOBpcBsd9/Z7DNuAR52b62PBTkUSQlx3H/lRABunL+MugZtXhGJLFohUQ6Ed+KTGRq3n7sXuftMYBQw08xy9k0zs8uAJHf/c5Tq69GGZKZy56XjWL5xJ7/++6exLkdEOrFohcRi4Bw70LPcxcAbLc3o7g1APJAEYGbTgWPd/a4o1SbABccN4hsnFjD7jbW8smpr6wuISI8UlZAIHTqaA8w3s0eBFe6+at90M5tkZo+a2W/NbB6wwN2LzawAmA3khKb91sxGR6NGgf/40jEcMyidf/3zcjbv+typHxGRju27ycwWAJe5e7tdqK++m47M2rI9XHjfW4zN7cu8q6aSEK9bZ0R6gk7Zd5O7X9qeASFHbnh2H37+5bG8v76Ce15e3foCItKj6Guj8OWJeXzt+Dzuf7WIt4u2x7ocEelEFBICwO3TxzAiuw+3PFZIWWVtrMsRkU6iTSFhZkeFfvYys2vNbFR0y5KOlpqUwAMzJlFZU8+tjxfS2Ni1nzMiIu2jrS2Jq0I//wPYBdwRnXIklo7OSWPWxWN4q2g7D72+JtbliEgn0NaQ6G1mWUCju88HtkWxJomhK04YwkXjc7n7xc/4x/qK1hcQkW6trSGxEvgjcH9ouD4q1UjMmRl3fHkseRm9uHn+MnZU1cW6JBGJobaGxIPufpG7bwvd3PaDaBYlsZWWksgDMyZRvqeO7z6xnK7+HHQROXxtDYk7AczseuBWgndFSzc2dnBf/u2C0by8ahv/+9a6WJcjIjHS1pBIMLN4YIS7XwNUtbaAdH3fOnko5x47kDv/torlG5t30isiPUFbQyIReIZgf0wAadEpRzoTM+NXXx3HgLQUbpy/lN01OhUl0tO0KSTc/Ubga+5eGBr1neiVJJ1Jv9Qk7r1yIqU7a/jhghU6PyHSw7T1ZrpRwJ9DjxNdRNNnRUg3d3xBBt/9wtE8/+EWvv+XFZTs0BPtRHqKtj7jehbwT+5ebmb9gd8AX49aVdLpXHP6cMoqa5nz3nqeWraJr0wazA3TRlKQ1TvWpYlIFLX1nMRudy8HcPftQGX0SpLOKC7O+K+LjuWN70/j6ycWsLCwlLP++3Vue7yQom17Yl2eiERJW0Miw8z2PTkuGegXvZKkMxvUt1ew647vT+PbJw/lrx9t4dz/eZ0b5y3l0y367iDS3bTpoUNmdhLBeyWKgTzgB+6+OMq1tYkeOhRb5Xtq+cNb6/jTO+upqgtw3pgcbjxrJGMH9411aSJyEG196NBBQ8LMfsuB1kYcYIAD9e5+XXsUeqQUEp3Dzr11/N/b63n47XVU1jRw9ugB3HT2KCYMUaNTpDNqr5DIA+JbmBRw95IjqK/dKCQ6l9019Tzy9nr+9+117Nxbz2mj+nPz2aM4YWhmrEsTkTDtEhJdgUKic9pT28Cj723g92+spbyqjhOHZ3Lz2aM4aXgWZhbr8kR6PIWEdArVdQHmvV/M715fw7bKWiYXZHDT2aM4fVR/hYVIDCkkpFOpqQ/w5yUb+e1rayjdVcP4If24adpIzj5mgMJCJAYUEtIp1TU0smBpCQ++VsTGimrG5KZz01kj+cKxOcTFKSxEOopCQjq1+kAjiwpLeeDVItZtr+LogWnccNZIvnTcIOIVFiJRp5CQLiHQ6Dy7opT7Xyli9bY9DM/uzY3TRnLx+FwS4tt6r6eIHCqFhHQpjY3O3z7ewn2vFPHJ5t0UZKVy/Zkj+PLEPJISFBYi7S3mIWFmM4HLgQDwrrvf1Wz6gwSfU9Eb+MzdZ4XGn0Pw6XdVQIm733aw9Sgkuhd356VPtnHfK6tZUbKLwf16ce2ZI7hsch7JCS3dsiMihyOmIWFmacATwPnu7mY2B/iJu6+OMP8jwB3AZ8BLwAXuXmtmPwNed/cXI61LIdE9uTuvfVbGfS+vZmnxTgamJ3PtGSO4cko+KYkKC5Ej1daQiFY7/mTgRT+QQIuAaS3NaGYZQDawFTgKWOnutaHJC1tazsyuNrMlZrakrKys3YuX2DMzph09gAXXnczc/zeVoVm9uf2ZlZx656vMfmMNVbUNsS5RpEeIVkhkARVhwxWhcfuZ2UgzmwssBWa7+862LAfg7rPdfbK7T87Ozm734qXzMDNOGdmfx685icevPpHROWnc8fwqTr3zFR54tYhKPVJVJKra+tChQ1UOjAkbzgyN28/di4CZZpYAzDez90LzZBxsOem5pg7PYurwLJYW7+C+l1fzqxc+5Xevr+GfTx3Gt08eRt/UxFiXKNLtRKslsRg4xw7cSnsx8EZLM7p7A8FOBJOAImBs6JkVANOB16NUo3RRk/IzePjbU3jmxlM5cXgWv3lpNafc+Qq/emEVFVV1sS5PpFuJ5tVNVxLcyTcAhe7+67Bpk4DbgD1AOvCcu88NTZsG3BCaVgZ83w9SpE5cyyebd3P/q0U8/+FmeiXG8/UTC7jqtOFkpyW3vrBIDxXzS2BbXJnZAuAydw+012cqJGSfom2V3P9KEU8vLyUxPo4ZU/O55vQR5PRNiXVpIp1OpwyJaFBISHPrtlfx4KtFPLVsE3FmXHZCHteeMYK8jNRYlybSaSgkpMfbWLGXB19bw18+2Ig7XDopj+unjaAgq3esSxOJOYWESEjpzmp+9/oa5v9jI4FGZ/qEXG6YNpIR2X1iXZpIzCgkRJrZtruG2W+sZe7iYmoaAlw4Lpcbp43k6Jy0WJcm0uEUEiIRlO+p5Q9vreNP76ynqi7AeWNyuOnskYzJ7Rvr0kQ6jEJCpBU7qup4+O11PPzOeiprGjjnmAHcdNYoxg/pF+vSRKJOISHSRruq6/nTO+v537fXsXNvPacflc3NZ41k8tDMWJcmEjUKCZFDtKe2gTnvbuAPb66lvKqOk4ZncfPZozhxeKaewy3djkJC5DDtrWtg3uJifvfGWsoqazlhaAY3nTWK00b1V1hIt6GQEDlCNfUB/rxkIw+9tobNu2qYMKQfN501krNGD1BYSJenkBBpJ7UNARZ8sIkHXyuiZEc1Y3LTuemskXzh2Bzi4hQW0jUpJETaWX2gkYXLNvHga2tYt72KowemceNZI7nguEHEKyyki1FIiERJQ6CR5z7czH2vFFG0bQ8jsntzw7SRXDw+l4T4aPW+L9K+FBIiUdbY6Pz1oy3c98pqVm2ppCArlRvOHMmXJw0mUWEhnZxCQqSDNDY6L32ylfteKeLDTbsY3K8X1505gq9NziM5IT7W5Ym0SCEh0sHcndc+K+O+l1eztHgnOekpfPuUoXz1+Dyy+ugBSNK5KCREYsTdeWdNOfe9spr31laQFB/H+cflMHNqAScMzdDls9IptDUkEjqiGJGexMw4ZWR/ThnZn9VbK5mttcROAAAQg0lEQVS7uJgFS0tYVFjKqAF9mDk1ny9PyqNvr8RYlyrSKrUkRDpAdV2AZ1aUMndxMcs37iQlMY6LxuUy88QCxuf1VetCOpwON4l0Uh9t2sXcxcUsKtzE3roAY3LTmTm1gOkTcumdrMa9dAyFhEgnV1lTz6LCUh59bwOrtlTSJzmBSybmMmNKAcfmpse6POnmFBIiXYS7s2zjTua+V8yzK0qpbWhkYn4/Zk4t4MJxg0hJ1GW00v4UEiJd0M69dSxYuol5izewpqyKvr0SuXRSHjOm5jNygJ7JLe1HISHShbk7i9dVMHdxMX/7aDP1AefE4ZnMnFrAF8fkkJSgO7rlyCgkRLqJ7XtqeWJJCfPe38DGimqyeifxtclDmDEln/ys1FiXJ12UQkKkm2lsdN4s2s68xRt46ZNtBBqd04/KZubUfM4ePUCdC8ohUUiIdGNbdtXw+D828tg/itm8q4aB6clccUI+V0wZwqC+vWJdnnQBMQ8JM5sJXA4EgHfd/a5m0x8CGoFM4Dl3fzQ0/kpgOlAJZADXuXtZpPUoJKQnawg08uqnZcxdvIHXPyvDgLNGD2TmifmcPipbz7mQiGIaEmaWBjwBnO/ubmZzgJ+4++oW5jXgDXc/LTT8FnBaaLnLgRx3vyfSuhQSIkEbK/by2D+KefwfG9m+p468jF5cOSWfyyYPITtNHQxKU20NiWgdxDwZeNEPJNAiYFqEeZOBirDh94HRZhYPHA8833wBM7vazJaY2ZKysoiNDJEeZUhmKt/74mje+eHZPDBjEvmZqfzqhU856Rcvc8O8pbyzZjtd/fCydLxo9QGQRdMdfwUwKsK8PwPCD0X9L/AvwCdACbC2+QLuPhuYDcGWRDvUK9JtJCXE8aVxg/jSuEGsKdvD/MXF/GVpCc+t2Mzw/r2ZMTWfSyflkdE7KdalShcQrcNNXwTGuPvdoeGvApmhnXv4fLcC29x9bmh4IPALd//n0PAk4Cvu/p+R1qXDTSKtq6kP8PyHm5m7uJgPNuwgKSGOC48bxMwT85mUr+7Le6JYdxW+GLjFzP4ndMjpYuCOZgVeD1TtC4iQfkD4hd/VwNAo1SjSY6QkxvOVSXl8ZVIeq7bsZt7iYp5cuoknl21idE4aM6fmM33iYNJT1H25NBXNq5v2XaXUABS6+6/Dpp0MPEbT8w0/cvcyM/sOMBXYTfDKpx+4++cOOe2jloTI4amqbeCZ5aU8ungDH23aTa/EeKZPyGXm1AKOy+sb6/IkymJ+CWyLKzNbAFzm7oH2+kyFhMiRW1ES7GDw6eWlVNcHGJfXl5lT87lofC6pSeq+vDvqlCERDQoJkfazu6aehcs28eh7G/hs6x7SkhP4yqTBzJhawNE5abEuT9qRQkJEDpu788GGHcxdXMxzH26mrqGRyQUZzDwxn/PHqvvy7kAhISLtoqKqjgUflDDv/WLWba+iX2oiXzs+jyun5DM8W92Xd1UKCRFpV42Nzrtry5m3uJgXPt5CQ6NzysgsZkwp4NxjB6r78i5GISEiUbOtsibYffniYjbtrKZ/n2QuPyGPK07IZ0imui/vChQSIhJ1gUbnjc+CHQy+smobDpx5VDYzpxYwbfQAdTDYiSkkRKRDle6s5rF/bOSx94vZVllLbt8UrpiSz+UnDGFgekqsy5NmFBIiEhP1gUZe/mQbcxdv4M3V24mPM849ZiAzpuZz6sj+xKl10SnEulsOEemhEuPjOG9sDueNzWFDeRXz3i/miSUl/O3jLeRnpjJjaj5fOz6PrD7qvrwrUEtCRKKutiHACx9vZe57G1i8roKkUJDMnJrPlGGZ6mAwBnS4SUQ6paJtlcxdXMyCD0rYXdPAyAF9mDEl2H1531R1MNhRFBIi0qlV1wV4dkUpcxcXU7hxJ8kJcZw/NofpEwdz2sj+JMTrvotoUkiISJfxceku5i0u5pnlpeyuaSCrdxIXjhvE9ImDmTiknw5HRYFCQkS6nNqGAK9/WsaiwlJe+mQrtQ2N5GemMn1CLtMnDGbkAHUD0l4UEiLSpVXW1PPCx1tZVLiJt4u20+gwdnA608cP5qLxueT01b0XR0IhISLdxrbdNTyzYjNPF25ieckuzODEYVlcMjGX88YOom8vnfA+VAoJEemW1pbtYVFhKYsKN7G+fC9J8XFMG53NJRMGM230AHVj3kYKCRHp1tydFSW7WFi4iWeWb2b7nlrSUhKCV0hNGMyJw7PUd9RBKCREpMdoCDTy7tpyFi4r5YWPt7CntoGB6clcNC54wnvs4HRdIdWMQkJEeqSa+gAvf7KNhYWbeO3TbdQHnOHZvblkwmCmT8ilIKt3rEvsFBQSItLj7dxbx18/2sLCZZtYvK4CgAlD+nHJhFy+NC6X7LSe23+UQkJEJEzpzmqeWV7KwsJSPtm8m/g445SR/blkQi5fGJNDn+Se1d+pQkJEJILPtlayqHATiwpLKdlRTUpiHOccM5BLJgzm9KOye8SjWBUSIiKtcHc+2LCDhYWbeG7FZnbsradfaiIXHDeISyYMZnJBRrd9/oVCQkTkENQHGnlzdRkLl5Xy4sqtVNcHGNyvFxeNz+WSibmMzkmPdYntSiEhInKYqmobeHFlsEuQN1ZvJ9DojM5J4+JQH1KD+/WKdYlHTCEhItIOyvfU8tyHm1m4bBNLi3cCMGVoJhdPyOVLxw0io3dSjCs8PDEPCTObCVwOBIB33f2uZtMfAhqBTOA5d380ND4b+CmQAtQB97v7ikjrUUiISEcpLt/L08s3sbCwlKJte0iMN844KpuLJwzm3GMG0iup63QJEtOQMLM04AngfHd3M5sD/MTdV7cwrwFvuPtpoeFHgB+5e3Fb1qWQEJGO5u6s3LybRYWlPF1YypbdNfROiueLY4IPTTplRFanf2hSW0MiWhcGnwy86AcSaBEwDfhcSADJQAWAmQ0MjbvNzDKBT9z9F80XMLOrgasB8vPz27l0EZGDMzPG5PZlTG5ffnDeaN5fV8Giwk08/+Fmnly2if59krhwXC7TJ+QyoYs/NClaLYkZQLK7PxwaPguYGmGH/2vgKXd/28ymAH8ATnf3nWb2Q2CTu8+JtC61JESks6htCPDqqjKeXr6Jlz7ZRl1DIwVZqUwfn8v0iYMZkd15HpoU65ZEOTAmbDgzNK4JM7sVWObub4dG7QXedPedoeFFwDVAxJAQEekskhPiOW9sDueNzWF3TT0vfLSFRYWl3P9qEfe+UsTYwelcMiH40KSB6V3joUnRakn0A+YBXwqdk/gTcIe7rwqb53qgwd1nh41LBp4GLnD3gJl9C4hz9/+LtC61JESks9u2u4anl5fy9PJSVoQemnTyiCymjx/MecflkJ7S8Q9N6gxXN10JTAcagEJ3/3XYtJOBx4Dnwxb5kbuXmdmFwLcItjz2At9190Ck9SgkRKQrWRP20KQN5XtJSojjrKMHcMnEXM48uuMemhTzkGhxZWYLgMsOttM/VAoJEemK3J3lJbtYuGwTz64oZfueuv0PTbpkwmCmRvmhSZ0yJKJBISEiXV1DoJF31pSzsHATL3y0haq6AAPTk7l4fPAO7zG57f/QJIWEiEgXVF0X4OVVW1m4rJTXPws+NGlE6KFJF7fjQ5MUEiIiXdzOvXU8/+EWFhZu4v3QQ5Mm5vfjkgmD+dK4QfTvc/gPTVJIiIh0I5v2PTRp2SZWbakkPs44f2wO98+YdFifF+v7JEREpB0N7teLa88YwbVnjODTLcGHJnXEjdwKCRGRLubonDS+f97oDllX5+6BSkREYkohISIiESkkREQkIoWEiIhEpJAQEZGIFBIiIhKRQkJERCJSSIiISERdvlsOMysDNhzBR/QHtrdTOe1JdR0a1XVoVNeh6Y51Fbh7dmszdfmQOFJmtqQt/Zd0NNV1aFTXoVFdh6Yn16XDTSIiEpFCQkREIlJIwOxYFxCB6jo0quvQqK5D02Pr6vHnJEREJDK1JEREJCKFhIiIRKSQEBGRiHrMk+nMbCZwORAA3nX3uw5legzrWgYsDg02ADd5B5xIMrN44CfA8e5+XgvTzwFuBaqAEne/Ldo1tbGul4CisFE/dPedHVDXQ0AjkAk85+6PNpseq+3VWl2x2l4PAolAb+Azd5/VbHqstldrdcVke4XWnQD8Cah092uaTYve9nL3bv8C0oC/ceBE/RxgVFunx6qu0LiXYrTNpgNTW1o/YMDLQHJo+GfAubGuK5bbq9m2ebOzbK+D1dUZtleohkeAozvT9mqprlhvL2AW8AXgDy3820Zte/WUw00nAy96aAsCi4BphzA9VnUBxJvZL8xsrpld0gE1AeDui9x9cYTJRwEr3b02NLyQjtlerdUFsMfMfmpmc8zsqo6oqZlkoKLZuJhtrzAt1QUx3l5mlgFkA1vDRsd8e0WoC2K0vcxsBrAE+KyFyVHdXj3lcFMWTf+DVACjDmF6rOrC3acBmFki8ISZfezuqzugtoNpqe6sGNXShLtfAmBmBjxkZmvc/ZUOLOFnQPNDlZ1he7VUV8y2l5mNBG4n+EXpVm96yCZm26uVumKyvcxsIpDj7vPMbGgLs0R1e/WUlkQ5kBE2nBka19bpsaprP3evB14ExnRAXa2J1fZqs1Dr7BlgXEet08xuBZa5+9vNJsV0ex2krv06enu5e5G7zyT4pWimmeWETY7Z9mqlrvD5OnJ7XQEcbWa/BX4OnGJm14dNj+r26ikhsRg4J5T+ABcDbxzC9FjV1dxJQGHUq2pdETDWzJJDw9OB12NYTySnA//oiBWF/tNWufvcFibHbHu1UldzHba99nH3BiAeSAobHfO/rwh1Ndch28vdf+Du17j7tcB/AG+7+4Nhs0R1e/WIw03uvtPM5gDzzawBKHT3VW2dHqu6AMzsEaAa6AMsdPf10a6rmfrmI9w9YGY/Beaa2R6gDPh7rOsCMLP/JritUoDFB/v23F7M7GTgh8DzoW97AD9y9zKI3fZqra7QPLHYXpOA24A9QDqwwN2L902P4fY6aF2heTp8ezUTIHiV437R3l49ulsOM1sAXObugVjXEq4T1/U74L/cvfnJvJhSXYdGdR2anl5Xjw4JERE5uJ5yTkJERA6DQkJERCJSSIiISEQKCZEYMrO/xroGkYNRSIjEVmKsCxA5GF3dJNJGZvZzoC/B6+T/AHwXWAnsIniH7t3uvtLMRhPsjG0X0A/4H3d/z8yGAT8KjQ+4+3fNbAnwHsF7YYYB17h7p7p7XXo2hYRIG5jZ+cB4d/9lqMvmZ4BewPWhYMgEfufuXzOzF4Cvu3tZ6C7YvwNnAk8CV7n79rDPXQqc5O61oW7j+7j77zr41xOJqEfccS3SDo4DJpjZL0PDtQTvuv0MwN0rzKxvaFp82N3WtWZWSrDDtd7hARFSEdZ75ybglGj+EiKHSiEh0jargTp3/82+EWb2GjAJeD/UO+fm0KR6M8sOa0nkuPt2M6s3s0HuvpnI7CDTRDqcQkKkbRYBvzGz/yPYiniLYB86F5jZV4ECgv0kQfAJYfeY2W6C5yT2jf8OcLeZlQO17v6vNO2DKhB6iXQaOichcpjM7CV3PyfWdYhEky6BFTl8LfZEK9KdqCUhIiIRqSUhIiIRKSRERCQihYSIiESkkBARkYgUEiIiEtH/B7ZXWd1XGLscAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tot_train_loss)\n",
    "plt.plot(tot_test_loss)\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'valid'], loc='upper left')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}