Skip to content

Instantly share code, notes, and snippets.

@georgehc

georgehc/95-865 Recurrent neural nets.ipynb Secret

Last active December 7, 2022 04:55
Show Gist options
  • Save georgehc/8f26c4c13e1274d64eaa3fba8d03cba7 to your computer and use it in GitHub Desktop.
Save georgehc/8f26c4c13e1274d64eaa3fba8d03cba7 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
95-865 Recurrent neural nets.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 95-865: Sentiment Analysis with IMDb Reviews\n",
"\n",
"Author: George H. Chen (georgechen [at symbol] cmu.edu)\n",
"\n",
"This demo shows how to train an LSTM model for sentiment analysis with IMDb reviews. This is a binary classification task: for each review, we classify it as having positive or negative sentiment."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import random\n",
"import os\n",
"\n",
"os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8' # to help make code deterministic\n",
"\n",
"from glob import glob\n",
"\n",
"import torch\n",
"torch.use_deterministic_algorithms(True) # to help make code deterministic\n",
"torch.backends.cudnn.benchmark = False # to help make code deterministic\n",
"import torch.nn as nn\n",
"from torchinfo import summary\n",
"\n",
"np.random.seed(0) # to help make code deterministic\n",
"torch.manual_seed(0) # to help make code deterministic\n",
"random.seed(0) # to help make code deterministic\n",
"\n",
"from UDA_pytorch_utils import UDA_pytorch_classifier_fit, \\\n",
" UDA_plot_train_val_accuracy_vs_epoch, UDA_pytorch_classifier_predict, \\\n",
" UDA_compute_accuracy, UDA_get_rnn_last_time_step_outputs"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Load the dataset\n",
"\n",
"Here, we downloaded the IMDb dataset from: http://ai.stanford.edu/~amaas/data/sentiment/\n",
"\n",
"We place the file `aclImdb_v1.tar.gz` into `./data/` and uncompress the file within that directory so that after uncompressing, you should have access to the directories `./data/aclImdb/train`, `./data/aclImdb/test`, and other files such as the \"README\" file `./data/aclImdb/README`."
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"train_dataset = []\n",
"\n",
"for filename in sorted(glob('./data/aclImdb/train/pos/*.txt')):\n",
" with open(filename, 'r', encoding='utf-8') as f:\n",
" train_dataset.append((f.read(), 1)) # 1 means `positive` sentiment\n",
"\n",
"for filename in sorted(glob('./data/aclImdb/train/neg/*.txt')):\n",
" with open(filename, 'r', encoding='utf-8') as f:\n",
" train_dataset.append((f.read(), 0)) # 0 means `negative` sentiment"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"# proper training data points: 20000\n",
"# validation data points: 5000\n"
]
}
],
"source": [
"proper_train_size = int(len(train_dataset) * 0.8)\n",
"val_size = len(train_dataset) - proper_train_size\n",
"print('# proper training data points:', proper_train_size)\n",
"print('# validation data points:', val_size)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"proper_train_dataset, val_dataset = torch.utils.data.random_split(train_dataset,\n",
" [proper_train_size,\n",
" val_size])"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(\"Master cinéaste Alain Resnais likes to work with those actors who are a part of his family.In this film too we see Resnais' family members like Pierre Arditi, Sabine Azema, André Dussolier and Fanny Ardant dealing with serious themes like death,religion,suicide,love and their overall implications on our daily lives.The formal nature of relationship shared by these people is evident as even friends, they address each other using a formal you.In 1984,while making L'amour à mort,Resnais dealt with time,memory and space to unravel the mysteries of a fundamental question of human existence :Is love stronger than death ? It was 16 years ago in 1968 that Resnais made a somewhat similar film Je t'aime Je t'aime which was also about love and memories.Message of this film is loud and clear :true and deep love can even put science to shame as dead lovers regain their lost lives leaving doctors to care for their reputation.L'amour à mort is like a game which is not at all didactic.It is a film in which the musical score is in perfect tandem with its images.This is one of the reasons why this film can easily be grasped.\",\n",
" 1)"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"proper_train_dataset[0] # this is a tuple of the format (text, label)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"from torchtext.data import get_tokenizer"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/george/anaconda3_UDA/lib/python3.9/site-packages/torch/cuda/__init__.py:497: UserWarning: Can't initialize NVML\n",
" warnings.warn(\"Can't initialize NVML\")\n"
]
}
],
"source": [
"tokenizer = get_tokenizer('spacy', language='en_core_web_sm')"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['Master',\n",
" 'cinéaste',\n",
" 'Alain',\n",
" 'Resnais',\n",
" 'likes',\n",
" 'to',\n",
" 'work',\n",
" 'with',\n",
" 'those',\n",
" 'actors',\n",
" 'who',\n",
" 'are',\n",
" 'a',\n",
" 'part',\n",
" 'of',\n",
" 'his',\n",
" 'family',\n",
" '.',\n",
" 'In',\n",
" 'this',\n",
" 'film',\n",
" 'too',\n",
" 'we',\n",
" 'see',\n",
" 'Resnais',\n",
" \"'\",\n",
" 'family',\n",
" 'members',\n",
" 'like',\n",
" 'Pierre',\n",
" 'Arditi',\n",
" ',',\n",
" 'Sabine',\n",
" 'Azema',\n",
" ',',\n",
" 'André',\n",
" 'Dussolier',\n",
" 'and',\n",
" 'Fanny',\n",
" 'Ardant',\n",
" 'dealing',\n",
" 'with',\n",
" 'serious',\n",
" 'themes',\n",
" 'like',\n",
" 'death',\n",
" ',',\n",
" 'religion',\n",
" ',',\n",
" 'suicide',\n",
" ',',\n",
" 'love',\n",
" 'and',\n",
" 'their',\n",
" 'overall',\n",
" 'implications',\n",
" 'on',\n",
" 'our',\n",
" 'daily',\n",
" 'lives',\n",
" '.',\n",
" 'The',\n",
" 'formal',\n",
" 'nature',\n",
" 'of',\n",
" 'relationship',\n",
" 'shared',\n",
" 'by',\n",
" 'these',\n",
" 'people',\n",
" 'is',\n",
" 'evident',\n",
" 'as',\n",
" 'even',\n",
" 'friends',\n",
" ',',\n",
" 'they',\n",
" 'address',\n",
" 'each',\n",
" 'other',\n",
" 'using',\n",
" 'a',\n",
" 'formal',\n",
" 'you',\n",
" '.',\n",
" 'In',\n",
" '1984,while',\n",
" 'making',\n",
" \"L'amour\",\n",
" 'à',\n",
" 'mort',\n",
" ',',\n",
" 'Resnais',\n",
" 'dealt',\n",
" 'with',\n",
" 'time',\n",
" ',',\n",
" 'memory',\n",
" 'and',\n",
" 'space',\n",
" 'to',\n",
" 'unravel',\n",
" 'the',\n",
" 'mysteries',\n",
" 'of',\n",
" 'a',\n",
" 'fundamental',\n",
" 'question',\n",
" 'of',\n",
" 'human',\n",
" 'existence',\n",
" ':',\n",
" 'Is',\n",
" 'love',\n",
" 'stronger',\n",
" 'than',\n",
" 'death',\n",
" '?',\n",
" 'It',\n",
" 'was',\n",
" '16',\n",
" 'years',\n",
" 'ago',\n",
" 'in',\n",
" '1968',\n",
" 'that',\n",
" 'Resnais',\n",
" 'made',\n",
" 'a',\n",
" 'somewhat',\n",
" 'similar',\n",
" 'film',\n",
" 'Je',\n",
" \"t'aime\",\n",
" 'Je',\n",
" \"t'aime\",\n",
" 'which',\n",
" 'was',\n",
" 'also',\n",
" 'about',\n",
" 'love',\n",
" 'and',\n",
" 'memories',\n",
" '.',\n",
" 'Message',\n",
" 'of',\n",
" 'this',\n",
" 'film',\n",
" 'is',\n",
" 'loud',\n",
" 'and',\n",
" 'clear',\n",
" ':',\n",
" 'true',\n",
" 'and',\n",
" 'deep',\n",
" 'love',\n",
" 'can',\n",
" 'even',\n",
" 'put',\n",
" 'science',\n",
" 'to',\n",
" 'shame',\n",
" 'as',\n",
" 'dead',\n",
" 'lovers',\n",
" 'regain',\n",
" 'their',\n",
" 'lost',\n",
" 'lives',\n",
" 'leaving',\n",
" 'doctors',\n",
" 'to',\n",
" 'care',\n",
" 'for',\n",
" 'their',\n",
" 'reputation',\n",
" '.',\n",
" \"L'amour\",\n",
" 'à',\n",
" 'mort',\n",
" 'is',\n",
" 'like',\n",
" 'a',\n",
" 'game',\n",
" 'which',\n",
" 'is',\n",
" 'not',\n",
" 'at',\n",
" 'all',\n",
" 'didactic',\n",
" '.',\n",
" 'It',\n",
" 'is',\n",
" 'a',\n",
" 'film',\n",
" 'in',\n",
" 'which',\n",
" 'the',\n",
" 'musical',\n",
" 'score',\n",
" 'is',\n",
" 'in',\n",
" 'perfect',\n",
" 'tandem',\n",
" 'with',\n",
" 'its',\n",
" 'images',\n",
" '.',\n",
" 'This',\n",
" 'is',\n",
" 'one',\n",
" 'of',\n",
" 'the',\n",
" 'reasons',\n",
" 'why',\n",
" 'this',\n",
" 'film',\n",
" 'can',\n",
" 'easily',\n",
" 'be',\n",
" 'grasped',\n",
" '.']"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"tokenizer(proper_train_dataset[0][0])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"proper_train_dataset_as_tokens_without_labels = [tokenizer(text) for text, _ in proper_train_dataset]"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"from torchtext.vocab import build_vocab_from_iterator"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"vocab = build_vocab_from_iterator(proper_train_dataset_as_tokens_without_labels,\n",
" specials=[\"<unk>\"])"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"vocab.set_default_index(vocab[\"<unk>\"])"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[3752,\n",
" 87615,\n",
" 12966,\n",
" 17640,\n",
" 1240,\n",
" 7,\n",
" 178,\n",
" 20,\n",
" 172,\n",
" 173,\n",
" 43,\n",
" 31,\n",
" 5,\n",
" 194,\n",
" 6,\n",
" 34,\n",
" 250,\n",
" 3,\n",
" 158,\n",
" 15,\n",
" 23,\n",
" 115,\n",
" 98,\n",
" 78,\n",
" 17640,\n",
" 49,\n",
" 250,\n",
" 1093,\n",
" 46,\n",
" 6444,\n",
" 63116,\n",
" 2,\n",
" 39058,\n",
" 63412,\n",
" 2,\n",
" 16573,\n",
" 32452,\n",
" 4,\n",
" 13479,\n",
" 36868,\n",
" 2001,\n",
" 20,\n",
" 642,\n",
" 1339,\n",
" 46,\n",
" 419,\n",
" 2,\n",
" 2416,\n",
" 2,\n",
" 1804,\n",
" 2,\n",
" 142,\n",
" 4,\n",
" 79,\n",
" 872,\n",
" 11585,\n",
" 27,\n",
" 304,\n",
" 3595,\n",
" 494,\n",
" 3,\n",
" 24,\n",
" 12327,\n",
" 917,\n",
" 6,\n",
" 653,\n",
" 5512,\n",
" 40,\n",
" 168,\n",
" 99,\n",
" 8,\n",
" 3863,\n",
" 19,\n",
" 80,\n",
" 399,\n",
" 2,\n",
" 44,\n",
" 5998,\n",
" 287,\n",
" 96,\n",
" 835,\n",
" 5,\n",
" 12327,\n",
" 30,\n",
" 3,\n",
" 158,\n",
" 61332,\n",
" 255,\n",
" 47326,\n",
" 15138,\n",
" 54119,\n",
" 2,\n",
" 17640,\n",
" 3446,\n",
" 20,\n",
" 67,\n",
" 2,\n",
" 1964,\n",
" 4,\n",
" 1060,\n",
" 7,\n",
" 9490,\n",
" 1,\n",
" 4811,\n",
" 6,\n",
" 5,\n",
" 9632,\n",
" 923,\n",
" 6,\n",
" 448,\n",
" 2108,\n",
" 90,\n",
" 863,\n",
" 142,\n",
" 3669,\n",
" 86,\n",
" 419,\n",
" 57,\n",
" 53,\n",
" 18,\n",
" 3290,\n",
" 177,\n",
" 636,\n",
" 9,\n",
" 5389,\n",
" 12,\n",
" 17640,\n",
" 103,\n",
" 5,\n",
" 648,\n",
" 747,\n",
" 23,\n",
" 14617,\n",
" 14452,\n",
" 14617,\n",
" 14452,\n",
" 69,\n",
" 18,\n",
" 107,\n",
" 50,\n",
" 142,\n",
" 4,\n",
" 1941,\n",
" 3,\n",
" 29251,\n",
" 6,\n",
" 15,\n",
" 23,\n",
" 8,\n",
" 1337,\n",
" 4,\n",
" 799,\n",
" 90,\n",
" 331,\n",
" 4,\n",
" 1013,\n",
" 142,\n",
" 68,\n",
" 80,\n",
" 291,\n",
" 1289,\n",
" 7,\n",
" 1041,\n",
" 19,\n",
" 504,\n",
" 1882,\n",
" 10141,\n",
" 79,\n",
" 501,\n",
" 494,\n",
" 1257,\n",
" 6468,\n",
" 7,\n",
" 491,\n",
" 22,\n",
" 79,\n",
" 2764,\n",
" 3,\n",
" 47326,\n",
" 15138,\n",
" 54119,\n",
" 8,\n",
" 46,\n",
" 5,\n",
" 549,\n",
" 69,\n",
" 8,\n",
" 32,\n",
" 39,\n",
" 41,\n",
" 18811,\n",
" 3,\n",
" 53,\n",
" 8,\n",
" 5,\n",
" 23,\n",
" 9,\n",
" 69,\n",
" 1,\n",
" 668,\n",
" 656,\n",
" 8,\n",
" 9,\n",
" 454,\n",
" 23528,\n",
" 20,\n",
" 112,\n",
" 1276,\n",
" 3,\n",
" 65,\n",
" 8,\n",
" 37,\n",
" 6,\n",
" 1,\n",
" 1051,\n",
" 190,\n",
" 15,\n",
" 23,\n",
" 68,\n",
" 754,\n",
" 35,\n",
" 21483,\n",
" 3]"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# we can now convert any piece of text first into tokens and then from tokens into indices into the vocabulary\n",
"vocab(tokenizer(proper_train_dataset[0][0]))"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Master'"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# we can also look up what any specific word index refers to\n",
"vocab.lookup_token(3752)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"proper_train_encoded = [vocab(tokens) for tokens in proper_train_dataset_as_tokens_without_labels]\n",
"# note that another way to have written the above line is to instead write the line below (but this would repeat the work of tokenization which we already did):\n",
"# proper_train_encoded = [vocab(tokenizer(text)) for text, label in proper_train_dataset]"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[3752,\n",
" 87615,\n",
" 12966,\n",
" 17640,\n",
" 1240,\n",
" 7,\n",
" 178,\n",
" 20,\n",
" 172,\n",
" 173,\n",
" 43,\n",
" 31,\n",
" 5,\n",
" 194,\n",
" 6,\n",
" 34,\n",
" 250,\n",
" 3,\n",
" 158,\n",
" 15,\n",
" 23,\n",
" 115,\n",
" 98,\n",
" 78,\n",
" 17640,\n",
" 49,\n",
" 250,\n",
" 1093,\n",
" 46,\n",
" 6444,\n",
" 63116,\n",
" 2,\n",
" 39058,\n",
" 63412,\n",
" 2,\n",
" 16573,\n",
" 32452,\n",
" 4,\n",
" 13479,\n",
" 36868,\n",
" 2001,\n",
" 20,\n",
" 642,\n",
" 1339,\n",
" 46,\n",
" 419,\n",
" 2,\n",
" 2416,\n",
" 2,\n",
" 1804,\n",
" 2,\n",
" 142,\n",
" 4,\n",
" 79,\n",
" 872,\n",
" 11585,\n",
" 27,\n",
" 304,\n",
" 3595,\n",
" 494,\n",
" 3,\n",
" 24,\n",
" 12327,\n",
" 917,\n",
" 6,\n",
" 653,\n",
" 5512,\n",
" 40,\n",
" 168,\n",
" 99,\n",
" 8,\n",
" 3863,\n",
" 19,\n",
" 80,\n",
" 399,\n",
" 2,\n",
" 44,\n",
" 5998,\n",
" 287,\n",
" 96,\n",
" 835,\n",
" 5,\n",
" 12327,\n",
" 30,\n",
" 3,\n",
" 158,\n",
" 61332,\n",
" 255,\n",
" 47326,\n",
" 15138,\n",
" 54119,\n",
" 2,\n",
" 17640,\n",
" 3446,\n",
" 20,\n",
" 67,\n",
" 2,\n",
" 1964,\n",
" 4,\n",
" 1060,\n",
" 7,\n",
" 9490,\n",
" 1,\n",
" 4811,\n",
" 6,\n",
" 5,\n",
" 9632,\n",
" 923,\n",
" 6,\n",
" 448,\n",
" 2108,\n",
" 90,\n",
" 863,\n",
" 142,\n",
" 3669,\n",
" 86,\n",
" 419,\n",
" 57,\n",
" 53,\n",
" 18,\n",
" 3290,\n",
" 177,\n",
" 636,\n",
" 9,\n",
" 5389,\n",
" 12,\n",
" 17640,\n",
" 103,\n",
" 5,\n",
" 648,\n",
" 747,\n",
" 23,\n",
" 14617,\n",
" 14452,\n",
" 14617,\n",
" 14452,\n",
" 69,\n",
" 18,\n",
" 107,\n",
" 50,\n",
" 142,\n",
" 4,\n",
" 1941,\n",
" 3,\n",
" 29251,\n",
" 6,\n",
" 15,\n",
" 23,\n",
" 8,\n",
" 1337,\n",
" 4,\n",
" 799,\n",
" 90,\n",
" 331,\n",
" 4,\n",
" 1013,\n",
" 142,\n",
" 68,\n",
" 80,\n",
" 291,\n",
" 1289,\n",
" 7,\n",
" 1041,\n",
" 19,\n",
" 504,\n",
" 1882,\n",
" 10141,\n",
" 79,\n",
" 501,\n",
" 494,\n",
" 1257,\n",
" 6468,\n",
" 7,\n",
" 491,\n",
" 22,\n",
" 79,\n",
" 2764,\n",
" 3,\n",
" 47326,\n",
" 15138,\n",
" 54119,\n",
" 8,\n",
" 46,\n",
" 5,\n",
" 549,\n",
" 69,\n",
" 8,\n",
" 32,\n",
" 39,\n",
" 41,\n",
" 18811,\n",
" 3,\n",
" 53,\n",
" 8,\n",
" 5,\n",
" 23,\n",
" 9,\n",
" 69,\n",
" 1,\n",
" 668,\n",
" 656,\n",
" 8,\n",
" 9,\n",
" 454,\n",
" 23528,\n",
" 20,\n",
" 112,\n",
" 1276,\n",
" 3,\n",
" 65,\n",
" 8,\n",
" 37,\n",
" 6,\n",
" 1,\n",
" 1051,\n",
" 190,\n",
" 15,\n",
" 23,\n",
" 68,\n",
" 754,\n",
" 35,\n",
" 21483,\n",
" 3]"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"proper_train_encoded[0]"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['Master', 'cinéaste', 'Alain', 'Resnais', 'likes', 'to', 'work', 'with', 'those', 'actors', 'who', 'are', 'a', 'part', 'of', 'his', 'family', '.', 'In', 'this', 'film', 'too', 'we', 'see', 'Resnais', \"'\", 'family', 'members', 'like', 'Pierre', 'Arditi', ',', 'Sabine', 'Azema', ',', 'André', 'Dussolier', 'and', 'Fanny', 'Ardant', 'dealing', 'with', 'serious', 'themes', 'like', 'death', ',', 'religion', ',', 'suicide', ',', 'love', 'and', 'their', 'overall', 'implications', 'on', 'our', 'daily', 'lives', '.', 'The', 'formal', 'nature', 'of', 'relationship', 'shared', 'by', 'these', 'people', 'is', 'evident', 'as', 'even', 'friends', ',', 'they', 'address', 'each', 'other', 'using', 'a', 'formal', 'you', '.', 'In', '1984,while', 'making', \"L'amour\", 'à', 'mort', ',', 'Resnais', 'dealt', 'with', 'time', ',', 'memory', 'and', 'space', 'to', 'unravel', 'the', 'mysteries', 'of', 'a', 'fundamental', 'question', 'of', 'human', 'existence', ':', 'Is', 'love', 'stronger', 'than', 'death', '?', 'It', 'was', '16', 'years', 'ago', 'in', '1968', 'that', 'Resnais', 'made', 'a', 'somewhat', 'similar', 'film', 'Je', \"t'aime\", 'Je', \"t'aime\", 'which', 'was', 'also', 'about', 'love', 'and', 'memories', '.', 'Message', 'of', 'this', 'film', 'is', 'loud', 'and', 'clear', ':', 'true', 'and', 'deep', 'love', 'can', 'even', 'put', 'science', 'to', 'shame', 'as', 'dead', 'lovers', 'regain', 'their', 'lost', 'lives', 'leaving', 'doctors', 'to', 'care', 'for', 'their', 'reputation', '.', \"L'amour\", 'à', 'mort', 'is', 'like', 'a', 'game', 'which', 'is', 'not', 'at', 'all', 'didactic', '.', 'It', 'is', 'a', 'film', 'in', 'which', 'the', 'musical', 'score', 'is', 'in', 'perfect', 'tandem', 'with', 'its', 'images', '.', 'This', 'is', 'one', 'of', 'the', 'reasons', 'why', 'this', 'film', 'can', 'easily', 'be', 'grasped', '.']\n"
]
}
],
"source": [
"# we can reconstruct any original review from the encoded version of the review\n",
"print([vocab.lookup_token(word_idx) for word_idx in proper_train_encoded[0]])"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"proper_train_labels = [label for text, label in proper_train_dataset]"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"val_encoded = [vocab(tokenizer(text)) for text, label in val_dataset]"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"val_labels = [label for text, label in val_dataset]"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"proper_train_dataset_encoded = list(zip(proper_train_encoded, proper_train_labels))\n",
"val_dataset_encoded = list(zip(val_encoded, val_labels))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Setting up a recurrent neural net for sentiment analysis that uses pre-trained word embeddings"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We first load in pre-trained GloVe embeddings only for tokens that we encountered in the proper training data."
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"from torchtext.vocab import GloVe\n",
"pretrained_embedding = GloVe(name='6B', dim=100)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"torchtext.vocab.vectors.GloVe"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"type(pretrained_embedding)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([ 0.2309, 0.2828, 0.6318, -0.5941, -0.5860, 0.6326, 0.2440, -0.1411,\n",
" 0.0608, -0.7898, -0.2910, 0.1429, 0.7227, 0.2043, 0.1407, 0.9876,\n",
" 0.5253, 0.0975, 0.8822, 0.5122, 0.4020, 0.2117, -0.0131, -0.7162,\n",
" 0.5539, 1.1452, -0.8804, -0.5022, -0.2281, 0.0239, 0.1072, 0.0837,\n",
" 0.5501, 0.5848, 0.7582, 0.4571, -0.2800, 0.2522, 0.6896, -0.6097,\n",
" 0.1958, 0.0442, -0.3114, -0.6883, -0.2272, 0.4618, -0.7716, 0.1021,\n",
" 0.5564, 0.0674, -0.5721, 0.2374, 0.4717, 0.8277, -0.2926, -1.3422,\n",
" -0.0993, 0.2814, 0.4160, 0.1058, 0.6220, 0.8950, -0.2345, 0.5135,\n",
" 0.9938, 1.1846, -0.1636, 0.2065, 0.7385, 0.2406, -0.9647, 0.1348,\n",
" -0.0072, 0.3302, -0.1236, 0.2719, -0.4095, 0.0219, -0.6069, 0.4076,\n",
" 0.1957, -0.4180, 0.1864, -0.0327, -0.7857, -0.1385, 0.0440, -0.0844,\n",
" 0.0491, 0.2410, 0.4527, -0.1868, 0.4618, 0.0891, -0.1819, -0.0152,\n",
" -0.7368, -0.1453, 0.1510, -0.7149])"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pretrained_embedding['cat']\n",
"# note that if you ask for a word embedding for a word that GloVe does not keep\n",
"# track of, you'll get all zeros"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
"embedding_matrix = torch.zeros(len(vocab), pretrained_embedding.dim)\n",
"for i, token in enumerate(vocab.lookup_tokens(range(len(vocab)))):\n",
" embedding_matrix[i] = pretrained_embedding[token]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The next code cell constructs a PyTorch recurrent neural net model for sentiment analysis. Unfortunately, the code involved doesn't readily work with ``nn.Sequential`` that we have previously been using (for somewhat complicated reasons; in short, it has to do with an input batch of time series possibly having varying lengths, and accounting for varying lengths properly is not easy to do with ``nn.Sequential``). We will instead build the PyTorch model using another standard approach by creating a Python class that inherits from the `nn.Module` class.\n",
"\n",
"To illustrate how this works, here was how we created the multilayer perceptron model for MNIST digits in the previous demo:\n",
"\n",
"```\n",
"deeper_model = nn.Sequential(nn.Flatten(),\n",
" nn.Linear(in_features=784, out_features=512),\n",
" nn.ReLU(),\n",
" nn.Linear(in_features=512, out_features=10))\n",
"```\n",
"\n",
"An alternative way to code the same model is as follows:\n",
"\n",
"```\n",
"class DeeperModel(nn.Module):\n",
" def __init__(self, num_in_features, num_intermediate_features, num_out_features):\n",
" super().__init__()\n",
" self.flatten = nn.Flatten()\n",
" self.linear1 = nn.Linear(num_in_features, num_intermediate_features)\n",
" self.relu = nn.ReLU()\n",
" self.linear2 = nn.Linear(num_intermediate_features, num_out_features)\n",
"\n",
" def forward(self, inputs):\n",
" flatten_output = self.flatten(inputs)\n",
" linear1_output = self.linear1(flatten_output)\n",
" relu_output = self.relu(linear1_output)\n",
" linear2_output = self.linear2(relu_output)\n",
" return linear2_output\n",
"\n",
"deeper_model = DeeperModel(784, 512, 10)\n",
"```\n",
"\n",
"**Importantly, in the above code, the `forward` function specifies how the neural net processes input data. Note that the only argument it takes (aside from `self`) is the input data (`inputs`), which for MNIST digits we saw will be of the format (batch size, 1, 28, 28). Consequently, the example input data batch supplied to the `summary` function only needs this 4D table.**\n",
"\n",
"The code below creates the PyTorch neural net model corresponding to the architecture:\n",
"\n",
"1. `Embedding` layer (for every time series: convert the word index at every time step into a 100-dimensional GloVe word embedding)\n",
"2. `LSTM` layer with 64 output nodes (for every time series: put the time series through the LSTM's `for` loop and grab only the last time step's output, which has 64 numbers)\n",
"3. `Linear` layer with 2 output nodes (now every input data point to the linear layer is just a 1D table of 64 numbers, which this linear layer converts to 2 output numbers corresponding to the two classes)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
"class EmbeddingLSTMLinearModel(nn.Module):\n",
" def __init__(self, embedding_matrix, num_lstm_output_nodes, num_final_output_nodes):\n",
" super().__init__()\n",
" self.embedding_layer = nn.Embedding.from_pretrained(embedding_matrix)\n",
" self.lstm_layer = nn.LSTM(embedding_matrix.shape[1], num_lstm_output_nodes)\n",
" self.linear_layer = nn.Linear(num_lstm_output_nodes, num_final_output_nodes)\n",
"\n",
" def forward(self, text_encodings, lengths):\n",
" embeddings = self.embedding_layer(text_encodings)\n",
"\n",
" rnn_last_time_step_outputs = \\\n",
" UDA_get_rnn_last_time_step_outputs(embeddings, lengths, self.lstm_layer)\n",
"\n",
" return self.linear_layer(rnn_last_time_step_outputs)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Notice that the `forward` function takes _two_ inputs (aside from `self`): `text_encodings` and `lengths`.** Keep in mind that `forward` takes in a batch of input time series, and these input time series could have different lengths. These different lengths are precisely what is stored in `lengths` as a 1D table of integers. Meanwhile, `text_encodings` is a 2D table where the number of rows is the maximum number of time steps in the input batch of time series, and the number of columns is the number of time series in the input batch. See the lecture slides for how `text_encodings` gets filled in (basically we pad all the time series in the batch to be of the same length as the longest time series; the padded entries will of course get ignored by the LSTM layer since it will know the correct length to use per time series). **When we give this neural net model an example data batch using the `summary` function, we need to specify two inputs corresponding to `text_encodings` and `lengths`.**"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"simple_lstm_model = EmbeddingLSTMLinearModel(embedding_matrix, 64, 2)"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"==========================================================================================\n",
"Layer (type:depth-idx) Output Shape Param #\n",
"==========================================================================================\n",
"EmbeddingLSTMLinearModel [5, 2] --\n",
"├─Embedding: 1-1 [7, 5, 100] (10,814,200)\n",
"├─LSTM: 1-2 [18, 64] 42,496\n",
"├─Linear: 1-3 [5, 2] 130\n",
"==========================================================================================\n",
"Total params: 10,856,826\n",
"Trainable params: 42,626\n",
"Non-trainable params: 10,814,200\n",
"Total mult-adds (M): 124.66\n",
"==========================================================================================\n",
"Input size (MB): 0.00\n",
"Forward/backward pass size (MB): 0.04\n",
"Params size (MB): 43.43\n",
"Estimated Total Size (MB): 43.47\n",
"=========================================================================================="
]
},
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# example where there are 5 input time series of lengths 3, 2, 5, 1, 7;\n",
"# we specify these time series using a 2D table that is padded and a\n",
"# 1D table of lengths (see lecture slides for details)\n",
"summary(simple_lstm_model,\n",
" input_data=[torch.zeros((7, 5), dtype=torch.long),\n",
" torch.tensor([3, 2, 5, 1, 7], dtype=torch.long)])\n",
"\n",
"# note: the LSTM's output is in a compressed format (called a \"packed sequence\")\n",
"# that appears to put all the outputs of all the time series together (the output\n",
"# shape in this case appears to be 18 by 64) but it actually does keep track of\n",
"# which of the time steps correspond to which input time series (i.e., it knows\n",
"# that 3 of the rows correspond to the 0-th time series, 2 of the rows correspond\n",
"# to the 1st time series, 5 of the rows correspond to the 2nd time series, 1 row\n",
"# corresponds to the 3rd time series, and 7 rows correspond to the 4th time\n",
"# series); my helper code automatically maps these rows back to the correct\n",
"# format so that the final linear layer recognizes that there are 5 input data\n",
"# points"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"os.makedirs('./saved_model_checkpoints', exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
" 0%| | 0/10 [00:00<?, ?it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Train accuracy: 0.7784\n",
" Validation accuracy: 0.7780\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 10%|████████████████▊ | 1/10 [00:14<02:08, 14.27s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Train accuracy: 0.8530\n",
" Validation accuracy: 0.8456\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 20%|█████████████████████████████████▌ | 2/10 [00:28<01:53, 14.25s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Train accuracy: 0.8858\n",
" Validation accuracy: 0.8654\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 30%|██████████████████████████████████████████████████▍ | 3/10 [00:42<01:39, 14.21s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Train accuracy: 0.9025\n",
" Validation accuracy: 0.8704\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 40%|███████████████████████████████████████████████████████████████████▏ | 4/10 [00:56<01:25, 14.22s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Train accuracy: 0.9155\n",
" Validation accuracy: 0.8716\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 50%|████████████████████████████████████████████████████████████████████████████████████ | 5/10 [01:11<01:11, 14.27s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Train accuracy: 0.9301\n",
" Validation accuracy: 0.8770\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 60%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 6/10 [01:25<00:56, 14.24s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Train accuracy: 0.9284\n",
" Validation accuracy: 0.8666\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 70%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 7/10 [01:39<00:42, 14.30s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Train accuracy: 0.9562\n",
" Validation accuracy: 0.8776\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 8/10 [01:53<00:28, 14.24s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Train accuracy: 0.9637\n",
" Validation accuracy: 0.8744\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 9/10 [02:08<00:14, 14.20s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Train accuracy: 0.9705\n",
" Validation accuracy: 0.8728\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10/10 [02:22<00:00, 14.23s/it]\n"
]
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAGwCAYAAAC99fF4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB10klEQVR4nO3deVxVdf7H8dflsiOCigJuiPuCO4pilpW5ZE62qqVpLmXTxjj1K6csdZwsW2cymTQ1LUuyvck0bXFX3HPLfRdEUAFFtsv5/XHkKgIKCFyW9/PxOA/ge88953PSvG++53u+X4thGAYiIiIikoOTowsQERERKYsUkkRERETyoJAkIiIikgeFJBEREZE8KCSJiIiI5EEhSURERCQPCkkiIiIieXB2dAHlVVZWFidPnsTb2xuLxeLockRERKQADMMgOTmZ2rVr4+R07b4ihaQiOnnyJPXq1XN0GSIiIlIEx44do27dutfcRyGpiLy9vQHzP3LVqlUdXI2IiIgURFJSEvXq1bN/jl+LQlIRZd9iq1q1qkKSiIhIOVOQoTIauC0iIiKSB4UkERERkTw4PCRNnz6d4OBg3N3d6dixIytXrrzm/h988AEtWrTAw8ODZs2aMW/evByv9+jRA4vFkmvr16+ffZ8JEybkej0gIKBErk9ERETKJ4eOSYqKiiIiIoLp06fTrVs3PvzwQ/r27cuuXbuoX79+rv0jIyMZN24cM2fOpFOnTkRHRzN69GiqVatG//79Afj6669JT0+3vychIYG2bdvywAMP5DhWq1atWLZsmf1nq9VaItdos9nIyMgokWNL6XJxcSmxvyciIlL2ODQkvfPOO4wcOZJRo0YB8N5777FkyRIiIyOZMmVKrv0/+eQTHn/8cQYOHAhAw4YNWbduHW+88YY9JFWvXj3HexYsWICnp2eukOTs7FyivUeGYRAbG8u5c+dK7BxS+nx9fQkICNDcWCIilYDDQlJ6ejqbNm3ixRdfzNHeq1cv1qxZk+d70tLScHd3z9Hm4eFBdHQ0GRkZuLi45HrPrFmzGDRoEF5eXjna9+3bR+3atXFzcyMsLIzXXnuNhg0b5ltvWloaaWlp9p+TkpKueX3ZAalWrVp4enrqQ7WcMwyDlJQU4uLiAAgMDHRwRSIiUtIcFpLi4+Ox2Wz4+/vnaPf39yc2NjbP9/Tu3ZuPPvqIAQMG0KFDBzZt2sTs2bPJyMggPj4+1wdXdHQ0O3bsYNasWTnaw8LCmDdvHk2bNuXUqVNMnjyZ8PBwdu7cSY0aNfI895QpU5g4cWKBrs1ms9kDUn7Hk/LHw8MDgLi4OGrVqqVbbyIiFZzDB25f3cNiGEa+vS7jx4+nb9++dOnSBRcXF+6++26GDx8O5D2maNasWYSEhNC5c+cc7X379uW+++6jdevW9OzZkx9//BGAuXPn5lvnuHHjSExMtG/Hjh3Ld9/sMUienp757iPlU/afqcaZiYhUfA4LSX5+flit1ly9RnFxcbl6l7J5eHgwe/ZsUlJSOHz4MEePHqVBgwZ4e3vj5+eXY9+UlBQWLFhgH+90LV5eXrRu3Zp9+/blu4+bm5t94siCTiCpW2wVj/5MRUQqD4eFJFdXVzp27MjSpUtztC9dupTw8PBrvtfFxYW6detitVpZsGABd911V65F6r744gvS0tIYMmTIdWtJS0tj9+7dGmciIiJSBtiyDNYeSOC7rSdYeyABW5bhkDoc+nTb2LFjGTp0KKGhoXTt2pUZM2Zw9OhRxowZA5i3uE6cOGGfC2nv3r1ER0cTFhbG2bNneeedd9ixY0eet8lmzZrFgAED8hwT9Nxzz9G/f3/q169PXFwckydPJikpiWHDhpXsBYuIiMg1Ld4Rw8QfdhGTmGpvC/Rx59X+LekTUrqdGQ4NSQMHDiQhIYFJkyYRExNDSEgIixYtIigoCICYmBiOHj1q399ms/H222+zZ88eXFxcuPXWW1mzZg0NGjTIcdy9e/eyatUqfv755zzPe/z4cQYPHkx8fDw1a9akS5curFu3zn7essSWZRB96AxxyanU8nanc3B1rE7l65ZPjx49aNeuHe+9916B9j98+DDBwcFs2bKFdu3alWhtIiJSdizeEcMTn27m6n6j2MRUnvh0M5FDOpRqULIYhuGYPqxyLikpCR8fHxITE3ONT0pNTeXQoUP2mcSLqrTT9PXG2wwbNoyPP/640Mc9c+YMLi4uBVpxGcwwfPr0afz8/HB2LltrMBfXn62IiORkyzK46Y1fc3zmXckCBPi4s+qF226os+Ban99XK1ufQGLniDQdExNj/z4qKopXXnmFPXv22NuyH4HPlt/cVFe7eoLP67FarVomRkSkErFlGXyx8Wi+AQnAAGISU4k+dIaujUpneh2HTwFQWRiGQUp6ZoG25NQMXv1+Z66ABNjbJny/i+TUjOseqzAdhQEBAfbNx8fHvqZdQEAAqamp+Pr68sUXX9CjRw/c3d359NNPSUhIYPDgwdStWxdPT09at27N559/nuO4PXr0ICIiwv5zgwYNeO211xgxYgTe3t7Ur1+fGTNm2F8/fPgwFouFrVu3AvD7779jsVj45ZdfCA0NxdPTk/Dw8BwBDmDy5MnUqlULb29vRo0axYsvvqjbdSIiZVBqho11BxOY9us+hs2Opt3Enxn39Y4CvTcuOf8gVdzUk1RKLmbYaPnKkmI5lgHEJqXSekLeY66utGtSbzxdi++P+YUXXuDtt99mzpw5uLm5kZqaSseOHXnhhReoWrUqP/74I0OHDqVhw4aEhYXle5y3336bf/7zn/zjH//gyy+/5IknnuDmm2+mefPm+b7npZde4u2336ZmzZqMGTOGESNGsHr1agDmz5/Pv/71L/s6gAsWLODtt98mODi42K5dRESK5uyFdDYeOcvGw2eIPnyGHScSybDl/CXe3dmJ1Mys6x6rlnfpDXVQSJJCiYiI4N57783R9txzz9m/f/rpp1m8eDELFy68Zki68847+etf/wqYwevdd9/l999/v2ZI+te//sUtt9wCwIsvvki/fv1ITU3F3d2d999/n5EjR/Loo48C8Morr/Dzzz9z/vz5Il+riIgUnmEYHD97kQ2Hz1zazrI/Lve/xTW93ejcoDqhDarRqUF1mtSqQo+3fic2MTXPOynZY5I6BxduCMeNUEgqJR4uVnZN6l2gfaMPnWH4nA3X3e/jRztd9y+Lh0vxLp0RGhqa42ebzcbrr79OVFQUJ06csK9xd/VaeVdr06aN/fvs23rZ66IV5D3Zc1rFxcVRv3599uzZYw9d2Tp37syvv/5aoOsSEZGisWUZ/BmbxMbDZ4k+fIaNh89wKikt136NanrRqUF1+1avukeuB4Ze7d+SJz7djAVyBCXLFa+X5hPeCkmlxGKxFPi2V/cmNQn0cb9umu7epGapTwdwdfh5++23effdd3nvvfdo3bo1Xl5eREREkJ6efs3jXD3g22KxkJV17W7WK9+T/T/Wle/Ja4kbEREpXhfTbWw9do6Nh8+w4chZNh85y/m0zBz7ODtZaF3Xh04NqhMaVI2OQdWoUcXtusfuExJI5JAOuZ7sDqiM8yRJ3qxOljKXpvOzcuVK7r77bvvM5llZWezbt48WLVqUah3NmjUjOjqaoUOH2ts2btxYqjWIiFREZy6ks/HwGTYeOcuGfMYTVXFzpkNQNToFVSO0QXXa1fPFw7VodzL6hARyR8uAMjFHoEJSGVXW0nR+GjduzFdffcWaNWuoVq0a77zzDrGxsaUekp5++mlGjx5NaGgo4eHhREVF8ccff9CwYcNSrUNEpDwzDINjZ8zxRBuP5D+eqJa3G52Cq9tDUYvAqsUaYqxOllJ7zP9aFJLKsLKUpvMzfvx4Dh06RO/evfH09OSxxx5jwIABJCYmlmodDz/8MAcPHuS5554jNTWVBx98kOHDhxMdHV2qdYiIOFJhV2mwZRnsjkmy3zrLbzxR41pV6NSgGqFB1ekcXJ261XKPJ6qINON2EZXGjNtyY+644w4CAgL45JNPiu2Y+rMVkbKqIKs0XDmeKPrwGbYcPZdrPJGL1UJIHR/7AOuOQdWo7uVaqtdSkjTjtlQ6KSkp/Pe//6V3795YrVY+//xzli1bxtKlSx1dmohIibvWKg1jPt1Mzxa1iD+fzo4TiWRm5dzLO3s8UQPz1lnbukUfT1TRKCRJhWCxWFi0aBGTJ08mLS2NZs2a8dVXX9GzZ09HlyYiUqJsWQYTf9h1zVUalu2+PMWKf1U3ey9RaINqNA8o3vFEFYlCklQIHh4eLFu2zNFliIiUqnMp6cxdc/iaa55l+2uPRgzuXL/SjCcqDgpJIiIi5cTFdBsbDp9h9YF41uxPYMfJRAo6srhZgDf1qnuWbIEVjEKSiIhIGZVpy+KPE4ms3hfP6gPxbD5yjnRbzol36/i6c+Lc9XuSSnPNs4pCIUlERKSMMAyDfXHnWb0/ntX741l/8AzJVz19FujjTrfGfnRrXIPwRn74VXHjpjd+LVNrnlUUCkkiIiIOdOLcRVbvj2fN/nhWH0jgdHLOeYp8PFzo2rAG3Zr40a1RDYL9vMr8mmcVhUKSiIhIKTqXks7aAwms2h/PmgMJHIq/kON1N2cnOgdXJ7yRHzc19qNl7es/fVZeVmkobxSSpFj16NGDdu3a8d577wHQoEEDIiIiiIiIyPc9FouFb775hgEDBtzQuYvrOCIixck+2Hq/Oa5o58mkHIOtnSzQpq4vNzX2I7xxDTrUr4a7S+HnKSoPqzSUNwpJYte/f38uXryY56P0a9euJTw8nE2bNtGhQ4cCH3PDhg14eXkVZ5lMmDCBb7/9lq1bt+Zoj4mJoVq1asV6LhGRwsq0ZbHteCJr9sezan88W47mHmzdpFaVS+OK/AhrWJ2q7i7Fcu6ysuZZRaGQVB4kx8LGORD6KHgHlNhpRo4cyb333suRI0cICgrK8drs2bNp165doQISQM2aNYuzxGsKCCi5/zYiIvnJHmy9al88aw7Es+7gmVxLfdT2cSe8sXn7LLxRDWpV1ZNm5YGTowuQAkiOheWvm19L0F133UWtWrX4+OOPc7SnpKQQFRXFgAEDGDx4MHXr1sXT05PWrVvz+eefX/OYDRo0sN96A9i3bx8333wz7u7utGzZMs9lQ1544QWaNm2Kp6cnDRs2ZPz48WRkZADw8ccfM3HiRLZt24bFYsFisdjrtVgsfPvtt/bjbN++ndtuuw0PDw9q1KjBY489xvnzl1ezHj58OAMGDOCtt94iMDCQGjVq8OSTT9rPJSKVgy3LYO2BBL7beoK1BxKwZV1/4qET5y7yxcZjPLtgC51f+4Ve765g0v92sWx3HOfTMvHxcKFvSAD/HBDCb8/1YPWLt/HWA20Z0L6OAlI5op6k0mIYkJFStPdmXrz8Nf3Ctfe9mosnFHBmVWdnZx555BE+/vhjXnnlFfvTEwsXLiQ9PZ1Ro0bx+eef88ILL1C1alV+/PFHhg4dSsOGDQkLC7vu8bOysrj33nvx8/Nj3bp1JCUl5TlWydvbm48//pjatWuzfft2Ro8ejbe3N//3f//HwIED2bFjB4sXL7bfFvTx8cl1jJSUFPr06UOXLl3YsGEDcXFxjBo1iqeeeipHCPztt98IDAzkt99+Y//+/QwcOJB27doxevToAv03E5HyrSCLwgKcvZDO2oMJ9kfzDyfk/Pfc3cWJTg2qm7fQGhVssLWUfQpJpSUjBV6rfWPHmN2n8O/5x0lwLfiYoBEjRvDmm2/y+++/c+utt5qnnT2be++9lzp16vDcc8/Z93366adZvHgxCxcuLFBIWrZsGbt37+bw4cPUrVsXgNdee42+ffvm2O/ll1+2f9+gQQP+/ve/ExUVxf/93//h4eFBlSpVcHZ2vubttfnz53Px4kXmzZtnHxM1bdo0+vfvzxtvvIG/vz8A1apVY9q0aVitVpo3b06/fv345ZdfFJJEKoFrLQr7xKebeeq2xqRnZuU52NrqZKFNXR+6NTLHFXUI8sXNWYvCVjQKSZJD8+bNCQ8PZ/bs2dx6660cOHCAlStX8vPPP2Oz2Xj99deJiorixIkTpKWlkZaWVuCB2bt376Z+/fr2gATQtWvXXPt9+eWXvPfee+zfv5/z58+TmZlJ1apVC3Udu3fvpm3btjlq69atG1lZWezZs8ceklq1aoXVevkftsDAQLZv316oc4lI+VOQRWHf/3V/jvam/lXsj+V3LsbB1lJ2KSSVFhdPs1enoM6fMjeA2O2w6Hm4800IaG22VfE3t4Kct5BGjhzJU089xQcffMCcOXMICgri9ttv58033+Tdd9/lvffeo3Xr1nh5eREREUF6enqBjmvkscDQ1ROirVu3jkGDBjFx4kR69+6Nj48PCxYs4O233y7UNRiGke8Cjle2u7i45HotKyvr6reISAUTfehMgRaFvaWpH/e0r6vB1pWUQlJpsVgKdduL6g3NDcDZw/xatzPUblfspV3twQcf5Nlnn+Wzzz5j7ty5jB49GovFwsqVK7n77rsZMmQIYI4x2rdvHy1atCjQcVu2bMnRo0c5efIktWubtx7Xrl2bY5/Vq1cTFBTESy+9ZG87cuRIjn1cXV2x2WzXPdfcuXO5cOGCvTdp9erVODk50bRp0wLVKyIVV1zy9QMSwL0d6nJ3uzolXI2UVXq6TXKpUqUKAwcO5B//+AcnT55k+PDhADRu3JilS5eyZs0adu/ezeOPP05sbMGfuOvZsyfNmjXjkUceYdu2baxcuTJHGMo+x9GjR1mwYAEHDhzgP//5D998802OfRo0aMChQ4fYunUr8fHxpKXlnMIf4OGHH8bd3Z1hw4axY8cOfvvtN55++mmGDh1qv9UmIpVXQRd71aKwlZtCUnngHQC3vFiicyRdbeTIkZw9e5aePXtSv359AMaPH0+HDh3o3bs3PXr0ICAgoFCzWzs5OfHNN9+QlpZG586dGTVqFP/6179y7HP33Xfzt7/9jaeeeop27dqxZs0axo8fn2Of++67jz59+nDrrbdSs2bNPKch8PT0ZMmSJZw5c4ZOnTpx//33c/vttzNt2rTC/8cQkQqnVe2quFrzf/rMgvmUmxaFrdwsRl4DReS6kpKS8PHxITExMdeg4tTUVA4dOkRwcDDu7votpCLRn61I+ZeUmsGjczaw6cjZPF/Pjk6RQzpozbMK6Fqf31dTT5KIiFQa51LSGfLRejYdOUtVd2de6NOcQJ+cv/AE+LgrIAmggdsiIlJJnLlgBqRdMUlU83Thk5FhhNTx4bGbG2pRWMmTQpKIiFR4p5PTePijdew9dR6/Kq7MH9WFZgHegBaFlfw5/Hbb9OnT7eM7OnbsyMqVK6+5/wcffECLFi3w8PCgWbNmzJs3L8frH3/8sX1Nryu31NScj3sW9rwiIlI+xSamMnDGWvaeOo9/VTcWPNbVHpBErsWhISkqKoqIiAheeukltmzZQvfu3enbty9Hjx7Nc//IyEjGjRvHhAkT2LlzJxMnTuTJJ5/khx9+yLFf1apViYmJybFdOci2sOctKo2Jr3j0ZypSvhw/m8KDH67l4OkL1PH14IvHu9K4VhVHlyXlhEOfbgsLC6NDhw5ERkba21q0aMGAAQOYMmVKrv3Dw8Pp1q0bb775pr0tIiKCjRs3smrVKsDsSYqIiODcuXPFdt68XGt0vM1mY+/evdSqVYsaNdSFW5EkJCQQFxdH06ZNcyxnIiJlz5GECzw0cz0nzl2kXnUPPh/dhbrVCr8KgVQshXm6zWFjktLT09m0aRMvvvhijvZevXqxZs2aPN+TlpaW67FrDw8PoqOjycjIsC8xcf78eYKCgrDZbLRr145//vOftG/fvsjnzT73lZMWJiUl5buv1WrF19eXuLg4wJyzJ78lMqR8MAyDlJQU4uLi8PX1VUASKeMOnD7PwzPXE5uUSrCfF5+NDiPQx8PRZUk547CQFB8fj81myzX7sb+/f76zOPfu3ZuPPvqIAQMG0KFDBzZt2sTs2bPJyMggPj6ewMBAmjdvzscff0zr1q1JSkri3//+N926dWPbtm00adKkSOcFmDJlChMnTizw9WWvUJ8dlKRi8PX1tf/ZikjZtPdUMg/NXE/8+TSa1KrC/FFhWndNisThT7dd3cNyrYVJx48fT2xsLF26dMEwDPz9/Rk+fDhTp061/2bfpUsXunTpYn9Pt27d6NChA++//z7/+c9/inRegHHjxjF27Fj7z0lJSdSrV++a1xUYGEitWrXIyMjIdz8pP1xcXNSDJFLG7TqZxJBZ6zlzIZ0WgVX5dGRnalRxc3RZUk45LCT5+flhtVpz9d7ExcXlu7aWh4cHs2fP5sMPP+TUqVMEBgYyY8YMvL298fPzy/M9Tk5OdOrUiX379hX5vABubm64uRX+fzSr1aoPVhGRUvDH8XMMnRVN4sUMWtfx4ZORnfH1dHV0WVKOOezpNldXVzp27MjSpUtztC9dupTw8PBrvtfFxYW6detitVpZsGABd911F05OeV+KYRhs3bqVwMDAGz6viIiUTZuOnOXhmetJvJhB+/q+zB8dpoAkN8yht9vGjh3L0KFDCQ0NpWvXrsyYMYOjR48yZswYwLzFdeLECftcSHv37iU6OpqwsDDOnj3LO++8w44dO5g7d679mBMnTqRLly40adKEpKQk/vOf/7B161Y++OCDAp9XRETKj3UHExjx8QZS0m10blCd2Y92ooqbw0eTSAXg0L9FAwcOJCEhgUmTJhETE0NISAiLFi0iKCgIgJiYmBxzF9lsNt5++2327NmDi4sLt956K2vWrKFBgwb2fc6dO8djjz1GbGwsPj4+tG/fnhUrVtC5c+cCn1dERMqHVfviGTVvA6kZWXRrXIOZj4Ti6aqAJMXDofMklWeFmWdBRESK329/xvH4p5tIz8yiR7Oa/HdIR9xdNAZUrq1czJMkIiJSVEt2xvLUZ5vJsBnc0dKfaQ+1x81ZAUmKl0KSiIiUK//74yQRC7aSmWXQr3Ug7w1qh4vV4UuRSgWkkCQiIuXGN1uO8/cvtpFlwD3t6/Dm/W1wVkCSEqKQJCIi5ULUhqO8+PV2DAMeDK3LlHvbYHXSkk9SchSSRESkzPtk7WHGf7cTgCFd6jPpLyE4KSBJCVNIEhGRMu2jlQeZ/ONuAEZ0C2b8XS20aLiUCoUkEREpsz74bT9vLtkDwBM9GvF/vZspIEmpUUgSEZEyxzAM3lu2j3//Yq67GdGzCc/e3kQBSUqVQpKIiJQphmEwdckeIn8/AMD/9WnGX3s0dnBVUhkpJImISJlhGAb//N9uZq8+BMDL/VowqntDB1cllZVCkoiIlAlZWQavfL+DT9eZa3b+8+5WDO3awLFFSaWmkCQiIg5nyzIY9/UffLHxOBYLvH5vawZ2qu/osqSSU0gSERGHyrRl8dzCbXy79SROFnj7wbbc076uo8sSUUgSERHHybBlEbFgKz9uj8HZycK/B7WnX5tAR5clAigkiYiIg6Rl2njqsy0s3XUKF6uFDx7qQK9WAY4uS8ROIUlEREpdaoaNMZ9u4vc9p3F1duLDIR25tXktR5clkoNCkoiIlKqU9ExGz9vI6v0JuLs48dEjnbipiZ+jyxLJRSFJRERKzfm0TEbM2UD04TN4uVqZPbwTYQ1rOLoskTwpJImISKlIvJjB8DnRbDl6Dm83Zz4e0ZmOQdUcXZZIvhSSRESkxJ29kM7Q2evZcSIJHw8XPhnZmTZ1fR1dlsg1KSSJiEiJij+fxpCP1vNnbDLVvVz5dGQYLWtXdXRZItelkCQiIiUmLimVhz9az7648/hVceOz0WE09fd2dFkiBaKQJCIiJSIm8SIPzVzPofgLBFR157PRYTSsWcXRZYkUmEKSiEgps2UZRB86Q1xyKrW83ekcXB2rk8XRZRWrY2dSeOijdRw7c5E6vh58ProL9Wt4OroskUJRSBIRKUWLd8Qw8YddxCSm2tsCfdx5tX9L+oRUjOU4Dsdf4KGZ6ziZmEpQDU/mjwqjbjUFJCl/nBxdgIhIZbF4RwxPfLo5R0ACiE1M5YlPN7N4R4yDKisaW5bB2gMJfLf1BGsPJGDLMtgfd54HP1zLycRUGtb0IuqxrgpIUm6pJ0lEpBTYsgwm/rALI4/Xstv+8c0Oanm74+vpgre7C97uzri7WEuzzALLq0fMr4oraZlZJKdm0tS/Cp+OCqOWt7sDqxS5MQpJIiKlIPrQmVw9SFc7cyGdeyPX5GhztTrh7e5MVQ8zNHm7O+Ptlv29C1U9nO2Bqqr7ld9f3sfVuXhvGmT3iF0d+OLPpwNQ19eDBY91pbqXa7GeV6S0KSSJiJQwwzBYtf90gfb19XDBlmWQnJYJQLoti4QL6SRcSC/y+d2cncxA5e6Mt8elr9cIW1eGrOyvzlYzaF2rRyxbRlYWPh4uRa5XpKxQSBIRKUHrDibwztK9RB86U6D9I4d0pGujGmRlGZxPzyTpYgbJqZmXtgz716TUTJJS83ntovn1QroNgLTMLNLOpxF/Pq3I1+HhYjXDkpPluj1ip5LSiD50hq6NtCablG8KSSIiJWDD4TO88/Ne1h5MAMDFyYKLsxMpl4LL1SxAgI85HQCAk5OFqu4uVHUveo+MLcvgfI4wZYarKwNVcmrmVYEr59fsei9m2LiYkXfteYlLvnaQEikPFJJERIrRpiNneW/ZXlbuiwfAxWphUKf6/PXWRmw7do4nPt0MkON2VfYMSa/2b1ms8yVZnSz4eLrg41n0oJVhy+L8pd6qpNQM1h9M4J8/7r7u+zRgWyoChSQRkWKw9dg53l26l+V7zbFHzk4WHgitx1O3NaaOrwcAgT4eRA7pkOupsIAyPE+Si9WJal6uVLs0CLtFYFU+WnWI2MTUPMclXd0jJlKeOXyepOnTpxMcHIy7uzsdO3Zk5cqV19z/gw8+oEWLFnh4eNCsWTPmzZuX4/WZM2fSvXt3qlWrRrVq1ejZsyfR0dE59pkwYQIWiyXHFhAQUOzXJiIV344TiYz8eAMDPljN8r2nsTpZGBhaj9+e68GUe1vbA1K2PiGBrHrhNj4f3YV/D2rH56O7sOqF28pkQMqL1cnCq/1bApd7wLKVVI+YiKM4tCcpKiqKiIgIpk+fTrdu3fjwww/p27cvu3bton79+rn2j4yMZNy4ccycOZNOnToRHR3N6NGjqVatGv379wfg999/Z/DgwYSHh+Pu7s7UqVPp1asXO3fupE6dOvZjtWrVimXLltl/tlrL5lwkIlI27TyZyHvL9rF01ykAnCxwT/u6PHN7Y4JqeF3zvVYnS7ke1NwnJLDc9YiJFIXFMIxrPclZosLCwujQoQORkZH2thYtWjBgwACmTJmSa//w8HC6devGm2++aW+LiIhg48aNrFq1Ks9z2Gw2qlWrxrRp03jkkUcAsyfp22+/ZevWrQWuNS0tjbS0y0+GJCUlUa9ePRITE6latWqBjyMi5due2GTeW7aXn3bEAmCxwIB2dXj6tsaVbvHWyrAGnVQ8SUlJ+Pj4FOjz22E9Senp6WzatIkXX3wxR3uvXr1Ys2ZNnu9JS0vD3T3nYEAPDw+io6PJyMjAxSX34MSUlBQyMjKoXj3n/fF9+/ZRu3Zt3NzcCAsL47XXXqNhw4b51jtlyhQmTpxY0MsTkQpmf1wy7y3bx4/bYzAMMxzd1aY2z97emMa1vB1dnkOU9x4xketx2Jik+Ph4bDYb/v7+Odr9/f2JjY3N8z29e/fmo48+YtOmTRiGwcaNG5k9ezYZGRnEx8fn+Z4XX3yROnXq0LNnT3tbWFgY8+bNY8mSJcycOZPY2FjCw8NJSEjIt95x48aRmJho344dO1aEqxaR8ubA6fM8u2ALd7y7gv/9YQakfq0DWRJxM+8Pbl9pA5JIZeDwp9sslpxds4Zh5GrLNn78eGJjY+nSpQuGYeDv78/w4cOZOnVqnmOKpk6dyueff87vv/+eoweqb9++9u9bt25N165dadSoEXPnzmXs2LF5ntvNzQ03N7eiXKKIlEOH4y/wn1/38e2WE2RdGpTQu5U/ET2b0iJQt9hFKgOHhSQ/Pz+sVmuuXqO4uLhcvUvZPDw8mD17Nh9++CGnTp0iMDCQGTNm4O3tjZ+fX45933rrLV577TWWLVtGmzZtrlmLl5cXrVu3Zt++fTd2USJS7h1NSOH9X/fx9ZYT2C6lo54tahHRsykhdXwcXJ2IlCaHhSRXV1c6duzI0qVLueeee+ztS5cu5e67777me11cXKhbty4ACxYs4K677sLJ6fKdwzfffJPJkyezZMkSQkNDr1tLWloau3fvpnv37kW8GhEp746fTeGD3/azcONxMi+Fo1ub1SSiZ1Pa1vN1bHEi4hAOvd02duxYhg4dSmhoKF27dmXGjBkcPXqUMWPGAOY4oBMnTtjnQtq7dy/R0dGEhYVx9uxZ3nnnHXbs2MHcuXPtx5w6dSrjx4/ns88+o0GDBvaeqipVqlClivnkyXPPPUf//v2pX78+cXFxTJ48maSkJIYNG1bK/wVExNFOnrvIB7/t54uNx8iwmeGoexM//nZHUzrUr+bg6kTEkRwakgYOHEhCQgKTJk0iJiaGkJAQFi1aRFBQEAAxMTEcPXrUvr/NZuPtt99mz549uLi4cOutt7JmzRoaNGhg32f69Omkp6dz//335zjXq6++yoQJEwA4fvw4gwcPJj4+npo1a9KlSxfWrVtnP6+IVHynklKZ/tt+Po8+RrotC4BujWvwt55NCW2g2aJFxMHzJJVnhZlnQUTKjrjkVP77+0Hmrz9CWqYZjjoHV2fsHU3p0lCPs4tUdOViniQRkdIUfz6ND5cf4JN1R0jNMMNRaFA1xt7RlK6NauT7VK2IVF4KSSJSoZ25kM6MFQeZu+YwFzNsALSv78vYO5pyU2M/hSMRyZdCkohUSOdS0vlo5SHmrD7EhXQzHLWt60PEHU3p0bSmwpGIXJdCkohUKIkXM5i16hBzVh0iOS0TgFa1qzL2jqbc1ryWwpGIFJhCkoiUG9daUDU5NYM5qw8zc+VBklPNcNQ8wJu/3dGUXi39FY5EpNAUkkSkXFi8I4aJP+wiJjHV3hbo487/9WnOyXMXmbHiIIkXMwBo6l+Fv/VsSu9WAThpVXoRKSKFJBEp8xbviOGJTzdz9XwlMYmp/C1qq/3nRjW9iOjZlH6tAxWOROSGKSSJSJlmyzKY+MOuXAHpSlYnC2/d34a/tKtjv/0mInKjnK6/i4iI40QfOpPjFltebFkGAT4eCkgiUqwUkkSkTNtxIrFA+8UlXztIiYgUlm63iUiZYxgGm46cZcaKg/y861SB3lPL272EqxKRykYhSUTKDFuWwZKdscxYcZCtx87Z292cnezrrF3NAgT4mNMBiIgUJ4UkEXG4lPRMFm48zqxVhzh6JgUAV2cn7utQh5E3BbM/7jxPfLoZIMcA7uwRSK/2b6nxSCJS7BSSRMRh4pJTmbfmCJ+sO2Kf48jX04VHugQxtGsDanq7AdC4ljeRQzrkmicpwMedV/u3pE9IoEPqF5GKTSFJRErdvlPJfLTyEN9sOUG6zbyNFlTDk1E3BXNfx7p4uub+p6lPSCB3tAzId8ZtEZHippAkIqXCMAzWHTzDzJUH+fXPOHt7h/q+PHZzQ+5oGXDdwGN1stC1UY2SLlVEBFBIEpESlmnLYtGOWGauOMj2S4/zWyzQu2UAo28OpmOQBlyLSNmkkCQiJeJ8WiZRG44xe9UhTpy7CIC7ixMPdKzHiJuCCfbzcnCFIiLXppAkIsUqNjGVOWsO8dn6oySnZgJQw8uVYeENGNIliOperg6uUESkYBSSRKRY7I5JYubKg3y/9SSZWeaD+g1rejG6e0PuaV8HdxergysUESkchSQRKTLDMFi1P54ZKw6ycl+8vb1zcHUe696Q25rXwklPn4lIOaWQJCKFlp6Zxf/+OMmMFQf5MzYZACcL9G0dyOjuDWlXz9exBYqIFAOFJBEpsKTUDD5ff5Q5qw8Tm2RO6ujpamVgp3qM6BZMveqeDq5QRKT4KCSJyHWdOHeROasOsWDDMc6nmYOxa3q78Wi3BjzcOQgfTxcHVygiUvwUkkQkXztOJDJjxUF+3B6D7dJg7Kb+VRjdvSF/aVcbN2cNxhaRikshSURyyMoyWL73NDNWHGTtwQR7e7fGNRjdvSG3NK2JxaLB2CJS8SkkiQgAaZk2vttykpkrD7Iv7jxgLgPSv00go7o3JKSOj4MrFBEpXQpJIpWALcvId2HYcynpzF9/lI/XHOZ0choAVdycGdy5HsO7BVPH18ORpYuIOIxCkkgFt3hHDBN/2EVMYqq9LdDHnb/2aMyB0+eJ2nCMixk2AAKqujPipgYM6lyfqu4ajC0ilZtCkkgFtnhHDE98uhnjqvaYxFTGf7fD/nOLwKo8fnND+rUJxMXqVLpFioiUUQpJIhWULctg4g+7cgWkK7k5O/HRI6Hc1MRPg7FFRK6iXxlFKqjoQ2dy3GLLS1pmFs5WJwUkEZE8ODwkTZ8+neDgYNzd3enYsSMrV6685v4ffPABLVq0wMPDg2bNmjFv3rxc+3z11Ve0bNkSNzc3WrZsyTfffHPD5xUpb46cuVCg/eKSrx2kREQqK4eGpKioKCIiInjppZfYsmUL3bt3p2/fvhw9ejTP/SMjIxk3bhwTJkxg586dTJw4kSeffJIffvjBvs/atWsZOHAgQ4cOZdu2bQwdOpQHH3yQ9evXF/m8IuVJVpZB1IajvPbj7gLtX8vbvYQrEhEpnyyGYVxryEKJCgsLo0OHDkRGRtrbWrRowYABA5gyZUqu/cPDw+nWrRtvvvmmvS0iIoKNGzeyatUqAAYOHEhSUhI//fSTfZ8+ffpQrVo1Pv/88yKdNy9JSUn4+PiQmJhI1apVC3fhIiVk05EzTPh+F9tPJALmPEfZM2VfzQIE+Liz6oXb7NMBiIhUdIX5/HZYT1J6ejqbNm2iV69eOdp79erFmjVr8nxPWloa7u45f+v18PAgOjqajIwMwOxJuvqYvXv3th+zKOfNPndSUlKOTaSsOJWUyt+itnJf5Fq2n0jE282Zl/u14L2B7bBgBqIrZf/8av+WCkgiIvlw2NNt8fHx2Gw2/P39c7T7+/sTGxub53t69+7NRx99xIABA+jQoQObNm1i9uzZZGRkEB8fT2BgILGxsdc8ZlHOCzBlyhQmTpxYlEsVKTFpmTZmrTrEtF/3k5Juw2KBBzvW47nezajp7QaAi9WSa56kAB93Xu3fkj4hgY4qXUSkzHP4FABXP1VjGEa+T9qMHz+e2NhYunTpgmEY+Pv7M3z4cKZOnYrVenmhzYIcszDnBRg3bhxjx461/5yUlES9evWufXEiJcQwDH7ZHcc/f9zFkYQUADrU92XCX1rRpq5vjn37hARyR8uAfGfcFhGRvDksJPn5+WG1WnP13sTFxeXq5cnm4eHB7Nmz+fDDDzl16hSBgYHMmDEDb29v/Pz8AAgICLjmMYtyXgA3Nzfc3NwKfZ0ixW1/3Hkm/W8XK/aeBqCWtxsv9m3OgHZ1cMon+FidLHRtVKM0yxQRKfccNibJ1dWVjh07snTp0hztS5cuJTw8/JrvdXFxoW7dulitVhYsWMBdd92Fk5N5KV27ds11zJ9//tl+zBs5r4gjJaVmMPl/u+jz3gpW7D2Nq9WJJ3o04tfnenBvh7r5BiQRESkah95uGzt2LEOHDiU0NJSuXbsyY8YMjh49ypgxYwDzFteJEyfscyHt3buX6OhowsLCOHv2LO+88w47duxg7ty59mM+++yz3HzzzbzxxhvcfffdfPfddyxbtsz+9FtBzitSlmRlGXy56ThTl/xJ/Pl0AHq2qMXL/VrSwM/LwdWJiFRcDg1JAwcOJCEhgUmTJhETE0NISAiLFi0iKCgIgJiYmBxzF9lsNt5++2327NmDi4sLt956K2vWrKFBgwb2fcLDw1mwYAEvv/wy48ePp1GjRkRFRREWFlbg84qUFZuOnGXiDzv547j5SH/Dml68cldLejSr5eDKREQqPofOk1SeaZ4kKUmnklJ546c/+XrLCQC83Zx5tmcTHunaAFdnh0+ULyJSbhXm89vhT7eJyGVpmTZmrzrMtF/3cSHdBsCDoXV5vndz+yP9IiJSOhSSRMqA7Ef6J/+4i8OXHulvX9+XCf1b0baer2OLExGppBSSRBxsf9x5/vm/XSwvxCP9IiJS8hSSRBwkKTWD93/Zx5zVh8nMMnCxWhh5U0Oeuq0xVdz0v6aIiKPpX2KRUpbXI/23N6/Fy3e1JFiP9IuIlBkKSSKlaPPRs0z8fifb9Ei/iEiZp5AkUgriklJ5ffGffL3ZfKS/ipszz97ehGHheqRfRKSsUkgSKUF6pF9EpPxSSBIpAYZh8Oufcfzzf3qkX0SkvFJIEilmB06fZ9IPlx/pr+ntxot9mnNPez3SLyJSnhQ6JDVo0IARI0YwfPhw6tevXxI1iZRLyakZ/EeP9IuIVBiFHjH697//ne+++46GDRtyxx13sGDBAtLS0kqiNpFyISvL4IuNx7j1reXMXHmIzCyD25vX4ue/3cKLfZsrIImIlFNFXuB227ZtzJ49m88//5zMzEweeughRowYQYcOHYq7xjJJC9wK5PFIv58X4/u35FY90i8iUiYV5vO7yCEpW0ZGBtOnT+eFF14gIyODkJAQnn32WR599FEsloo7/kIhqXKwZRlEHzpDXHIqtbzd6RxcHauTRY/0i4iUU4X5/C7yfYCMjAy++eYb5syZw9KlS+nSpQsjR47k5MmTvPTSSyxbtozPPvusqIcXcbjFO2KY+MMuYhJT7W0BVd3o2siPn3fG2h/pf6BjXZ7v04xa3u6OKlVEREpAoUPS5s2bmTNnDp9//jlWq5WhQ4fy7rvv0rx5c/s+vXr14uabby7WQkVK0+IdMTzx6Wau7maNTUrjmy1m71G7er5M+Esr2umRfhGRCqnQIalTp07ccccdREZGMmDAAFxcXHLt07JlSwYNGlQsBYqUNluWwcQfduUKSFfy9XBh4eNdcdGtNRGRCqvQIengwYMEBQVdcx8vLy/mzJlT5KJEHCn60Jkct9jycu5iBhuPnKVroxqlVJWIiJS2Qv8aHBcXx/r163O1r1+/no0bNxZLUSKOFJd87YBU2P1ERKR8KnRIevLJJzl27Fiu9hMnTvDkk08WS1EijlTQAdgaqC0iUrEVOiTt2rUrz7mQ2rdvz65du4qlKBFHal3HBxdr/tNXWIBAH3M6ABERqbgKHZLc3Nw4depUrvaYmBicnTWzsJRvtiyDsV9sJcOW97Dt7Oj0av+WWLUOm4hIhVbokHTHHXcwbtw4EhMT7W3nzp3jH//4B3fccUexFidSmgzD4J//28XPu07h6uzEc72aEuiT85ZagI87kUM60Cck0EFViohIaSn0jNsnTpzg5ptvJiEhgfbt2wOwdetW/P39Wbp0KfXq1SuRQssazbhd8cxccZB/LdqNxQLTBnegX5vAfGfcFhGR8qnElyW5cOEC8+fPZ9u2bXh4eNCmTRsGDx6c55xJFZVCUsXy4x8xPPnZZgBe7teCUd0bOrgiEREpCSW+LImXlxePPfZYkYoTKWs2HD7D377YCsDw8AaMvCnYsQWJiEiZUOSR1rt27eLo0aOkp6fnaP/LX/5yw0WJlJb9cecZNXcj6ZlZ9G7lz/i7WlbohZlFRKTgijTj9j333MP27duxWCxk363L/mCx2WzFW6FICYlLTmX4nGgSL2bQvr4v7w1sr/FGIiJiV+in25599lmCg4M5deoUnp6e7Ny5kxUrVhAaGsrvv/9eAiWKFL+U9ExGfryR42cv0qCGJx89EoqHq9XRZYmIoyTHwm9TzK8ilxQ6JK1du5ZJkyZRs2ZNnJyccHJy4qabbmLKlCk888wzJVGjSLHKtGXx1Gdb2H4ikepernz8aGdqVHFzdFlS2ehDuWxJjoXlr+vPQ3IodEiy2WxUqVIFAD8/P06ePAlAUFAQe/bsKd7qRIqZYRi88v1Ofv0zDncXJz4aFkoDPy9HlyWVkT6URcq8Qo9JCgkJ4Y8//qBhw4aEhYUxdepUXF1dmTFjBg0b6rFpKdum/36Az9YfxWKBfw9qT4f61Rxdkog4SnIsJB6HI2vhjwVm26f3gU898KoJNYLBrylUrQs+daBqHfCoBnq4o9IodEh6+eWXuXDhAgCTJ0/mrrvuonv37tSoUYOoqKhiL1CkuHy75QRvLjF7O1+9qyW9WwU4uCKpVFIT4eAKOLYeEvZD7B9m+2+Tof0j4FsfvAPMTUpe3G74/mk4viFne0q8uQHsz+N9zh5Qtfbl0FS1zqWf65pfFaQqlCJNJnm1M2fOUK1atSI9Oj19+nTefPNNYmJiaNWqFe+99x7du3fPd//58+czdepU9u3bh4+PD3369OGtt96iRo0aAPTo0YPly5fnet+dd97Jjz/+CMCECROYOHFijtf9/f2JjS14t7cmkyxf1hyIZ9jsaDJsBqO7B/NSv5aOLkkqKsOApBMQ8wfEbjfDUOx2OHfk+u9t1BPu+S9UqVnydVZGqYmw42vY8imc2Hi53b0a1OkAB36B8GfArSpciIO0C5B6DpKOQ9JJuHC6YOdx8bwUmGpf0Qt16fvsgOXuqyDlICU2mWRmZibu7u5s3bqVkJAQe3v16kVbDT0qKoqIiAimT59Ot27d+PDDD+nbty+7du2ifv36ufZftWoVjzzyCO+++y79+/fnxIkTjBkzhlGjRvHNN98A8PXXX+eYuykhIYG2bdvywAMP5DhWq1atWLZsmf1nq1VPNlVUe2KTefyTTWTYDPq1CWRc3xaOLkkqClsGxO+9FIauCEQXz+a9v3dtqB4MNZqAkzNs/Aj8W0Hcn2DY4MAyeLsZNL4d2gyE5v3AxaN0r6miycqCI6vNYLTrO8i8aLY7OUPTPtB+CDTuCad2miEp5D6o3S7vY2WkQvJJMzAlnjDDcNKJSz9fClIp8ZCRYvYWJuTVFXWJi2fevVBV61zupXL3KVqQSo6FjXMg9FH1TN6gQoUkZ2dngoKCim0upHfeeYeRI0cyatQoAN577z2WLFlCZGQkU6ZMybX/unXraNCggf0puuDgYB5//HGmTp1q3+fqwLZgwQI8PT1zhSRnZ2cCAvSXp6I7lZTKo3OiSU7NpFODarz9QFucNBeSFEVqkvlBemUYitsNtrTc+zo5Q83mEND68uYfAp5X/Pt0cqsZku6ebn5I7vgKti2Ak5th38/m5uoNLe+GNg9Cg+7gVOhnbSqvxBOw7TMzHJ09fLndrxl0GGqG0Cq1CndMF3eo3tDc8pORejk4ZYeoxOyfs4NUwqUgtc/c8j2fV969UFfe5ssrSGU/FNCsr0LSDSrSmKRx48bx6aefFrkHCSA9PZ1Nmzbx4osv5mjv1asXa9asyfM94eHhvPTSSyxatIi+ffsSFxfHl19+Sb9+/fI9z6xZsxg0aBBeXjmfYNq3bx+1a9fGzc2NsLAwXnvttWsOPE9LSyMt7fI/hklJSQW5THGg82mZPDpnAycTU2lY04uZj4Ti7qIeQ7kOw4DkmJxhKHY7nDmY9/6u3jnDUEBrqNUCnAsxrYSXH4Q9bm7x++CPKHM7dxS2fmpuVetA6weg7SDz+JJbZhrsWWQGowO/gpFltrt6Q8i90H4o1A3Nu3fGOwBuefHGQ4WLO9RoZG75ybh4OUTl6pG69P3FM5BxweypjN+b/7Fcq1zuhcoOUFmZ5mvHos3eTjdvc3Ovagav8hK2y0CPWKHHJLVv3579+/eTkZFBUFBQrvCxefPmAh3n5MmT1KlTh9WrVxMeHm5vf+2115g7d26+0wl8+eWXPProo6SmppKZmclf/vIXvvzyyzwX142OjiYsLIz169fTuXNne/tPP/1ESkoKTZs25dSpU0yePJk///yTnTt32sc2XS2vcUyAxiSVURm2LEZ8vIGV++Lxq+LGN38Np151T0eXJWWNLfPSIOqrAlH2wN2rVa1zVSBqA75BRfvQud4HQFYWHFtnhqWd35jjabIFtIY2g6D1/eopAPPPbMun5n+rK291Bt1k3k5r+RdwLWdTfaSnmGE98XgePVLZQSqf27rXZLkUmqpeDk/ZAcrentdrVXO+z9Wr5MdUndwKM26Bx5bnfwu0CEp0gdsBAwYUta48XT3Y2zCMfAeA79q1i2eeeYZXXnmF3r17ExMTw/PPP8+YMWOYNWtWrv1nzZpFSEhIjoAE0LdvX/v3rVu3pmvXrjRq1Ii5c+cyduzYPM89bty4HK8lJSVRr169Al+nlB7DMPjH19tZuS8eDxcrs4eHKiAJpJ2HuF05w9CpnZCZmntfi9V89DugNQS2uXS7rDV45f1LVJF4B8Ct4/J/3ckJgsLNrc8bsG8J/PEF7F1yuf6l46HhreatoxZ3lb8gcCMunoXtX5rhKGbr5Xbv2tDuIXO7Vm9OWefqef0eqfQLkBRj3sZbPwP2/HiNA1oAw9zSksztRliczB4696sC1ZUBy90nj9eq5gxlLp5legB7oUPSq6++Wiwn9vPzw2q15nqiLC4uDn9//zzfM2XKFLp168bzzz8PQJs2bfDy8qJ79+5MnjyZwMBA+74pKSksWLCASZMmXbcWLy8vWrduzb59+d8bdnNzw81NszKXB//5ZT8LNx3HyQLTHmpPm7q+ji5JilNBuuCTT13VO/QHJBzA/JC4iosXBISYvUJX3i4rSwOmXdzNsUkt74aUM7Dza9gWBcejzcHGB36B/3lBi/7QdiAE3wJOFfDWclYWHFpuBqPdP1weD+bkYg5ybz8UGt1aMa89L65e4NfY3Go2h1v+z2yP2QY/PAP9/wOBbc22Kv7g4WuOrUtLvhyU0pIvb6lXt131ffZ7DZt5KzMt0dxuhMXpUoC6FKic3cDqaobE7F9gYrZd3r+Up8kodEgqLq6urnTs2JGlS5dyzz332NuXLl3K3Xffned7UlJScHbOWXL2U2lX3zX84osvSEtLY8iQIdetJS0tjd27d19z6gEpHxZuPMa7y8z79/8cEMLtLfIO3FKOXTko1aumOVYoOwxlP3Z/IS7v93oH5r5dVi24/IzRAHPwd6dR5pZwALYvNAd8nz1kToj4xwKoEmDeims7yLzO8u7sEdj6mbklHr3c7h9iBqPWDxRvL195lFd4CGyb+zaViwd438C/i4Zhjqm6OmilXhW40hLzCGDJOd9nZJlbamLO28lX++GKJc9uefHaPbDFrNAhycnJ6ZrzIRXmybexY8cydOhQQkND6dq1KzNmzODo0aOMGTMGMG9xnThxgnnz5gHQv39/Ro8eTWRkpP12W0REBJ07d6Z27do5jj1r1iwGDBiQ5xij5557jv79+1O/fn3i4uKYPHkySUlJDBs2rMC1S9mzYu9pxn29HYC/9mjEw2FBDq6ojCkDgyDzlWUzbx1kXDQHq6anmE//ZKTk/j77sepvnzCfWspIyX08i5P5mP3VA6oL+zRTWVejEfR4EW55wZwUcdsCs5fpfCysnWZutVqZvUutHzAH+JYXGRfhzx9h8zyz9yibu495Le2HQGC7Mn2rpkKyWMxeHlfPYghbKVcFqCQ4d8y8fZieAmcOmAPxr+wRK+V/uwodkrLnI8qWkZHBli1bmDt3bp4Dm69l4MCBJCQkMGnSJGJiYggJCWHRokUEBZkfbjExMRw9evm3huHDh5OcnMy0adP4+9//jq+vL7fddhtvvPFGjuPu3buXVatW8fPPP+d53uPHjzN48GDi4+OpWbMmXbp0Yd26dfbzSvmz62QSf52/mcwsg7vb1eb53s0cXVLZcyOPBWfZ8g4sGZeCTfqFS+35fG8PQPl8n9dj9NcTt8v86uwONZtBnY6Xe4dqtTT/Ea8sLBao19nc+rwO+5eagWnvYojbCUtfgaWvQvDNZu9Si/7mrY2yxjDM8UVbPjV7yK7sXWjYw+w10txR11dcT+qVJIvFvF3o6pV/nSe3miEprx6xUlIsM24DfPbZZ0RFRfHdd98Vx+HKPM24XXacPHeRe6av5lRSGl0aVmfuiM64OVeSMQkFYcs0b0ntWQTLXoXOj5mz/V4ddtJTrurJueL7ooSYIrn0D6eLhzmgM/v786dz3ma5Wil3wZcbF8+aEyhui4KjV0yt4uxhho22g8yB31aHjbwwXUiA7V+Y4ejUjsvtPvWh/cPQdjBU0y+xlU4ZeLqt2ELSgQMHaNOmjX1dt4pOIalsSLyYwYP/XcueU8k09a/CwjHh+Hjkng6iUjAM8/HguF3mdmqX+Vt5wgHIyiimk1guhRfPS0HG69L3nle0Xwo2+X7vdcW+njnDkLN73rdPkmPNDfIelKo1z67v7GH4Y6E5ZunKmaC9akLI/eYtudK8fZVlM+cy2vIJ/Lno8t9Rq5v5yH77IdDg5vI1XkyKVwkNESjRKQDycvHiRd5//33q1q1bHIcTKZD0zCzGfLKJPaeSqeXtxpxHO1eegHTxrDnb86mdl0LRbvPrtQY/5iXoJmja+6rwkkdPTnYYyi/ElLSCDkqV/FVrALc8Dzc/Z87qvS3KnOX7wmlYH2lufs0ujV96EHxLaIqThAOXB2Enn7zcHtjODEat7zcXiBW53jQZpaDQIenqhWwNwyA5ORlPT08+/fTTYi1OJD+GYfDCV3+w9mACXq5W5jzaiTq+FXCcQsZFOL3nUgjaafYOxe3O+eFyJYsV/JqYY3L8W5pPc3lUN7/G/qEeGDFDbp2O5tb7X2ZvzrYF5u3Y+D3wyyRzC7rJDEwt7zYHS9+I9Auw63vzdtqRVZfbPaqbczy1f7hiPIUnFU6hQ9K7776bIyQ5OTlRs2ZNwsLCqFZN6V9Kx9s/7+WbLSewOlmYPqQjrWrf4D/ijpZlgzOHzCB0ZQ/RmYOXl1a4mk99cz4f/5bmE0y1WpgBKb/lMLL/vy3vPTDlYVBqeWF1MXsSm/Y2eyF3fW/OWn14lRlmjqyCH58zB/u3HWQuBGu9qrc2v1sihgHHN5q303Z8DenJZrvFCRrdbvYaNetbuOVbREpZsY1Jqmw0JslxPlt/lH98Yz7qP/X+NjwYWo5mPs9eFyx7zFB2D9HpPXnP/Azmb9v+l0JQrZbm9zWbmzPWFkYJDYKUCijxuDm79x9RcPrPy+2eNSDkPnNJlDodzOB99d+r83Fmz9SWT82eqWzVGpjBqO1D5hpjIg5SomOS5syZQ5UqVXjggQdytC9cuJCUlBTNNSQl6rc/4xj/nfn0y7O3NymdgFTUwYMXz10eK2QPRbsg9Vze+zt7QK3ml3uFsnuIqtQqnnFA6oGRgvKpC93Hwk1/M2/TbosyH8m/EAfRM8ytRmPzVln2bbIja2DFm+a0A9kLrDp7QKsBZjiqH65B2FLuFLonqVmzZvz3v//l1ltvzdG+fPlyHnvssXwXpq1o1JNU+rYfT2TgjLWkpNu4v2Nd3ry/zTUnNi021+uByUg1V+m+Ogwlncj7eBar+QFTq8WlHqKW5vfVGlSe5RSk/LFlwsHfzd6l3T9A5sX89w1sC6EjoNW9he/xFClhJdqTdOTIEYKDg3O1BwUF5Zj4UaQ4HTuTwqMfbyAl3Ub3Jn5Mubd16QSkKxlZ5pM59iB0afxQwgFzLaO8VK17qUeo5eXB1DWamGtxiZQnVmdo0tPclk2AVe/mv2/TvtBxeGlVJlJiCh2SatWqxR9//EGDBg1ytG/bti3PJUBEbtS5lHSGz4km/nwazQO8mf5wB1ysJdxtnz0vT/p5WPGW2TarV/7zDbn7Xu4VsoeiFjf+VJBIWRQ2BloOML8/tNyc0fuud6F2B7NNt3Slgih0SBo0aBDPPPMM3t7e3HzzzYB5q+3ZZ59l0KBBxV6gVG5pmTYe+2QTB05fINDHnY8f7Yy3eynMhbRxjrmEx5WuDEj+IebTPtk9RN4BWkNKKo+8po6o3UEPBEiFU+iQNHnyZI4cOcLtt9+Os7P59qysLB555BFee+21Yi9QKq+sLIO/f7GN6ENn8HZzZs6jnQjwKYXbVBkXL8/uDOYTPSkJcNt4aHSb+Qiz5hcSEanwCh2SXF1diYqKYvLkyWzduhUPDw9at26txWGl2L2x+E/+90cMLlYLHw7tSPOAUhgAenIrfPP45ceeOw43H3ee08ecI0a/KYvkpKcmpQIr8rIkTZo0oUmTJsVZi4jdvLWH+XDFQcCcCym8sV/JntCWaQ5EXf66+fiyVy24e5o5yd7JrSV7bpHyrAwsHSFSUgo9+vX+++/n9ddfz9X+5ptv5po7SaQoft4Zy4TvdwLwXK+m3NO+hNcEjN8Ps3vDb5PNgNTybvjrOjMggX5TFhGppAodkpYvX06/fv1ytffp04cVK1YUS1FSeW05epZnFmwhy4DBnevx5K2NS+5khgHRM+G/N8GJjeDmA/fMgAfmgtcVT2pm/6askCQiUqkU+nbb+fPncXV1zdXu4uJCUlJSsRQlldORhAuMmruR1Iwsbm1Wk3/eHVJycyElnYRv/woHfzN/Dr4FBkw3ZxoWERGhCD1JISEhREVF5WpfsGABLVu2LJaipPI5cyGd4XM2kHAhnZA6VZn2UAecS2oupO1fwvQuZkBydoc+b8DQbxWQREQkh0L3JI0fP5777ruPAwcOcNtttwHwyy+/8Nlnn/Hll18We4FS8aVm2Bg1dwOH4i9Qx9eD2cM74eVW5GcK8pdyBn78O+z82vy5dnvz9lrNpsV/LhERKfcK/Un0l7/8hW+//ZbXXnuNL7/8Eg8PD9q2bcuvv/6qNcyk0GxZBhELtrL56Dl8PFyYO6ITtbxLYC6kfcvguyfhfKy5dtot/wfd/w7WUpiYUkREyqVCL3B7tXPnzjF//nxmzZrFtm3bsNnyWcOqgtECt8Vj4g87mbP6MK5WJz4Z2ZmwhsW8tE36Bfj5Zdg42/zZrync8yHU6VC85xERkXKhMJ/fRR708euvvzJkyBBq167NtGnTuPPOO9m4cWNRDyeV0KxVh5iz+jAAbz3YtvgD0rFo88m17IAUNgYeX6GAJCIiBVKo223Hjx/n448/Zvbs2Vy4cIEHH3yQjIwMvvrqKw3alkL5aXsMk3/cBcC4vs35S9vaxXfwzHRzUshV74KRBVXrmE+uNexRfOcQEZEKr8A9SXfeeSctW7Zk165dvP/++5w8eZL333+/JGuTCmrj4TM8G7UVw4BHugbx2M0Ni+/gp3bBR7fByrfNgNRmIDyxRgFJREQKrcA9ST///DPPPPMMTzzxhJYjkSI7cPo8o+ZtJD0zi54t/Hm1f6vimQspywbrpsMvk8CWDh7V4a53odWAGz+2iIhUSgXuSVq5ciXJycmEhoYSFhbGtGnTOH36dEnWJhXM6eQ0hs+J5lxKBm3r+fL+4PZYnYohIJ09AnP7mwO0benQpBf8da0CkoiI3JACh6SuXbsyc+ZMYmJiePzxx1mwYAF16tQhKyuLpUuXkpycXJJ1SjmXkp7JqLkbOHbmIvWrezJrWCgertYbO6hhwJZPIbIbHFkNLl7Q/9/w0BdaQkRERG7YDU0BsGfPHmbNmsUnn3zCuXPnuOOOO/j++++Ls74yS1MAXJstyyD60BniklOp4eXKnNWH+OXP01TzdOGrJ8JpWLPKjZ3g/Gn44VnY86P5c70ucM9/oXrwjRcvIiIVVmE+v294niQAm83GDz/8wOzZsxWShMU7Ypj4wy5iElNztDs7WYh6vAsdg6rf2An+/BG+fwZS4sHJBW57CcKfAacb7JkSEZEKrzCf38Wy9oPVamXAgAEMGDCgOA4n5djiHTE88elm8kremVkGp5PTin7w1CRY/CJsnW/+XKsV3DsDAkKKfkwREZF8lNAKolIZ2bIMJv6wK8+ABGABJv6wC1tWETovD600xx5tnW8eqVsEPPabApKIiJSYElhFVCqr6ENnct1iu5IBxCSmEn3oDF0bFXB27YxU87H+dR+YP/sGmcuKBHW98YJFRESuQSFJik1ccv4BqSj7cXIrfPM4nP7T/LnDMOj9L3DzLlqBIiIihaCQJMWmlrd78exnyzSXFFn+OmRlglct+Mv70KxPMVQpIiJSMA4fkzR9+nSCg4Nxd3enY8eOrFy58pr7z58/n7Zt2+Lp6UlgYCCPPvooCQkJ9tc//vhjLBZLri01NWfvRWHPK9fXObg6gT75ByALEOjjTufgazzdlnAA5vSB3yabAanFX+Cv6xSQRESk1Dk0JEVFRREREcFLL73Eli1b6N69O3379uXo0aN57r9q1SoeeeQRRo4cyc6dO1m4cCEbNmxg1KhROfarWrUqMTExOTZ398sf3oU9rxSM1cnCq/3zXug4e17tV/u3zHuWbcOA6Jnw35vg+AZw84F7ZsCD88CrgOOXREREilGxzJNUVGFhYXTo0IHIyEh7W4sWLRgwYABTpkzJtf9bb71FZGQkBw4csLe9//77TJ06lWPHjgFmT1JERATnzp0rtvMCpKWlkZZ2+fH1pKQk6tWrp3mSrhKXlEqXKb9w9QNsgT7uvNq/JX1CAnO/KekkfPcUHPjF/Dn4ZhgQCT51S75gERGpVAozT5LDepLS09PZtGkTvXr1ytHeq1cv1qxZk+d7wsPDOX78OIsWLcIwDE6dOsWXX35Jv379cux3/vx5goKCqFu3LnfddRdbtmy5ofMCTJkyBR8fH/tWr169wl5ypfD1lhNkGdChvi+fj+7Cvwe14/PRXVj1wm15B6TtX8L0rmZAcnaHPm/A0O8UkERExOEcFpLi4+Ox2Wz4+/vnaPf39yc2NjbP94SHhzN//nwGDhyIq6srAQEB+Pr68v7779v3ad68OR9//DHff/89n3/+Oe7u7nTr1o19+/YV+bwA48aNIzEx0b5l91zJZYZh8MUG87/LoE716dqoBne3q0PXRjVy32JLOQMLH4WvRkLqOQhsB4+vgC5jwMnhQ+VEREQcP3DbYsn54WkYRq62bLt27eKZZ57hlVdeYdOmTSxevJhDhw4xZswY+z5dunRhyJAhtG3blu7du/PFF1/QtGnTHEGqsOcFcHNzo2rVqjk2yWnTkbMcjL+Ap6uVO9vk0WuUbd8ys/do59dgscItL8KoZVCzWekVKyIich0OmwLAz88Pq9Waq/cmLi4uVy9PtilTptCtWzeef/55ANq0aYOXlxfdu3dn8uTJBAbm/mB2cnKiU6dO9p6kopxXCibqUi9Sv9aBVHHL469W+gX4eTxsnGX+XKOxOTi7bsdSrFJERKRgHNaT5OrqSseOHVm6dGmO9qVLlxIeHp7ne1JSUnC66laM1Wouaprf+HPDMNi6das9QBXlvHJ959My+XF7DAADO10ar5UcC79NMb8e22A+uZYdkMLGwOMrFZBERKTMcuhkkmPHjmXo0KGEhobStWtXZsyYwdGjR+23z8aNG8eJEyeYN28eAP3792f06NFERkbSu3dvYmJiiIiIoHPnztSuXRuAiRMn0qVLF5o0aUJSUhL/+c9/2Lp1Kx988EGBzyuF9+MfJ0lJt9Gwphcdg6qZjcmx5oSQSSfMNdeMLKhaB+7+ABrd6tiCRURErsOhIWngwIEkJCQwadIkYmJiCAkJYdGiRQQFBQEQExOTY+6i4cOHk5yczLRp0/j73/+Or68vt912G2+88YZ9n3PnzvHYY48RGxuLj48P7du3Z8WKFXTu3LnA55XC+2LjcQAeDK13eWzX2UPm1y2fmF/bDIS+U8HDt/QLFBERKSSHzpNUnhVmnoWKbn/ceXq+sxyrk4X1T7fCzzgLicfhy0fBlg4unnDLC9CwB3gHmJuIiIgDFObzW2u3yQ1buNEcsH1rs1r4/fmZeYvtShkpsOxV8/tbXoRbx5VyhSIiIoWnkCQ3JMOWxVebTwDwYGhdqP+oOWP2p/dB5kVzp/7/gcC25vfqRRIRkXJCIUluyG9/xhF/Pg2/Km7c2rwWWJ1g13dmQPINgnNHzIBUu52jSxURESkUh08mKeVb9oDt+zrUwcXqBFlZED3DfLHVPQ6sTERE5MaoJ0mKLC4pld/2xAHwQOiluZEO/goJ+8GtKnR4xFyPTbfYRESkHFJIkiL7essJbFkGHYOq0bhWFbNx/aVepHYPQY1GGqQtIiLllm63SZEYhsEXl55qezC0rtl45iDs+9n8vtNoB1UmIiJSPBSSpEg2HTnLwdPmYrb92piznbNhFmBA457g19ih9YmIiNwohSQpkuxeJPtitukXLs+s3fkxB1YmIiJSPBSSpNDOp2Xyvz/MxWwfzF7M9o8vIDURqgVD4zscWJ2IiEjxUEiSQlv0R4y5mK2fF6FB1cAwLj/233k0OOmvlYiIlH/6NJNCi7p0q+2B7MVsD6+CuF3mGm3tHnZwdSIiIsVDIUkKZX/ceTYdOYvVycJ9HeqYjdm9SG0Ggoevw2oTEREpTgpJUigLN2UvZluTWlXdIfE4/Pmj+aIGbIuISAWikCQFlmHL4qtN2YvZXhqwvWEWGDZo0B38WzqwOhERkeKlkCQF9vue05cWs3U1F7PNSIXNc80X1YskIiIVjEKSFFjUBvNW270d6pqL2e78GlISoGpdaHang6sTEREpXgpJUiBxyZcXs30wtK752P/6D80XO40Eq5YBFBGRikUhSQrk683mYrYd6vvSuJY3HN8AMVvB6gYdhjm6PBERkWKnkCTXdeVitgOzZ9jOfuy/9f3gVcNBlYmIiJQchSS5rlyL2Safgp3fmi9qwLaIiFRQCklyXbkWs900B7IyoF4Y1G7n2OJERERKiEKSXNOFqxezzUyHjbPNF9WLJCIiFZhCklzTj1cvZrv7ezh/Cqr4Q4u/OLo8ERGREqOQJNf0xdWL2WYP2A4dAc6uDqxMRESkZCkkSb72x51n45WL2Z7cCsfWg5MzdBzu6PJERERKlEKS5CvXYrbRM80XWg4A7wDHFSYiIlIKFJIkT1cuZvtAaD24kADbF5ovhj3uwMpERERKh0KS5OnKxWxva17LXMjWlgaBbaFuJ0eXJyIiUuIUkiRP2QO27+1QFxeyrnjs/3GwWBxYmYiISOlQSJJc4pJT+fXPKxaz3fsTJB4DzxoQcp+DqxMRESkdCkmSyzdXL2a7/kPzhQ7DwMXdscWJiIiUEoeHpOnTpxMcHIy7uzsdO3Zk5cqV19x//vz5tG3bFk9PTwIDA3n00UdJSEiwvz5z5ky6d+9OtWrVqFatGj179iQ6OjrHMSZMmIDFYsmxBQToaS0wF7ONunSr7cHQehC3Gw6vBIuTOTeSiIhIJeHQkBQVFUVERAQvvfQSW7ZsoXv37vTt25ejR4/muf+qVat45JFHGDlyJDt37mThwoVs2LCBUaNG2ff5/fffGTx4ML/99htr166lfv369OrVixMnTuQ4VqtWrYiJibFv27dvL9FrLS82HzUXs/VwsXJX29qXJ49s3g986zm2OBERkVLk0JD0zjvvMHLkSEaNGkWLFi147733qFevHpGRkXnuv27dOho0aMAzzzxDcHAwN910E48//jgbN2607zN//nz++te/0q5dO5o3b87MmTPJysril19+yXEsZ2dnAgIC7FvNmjVL9FrLi6gNlxazbRNIlazzsG2B+UJnPfYvIiKVi8NCUnp6Ops2baJXr1452nv16sWaNWvyfE94eDjHjx9n0aJFGIbBqVOn+PLLL+nXr1++50lJSSEjI4Pq1avnaN+3bx+1a9cmODiYQYMGcfDgwWvWm5aWRlJSUo6torlyMduBnerB1vmQkQK1WkKDmxxcnYiISOlyWEiKj4/HZrPh7++fo93f35/Y2Ng83xMeHs78+fMZOHAgrq6uBAQE4Ovry/vvv5/veV588UXq1KlDz5497W1hYWHMmzePJUuWMHPmTGJjYwkPD88xtulqU6ZMwcfHx77Vq1fxbj1lL2Yb7OdFaH2fyzNsdx6tx/5FRKTScfjAbctVH76GYeRqy7Zr1y6eeeYZXnnlFTZt2sTixYs5dOgQY8aMyXP/qVOn8vnnn/P111/j7n75qay+ffty33330bp1a3r27MmPP/4IwNy5c/Otc9y4cSQmJtq3Y8eOFfZSy7zLi9nWxbL/Fzh7CNx8oM1AB1cmIiJS+pwddWI/Pz+sVmuuXqO4uLhcvUvZpkyZQrdu3Xj++ecBaNOmDV5eXnTv3p3JkycTGBho3/ett97itddeY9myZbRp0+aatXh5edG6dWv27duX7z5ubm64ubkV9PLKnQOnLy9me3+HuvD9C+YL7YeAq5djixMREXEAh/Ukubq60rFjR5YuXZqjfenSpYSHh+f5npSUFJyccpZstVoBswcq25tvvsk///lPFi9eTGho6HVrSUtLY/fu3TlCVmWT3YvUo2lNamWcgP3LAAt0HnXtN4qIiFRQDr3dNnbsWD766CNmz57N7t27+dvf/sbRo0ftt8/GjRvHI488Yt+/f//+fP3110RGRnLw4EFWr17NM888Q+fOnalduzZg3mJ7+eWXmT17Ng0aNCA2NpbY2FjOnz9vP85zzz3H8uXLOXToEOvXr+f+++8nKSmJYcOGle5/gDLiysVsH+xU7/JYpCa9oHpDB1YmIiLiOA673QYwcOBAEhISmDRpEjExMYSEhLBo0SKCgoIAiImJyTFn0vDhw0lOTmbatGn8/e9/x9fXl9tuu4033njDvs/06dNJT0/n/vvvz3GuV199lQkTJgBw/PhxBg8eTHx8PDVr1qRLly6sW7fOft7KJsditg094fv55gthjzm2MBEREQeyGFfep5ICS0pKwsfHh8TERKpWrerocm7I6HkbWbrrFI/d3JB/1FwNP/4dqjeCpzaCk8PH9ouIiBSbwnx+6xOwksuxmG3HOlc89v+YApKIiFRq+hSs5HIsZnthM5z+E1yrQLuHHF2aiIiIQykkVWKGYdifanswtB6sv7ROW9tB4F6+byGKiIjcKIWkSmzz0bMcyF7Mtn4G7P3JfKGzBmyLiIgoJFViX2w4DlxazHb7XDCyoGEPqNnMsYWJiIiUAQpJlZS5mO1JAAa2qwmb55kvqBdJREQEUEiqtH7cHsOF7MVsk5fBxbPgWx+a9nF0aSIiImWCQlIl9cWGS4vZdqyDJfrSgO1Oo8DJ6sCqREREyg6FpEroysVsBwechNjt4OwO7Yc6ujQREZEyQyGpElq40Ryw3aNpTartmGM2tn4APKs7sCoREZGyRSGpksm0ZfHVZjMkDWnlCrt/MF8Ie9yBVYmIiJQ9CkmVzO97TnM62VzM9uakHyArE+qHQ0BrR5cmIiJSpigkVTJRl2bYvr9tTaybPzYbO492XEEiIiJllEJSJXLlYrbDfbfBhdPgXRta9HdwZSIiImWPQlIlkr2Ybfv6vgT8eWnyyNARYHVxbGEiIiJlkEJSJXHlYrZPNE6EExvB6godhzm4MhERkbJJIamSuHIx21uTvjUbW90DVWo5tC4REZGySiGpkshezPbBFu647P7GbOysx/5FRETyo5BUCVy5mO0orxVgS4c6HaFuRwdXJiIiUnYpJFUC2YvZNq7hRt39n5uNnR9zbFEiIiJlnEJSJbDw0oDt54L2Y0k+CV41zfFIIiIiki+FpAruwOnzbDh8FicL3Jb4rdnYcTg4uzmyLBERkTJPIamCy17MdmjweVxPrAWLFTo+6uCqREREyj6FpArsysVsR7ktMxtb9AefOg6sSkREpHxQSKrAshezDfZMo+6xH8zGMD32LyIiUhAKSRVY9gzbL9XehCXzIviHQP2uDq5KRESkfFBIqqBOJ6fx659xOJHFLYnfm42dHwOLxbGFiYiIlBMKSRXUN1uOk5llMNJ/Hy7JR8HdF1o/4OiyREREyg2FpArIMAyiNpi32ka4/Gw2dngEXD0dWJWIiEj5opBUAW0+eo4Dpy/Q0iWWwPi1gAU6jXR0WSIiIuWKQlIF9MWlXqR/+K0yG5r1hWoNHFeQiIhIOaSQVMFkL2ZbhRS6Ji8xG7VOm4iISKEpJFUw2YvZjq66DmvmBfBrCg17OLosERGRcsfhIWn69OkEBwfj7u5Ox44dWbly5TX3nz9/Pm3btsXT05PAwEAeffRREhIScuzz1Vdf0bJlS9zc3GjZsiXffPPNDZ+3vFi48RgWsnjEutRs0GP/IiIiReLQkBQVFUVERAQvvfQSW7ZsoXv37vTt25ejR4/muf+qVat45JFHGDlyJDt37mThwoVs2LCBUaNG2fdZu3YtAwcOZOjQoWzbto2hQ4fy4IMPsn79+iKft7w4eGkx2+5OO6h28Qi4ekPbQY4uS0REpFyyGIZhOOrkYWFhdOjQgcjISHtbixYtGDBgAFOmTMm1/1tvvUVkZCQHDhywt73//vtMnTqVY8fMwcoDBw4kKSmJn376yb5Pnz59qFatGp9//nmRzpuXpKQkfHx8SExMpGrVqoW78BLy+k9/8t/lB/i22n9od3EddH4c7pzq6LJERETKjMJ8fjusJyk9PZ1NmzbRq1evHO29evVizZo1eb4nPDyc48ePs2jRIgzD4NSpU3z55Zf069fPvs/atWtzHbN37972YxblvABpaWkkJSXl2MqS7MVs61lO0fbipV4zDdgWEREpMoeFpPj4eGw2G/7+/jna/f39iY2NzfM94eHhzJ8/n4EDB+Lq6kpAQAC+vr68//779n1iY2OvecyinBdgypQp+Pj42Ld69eoV6npLWvZito+5/4oFAxrdDn6NHV2WiIhIueXwgduWqwYVG4aRqy3brl27eOaZZ3jllVfYtGkTixcv5tChQ4wZM6bQxyzMeQHGjRtHYmKifcu+vVdWfLHxGB6kcr/T72aDepFERERuiLOjTuzn54fVas3VexMXF5erlyfblClT6NatG88//zwAbdq0wcvLi+7duzN58mQCAwMJCAi45jGLcl4ANzc33NzcCn2dpSF7MdsHrKvxsCWbE0c2ucPRZYmIiJRrDutJcnV1pWPHjixdujRH+9KlSwkPD8/zPSkpKTg55SzZarUCZk8QQNeuXXMd8+eff7YfsyjnLevMxWyzeNz9F7Oh02hwsjq2KBERkXLOYT1JAGPHjmXo0KGEhobStWtXZsyYwdGjR+23z8aNG8eJEyeYN28eAP3792f06NFERkbSu3dvYmJiiIiIoHPnztSuXRuAZ599lptvvpk33niDu+++m++++45ly5axatWqAp+3PDEMgy82HifM8icNbIfBxRPaP+zoskRERMo9h4akgQMHkpCQwKRJk4iJiSEkJIRFixYRFBQEQExMTI65i4YPH05ycjLTpk3j73//O76+vtx222288cYb9n3Cw8NZsGABL7/8MuPHj6dRo0ZERUURFhZW4POWJ5uPnmN/3Hmed/vZbGjzIHhUc2xRIiIiFYBD50kqz8rKPEkvfvUHyzdsZZV7BFZs8MQa8G/lsHpERETKsnIxT5LcuAtpmfyw7SQPOy8zA1KD7gpIIiIixUQhqRxbtD2GzPSLDHH+zWzoPNqxBYmIiFQgCknl2Bcbj9HPaR2+JEHVutCs3/XfJCIiIgWikFROmYvZnmG486UB251GgNWh4/BFREQqFIWkcmrhpuO0t+ynjdNBsLpBh2GOLklERKRCUUgqhzJtWXy16TjDnJeYDSH3gZefY4sSERGpYBSSyqHle09jJJ+inzXabAjTOm0iIiLFTSGpHIracIzB1l9xIRPqdoba7R1dkoiISIWjkFTOnE5OY+Wf5txIAHRWL5KIiEhJUEgqZ77Zcpzbicbfcg6q+EPLux1dkoiISIWkkFSOZC9max+w3fFRcHZ1bFEiIiIVlEJSObL56DncTu+gk9NeDCdn6Djc0SWJiIhUWApJ5cjCjcd4xGpOHmlpeTdUDXRwRSIiIhWXQlI5kZKeyYpte7jbutps6Py4YwsSERGp4BSSyokf/4jhL7ZluFsyMALaQL3Oji5JRESkQlNIKie+2nCEIZce+7eEPQ4Wi4MrEhERqdgUksqBg6fPU/XYMupa4slyr24uQyIiIiIlSiGpHFi46TjDLg3YdgodBi4eDq5IRESk4lNIKuMybVls2biGbtadGDhB6AhHlyQiIlIpKCSVccv3nqZf6o8AGM3uBN/6Dq5IRESkclBIKuO+X7+be60rAXDqosf+RURESotCUhl2OjmNmvu/xMuSRlq1ZtCgu6NLEhERqTQUksqwbzcf42Enc8C2W7ge+xcRESlNCklllGEYHFz3PcFOp0h39oY2Ax1dkoiISKWikFRGbTl2jl7nvwXAaPcQuFVxbEEiIiKVjEJSGWPLMlh7IIFP/vcLt1q3kYUFt64asC0iIlLanB1dgFy2eEcME3/YRUxiKuOdvwRnWE17LsR40qeGo6sTERGpXNSTVEYs3hHDE59uJiYxlfrEMsS6FICP0u/giU83s3hHjIMrFBERqVwUksoAW5bBxB92YVz6eaD1N9wsmRzPqsGKrNYATPxhF7YsI/+DiIiISLFSSCoDog+dISYx9dJPBndZ1wHwoy0MAycMICYxlehDZxxWo4iISGWjMUllQFxyKjU5Sy3LOdo4HSDI6TQAJ/GjleWQuY/hS1xy6rUOIyIiIsVIIakMqOXtzsPOvxDh/HWO9oku8+zfv5d5L7W8+5Z2aSIiIpWWQlIZ0Dm4Ov/yvJNlyR0xgPZO+5js8jEvZIxmR1YDLADe/jwdXN3BlYqIiFQeDh+TNH36dIKDg3F3d6djx46sXLky332HDx+OxWLJtbVq1cq+T48ePfLcp1+/fvZ9JkyYkOv1gICAEr3Oa7E6WXjqLzex0whmlxHMlqwmAOzIasAuI5idRjBP/eUmrE5alkRERKS0ODQkRUVFERERwUsvvcSWLVvo3r07ffv25ejRo3nu/+9//5uYmBj7duzYMapXr84DDzxg3+frr7/Osc+OHTuwWq059gFo1apVjv22b99eotd6PX1CAokc0oEAH/cc7QE+7kQO6UCfkEAHVSYiIlI5OfR22zvvvMPIkSMZNWoUAO+99x5LliwhMjKSKVOm5Nrfx8cHHx8f+8/ffvstZ8+e5dFHH7W3Va+e85bUggUL8PT0zBWSnJ2dC9V7lJaWRlpamv3npKSkAr+3oPqEBHJHywC27KzNnztOMDHkdtq3aq4eJBEREQdwWE9Seno6mzZtolevXjnae/XqxZo1awp0jFmzZtGzZ0+CgoKuuc+gQYPw8vLK0b5v3z5q165NcHAwgwYN4uDBg9c815QpU+whzcfHh3r16hWoxsKyOlkIbd2C5oOnENq6hQKSiIiIgzgsJMXHx2Oz2fD398/R7u/vT2xs7HXfHxMTw08//WTvhcpLdHQ0O3bsyLVPWFgY8+bNY8mSJcycOZPY2FjCw8NJSEjI91jjxo0jMTHRvh07duy6NYqIiEj55fCn2yyWnD0lhmHkasvLxx9/jK+vLwMGDMh3n1mzZhESEkLnzp1ztPfte/lR+tatW9O1a1caNWrE3LlzGTt2bJ7HcnNzw83N7bp1iYiISMXgsJ4kPz8/rFZrrl6juLi4XL1LVzMMg9mzZzN06FBcXV3z3CclJYUFCxZcs6cpm5eXF61bt2bfvn0FvwARERGp0BwWklxdXenYsSNLly7N0b506VLCw8Ov+d7ly5ezf/9+Ro4cme8+X3zxBWlpaQwZMuS6taSlpbF7924CA/UEmYiIiJgcertt7NixDB06lNDQULp27cqMGTM4evQoY8aMAcxxQCdOnGDevHk53jdr1izCwsIICQnJ99izZs1iwIAB1KhRI9drzz33HP3796d+/frExcUxefJkkpKSGDZsWPFeoIiIiJRbDg1JAwcOJCEhgUmTJhETE0NISAiLFi2yP60WExOTa86kxMREvvrqK/7973/ne9y9e/eyatUqfv755zxfP378OIMHDyY+Pp6aNWvSpUsX1q1bd82n5ERERKRysRiGYTi6iPIoKSkJHx8fEhMTqVq1qqPLERERkQIozOe3w5clERERESmLFJJERERE8qCQJCIiIpIHh08mWV5lD+UqiTXcREREpGRkf24XZEi2QlIRJScnA5TYGm4iIiJScpKTk/Hx8bnmPnq6rYiysrI4efIk3t7eBVpGpTCSkpKoV68ex44dK9dPzuk6yhZdR9mi6yhbKsp1QMW5lpK6DsMwSE5Opnbt2jg5XXvUkXqSisjJyYm6deuW6DmqVq1arv+CZ9N1lC26jrJF11G2VJTrgIpzLSVxHdfrQcqmgdsiIiIieVBIEhEREcmDQlIZ5Obmxquvvoqbm5ujS7khuo6yRddRtug6ypaKch1Qca6lLFyHBm6LiIiI5EE9SSIiIiJ5UEgSERERyYNCkoiIiEgeFJJERERE8qCQVIasWLGC/v37U7t2bSwWC99++62jSyq0KVOm0KlTJ7y9valVqxYDBgxgz549ji6rSCIjI2nTpo19IrOuXbvy008/ObqsGzJlyhQsFgsRERGOLqXQJkyYgMViybEFBAQ4uqwiOXHiBEOGDKFGjRp4enrSrl07Nm3a5OiyCqVBgwa5/jwsFgtPPvmko0srlMzMTF5++WWCg4Px8PCgYcOGTJo0iaysLEeXVmjJyclEREQQFBSEh4cH4eHhbNiwwdFlXdP1PvcMw2DChAnUrl0bDw8PevTowc6dO0utPoWkMuTChQu0bduWadOmObqUIlu+fDlPPvkk69atY+nSpWRmZtKrVy8uXLjg6NIKrW7durz++uts3LiRjRs3ctttt3H33XeX6v+gxWnDhg3MmDGDNm3aOLqUImvVqhUxMTH2bfv27Y4uqdDOnj1Lt27dcHFx4aeffmLXrl28/fbb+Pr6Orq0QtmwYUOOP4ulS5cC8MADDzi4ssJ54403+O9//8u0adPYvXs3U6dO5c033+T99993dGmFNmrUKJYuXconn3zC9u3b6dWrFz179uTEiROOLi1f1/vcmzp1Ku+88w7Tpk1jw4YNBAQEcMcdd9jXTy1xhpRJgPHNN984uowbFhcXZwDG8uXLHV1KsahWrZrx0UcfObqMQktOTjaaNGliLF261LjllluMZ5991tElFdqrr75qtG3b1tFl3LAXXnjBuOmmmxxdRrF79tlnjUaNGhlZWVmOLqVQ+vXrZ4wYMSJH27333msMGTLEQRUVTUpKimG1Wo3//e9/Odrbtm1rvPTSSw6qqnCu/tzLysoyAgICjNdff93elpqaavj4+Bj//e9/S6Um9SRJiUpMTASgevXqDq7kxthsNhYsWMCFCxfo2rWro8sptCeffJJ+/frRs2dPR5dyQ/bt20ft2rUJDg5m0KBBHDx40NElFdr3339PaGgoDzzwALVq1aJ9+/bMnDnT0WXdkPT0dD799FNGjBhR7At+l7SbbrqJX375hb179wKwbds2Vq1axZ133ungygonMzMTm82Gu7t7jnYPDw9WrVrloKpuzKFDh4iNjaVXr172Njc3N2655RbWrFlTKjVogVspMYZhMHbsWG666SZCQkIcXU6RbN++na5du5KamkqVKlX45ptvaNmypaPLKpQFCxawefPmMj824XrCwsKYN28eTZs25dSpU0yePJnw8HB27txJjRo1HF1egR08eJDIyEjGjh3LP/7xD6Kjo3nmmWdwc3PjkUcecXR5RfLtt99y7tw5hg8f7uhSCu2FF14gMTGR5s2bY7Vasdls/Otf/2Lw4MGOLq1QvL296dq1K//85z9p0aIF/v7+fP7556xfv54mTZo4urwiiY2NBcDf3z9Hu7+/P0eOHCmVGhSSpMQ89dRT/PHHH+X2txiAZs2asXXrVs6dO8dXX33FsGHDWL58ebkJSseOHePZZ5/l559/zvUbZnnTt29f+/etW7ema9euNGrUiLlz5zJ27FgHVlY4WVlZhIaG8tprrwHQvn17du7cSWRkZLkNSbNmzaJv377Url3b0aUUWlRUFJ9++imfffYZrVq1YuvWrURERFC7dm2GDRvm6PIK5ZNPPmHEiBHUqVMHq9VKhw4deOihh9i8ebOjS7shV/dOGoZRaj2WCklSIp5++mm+//57VqxYQd26dR1dTpG5urrSuHFjAEJDQ9mwYQP//ve/+fDDDx1cWcFs2rSJuLg4OnbsaG+z2WysWLGCadOmkZaWhtVqdWCFRefl5UXr1q3Zt2+fo0splMDAwFwhu0WLFnz11VcOqujGHDlyhGXLlvH11187upQief7553nxxRcZNGgQYAbwI0eOMGXKlHIXkho1asTy5cu5cOECSUlJBAYGMnDgQIKDgx1dWpFkP70aGxtLYGCgvT0uLi5X71JJ0ZgkKVaGYfDUU0/x9ddf8+uvv5bb/znzYxgGaWlpji6jwG6//Xa2b9/O1q1b7VtoaCgPP/wwW7duLbcBCSAtLY3du3fn+MezPOjWrVuuaTH27t1LUFCQgyq6MXPmzKFWrVr069fP0aUUSUpKCk5OOT8KrVZruZwCIJuXlxeBgYGcPXuWJUuWcPfddzu6pCIJDg4mICDA/uQkmOPfli9fTnh4eKnUoJ6kMuT8+fPs37/f/vOhQ4fYunUr1atXp379+g6srOCefPJJPvvsM7777ju8vb3t95R9fHzw8PBwcHWF849//IO+fftSr149kpOTWbBgAb///juLFy92dGkF5u3tnWs8mJeXFzVq1Ch348See+45+vfvT/369YmLi2Py5MkkJSWVu9/2//a3vxEeHs5rr73Ggw8+SHR0NDNmzGDGjBmOLq3QsrKymDNnDsOGDcPZuXx+nPTv359//etf1K9fn1atWrFlyxbeeecdRowY4ejSCm3JkiUYhkGzZs3Yv38/zz//PM2aNePRRx91dGn5ut7nXkREBK+99hpNmjShSZMmvPbaa3h6evLQQw+VToGl8gydFMhvv/1mALm2YcOGObq0AsurfsCYM2eOo0srtBEjRhhBQUGGq6urUbNmTeP22283fv75Z0eXdcPK6xQAAwcONAIDAw0XFxejdu3axr333mvs3LnT0WUVyQ8//GCEhIQYbm5uRvPmzY0ZM2Y4uqQiWbJkiQEYe/bscXQpRZaUlGQ8++yzRv369Q13d3ejYcOGxksvvWSkpaU5urRCi4qKMho2bGi4uroaAQEBxpNPPmmcO3fO0WVd0/U+97KysoxXX33VCAgIMNzc3Iybb77Z2L59e6nVZzEMwyidOCYiIiJSfmhMkoiIiEgeFJJERERE8qCQJCIiIpIHhSQRERGRPCgkiYiIiORBIUlEREQkDwpJIiIiInlQSBIRERHJg0KSiEgxsVgsfPvtt44uQ0SKiUKSiFQIw4cPx2Kx5Nr69Onj6NJEpJwqnysSiojkoU+fPsyZMydHm5ubm4OqEZHyTj1JIlJhuLm5ERAQkGOrVq0aYN4Ki4yMpG/fvnh4eBAcHMzChQtzvH/79u3cdttteHh4UKNGDR577DHOnz+fY5/Zs2fTqlUr3NzcCAwM5Kmnnsrxenx8PPfccw+enp40adKE77//vmQvWkRKjEKSiFQa48eP57777mPbtm0MGTKEwYMHs3v3bgBSUlLo06cP1apVY8OGDSxcuJBly5blCEGRkZE8+eSTPPbYY2zfvp3vv/+exo0b5zjHxIkTefDBB/njjz+48847efjhhzlz5kypXqeIFBNDRKQCGDZsmGG1Wg0vL68c26RJkwzDMAzAGDNmTI73hIWFGU888YRhGIYxY8YMo1q1asb58+ftr//444+Gk5OTERsbaxiGYdSuXdt46aWX8q0BMF5++WX7z+fPnzcsFovx008/Fdt1ikjp0ZgkEakwbr31ViIjI3O0Va9e3f59165dc7zWtWtXtm7dCsDu3btp27YtXl5e9te7detGVlYWe/bswWKxcPLkSW6//fZr1tCmTRv7915eXnh7exMXF1fUSxIRB1JIEpEKw8vLK9ftr+uxWCwAGIZh/z6vfTw8PAp0PBcXl1zvzcrKKlRNIlI2aEySiFQa69aty/Vz8+bNAWjZsiVbt27lwoUL9tdXr16Nk5MTTZs2xdvbmwYNGvDLL7+Uas0i4jjqSRKRCiMtLY3Y2Ngcbc7Ozvj5+QGwcOFCQkNDuemmm5g/fz7R0dHMmjULgIcffphXX32VYcOGMWHCBE6fPs3TTz/N0KFD8ff3B2DChAmMGTOGWrVq0bdvX5KTk1m9ejVPP/106V6oiJQKhSQRqTAWL15MYGBgjrZmzZrx559/AuaTZwsWLOCvf/0rAQEBzJ8/n5YtWwLg6enJkiVLePbZZ+nUqROenp7cd999vPPOO/ZjDRs2jNTUVN59912ee+45/Pz8uP/++0vvAkWkVFkMwzAcXYSISEmzWCx88803DBgwwNGliEg5oTFJIiIiInlQSBIRERHJg8YkiUiloJEFIlJY6kkSERERyYNCkoiIiEgeFJJERERE8qCQJCIiIpIHhSQRERGRPCgkiYiIiORBIUlEREQkDwpJIiIiInn4f6sbjzgsCsIXAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"num_epochs = 10 # during optimization, how many times we look at training data\n",
"batch_size = 64 # during optimization, how many training data to use at each step\n",
"learning_rate = 0.005 # during optimization, how much we nudge our solution at each step\n",
"\n",
"train_accuracies, val_accuracies = \\\n",
" UDA_pytorch_classifier_fit(simple_lstm_model,\n",
" torch.optim.Adam(simple_lstm_model.parameters(),\n",
" lr=learning_rate),\n",
" nn.CrossEntropyLoss(), # includes softmax\n",
" proper_train_dataset_encoded, val_dataset_encoded,\n",
" num_epochs, batch_size,\n",
" rnn=True,\n",
" save_epoch_checkpoint_prefix='./saved_model_checkpoints/imdb_lstm')\n",
"\n",
"UDA_plot_train_val_accuracy_vs_epoch(train_accuracies, val_accuracies)"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The model at the end of epoch 8 achieved the highest validation accuracy: 0.877600\n"
]
},
{
"data": {
"text/plain": [
"<All keys matched successfully>"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"best_epoch_idx = np.argmax(val_accuracies)\n",
"print('The model at the end of epoch %d achieved the highest validation accuracy: %f'\n",
" % (best_epoch_idx + 1, val_accuracies[best_epoch_idx]))\n",
"simple_lstm_model.load_state_dict(torch.load('./saved_model_checkpoints/imdb_lstm_epoch%d.pt' % (best_epoch_idx + 1)))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Finally evaluate on test data"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"test_dataset = []\n",
"\n",
"for filename in sorted(glob('./data/aclImdb/test/pos/*.txt')):\n",
" with open(filename, 'r', encoding='utf-8') as f:\n",
" test_dataset.append((f.read(), 1)) # 1 means `positive` sentiment\n",
"\n",
"for filename in sorted(glob('./data/aclImdb/test/neg/*.txt')):\n",
" with open(filename, 'r', encoding='utf-8') as f:\n",
" test_dataset.append((f.read(), 0)) # 0 means `negative` sentiment"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [],
"source": [
"test_encoded = [vocab(tokenizer(text)) for text, label in test_dataset]"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"test_labels = [label for text, label in test_dataset]"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
"predicted_test_labels = UDA_pytorch_classifier_predict(simple_lstm_model,\n",
" test_encoded,\n",
" rnn=True)"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Test accuracy: 0.87848\n"
]
}
],
"source": [
"print('Test accuracy:', UDA_compute_accuracy(predicted_test_labels, test_labels))"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([1], device='cuda:0')"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"UDA_pytorch_classifier_predict(simple_lstm_model,\n",
" [vocab(tokenizer('this movie rocks'))],\n",
" rnn=True)"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([0], device='cuda:0')"
]
},
"execution_count": 38,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"UDA_pytorch_classifier_predict(simple_lstm_model,\n",
" [vocab(tokenizer('this movie sucks'))],\n",
" rnn=True)"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([0], device='cuda:0')"
]
},
"execution_count": 39,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"UDA_pytorch_classifier_predict(simple_lstm_model,\n",
" [vocab(tokenizer('this sucks'))],\n",
" rnn=True)"
]
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
}
},
"nbformat": 4,
"nbformat_minor": 1
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment