Skip to content

Instantly share code, notes, and snippets.

@AwsafAlam

AwsafAlam/preprocessing_contrastive_learning_toxicr_bert-v2.ipynb

Last active August 19, 2023 06:49
Show Gist options
  • Save AwsafAlam/10a70361382dfcc72f1b62e5fd50439a to your computer and use it in GitHub Desktop.
Save AwsafAlam/10a70361382dfcc72f1b62e5fd50439a to your computer and use it in GitHub Desktop.
Download ZIP
Contrastive_Learning_ToxiCR_BERT-Basic.ipynb
Display the source blob
Display the rendered blob
Raw
preprocessing_contrastive_learning_toxicr_bert-v2.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"gpuType": "T4",
"collapsed_sections": [
"4JrUHXms16cn",
"lgmMzEm_ShK1",
"ORnn8VPz9uLC",
"so-0GYfcLuaf"
],
"toc_visible": true,
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/AwsafAlam/10a70361382dfcc72f1b62e5fd50439a/preprocessing_contrastive_learning_toxicr_bert-v2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "EKOTlwcmxmej"
},
"source": [
"# ToxiCR Dataset BERT Fine-Tuning using PyTorch\n",
"\n",
"~ *Modified: Md Awsaf Alam*"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "nSU7yERLP_66"
},
"source": [
"### Step 1: Steup\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "GI0iOY8zvZzL"
},
"source": [
"GPU Detection"
]
},
{
"cell_type": "code",
"metadata": {
"id": "DEfSbAA4QHas",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "1d792c59-b50e-4cc5-9c18-ff6bf748acbf"
},
"source": [
"import tensorflow as tf\n",
"\n",
"# Get the GPU device name.\n",
"device_name = tf.test.gpu_device_name()\n",
"\n",
"# The device name should look like the following:\n",
"if device_name == '/device:GPU:0':\n",
" print('Found GPU at: {}'.format(device_name))\n",
"else:\n",
" raise SystemError('GPU device not found')"
],
"execution_count": 36,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Found GPU at: /device:GPU:0\n"
]
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "cqG7FzRVFEIv"
},
"source": [
"In order for torch to use the GPU, we need to identify and specify the GPU as the device. Later, in our training loop, we will load data onto the device."
]
},
{
"cell_type": "code",
"metadata": {
"id": "oYsV4H8fCpZ-",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "db306625-7003-497e-e93c-fd368e5a3f71"
},
"source": [
"import torch\n",
"\n",
"# If there's a GPU available...\n",
"if torch.cuda.is_available():\n",
"\n",
" # Tell PyTorch to use the GPU.\n",
" device = torch.device(\"cuda\")\n",
"\n",
" print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
"\n",
" print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
"\n",
"# If not...\n",
"else:\n",
" print('No GPU available, using the CPU instead.')\n",
" device = torch.device(\"cpu\")"
],
"execution_count": 37,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"There are 1 GPU(s) available.\n",
"We will use the GPU: Tesla T4\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"!nvidia-smi"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "9gFnA5Ep8TOF",
"outputId": "5c90f2bf-b42f-44f7-c0cd-b1cc84bfef3b"
},
"execution_count": 38,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Sat Aug 19 06:08:22 2023 \n",
"+-----------------------------------------------------------------------------+\n",
"| NVIDIA-SMI 525.105.17 Driver Version: 525.105.17 CUDA Version: 12.0 |\n",
"|-------------------------------+----------------------+----------------------+\n",
"| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
"| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n",
"| | | MIG M. |\n",
"|===============================+======================+======================|\n",
"| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
"| N/A 75C P0 31W / 70W | 1353MiB / 15360MiB | 0% Default |\n",
"| | | N/A |\n",
"+-------------------------------+----------------------+----------------------+\n",
" \n",
"+-----------------------------------------------------------------------------+\n",
"| Processes: |\n",
"| GPU GI CI PID Type Process name GPU Memory |\n",
"| ID ID Usage |\n",
"|=============================================================================|\n",
"+-----------------------------------------------------------------------------+\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"## Clearing cache\n",
"import gc\n",
"# del variables\n",
"gc.collect()\n",
"# del model\n",
"# del tokenizer\n",
"torch.cuda.memory_summary(device=None, abbreviated=False)\n",
"torch.cuda.empty_cache()\n"
],
"metadata": {
"id": "bUmPt74q78kd"
},
"execution_count": 39,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "2ElsnSNUridI"
},
"source": [
"Installing the Hugging Face Library\n"
]
},
{
"cell_type": "code",
"metadata": {
"id": "0NmMdkZO8R6q",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "e7edb376-e1b0-44ab-bf95-c7ad100bd25f"
},
"source": [
"!pip install transformers\n",
"!pip install pytorch_metric_learning"
],
"execution_count": 40,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.31.0)\n",
"Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.2)\n",
"Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.16.4)\n",
"Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
"Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
"Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
"Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
"Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
"Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.13.3)\n",
"Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.3.2)\n",
"Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
"Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.6.0)\n",
"Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.7.1)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.2.0)\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.4)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
"Requirement already satisfied: pytorch_metric_learning in /usr/local/lib/python3.10/dist-packages (2.3.0)\n",
"Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from pytorch_metric_learning) (1.23.5)\n",
"Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from pytorch_metric_learning) (1.2.2)\n",
"Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from pytorch_metric_learning) (4.66.1)\n",
"Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from pytorch_metric_learning) (2.0.1+cu118)\n",
"Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->pytorch_metric_learning) (3.12.2)\n",
"Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->pytorch_metric_learning) (4.7.1)\n",
"Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->pytorch_metric_learning) (1.12)\n",
"Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->pytorch_metric_learning) (3.1)\n",
"Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->pytorch_metric_learning) (3.1.2)\n",
"Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->pytorch_metric_learning) (2.0.0)\n",
"Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->pytorch_metric_learning) (3.27.2)\n",
"Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->pytorch_metric_learning) (16.0.6)\n",
"Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->pytorch_metric_learning) (1.10.1)\n",
"Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->pytorch_metric_learning) (1.3.2)\n",
"Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->pytorch_metric_learning) (3.2.0)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.6.0->pytorch_metric_learning) (2.1.3)\n",
"Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.6.0->pytorch_metric_learning) (1.3.0)\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import torch\n",
"import os\n",
"import random\n",
"import time\n",
"import numpy as np\n",
"import pandas as pd\n",
"\n",
"from pytorch_metric_learning import losses as loss_fun\n",
"from pytorch_metric_learning.distances import CosineSimilarity\n",
"from pytorch_metric_learning.reducers import ThresholdReducer\n",
"from pytorch_metric_learning.regularizers import LpRegularizer\n",
"\n",
"from transformers import BertTokenizer, ElectraTokenizer, ElectraForSequenceClassification, ElectraModel\n",
"from transformers import BertForSequenceClassification, AdamW, get_linear_schedule_with_warmup, BertModel\n",
"\n",
"import torch.nn.functional as F\n",
"import torch.nn as nn\n",
"\n",
"from torch.utils.data import Dataset\n",
"from torch.utils.data.dataloader import DataLoader\n",
"from torch import nn, optim\n",
"\n",
"EPOCHS = 4\n",
"BATCH_SIZE = 16\n",
"# MAX_LEN_LIST = [200,160]\n",
"undersample = [False]\n",
"learn_rate_list = [5e-5,3e-5,1e-5,8e-6]\n",
"\n",
"# Set the seed value all over the place to make this reproducible.\n",
"seed_val = 42\n",
"\n",
"random.seed(seed_val)\n",
"np.random.seed(seed_val)\n",
"torch.manual_seed(seed_val)\n",
"torch.cuda.manual_seed_all(seed_val)\n",
"\n"
],
"metadata": {
"id": "BP3_FiOcesNX"
},
"execution_count": 41,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"Helper functions"
],
"metadata": {
"id": "e5CN-SsEzL2d"
}
},
{
"cell_type": "code",
"metadata": {
"id": "gpt6tR83keZD"
},
"source": [
"import time\n",
"import datetime\n",
"import numpy as np\n",
"\n",
"# Function to calculate the accuracy of our predictions vs labels\n",
"def flat_accuracy(preds, labels):\n",
" pred_flat = np.argmax(preds, axis=1).flatten()\n",
" labels_flat = labels.flatten()\n",
" return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
"\n",
"def format_time(elapsed):\n",
" '''\n",
" Takes a time in seconds and returns a string hh:mm:ss\n",
" '''\n",
" # Round to the nearest second.\n",
" elapsed_rounded = int(round((elapsed)))\n",
"\n",
" # Format as hh:mm:ss\n",
" return str(datetime.timedelta(seconds=elapsed_rounded))\n"
],
"execution_count": 42,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "4JrUHXms16cn"
},
"source": [
"### Step 2: Preprocessing"
]
},
{
"cell_type": "markdown",
"source": [
"##### Init Preprocessing Functions"
],
"metadata": {
"id": "lgmMzEm_ShK1"
}
},
{
"cell_type": "code",
"source": [
"import re\n",
"\n",
"url_regex = re.compile('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')\n",
"\n",
"\n",
"def remove_url(text):\n",
" return url_regex.sub(\" \", text)\n",
"\n",
"\n",
"def rem_special_sym(text):\n",
"# return re.sub('\\W+',' ', text)\n",
" pattern = re.compile('([^\\s\\w]|_)+')\n",
" return pattern.sub(' ', text)\n"
],
"metadata": {
"id": "-BxnpOeZSlfD"
},
"execution_count": 45,
"outputs": []
},
{
"cell_type": "code",
"source": [
"contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\n",
" \"can't\": \"cannot\", \"'cause\": \"because\",\n",
" \"could've\": \"could have\", \"couldn't\": \"could not\",\n",
" \"didn't\": \"did not\", \"doesn't\": \"does not\",\n",
" \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\",\n",
" \"haven't\": \"have not\", \"he'd\": \"he would\", \"he'll\": \"he will\",\n",
" \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\",\n",
" \"how'll\": \"how will\", \"how's\": \"how is\", \"I'd\": \"I would\",\n",
" \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\n",
" \"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\",\n",
" \"i'll\": \"i will\", \"i'll've\": \"i will have\", \"i'm\": \"i am\",\n",
" \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n",
" \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\n",
" \"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\",\n",
" \"might've\": \"might have\", \"mightn't\": \"might not\",\n",
" \"mightn't've\": \"might not have\", \"must've\": \"must have\",\n",
" \"mustn't\": \"must not\", \"mustn't've\": \"must not have\",\n",
" \"needn't\": \"need not\", \"needn't've\": \"need not have\",\n",
" \"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\",\n",
" \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\",\n",
" \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\",\n",
" \"she'd\": \"she would\", \"she'd've\": \"she would have\",\n",
" \"she'll\": \"she will\", \"she'll've\": \"she will have\",\n",
" \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\",\n",
" \"shouldn't've\": \"should not have\", \"so've\": \"so have\", \"so's\": \"so as\",\n",
" \"this's\": \"this is\", \"that'd\": \"that would\", \"that'd've\": \"that would have\",\n",
" \"that's\": \"that is\", \"there'd\": \"there would\",\n",
" \"there'd've\": \"there would have\", \"there's\": \"there is\",\n",
" \"here's\": \"here is\", \"they'd\": \"they would\", \"they'd've\": \"they would have\",\n",
" \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\",\n",
" \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\",\n",
" \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\",\n",
" \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\",\n",
" \"what'll\": \"what will\",\n",
" \"what'll've\": \"what will have\", \"what're\": \"what are\", \"what's\": \"what is\",\n",
" \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\",\n",
" \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\",\n",
" \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\",\n",
" \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\",\n",
" \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\",\n",
" \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\",\n",
" \"y'all\": \"you all\", \"y'all'd\": \"you all would\", \"y'all'd've\": \"you all would have\",\n",
" \"y'all're\": \"you all are\", \"y'all've\": \"you all have\", \"you'd\": \"you would\",\n",
" \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\",\n",
" \"you're\": \"you are\", \"you've\": \"you have\", \"aint\": \"is not\", \"arent\": \"are not\",\n",
" \"cant\": \"cannot\", \"cause\": \"because\",\n",
" \"couldve\": \"could have\", \"couldnt\": \"could not\",\n",
" \"didnt\": \"did not\", \"doesnt\": \"does not\",\n",
" \"dont\": \"do not\", \"hadnt\": \"had not\", \"hasnt\": \"has not\",\n",
" \"havent\": \"have not\", \"howdy\": \"how do you\",\n",
" \"its\": \"it is\", \"lets\": \"let us\", \"maam\": \"madam\", \"maynt\": \"may not\",\n",
" \"mightve\": \"might have\", \"mightnt\": \"might not\",\n",
" \"mightntve\": \"might not have\", \"mustve\": \"must have\",\n",
" \"mustnt\": \"must not\", \"mustntve\": \"must not have\",\n",
" \"neednt\": \"need not\", \"needntve\": \"need not have\",\n",
" \"oclock\": \"of the clock\", \"oughtnt\": \"ought not\",\n",
" \"shouldve\": \"should have\", \"shouldnt\": \"should not\",\n",
" \"werent\": \"were not\", \"yall\": \"you all\", \"youre\": \"you are\",\n",
" \"youve\": \"you have\"}\n",
"\n",
"def expand_contraction(text):\n",
" specials = [\"’\", \"‘\", \"´\", \"`\", \"'\"]\n",
"\n",
" for s in specials:\n",
" text = text.replace(s, \"'\")\n",
" text = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in text.split(\" \")])\n",
" return text\n"
],
"metadata": {
"id": "48Z_Ajo8SwPS"
},
"execution_count": 46,
"outputs": []
},
{
"cell_type": "code",
"source": [
"RE_PATTERNS = {\n",
"\n",
" ' fuck ':\n",
" [\n",
" '(f)(u|[^a-z0-9 ])(c|[^a-z0-9 ])(k|[^a-z0-9 ])([^ ])*',\n",
" '(f)([^a-z]*)(u)([^a-z]*)(c)([^a-z]*)(k)',\n",
" ' f[!@#\\$%\\^\\&\\*]*u[!@#\\$%\\^&\\*]*k', 'f u u c',\n",
" '(f)(c|[^a-z ])(u|[^a-z ])(k)', r'f\\*',\n",
" 'feck ', ' fux ', 'f\\*\\*',\n",
" 'f\\-ing', 'f\\.u\\.', 'f###', ' fu ', 'f@ck', 'f u c k', 'f uck', 'f ck'\n",
"\n",
" ],\n",
"\n",
" ' crap ':\n",
" [\n",
" ' (c)(r|[^a-z0-9 ])(a|[^a-z0-9 ])(p|[^a-z0-9 ])([^ ])*',\n",
" ' (c)([^a-z]*)(r)([^a-z]*)(a)([^a-z]*)(p)',\n",
" ' c[!@#\\$%\\^\\&\\*]*r[!@#\\$%\\^&\\*]*p', 'cr@p', ' c r a p',\n",
"\n",
" ],\n",
"\n",
" ' ass ':\n",
" [\n",
" '[^a-z]ass ', '[^a-z]azz ', 'arrse', ' arse ', '@\\$\\$'\n",
" '[^a-z]anus', ' a\\*s\\*s', '[^a-z]ass[^a-z ]',\n",
" 'a[@#\\$%\\^&\\*][@#\\$%\\^&\\*]', '[^a-z]anal ', 'a s s'\n",
" ],\n",
"\n",
" ' ass hole ':\n",
" [\n",
" ' a[s|z]*wipe', 'a[s|z]*[w]*h[o|0]+[l]*e', '@\\$\\$hole'\n",
" ],\n",
"\n",
" ' bitch ':\n",
" [\n",
" 'bitches', ' b[w]*i[t]*ch', ' b!tch',\n",
" ' bi\\+ch', ' b!\\+ch', ' (b)([^a-z]*)(i)([^a-z]*)(t)([^a-z]*)(c)([^a-z]*)(h)',\n",
" ' biatch', ' bi\\*\\*h', ' bytch', 'b i t c h'\n",
" ],\n",
"\n",
" ' bastard ':\n",
" [\n",
" 'ba[s|z]+t[e|a]+rd'\n",
" ],\n",
"\n",
" ' transgender':\n",
" [\n",
" 'transgender'\n",
" ],\n",
"\n",
" ' gay ':\n",
" [\n",
" 'gay', 'homo'\n",
" ],\n",
"\n",
" ' cock ':\n",
" [\n",
" '[^a-z]cock', 'c0ck', '[^a-z]cok ', 'c0k', '[^a-z]cok[^aeiou]', ' cawk',\n",
" '(c)([^a-z ])(o)([^a-z ]*)(c)([^a-z ]*)(k)', 'c o c k'\n",
" ],\n",
"\n",
" ' dick ':\n",
" [\n",
" ' dick[^aeiou]', 'd i c k'\n",
" ],\n",
"\n",
" ' suck ':\n",
" [\n",
" 'sucker', '(s)([^a-z ]*)(u)([^a-z ]*)(c)([^a-z ]*)(k)', 'sucks', '5uck', 's u c k'\n",
" ],\n",
"\n",
" ' cunt ':\n",
" [\n",
" 'cunt', 'c u n t'\n",
" ],\n",
"\n",
" ' bull shit ':\n",
" [\n",
" 'bullsh\\*t', 'bull\\$hit', 'bull sh.t'\n",
" ],\n",
"\n",
" ' jerk ':\n",
" [\n",
" 'jerk'\n",
" ],\n",
"\n",
" ' idiot ':\n",
" [\n",
" 'i[d]+io[t]+', '(i)([^a-z ]*)(d)([^a-z ]*)(i)([^a-z ]*)(o)([^a-z ]*)(t)', 'idiots' 'i d i o t'\n",
" ],\n",
"\n",
" ' dumb ':\n",
" [\n",
" '(d)([^a-z ]*)(u)([^a-z ]*)(m)([^a-z ]*)(b)'\n",
" ],\n",
"\n",
" ' shit ':\n",
" [\n",
" 'shitty', '(s)([^a-z ]*)(h)([^a-z ]*)(i)([^a-z ]*)(t)', 'shite', '\\$hit', 's h i t', 'sh\\*tty',\n",
" 'sh\\*ty', 'sh\\*t'\n",
" ],\n",
"\n",
" ' shit hole ':\n",
" [\n",
" 'shythole', 'sh.thole'\n",
" ],\n",
"\n",
" ' retard ':\n",
" [\n",
" 'returd', 'retad', 'retard', 'wiktard', 'wikitud'\n",
" ],\n",
"\n",
" ' rape ':\n",
" [\n",
" 'raped'\n",
" ],\n",
"\n",
" ' dumb ass':\n",
" [\n",
" 'dumbass', 'dubass'\n",
" ],\n",
"\n",
" ' ass head':\n",
" [\n",
" 'butthead'\n",
" ],\n",
"\n",
" ' sex ':\n",
" [\n",
" 'sexy', 's3x', 'sexuality'\n",
" ],\n",
"\n",
" ' nigger ':\n",
" [\n",
" 'nigger', 'ni[g]+a', ' nigr ', 'negrito', 'niguh', 'n3gr', 'n i g g e r'\n",
" ],\n",
"\n",
" ' shut the fuck up':\n",
" [\n",
" ' stfu' '^stfu'\n",
" ],\n",
"\n",
" ' for your fucking information':\n",
" [\n",
" ' fyfi', '^fyfi'\n",
" ],\n",
" ' get the fuck off':\n",
" [\n",
" 'gtfo', '^gtfo'\n",
" ],\n",
"\n",
" ' oh my fucking god ':\n",
" [\n",
" ' omfg', '^omfg'\n",
" ],\n",
"\n",
" ' what the hell ':\n",
" [\n",
" ' wth', '^wth'\n",
" ],\n",
"\n",
" ' what the fuck ':\n",
" [\n",
" ' wtf', '^wtf'\n",
" ],\n",
" ' son of bitch ':\n",
" [\n",
" ' sob ', '^sob '\n",
" ],\n",
"\n",
" ' pussy ':\n",
" [\n",
" 'pussy[^c]', 'pusy', 'pussi[^l]', 'pusses', '(p)(u|[^a-z0-9 ])(s|[^a-z0-9 ])(s|[^a-z0-9 ])(y)',\n",
" ],\n",
"\n",
" ' faggot ':\n",
" [\n",
" 'faggot', ' fa[g]+[s]*[^a-z ]', 'fagot', 'f a g g o t', 'faggit',\n",
" '(f)([^a-z ]*)(a)([^a-z ]*)([g]+)([^a-z ]*)(o)([^a-z ]*)(t)', 'fau[g]+ot', 'fae[g]+ot',\n",
" ],\n",
"\n",
" ' mother fucker':\n",
" [\n",
" ' motha f', ' mother f', 'motherucker', ' mofo', ' mf ',\n",
" ],\n",
"\n",
" ' whore ':\n",
" [\n",
" 'wh\\*\\*\\*', 'w h o r e'\n",
" ],\n",
"\n",
" ' haha ':\n",
" [\n",
" 'ha\\*\\*\\*ha',\n",
" ],\n",
" # ' what the fuck ':\n",
" # [\n",
" # ' wtf',\n",
" # ],\n",
"}\n",
"\n",
"patterns = RE_PATTERNS\n",
"initial_filters=r\"[^a-z0-9!@#\\$%\\^\\*\\+\\?\\&\\_\\-,\\.' ]\"\n",
"lower=True\n",
"remove_repetitions=True"
],
"metadata": {
"id": "y3SY7-8kUPEf"
},
"execution_count": 47,
"outputs": []
},
{
"cell_type": "code",
"source": [
"def process_profanity(text):\n",
" # lower\n",
" if lower:\n",
" text = text.lower()\n",
"\n",
" # remove special chars\n",
" if initial_filters is not None:\n",
" text = re.sub(initial_filters, ' ', text)\n",
"\n",
" # neeeeeeeeeerd => nerd\n",
" if remove_repetitions:\n",
" pattern = re.compile(r\"(.)\\1{2,}\", re.DOTALL)\n",
" text = pattern.sub(r\"\\1\", text)\n",
"\n",
" x = text\n",
" for target, pattern in patterns.items():\n",
" for pat in pattern:\n",
" x = re.sub(pat, target, x)\n",
" x = re.sub(r\"[^a-z' ]\", ' ', x)\n",
" return x\n",
"\n"
],
"metadata": {
"id": "_3KTYbdoUhdr"
},
"execution_count": 48,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"#### Loading Data"
],
"metadata": {
"id": "0T1zN5BboiBp"
}
},
{
"cell_type": "markdown",
"metadata": {
"id": "3ZNVW6xd0T0X"
},
"source": [
"We'll use the `wget` package to download the dataset to the Colab instance's file system."
]
},
{
"cell_type": "code",
"metadata": {
"id": "5m6AnuFv0QXQ",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "291dbae1-227b-4e47-909d-9dd31126f8bc"
},
"source": [
"!pip install wget"
],
"execution_count": 43,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Requirement already satisfied: wget in /usr/local/lib/python3.10/dist-packages (3.2)\n"
]
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "08pO03Ff1BjI"
},
"source": [
"Downloading ToxiCR daatset: https://github.com/WSU-SEAL/ToxiCR"
]
},
{
"cell_type": "code",
"metadata": {
"id": "pMtmPMkBzrvs",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "642b461f-2e0b-4937-bdcf-f8c1167af2e6"
},
"source": [
"import wget\n",
"import os\n",
"\n",
"print('Downloading dataset...')\n",
"\n",
"# The URL for the dataset zip file.\n",
"url = 'https://github.com/WSU-SEAL/ToxiCR/blob/master/models/code-review-dataset-full.xlsx?raw=true'\n",
"\n",
"# Download the file (if we haven't already)\n",
"if not os.path.exists('./code-review-dataset-full.xlsx'):\n",
" wget.download(url, './code-review-dataset-full.xlsx')\n",
"\n"
],
"execution_count": 44,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Downloading dataset...\n"
]
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "0Yv-tNv20dnH",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 417
},
"outputId": "11558882-385d-43ab-8e0c-12f19539ea12"
},
"source": [
"import pandas as pd\n",
"\n",
"def read_dataframe_from_excel(file):\n",
" print(\"Reading dataframe\")\n",
" dataframe = pd.read_excel(file)\n",
" return dataframe\n",
"\n",
"\n",
"# training_data_df = read_dataframe_from_excel('./code-review-dataset-full.xlsx')\n",
"df = read_dataframe_from_excel('./code-review-dataset-full.xlsx')\n",
"\n",
"\n",
"# Report the number of sentences.\n",
"print('Number of training sentences: {:,}\\n'.format(df.shape[0]))\n",
"\n",
"\n",
"# Display 10 random rows from the data.\n",
"df.sample(10)"
],
"execution_count": 49,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Reading dataframe\n",
"Number of training sentences: 19,651\n",
"\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
" message is_toxic\n",
"10504 While you're here fixing, drop these whitespac... 0\n",
"10882 Put it inside #ifdef SUPPORT_CHECKSUM 0\n",
"18463 This metdata_signature_size should not be incl... 0\n",
"7567 Remove linebreak 0\n",
"8692 lol, \"sucks\" is probably not the right word fo... 1\n",
"12395 If there is one by SLO segment, will this effe... 1\n",
"1535 the !sURL.isEmpty() part is redundant now and ... 0\n",
"19285 '==' is not case insensitive, so that is not a... 0\n",
"10455 I'm trying to structure data and you ask me to... 1\n",
"1815 You probably don't want two Change-Id lines in... 0"
],
"text/html": [
"\n",
" <div id=\"df-4336a41f-af51-481d-8879-d2900844b854\" class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>message</th>\n",
" <th>is_toxic</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>10504</th>\n",
" <td>While you're here fixing, drop these whitespac...</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10882</th>\n",
" <td>Put it inside #ifdef SUPPORT_CHECKSUM</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18463</th>\n",
" <td>This metdata_signature_size should not be incl...</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7567</th>\n",
" <td>Remove linebreak</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8692</th>\n",
" <td>lol, \"sucks\" is probably not the right word fo...</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12395</th>\n",
" <td>If there is one by SLO segment, will this effe...</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1535</th>\n",
" <td>the !sURL.isEmpty() part is redundant now and ...</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19285</th>\n",
" <td>'==' is not case insensitive, so that is not a...</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10455</th>\n",
" <td>I'm trying to structure data and you ask me to...</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1815</th>\n",
" <td>You probably don't want two Change-Id lines in...</td>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <div class=\"colab-df-buttons\">\n",
"\n",
" <div class=\"colab-df-container\">\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4336a41f-af51-481d-8879-d2900844b854')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
"\n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
" <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
" </svg>\n",
" </button>\n",
"\n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" .colab-df-buttons div {\n",
" margin-bottom: 4px;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-4336a41f-af51-481d-8879-d2900844b854 button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-4336a41f-af51-481d-8879-d2900844b854');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
"\n",
"\n",
"<div id=\"df-ff583c6f-e3bd-49a2-a256-8eb78524aaf7\">\n",
" <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ff583c6f-e3bd-49a2-a256-8eb78524aaf7')\"\n",
" title=\"Suggest charts.\"\n",
" style=\"display:none;\">\n",
"\n",
"<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <g>\n",
" <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
" </g>\n",
"</svg>\n",
" </button>\n",
"\n",
"<style>\n",
" .colab-df-quickchart {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-quickchart:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-quickchart {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-quickchart:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
"</style>\n",
"\n",
" <script>\n",
" async function quickchart(key) {\n",
" const charts = await google.colab.kernel.invokeFunction(\n",
" 'suggestCharts', [key], {});\n",
" }\n",
" (() => {\n",
" let quickchartButtonEl =\n",
" document.querySelector('#df-ff583c6f-e3bd-49a2-a256-8eb78524aaf7 button');\n",
" quickchartButtonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
" })();\n",
" </script>\n",
"</div>\n",
" </div>\n",
" </div>\n"
]
},
"metadata": {},
"execution_count": 49
}
]
},
{
"cell_type": "code",
"source": [
"import torch.utils.data as data\n",
"\n",
"train_size = int(0.99 * df.shape[0])\n",
"test_size = df.shape[0] - train_size\n",
"\n",
"# Divide the dataset by randomly selecting samples.\n",
"train_subset, test_subset = data.random_split(df, [train_size, test_size])\n",
"\n",
"train_df = df.iloc[train_subset.indices]\n",
"test_df = df.iloc[test_subset.indices]\n",
"\n",
"\n",
"print('{:>5,} training samples'.format(train_size))\n",
"print('{:>5,} test samples'.format(test_size))\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "mbBoLYUpowPy",
"outputId": "a7cc8a89-9bb5-4c29-c3b9-8c249525695b"
},
"execution_count": 50,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"19,454 training samples\n",
" 197 test samples\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# writing to Excel\n",
"# determining the name of the file\n",
"file_name = 'test-set.xlsx'\n",
"datatoexcel = pd.ExcelWriter(file_name)\n",
"\n",
"\n",
"# saving the excel\n",
"# test_df.to_excel(file_name)\n",
"\n",
"# write DataFrame to excel\n",
"test_df.to_excel(datatoexcel)\n",
"\n",
"# save the excel\n",
"datatoexcel.save()\n",
"print('DataFrame is written to Excel File successfully.')"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "19ckndF9YeZQ",
"outputId": "344624fd-e871-43b1-bd5a-a681a23e6200"
},
"execution_count": 51,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"DataFrame is written to Excel File successfully.\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"<ipython-input-51-50224c78287f>:14: FutureWarning: save is not part of the public API, usage can give unexpected results and will be removed in a future version\n",
" datatoexcel.save()\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# Mount Google Drive to this Notebook instance.\n",
"from google.colab import drive\n",
"\n",
"drive.mount('/content/drive')\n",
"\n",
"# Copy the model files to a directory in your Google Drive.\n",
"!cp -r ./test-set.xlsx ./drive/MyDrive"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "qtkVPM3uZraA",
"outputId": "1fd5bb83-1e31-4187-999a-4a6974e1dd98"
},
"execution_count": 52,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"#### Preprocess training data"
],
"metadata": {
"id": "ob5PXap-eK2E"
}
},
{
"cell_type": "code",
"source": [
"def process_text(text):\n",
" processed_text = remove_url(text)\n",
" processed_text = expand_contraction(processed_text)\n",
" processed_text = process_profanity(processed_text)\n",
" processed_text = rem_special_sym(processed_text)\n",
" return processed_text\n",
"\n",
"\n",
"train_df['modified_message'] = train_df.astype(str).apply(lambda row : process_text(row['message']), axis = 1)\n",
"print(train_df.sample(10))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Qf2NiBdzSzm7",
"outputId": "505f5fcc-befd-456a-f8b9-9fa9e1294157"
},
"execution_count": 53,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
" message is_toxic \\\n",
"4250 These CREVIEW tags should not be here 0 \n",
"830 Yes we can LOL. *Smacks head*. That piece of c... 0 \n",
"1915 Should these be nodebugwaitattach instead? You... 0 \n",
"6646 Yeah, but it affects the outcome of the method... 0 \n",
"3133 Unfortunately yes.\\n(1) I want to time just Ru... 0 \n",
"16402 nit: alignment here is off, project_id should ... 0 \n",
"12308 This should not be needed, get_project raises ... 0 \n",
"1029 Should be a BoolOpt instead of StrOpt. 0 \n",
"2905 Should be a VolumeDriverException rather than ... 0 \n",
"988 amqp_names = $amqp_ipaddresses,\\n \\n amqp_ipad... 0 \n",
"\n",
" modified_message \n",
"4250 these creview tags should not be here \n",
"830 yes we can lol smacks head that piece of c... \n",
"1915 should these be nodebugwaitattach instead you... \n",
"6646 yeah but it affects the outcome of the method... \n",
"3133 unfortunately yes i want to time just run... \n",
"16402 nit alignment here is off project id should ... \n",
"12308 this should not be needed get project raises ... \n",
"1029 should be a boolopt instead of stropt \n",
"2905 should be a volumedriverexception rather than ... \n",
"988 amqp names amqp ipaddresses amqp ipaddresses... \n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"<ipython-input-53-bb8a7ebbb9b2>:9: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" train_df['modified_message'] = train_df.astype(str).apply(lambda row : process_text(row['message']), axis = 1)\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# Get the lists of sentences and their labels.\n",
"# sample = train_df.sample(1000)\n",
"sentences = train_df.modified_message.values\n",
"labels = train_df.is_toxic.values\n",
"\n",
"# sentences = sample.message.values\n",
"# labels = sample.is_toxic.values"
],
"metadata": {
"id": "01plXrb_7awh"
},
"execution_count": 54,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "oQUy9Tat2EF_"
},
"source": [
"### Step 3: Contrastive Learning"
]
},
{
"cell_type": "markdown",
"source": [
"#### Initialize BERT"
],
"metadata": {
"id": "AM28Zk7RcB_t"
}
},
{
"cell_type": "code",
"metadata": {
"id": "nskPzUM084zL",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "683e411f-91dd-4a5c-92c2-dfb140db9ea2"
},
"source": [
"# Load a trained model and vocabulary that you have fine-tuned\n",
"from transformers import BertTokenizer\n",
"from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
"\n",
"# Load the BERT tokenizer.\n",
"print('Loading BERT tokenizer...')\n",
"\n",
"tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
"# tokenizer = BertTokenizer.from_pretrained(output_dir)\n",
"\n",
"# Load BertForSequenceClassification, the pretrained BERT model with a single\n",
"# linear classification layer on top.\n",
"# Load BertForSequenceClassification, the pretrained BERT model with a single\n",
"# linear classification layer on top.\n",
"model = BertForSequenceClassification.from_pretrained(\n",
" \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
" num_labels = 2, # The number of output labels--2 for binary classification.\n",
" # You can increase this for multi-class tasks.\n",
" output_attentions = False, # Whether the model returns attentions weights.\n",
" output_hidden_states = True, # Whether the model returns all hidden-states.\n",
")\n",
"\n",
"\n",
"# Copy the model to the GPU.\n",
"model.to(device)"
],
"execution_count": 55,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Loading BERT tokenizer...\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"BertForSequenceClassification(\n",
" (bert): BertModel(\n",
" (embeddings): BertEmbeddings(\n",
" (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
" (position_embeddings): Embedding(512, 768)\n",
" (token_type_embeddings): Embedding(2, 768)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (encoder): BertEncoder(\n",
" (layer): ModuleList(\n",
" (0-11): 12 x BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" (intermediate_act_fn): GELUActivation()\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" )\n",
" )\n",
" (pooler): BertPooler(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (activation): Tanh()\n",
" )\n",
" )\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
")"
]
},
"metadata": {},
"execution_count": 55
}
]
},
{
"cell_type": "markdown",
"source": [
"We define model parameters"
],
"metadata": {
"id": "IW_IKAxudnZd"
}
},
{
"cell_type": "code",
"source": [
"param_optimizer = list(model.named_parameters())\n",
"no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
"optimizer_grouped_parameters = [\n",
" {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
" {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay':0.0}]\n",
"optimizer = AdamW(optimizer_grouped_parameters,\n",
" lr=learn_rate_list[0], # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
" eps = 1e-8 # args.adam_epsilon - default is 1e-8.\n",
" )\n",
"\n",
"\n",
"# optimizer = AdamW(model.parameters(),\n",
"# lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
"# eps = 1e-8 # args.adam_epsilon - default is 1e-8.\n",
"# )\n"
],
"metadata": {
"id": "5QUzeWzZajYh",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "3932f7e0-8b62-4173-896f-51831e986fd9"
},
"execution_count": 56,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
" warnings.warn(\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"Model parameters"
],
"metadata": {
"id": "sCvgmC7C1Deq"
}
},
{
"cell_type": "code",
"metadata": {
"id": "8PIiVlDYCtSq",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "ab305f08-11c5-4d98-e0d9-381812372a65"
},
"source": [
"# Get all of the model's parameters as a list of tuples.\n",
"params = list(model.named_parameters())\n",
"\n",
"print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
"\n",
"print('==== Embedding Layer ====\\n')\n",
"\n",
"for p in params[0:5]:\n",
" print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
"\n",
"print('\\n==== First Transformer ====\\n')\n",
"\n",
"for p in params[5:21]:\n",
" print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
"\n",
"print('\\n==== Output Layer ====\\n')\n",
"\n",
"for p in params[-4:]:\n",
" print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
],
"execution_count": 57,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"The BERT model has 201 different named parameters.\n",
"\n",
"==== Embedding Layer ====\n",
"\n",
"bert.embeddings.word_embeddings.weight (30522, 768)\n",
"bert.embeddings.position_embeddings.weight (512, 768)\n",
"bert.embeddings.token_type_embeddings.weight (2, 768)\n",
"bert.embeddings.LayerNorm.weight (768,)\n",
"bert.embeddings.LayerNorm.bias (768,)\n",
"\n",
"==== First Transformer ====\n",
"\n",
"bert.encoder.layer.0.attention.self.query.weight (768, 768)\n",
"bert.encoder.layer.0.attention.self.query.bias (768,)\n",
"bert.encoder.layer.0.attention.self.key.weight (768, 768)\n",
"bert.encoder.layer.0.attention.self.key.bias (768,)\n",
"bert.encoder.layer.0.attention.self.value.weight (768, 768)\n",
"bert.encoder.layer.0.attention.self.value.bias (768,)\n",
"bert.encoder.layer.0.attention.output.dense.weight (768, 768)\n",
"bert.encoder.layer.0.attention.output.dense.bias (768,)\n",
"bert.encoder.layer.0.attention.output.LayerNorm.weight (768,)\n",
"bert.encoder.layer.0.attention.output.LayerNorm.bias (768,)\n",
"bert.encoder.layer.0.intermediate.dense.weight (3072, 768)\n",
"bert.encoder.layer.0.intermediate.dense.bias (3072,)\n",
"bert.encoder.layer.0.output.dense.weight (768, 3072)\n",
"bert.encoder.layer.0.output.dense.bias (768,)\n",
"bert.encoder.layer.0.output.LayerNorm.weight (768,)\n",
"bert.encoder.layer.0.output.LayerNorm.bias (768,)\n",
"\n",
"==== Output Layer ====\n",
"\n",
"bert.pooler.dense.weight (768, 768)\n",
"bert.pooler.dense.bias (768,)\n",
"classifier.weight (2, 768)\n",
"classifier.bias (2,)\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"Define constants and imports"
],
"metadata": {
"id": "1II6BA7KepwN"
}
},
{
"cell_type": "markdown",
"metadata": {
"id": "l6w8elb-58GJ"
},
"source": [
"#### Tokenize Dataset"
]
},
{
"cell_type": "code",
"metadata": {
"id": "dLIbudgfh6F0",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "fc67b442-d2bd-45db-c00e-bbfeb972347f"
},
"source": [
"# Print the original sentence.\n",
"print(' Original: ', sentences[1])\n",
"\n",
"# Print the sentence split into tokens.\n",
"print('Tokenized: ', tokenizer.tokenize(sentences[1]))\n",
"\n",
"# Print the sentence mapped to token ids.\n",
"print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[1])))"
],
"execution_count": 58,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
" Original: these paren make no sense delete them\n",
"Tokenized: ['these', 'par', '##en', 'make', 'no', 'sense', 'del', '##ete', 'them']\n",
"Token IDs: [2122, 11968, 2368, 2191, 2053, 3168, 3972, 12870, 2068]\n"
]
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "cKsH2sU0OCQA"
},
"source": [
"## --- Testing tokenization\n",
"## --- We apply the tokenizer to just one sentence to see the output\n",
"\n",
"# max_len = 0\n",
"# For every sentence...\n",
"# for sent in sentences:\n",
"# # Tokenize the text and add `[CLS]` and `[SEP]` tokens.\n",
"# try:\n",
"# # Floor Division : Gives only Fractional Part as Answer\n",
"# input_ids = tokenizer.encode(str(sent), add_special_tokens=True)\n",
"# # input_ids = tokenizer.encode(str(sent), add_special_tokens=True, max_length=512, truncation=True)\n",
"\n",
"# except Exception as e:\n",
"# # By this way we can know about the type of error occurring\n",
"# print(sent)\n",
"# print(\"The error is: \",e)\n",
"# break;\n",
"\n",
"# # Update the maximum sentence length.\n",
"# max_len = max(max_len, len(input_ids))\n",
"\n",
"# print('Max sentence length: ', max_len)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "2bBdb3pt8LuQ",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "26d415f3-e23e-4992-d1b2-dc6a57921375"
},
"source": [
"# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
"input_ids = []\n",
"attention_masks = []\n",
"i = 0\n",
"# For every sentence...\n",
"for sent in sentences:\n",
" # `encode_plus` will:\n",
" # (1) Tokenize the sentence.\n",
" # (2) Prepend the `[CLS]` token to the start.\n",
" # (3) Append the `[SEP]` token to the end.\n",
" # (4) Map tokens to their IDs.\n",
" # (5) Pad or truncate the sentence to `max_length`\n",
" # (6) Create attention masks for [PAD] tokens.\n",
" encoded_dict = tokenizer.encode_plus(\n",
" str(sent), # Sentence to encode.\n",
" add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
" max_length = 512, # Pad & truncate all sentences.\n",
" truncation=True,\n",
" pad_to_max_length = True,\n",
" return_attention_mask = True, # Construct attn. masks.\n",
" return_tensors = 'pt', # Return pytorch tensors.\n",
" )\n",
"\n",
" # Add the encoded sentence to the list.\n",
" input_ids.append(encoded_dict['input_ids'])\n",
"\n",
" # And its attention mask (simply differentiates padding from non-padding).\n",
" attention_masks.append(encoded_dict['attention_mask'])\n",
"\n",
"\n",
"# Convert the lists into tensors.\n",
"input_ids = torch.cat(input_ids, dim=0)\n",
"attention_masks = torch.cat(attention_masks, dim=0)\n",
"labels = torch.tensor(labels)\n",
"\n",
"# Print sentence 0, now as a list of IDs.\n",
"print('Original: ', sentences[0])\n",
"print('Token IDs:', input_ids[0])"
],
"execution_count": 59,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2393: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
" warnings.warn(\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Original: you re not missing anything i was just being stupid sorry about that \n",
"Token IDs: tensor([ 101, 2017, 2128, 2025, 4394, 2505, 1045, 2001, 2074, 2108, 5236, 3374,\n",
" 2055, 2008, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0])\n"
]
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "aRp4O7D295d_"
},
"source": [
"#### Training & Validation Split\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "qu0ao7p8rb06"
},
"source": [
"Divide up our training set to use 90% for training and 10% for validation."
]
},
{
"cell_type": "code",
"metadata": {
"id": "GEgLpFVlo1Z-",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "3c12bbc7-6a61-4023-f473-5990892b8739"
},
"source": [
"from torch.utils.data import TensorDataset, random_split\n",
"\n",
"# Combine the training inputs into a TensorDataset.\n",
"dataset = TensorDataset(input_ids, attention_masks, labels)\n",
"\n",
"# Create a 90-10 train-validation split.\n",
"\n",
"# Calculate the number of samples to include in each set.\n",
"train_size = int(0.98 * len(dataset))\n",
"val_size = len(dataset) - train_size\n",
"\n",
"# Divide the dataset by randomly selecting samples.\n",
"train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n",
"\n",
"print('{:>5,} training samples'.format(train_size))\n",
"print('{:>5,} validation samples'.format(val_size))"
],
"execution_count": 60,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"19,064 training samples\n",
" 390 validation samples\n"
]
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "dD9i6Z2pG-sN"
},
"source": [
"We'll also create an iterator for our dataset using the torch DataLoader class. This helps save on memory during training because, unlike a for loop, with an iterator the entire dataset does not need to be loaded into memory."
]
},
{
"cell_type": "code",
"metadata": {
"id": "XGUqOCtgqGhP"
},
"source": [
"from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
"\n",
"# The DataLoader needs to know our batch size for training, so we specify it\n",
"# here. For fine-tuning BERT on a specific task, the authors recommend a batch\n",
"# size of 16 or 32.\n",
"\n",
"# Create the DataLoaders for our training and validation sets.\n",
"# We'll take training samples in random order.\n",
"train_dataloader = DataLoader(\n",
" train_dataset, # The training samples.\n",
" sampler = RandomSampler(train_dataset), # Select batches randomly\n",
" batch_size = BATCH_SIZE # Trains with this batch size.\n",
" )\n",
"\n",
"# For validation the order doesn't matter, so we'll just read them sequentially.\n",
"validation_dataloader = DataLoader(\n",
" val_dataset, # The validation samples.\n",
" sampler = SequentialSampler(val_dataset), # Pull out batches sequentially.\n",
" batch_size = BATCH_SIZE # Evaluate with this batch size.\n",
" )"
],
"execution_count": 61,
"outputs": []
},
{
"cell_type": "code",
"source": [
"from transformers import get_linear_schedule_with_warmup\n",
"\n",
"epochs = 8\n",
"\n",
"# Total number of training steps is [number of batches] x [number of epochs].\n",
"# (Note that this is not the same as the number of training samples).\n",
"total_steps = len(train_dataloader) * epochs\n",
"\n",
"# Create the learning rate scheduler.\n",
"scheduler = get_linear_schedule_with_warmup(optimizer,\n",
" num_warmup_steps = 0, # Default value in run_glue.py\n",
" num_training_steps = total_steps)\n"
],
"metadata": {
"id": "9fbomX_Oaqv8"
},
"execution_count": 62,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"#### Training Loop"
],
"metadata": {
"id": "_bTg5tTB9fhX"
}
},
{
"cell_type": "code",
"source": [
"training_stats = []\n",
"\n",
"def supervisedContrativeTraining(model, train_dataloader, epochs, device, optimizer, scheduler):\n",
"\n",
" #change here for using different loss function from https://kevinmusgrave.github.io/pytorch-metric-learning/\n",
" loss_function = loss_fun.SupConLoss(temperature=0.1,embedding_regularizer = LpRegularizer())\n",
"\n",
" # We'll store a number of quantities such as training and validation loss,\n",
" # validation accuracy, and timings.\n",
"\n",
" # Measure the total training time for the whole run.\n",
" total_t0 = time.time()\n",
"\n",
" # For each epoch...\n",
" for epoch in range(epochs):\n",
" # ========================================\n",
" # Training\n",
" # ========================================\n",
"\n",
" # Perform one full pass over the training set.\n",
"\n",
" print(\"\")\n",
" print('======== Epoch {:} / {:} ========'.format(epoch + 1, epochs))\n",
" print('Training...')\n",
"\n",
" # Measure how long the training epoch takes.\n",
" t0 = time.time()\n",
"\n",
" # Reset the total loss for this epoch.\n",
" total_train_loss = 0\n",
"\n",
" # Put the model into training mode. Don't be mislead--the call to\n",
" # `train` just changes the *mode*, it doesn't *perform* the training.\n",
" # `dropout` and `batchnorm` layers behave differently during training\n",
" # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
" losses = []\n",
" model.zero_grad()\n",
" model.train()\n",
"\n",
" for step, batch in enumerate(train_dataloader):\n",
"\n",
" # Progress update every 40 batches.\n",
" if step % 40 == 0 and not step == 0:\n",
" # Calculate elapsed time in minutes.\n",
" elapsed = format_time(time.time() - t0)\n",
"\n",
" # Report progress.\n",
" print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
"\n",
" # for d in train_dataloader:\n",
" input_ids = batch[0].to(device)\n",
" attention_mask = batch[1].to(device)\n",
" targets = batch[2].to(device)\n",
"\n",
" # input_ids = d[\"input_ids\"].reshape(32, 160).to(device)\n",
" # attention_mask = d[\"attention_mask\"].to(device)\n",
" # targets = d[\"targets\"].to(device)\n",
"\n",
" outputs = model(input_ids=input_ids, token_type_ids=None, attention_mask=attention_mask)\n",
"\n",
" # hidden_states = outputs[0]\n",
" hidden_states = outputs.hidden_states\n",
" # print(hidden_states)\n",
" # supcon_fea_cls = F.normalize(hidden_states[:,0,:],dim=1)\n",
" # supcon_fea_cls = F.normalize(hidden_states[::1],dim=1)\n",
" supcon_fea_cls = F.normalize(hidden_states[-1][:,0,:],dim=1)\n",
"\n",
" loss = loss_function(supcon_fea_cls, targets)\n",
"\n",
" # loss = outputs.loss\n",
" # logits = outputs.logits\n",
"\n",
"\n",
" if not torch.isnan(loss):\n",
" losses.append(loss.item())\n",
"\n",
" # Accumulate the training loss over all of the batches so that we can\n",
" # calculate the average loss at the end. `loss` is a Tensor containing a\n",
" # single value; the `.item()` function just returns the Python value\n",
" # from the tensor.\n",
" total_train_loss += loss.item()\n",
" # print(loss)\n",
"\n",
" # Perform a backward pass to calculate the gradients.\n",
" loss.backward()\n",
"\n",
" # Clip the norm of the gradients to 1.0.\n",
" # This is to help prevent the \"exploding gradients\" problem.\n",
" nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
"\n",
" # Update parameters and take a step using the computed gradient.\n",
" # The optimizer dictates the \"update rule\"--how the parameters are\n",
" # modified based on their gradients, the learning rate, etc.\n",
" optimizer.step()\n",
"\n",
" # Update the learning rate.\n",
" scheduler.step()\n",
" optimizer.zero_grad()\n",
"\n",
" print('Contrastive Loss Mean: ', np.mean(losses))\n",
"\n",
" # Calculate the average loss over all of the batches.\n",
" avg_train_loss = total_train_loss / len(train_dataloader)\n",
"\n",
" # Measure how long this epoch took.\n",
" training_time = format_time(time.time() - t0)\n",
" training_stats.append(\n",
" {\n",
" 'epoch': epoch + 1,\n",
" 'Training Loss': avg_train_loss,\n",
" # 'Valid. Loss': avg_val_loss,\n",
" # 'Valid. Accur.': avg_val_accuracy,\n",
" 'Training Time': training_time,\n",
" # 'Validation Time': validation_time\n",
" }\n",
" )\n",
"\n",
" print(\"\")\n",
" print(\" Average training loss: {0:.2f}\".format(avg_train_loss))\n",
" print(\" Training epcoh took: {:}\".format(training_time))\n",
"\n",
"\n",
" print(\"\")\n",
" print(\"Training complete!\")\n",
"\n",
" print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()- total_t0)))\n"
],
"metadata": {
"id": "jrcelmju9iGj"
},
"execution_count": 63,
"outputs": []
},
{
"cell_type": "code",
"source": [
"supervisedContrativeTraining(model, train_dataloader, epochs, device, optimizer, scheduler)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Iqr6ufbG84ZO",
"outputId": "05ab974e-9a4d-47bc-e946-2e544d8fff7a"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"======== Epoch 1 / 8 ========\n",
"Training...\n",
" Batch 40 of 1,192. Elapsed: 0:01:00.\n",
" Batch 80 of 1,192. Elapsed: 0:01:56.\n",
" Batch 120 of 1,192. Elapsed: 0:02:51.\n",
" Batch 160 of 1,192. Elapsed: 0:03:47.\n",
" Batch 200 of 1,192. Elapsed: 0:04:43.\n",
" Batch 240 of 1,192. Elapsed: 0:05:38.\n",
" Batch 280 of 1,192. Elapsed: 0:06:34.\n",
" Batch 320 of 1,192. Elapsed: 0:07:30.\n",
" Batch 360 of 1,192. Elapsed: 0:08:26.\n",
" Batch 400 of 1,192. Elapsed: 0:09:21.\n",
" Batch 440 of 1,192. Elapsed: 0:10:17.\n",
" Batch 480 of 1,192. Elapsed: 0:11:13.\n",
" Batch 520 of 1,192. Elapsed: 0:12:08.\n",
" Batch 560 of 1,192. Elapsed: 0:13:04.\n",
" Batch 600 of 1,192. Elapsed: 0:14:00.\n",
" Batch 640 of 1,192. Elapsed: 0:14:56.\n",
" Batch 680 of 1,192. Elapsed: 0:15:52.\n",
" Batch 720 of 1,192. Elapsed: 0:16:48.\n",
" Batch 760 of 1,192. Elapsed: 0:17:44.\n",
" Batch 800 of 1,192. Elapsed: 0:18:40.\n",
" Batch 840 of 1,192. Elapsed: 0:19:36.\n",
" Batch 880 of 1,192. Elapsed: 0:20:32.\n",
" Batch 920 of 1,192. Elapsed: 0:21:27.\n",
" Batch 960 of 1,192. Elapsed: 0:22:24.\n",
" Batch 1,000 of 1,192. Elapsed: 0:23:20.\n",
" Batch 1,040 of 1,192. Elapsed: 0:24:15.\n",
" Batch 1,080 of 1,192. Elapsed: 0:25:11.\n",
" Batch 1,120 of 1,192. Elapsed: 0:26:07.\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import gc\n",
"# del variables\n",
"gc.collect()\n",
"\n",
"torch.cuda.memory_summary(device=None, abbreviated=False)\n",
"torch.cuda.empty_cache()"
],
"metadata": {
"id": "ceqbZU1XhOQk"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "RqfmWwUR_Sox"
},
"source": [
"#### Training Results"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "VQTvJ1vRP7u4"
},
"source": [
"Let's view the summary of the training process."
]
},
{
"cell_type": "code",
"metadata": {
"id": "6O_NbXFGMukX",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 244
},
"outputId": "42013514-c334-4aa1-d165-e37f3bf4cdd1"
},
"source": [
"import pandas as pd\n",
"\n",
"# Display floats with two decimal places.\n",
"# pd.set_option('precision', 2)\n",
"pd.options.display.max_rows = 5\n",
"# pd.options.precision = 2\n",
"print(training_stats)\n",
"# Create a DataFrame from our training statistics.\n",
"df_stats = pd.DataFrame(data=training_stats)\n",
"\n",
"# Use the 'epoch' as the row index.\n",
"df_stats = df_stats.set_index('epoch')\n",
"\n",
"# A hack to force the column headers to wrap.\n",
"#df = df.style.set_table_styles([dict(selector=\"th\",props=[('max-width', '70px')])])\n",
"\n",
"# Display the table.\n",
"df_stats"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[{'epoch': 1, 'Training Loss': 3.426470624450037, 'Training Time': '0:29:18'}, {'epoch': 2, 'Training Loss': 3.328372689180596, 'Training Time': '0:29:33'}, {'epoch': 3, 'Training Loss': 3.2352232127094585, 'Training Time': '0:29:34'}, {'epoch': 4, 'Training Loss': 3.1862575237537145, 'Training Time': '0:29:42'}]\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
" Training Loss Training Time\n",
"epoch \n",
"1 3.426471 0:29:18\n",
"2 3.328373 0:29:33\n",
"3 3.235223 0:29:34\n",
"4 3.186258 0:29:42"
],
"text/html": [
"\n",
"\n",
" <div id=\"df-28ab133e-970f-4b16-93c0-f537c437cf39\">\n",
" <div class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Training Loss</th>\n",
" <th>Training Time</th>\n",
" </tr>\n",
" <tr>\n",
" <th>epoch</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>3.426471</td>\n",
" <td>0:29:18</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3.328373</td>\n",
" <td>0:29:33</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3.235223</td>\n",
" <td>0:29:34</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>3.186258</td>\n",
" <td>0:29:42</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-28ab133e-970f-4b16-93c0-f537c437cf39')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
"\n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
" </svg>\n",
" </button>\n",
"\n",
"\n",
"\n",
" <div id=\"df-dd6eab2d-3d43-40f4-b8c5-3f6ed52dad37\">\n",
" <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-dd6eab2d-3d43-40f4-b8c5-3f6ed52dad37')\"\n",
" title=\"Suggest charts.\"\n",
" style=\"display:none;\">\n",
"\n",
"<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <g>\n",
" <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
" </g>\n",
"</svg>\n",
" </button>\n",
" </div>\n",
"\n",
"<style>\n",
" .colab-df-quickchart {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-quickchart:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-quickchart {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-quickchart:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
"</style>\n",
"\n",
" <script>\n",
" async function quickchart(key) {\n",
" const containerElement = document.querySelector('#' + key);\n",
" const charts = await google.colab.kernel.invokeFunction(\n",
" 'suggestCharts', [key], {});\n",
" }\n",
" </script>\n",
"\n",
" <script>\n",
"\n",
"function displayQuickchartButton(domScope) {\n",
" let quickchartButtonEl =\n",
" domScope.querySelector('#df-dd6eab2d-3d43-40f4-b8c5-3f6ed52dad37 button.colab-df-quickchart');\n",
" quickchartButtonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"}\n",
"\n",
" displayQuickchartButton(document);\n",
" </script>\n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" flex-wrap:wrap;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-28ab133e-970f-4b16-93c0-f537c437cf39 button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-28ab133e-970f-4b16-93c0-f537c437cf39');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
" </div>\n"
]
},
"metadata": {},
"execution_count": 29
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "1-G03mmwH3aI"
},
"source": [
"Notice that, while the the training loss is going down with each epoch, the validation loss is increasing! This suggests that we are training our model too long, and it's over-fitting on the training data.\n",
"\n",
"(For reference, we are using 7,695 training samples and 856 validation samples).\n",
"\n",
"Validation Loss is a more precise measure than accuracy, because with accuracy we don't care about the exact output value, but just which side of a threshold it falls on.\n",
"\n",
"If we are predicting the correct answer, but with less confidence, then validation loss will catch this, while accuracy will not."
]
},
{
"cell_type": "code",
"source": [
"import matplotlib.pyplot as plt"
],
"metadata": {
"id": "I8B0VgTYzJLz"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"%matplotlib inline"
],
"metadata": {
"id": "6bFcIFLtzLmi"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "68xreA9JAmG5",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 431
},
"outputId": "69343855-725d-4a5f-e30f-81aaf59dc044"
},
"source": [
"import seaborn as sns\n",
"\n",
"# Use plot styling from seaborn.\n",
"sns.set(style='darkgrid')\n",
"\n",
"# Increase the plot size and font size.\n",
"sns.set(font_scale=1.5)\n",
"plt.rcParams[\"figure.figsize\"] = (12,6)\n",
"\n",
"# Plot the learning curve.\n",
"plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n",
"# plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n",
"\n",
"# Label the plot.\n",
"plt.title(\"Training Loss - COntrastive learning\")\n",
"plt.xlabel(\"Epoch\")\n",
"plt.ylabel(\"Loss\")\n",
"plt.legend()\n",
"# plt.xticks([1, 2, 3, 4])\n",
"plt.xticks(list(range(1, epochs)))\n",
"\n",
"plt.show()"
],
"execution_count": null,
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"<Figure size 1200x600 with 1 Axes>"
],
"image/png": "iVBORw0KGgoAAAANSUhEUgAABBMAAAI/CAYAAAAleJEqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACi8ElEQVR4nOzdd3hUZd7G8e+U9B5IQuiBUELvRRCkKYgFVGyoKIjuWtay7LqgiIKgu+oquroCihEVFQUEFQRBFKQlhBJKaIEESEJ6T0iZmfcPXrJGWhISJpPcn+viusiZc57zmzE8ztzzFIPNZrMhIiIiIiIiIlJBRnsXICIiIiIiIiKORWGCiIiIiIiIiFSKwgQRERERERERqRSFCSIiIiIiIiJSKQoTRERERERERKRSFCaIiIiIiIiISKUoTBARERERERGRSlGYICIiIiIiIiKVojBBRERERERERCpFYYKIiNQ727dvp127drRr167a2162bBnt2rVj6NCh1d62VI/777+fdu3a8e6771bqsStt+2oYOnQo7dq1Y9myZXa5v4iI1B9mexcgIiJ105V8UH/11Ve57bbbqrEaqaw9e/awcOFCoqKiyMrKwsfHh+DgYAYMGMCoUaNo3759ldpNSkpi6NChWK1W/v73vzNp0qQKXfftt9/y3HPPAWcDm44dO1bp/o5q2bJlJCQk0KdPH/r27WvvcqrdP/7xD5YvX06TJk34+eef7V2OiIhUgMIEERGpEQ0bNrzg8YKCAgoKCi55jqura43VBeDm5kZISEiNtO3l5UVISAhBQUE10v7V8M033zB9+nSsVitw9vUqKChg79697N27l507d/Lpp59Wqe3g4GCuueYafvvtN5YtW1bhMGHp0qUAhIWF1WiQEBwcTEhICH5+fjV2j6pYvnw5ERERPPHEE5cME5o1a4azszNeXl5XsToREamPFCaIiEiN2Lx58wWPv/vuu/znP/+55Dk1rUuXLvz444810vaIESMYMWJEjbR9NWRkZDBz5kysVithYWG88sordOrUCYCTJ0/y888/c+zYsSu6xx133MFvv/3G0aNH2bNnD127dr3k+SdPniQyMhKA22+//YrufTn/+te/arT9mvbJJ5/YuwQREaknFCaIiIhImR07dlBUVATA66+/Tps2bcoea9asGRMmTLjiewwbNgxfX1+ysrJYunTpZcOEZcuWYbPZcHZ25uabb77i+4uIiMiVU5ggIiK1yrm1FhYtWkRoaCjz58/nl19+4fTp05w5c4ZDhw4BUFhYyPr169m4cSOHDh0iOTmZvLw8fH196dKlC3fddReDBw++4D22b9/OAw88AFDW3jnLli1j6tSpZXO39+3bx4IFC8rWDggKCmL48OE89thj+Pj4nNf2H6//vXOjMvr06cOnn37K1q1b+fjjj4mOjiY/P5+mTZsyevRoJk+ejIuLy0Vfo3Xr1rFo0SIOHDiAxWKhWbNm3HzzzTz44IN88MEH5e5RWSaTqezvNTVVw9nZmVtvvZVPPvmEH374gWnTpl10aovVauXbb78Fzo768PX1BeDw4cOsWbOGyMhIEhMTSUlJwWw207x5cwYPHsyECRPw9/evdG33339/2XSCJ5988rzHLRYLixcvZtmyZRw/fhxnZ2fatWvH+PHjGTly5CXbPnnyJKtXr2b79u2cOnWK5ORkDAZD2VoUDz30EI0bNy53zbnfp3P+85//lI3sOWf9+vU0bdoUOLsAY0JCwkXXHbFYLCxfvpyVK1dy6NAh8vPz8fPzo3v37owfP/6iUyh+/7o88cQTfP3113z99dfExsZis9lo27Yt9957L7feeuslX4OakJqaysKFC9m4cSMJCQkANGnShMGDBzNx4sSLTqfKzs4mPDycX375hfj4eIqLi/Hx8cHf35/u3bszatQo+vfvX+6aM2fO8Pnnn7N27VqOHTtGQUEBXl5e+Pv707lzZ4YOHcoNN9xQ489ZRKQ2UJggIiK10okTJ3j22WdJS0vDxcUFs7n8/7JWr15d9iHLYDDg6emJ2WwmNTWV9evXs379eiZOnFi2aF9VfPfdd0ydOpWSkhK8vLywWCycOnWK8PBwNm/ezFdffYWHh0eV2v7www954403gLPrLJSUlHDs2DHeffddIiIi+Pjjj8t9sD/nn//8JwsXLiz72dvbm9jYWN544w1+/fVXevbsWbUn+//69++Pv78/GRkZLFq0iCeeeOKK2ruYO+64g08++YS8vDzWrFlz0Q+hW7duJTExESg/xeFPf/pT2QdHFxcX3NzcyM7OJiYmhpiYGJYvX054eDitWrWqtpqLi4v585//zG+//QaA0WjEycmJyMhIIiIimDx58iWvnzZtGhEREQA4OTnh4eFBTk4OsbGxxMbGsnz5cj744AN69epVdo2rqysNGzYkOzubkpIS3N3dcXd3L9fuhX5PLiQ3N5fHHnusrAaTyYSHhwepqamsWbOGNWvWXPbfjMVi4fHHH2f9+vWYzWZcXV3Jz89n9+7d7N69m/j4eP7yl79UqJ7qEBERweOPP05OTg5A2Wtz9OhRjh49yjfffMP7779f7jUFOH36NPfcc0/Z75bRaMTLy4vMzEzS0tI4fPgwx48fLxcm5OXlMX78eA4ePAic7Xe8vLzIzc0lMzOT2NhYIiMjFSaISL2hrSFFRKRWmjNnDl5eXoSHh7N792527txZbp0Db29vJk6cyOLFi9m1axc7duxg9+7dbNq0iSeffBInJycWLlzI+vXrq3T/jIwMpk2bxpgxY/jll1/YsWMHO3fu5MUXX8TJyYkjR47w4YcfVqntgwcP8uabb/LII4+wZcsWIiMj2bFjB48//jhwduTE8uXLz7vuhx9+KAsSbrrpJjZu3EhkZCQ7d+5k1qxZREdH88UXX1SppnPc3d3LPky+9957rFy58orau5i2bdvSpUsX4H+LK17IuceaNGlS7oNd7969ee2119iwYQPR0dFs376d6OhowsPD6dKlC8nJyUyZMqVaa37zzTf57bffMBgMPP3000RGRhIZGcnmzZu55557WLBgATExMRe9vn379rz44ousWbOmrOa9e/fy9ddfc+2115Kbm8szzzzDmTNnyq658cYb2bx5M927dwdg4sSJbN68udyf4ODgCtX//PPPExERgZOTEy+88AJRUVFERkayadOmsqBm4cKFl/wdWrx4MREREbz22mtERUURFRXFr7/+ypAhQwD473//S1xcXIXquVJJSUllQUJoaGhZX7Br1y4+//xzQkJCyM7O5vHHHyc5Obncte+++y6JiYk0adKE8PBw9u3bR0REBHv37uXnn3/mpZdeOm/6zaJFizh48CC+vr68++67REdHExkZyd69e9m4cSP//Oc/GTBgwFV57iIitYHCBBERqZWMRiPh4eH0798fo/Hs/65+vwPD8OHDee655+jZsydubm5lxwMDA3niiSd45plnAKq860BhYSGjR4/mlVdeKfuw5ubmxvjx47nvvvuAsx/uqyInJ4fHHnuMZ599tmwovqenJ3/5y1+4/vrrL9i2zWZj7ty5AAwYMIA33nijbBqCi4sLd955Jy+99BLZ2dlVqumchISEspDEarXyj3/845If9q/EHXfcAZz9dvnkyZPnPZ6dnc26desAuO2228p+D+DsCI2xY8eWmxbg7OxM//79CQ8Pp2HDhuzfv58dO3ZUS63Jycl89tlnAPz5z3/mz3/+M56engA0aNCAl156iZtuuonc3NyLtvH8888zfvx4WrZsWfZczGYzXbp0Yd68ebRr146UlBTWrFlTLTX/3p49e8ranT59Ovfff3/Zv5uAgADmzJlT9o363Llzy9bN+KPs7Gz+85//MHbs2LKpKY0aNeKdd94hMDAQq9XK6tWrq73+C/nggw/IycnBx8eH8PDwcqNyevXqRXh4OJ6enmRlZTFv3rxy1+7atQuAZ599lv79+5eN7jCZTDRp0oR77rnnvDDq3DUTJ07k+uuvx9nZGTjbVwUFBTFmzBhmzZpVY89XRKS2UZggIiK10q233kqjRo2qfP11110HwO7du7FYLFVq489//vMFjw8bNgyA+Ph4CgsLK92us7MzEydOvGTbf1zLISYmhvj4eAAeffRRDAbDedf+8cN1ZWVnZzNhwgSOHDnCPffcw9y5czEYDDz//PMXDWU+//xz2rVrV6Wh3aNHj8bNzQ2bzXbBkRjff/89RUVFGI1Gxo4dW+F2PTw86N27NwA7d+6sdF0XsmbNGkpLS3F1db3odpZXMiXEZDJx7bXXAhAVFVXldi5m1apVwNkP/uPGjbvgOU899RQAmZmZF91ppUePHvTr1++8487OzgwcOBA4/3e3JthstrKRSnfffTcBAQHnndOoUSPuvvtu4PxwztvbGzi73kJFVeUaEZG6TGsmiIhIrdSjR4/LnpOWlsbixYvZvHkzcXFx5ObmnhccFBYWkp2dXenF+Hx9fWnRosUFHwsMDCz7e05OTrmRERXRpk2bi661cK7tP44w2L9/P3B2rv25Ie9/ZDAY6N27NytWrKhUPee88sornDx5km7dujF9+nRMJhMWi4W//e1vvPLKKxQUFPDoo4+Wu+bc8PGwsLBK38/T05MbbriBb7/9lm+//ZYnnnii3OiDcyMi+vfvT5MmTc67fsOGDaxYsYK9e/eSnp5+wWDn9OnTla7rQvbt2wdAp06dykYk/FFISAhBQUHnDan/vR07dvDNN9+we/dukpOTKSgoOO+cS11fVefq79u3b7nX+Pdat25dVv++ffsYOnToeedcaueNi/3u1oRTp06RlZUFcN4iib83YMAAPvzwQ7Kysjh58iTNmjUDzoaNu3bt4s033+TYsWOMGDGCHj16XPS/7blrvv/+ez777DMyMjK48cYb6dGjR5UW+hQRqQsUJoiISK3UoEGDSz6+a9cuHnnkkbKF1+DsfH83NzcMBgMWi4XMzEyAKo0euNTCir9f8K6kpKRG2i4tLS13/Nxz8fX1LRtefSFV3YEhNTW17Nvrxx57rKyO0aNHU1JSwtSpU/n3v/9Nfn4+zz77bNl1kZGRAGVz5ivrjjvu4NtvvyUhIYGtW7eWzTk/ePBgWYBybjrEOVarlb/97W98//33ZcfMZjM+Pj44OTkBZxcbLCoqqtJ/+wtJT08HLv/6NmrU6KJhwOuvv15unQ2TyVSu5oKCgrI/1a2y9Z87/48u9bt7bpHUP/7u1oTf13ep5/T7xzIyMsrChEmTJnHw4EFWr17NkiVLWLJkCQaDgTZt2jBw4EDGjRt33uKdN998M9HR0Xz22Wf88MMPZaMdWrRowYABA7j99tvp1KlTdT5NEZFaTdMcRESkVrrYt6dw9sPKX//6V3JycggLC2P+/PlERUWxa9cutmzZwubNm1myZEnZ+Tab7WqU7NAOHDhQ9iHwjztCjBkzhldeeQWDwcC8efN45ZVXsNlsHDt2jF27duHj48Pw4cOrdN/evXvTsmVL4Ow2iOec+7uvr+95bX/zzTd8//33mEwmHn/8cdauXcvevXuJiIgoW5Tw3LSL2vLffvPmzWVBwr333st33313Xs0TJkywc5X1h5OTE2+//TYrVqzg8ccfp1+/fri5uXH48GEWLlzITTfdVG7XlHOef/55fvzxR5599lkGDRqEt7c38fHxLF68mNtvv53Zs2fb4dmIiNiHRiaIiIjD2b17NwkJCZhMJubNm3fBbybr2rxmPz8/ALKysiguLr7o6ISqDpHPz8+/5OO33347paWlzJgxg08//ZT8/HxycnKw2WxMmDChyltknmv7zTff5KeffiqbNnJuF4mbb775vOd67hvhO+6446LbEKalpVW5ngs5N1Lmcq/vxR4/V/PAgQOZMWPGBc+p7pp/r0GDBhw/fvyy0z7OPX65kUH29vv6kpOTL7oF6O//e1xoOkL79u1p3749cDakjIyM5L333iMyMpJ//etfXHPNNWWPn9OiRQseffRRHn30UaxWK9HR0SxYsIB169axaNEi+vXrV7b2iYhIXaaRCSIi4nCSkpKAsx8OLjbEeevWrVezpBrXsWNH4Oy0inOryv+RzWar8u4F54Z/A2zbtu2C59x1111Mnz4dODtyYN26dYSEhPDwww9X6Z7njBkzBpPJRFFREd999x0///xz2bSOP05xgP994O3QocMF28vPz2fPnj1XVNMfnRu+vm/fvosGL3FxcRf9sH65mm0220Vfd6Bswc2qjrQ4V//27duxWq0XPCc2Nrbsw3fnzp2rdJ+rpWnTpvj6+gKX/re+ZcsW4OwIl9//jl+I2Wymf//+zJs3D2dnZ2w2W9n1F2M0GunWrRvvvPNO2eKnl7tGRKSuUJggIiIOx8vLCzj7Te6Fvs09ffp0lbeErK3CwsLKFoScP3/+BT9UrlixgoSEhCq136lTJ5o3bw6cndt/7sP8H40fP55Ro0aV/dy+fXtcXFyqdM9zAgMDGTRoEHA2pDg3xaFjx47nfSsMlC2Sd/DgwQu29/777192pEVl3XDDDZhMJs6cOXPB4e8A77333kWvv1zNX3zxxQW3x/zj9b9fI6QyRo8eDZz9pv7rr7++4DnvvPMOcHYUzDXXXFOl+1wtBoOh7Pfwq6++uuBIpOTkZL766isAbrrppnKPFRcXX7RtZ2fnsjVDfj/d6lLXmEymsrUvLrTTiohIXaQwQUREHE7Pnj1xd3fHZrPx9NNPc/z4cQAsFgubNm3i/vvvt3OF1c9gMPDkk08C8Ntvv/Hcc8+VfYtcVFTE119/zYwZM/Dx8aly+y+++CImk4m4uDjGjRvHmjVrKCoqAs6+tjt37uQvf/kLq1evLvvAtHr1at56660rfn7nRiDs27ePjRs3AmenP1zIuS0Uv/76a7766quyD3mpqanMmTOHDz/8sOxb6+oSFBTEvffeC5wNK+bNm0deXh5wdmG/mTNnsnLlyrKg62I1b9y4kffee69skcWcnBw++OADXnnllUvW3KZNm7LrqzKVpUuXLmXrSMyaNYvPPvusbHHK1NRUXnjhhbKtFp966qkrDoiqymq1kpGRcck/5173P/3pT3h7e5OVlcVDDz1UbhvQqKgoHnroIXJycvD19eWRRx4pd58hQ4bw5ptvsnv37nIhQXx8PFOmTKGwsBCj0Vi23SXAuHHjeOWVV9i+fXu5RTKTk5OZNWtW2datgwcPrpHXRkSkttGaCSIi4nC8vLz4+9//zksvvURkZCQjR47E3d0di8VCUVERfn5+vPrqq/z5z3+2d6nV6uabb2bv3r188sknrFixgpUrV+Lt7U1BQQElJSX069ePrl27lg3Trqxrr72Wf//73zz//POcPHmSv/zlL5jNZjw9PcnPzy/buaJx48bMmTOHjRs3snDhQj744AMCAgK47777qvzcrrvuOho2bEhaWhpWqxUXFxduvvnmC547ceJE1qxZw7Fjx3jxxRd56aWX8PT0JDc3F5vNxl133UVxcTHLly+vcj0X8re//Y3Y2Fi2bNnCv//9b+bOnYunp2fZ2hGTJ09mz549REREnHftmDFj+Pbbb9mxYwfvvPMO7777Lt7e3uTm5mK1WrnuuusICwvjv//97wXvPXbsWD7++GPi4+O57rrr8Pf3L/vAv3jxYho1anTZ+mfPnk1mZiYRERHMmjWLV199FQ8Pj7L64exre88991zBq3RlkpKSLrnVI8CwYcN4//33adSoEe+99x6PPfYYR44c4Z577sHd3R2g7MO+t7c377333nnTodLS0pg/fz7z58/HaDTi5eXFmTNnysIzg8HAc889R2hoaNk1ubm5fPrpp3z66acYDAa8vLwoLS0tFyw8+OCDZcGRiEhdpzBBREQc0j333EPjxo358MMP2bdvHxaLhaCgIAYPHszkyZOrtGWjI5g2bRq9e/dm0aJFHDhwgOLiYlq1asWtt97KhAkTeO2114CzH6KqYuTIkfTo0YPFixezceNG4uPjyc/Px9fXl44dOzJixAhuueUWnJ2d6du3L3Fxcfz888/Mnj2bBg0alJsCURlms5kxY8aU7XgwYsSIiz4Hb29vvvzyS9577z3WrVtHSkoKJpOJPn36cNdddzF69Gj+8Y9/VKmOS3FxcWHBggUsXryYZcuWcfz4cWw2G7169Sqb/nGxUTFOTk4sXLiQ+fPn8/3335OQkIDNZqNLly6MGTOGu+6665LTJFq2bMmiRYuYN28e0dHRZGVlle2+UdGtGL28vAgPD2f58uWsWLGCQ4cOUVBQQMOGDenRowfjx4+nb9++lX9h7KhPnz6sWrWKjz/+mF9//ZWEhAQMBgOtW7dm8ODBTJw4kYCAgPOuW7hwIdu3bycqKoqkpKSy6VItWrSgZ8+ejB8//rxtHv/973/z22+/sWPHDk6dOkVaWhqlpaU0adKErl27cuedd142CBERqUsMttqyZ5KIiIhcsbvvvptdu3bxl7/8hccff9ze5YiIiEgdpTUTRERE6oiIiIiynR401FpERERqksIEERERB/Lyyy+zbNkyUlNTy+a55+Tk8OWXX/LYY48B0K9fP7p06WLPMkVERKSO0zQHERERB3LrrbeWbS/o7OyMm5tbuQX0QkNDWbhw4XkLzomIiIhUJ4UJIiIiDmT9+vWsW7eO6Oho0tLSyMvLw9PTk9DQUEaMGMFdd92Fm5ubvcsUERGROk5hgoiIiIiIiIhUitZMEBEREREREZFKUZggIiIiIiIiIpVitncBcmk2mw2rtfbPRDEaDQ5Rp4g4LvUzIlLT1M+ISE1ylD7GaDRgMBgue57ChFrOarWRkZFv7zIuyWw24ufnQU5OAaWlVnuXIyJ1kPoZEalp6mdEpCY5Uh/j7++ByXT5MEHTHERERERERESkUhQmiIiIiIiIiEilKEwQERERERERkUpRmCAiIiIiIiIilaIwQUREREREREQqRWGCiIiIiIiIiFSKwgQRERERERERqRSFCSIiIiIiIiJSKQoTRERERERERKRSzPYuQERERERExFHZbDYsllJsNpu9S5FazGo1cOaMieLiIiyWq/O7YjQaMZlq7iO/wgQREREREZFKKi0tITc3i+LiM9hsVnuXIw4gLc2I1Xp1f1fMZmc8PLxxc/Oo/rarvUUREREREZE6rLi4iMzMFIxGIx4eXjg5uWA0GgGDvUuTWsxkMly1UQlgw2KxUFCQR3Z2GkC1BwoKE0RERERERCohLy8Lk8mMv3/Q/4cIIpdnNhspLb16IxOcnMDFxY3MzFTy83OqPUzQb76IiIiIiEgFWSwWiovP4OHhpSBBaj2DwYC7uwelpcVYLKXV2rZGJsgVsVptxMRlUHI8EyeDjdaNfTAaNbxLREREROomq9UCgNnsZOdKRCrm3CKMVqsVk6n62lWYIFUWdSiFxeuOkJlbVHbMz8uFe4e3oWe7QDtWJiIiIiJS0/QFmjiKmvld1bgcqZKoQym8t3xfuSABIDO3iPeW7yPqUIqdKhMREREREZGapjBBKs1qtbF43ZFLnvPFuiNYrdprV0REREREpC5SmCCVdvhk1nkjEv4oI7eIwyezrk5BIiIiIiIiclUpTJBKy8q/dJBQ2fNERERERERq2uzZLzFwYC9Wrfqu2tp84olHGDiwFzt37qi2Nh2FFmCUSvP1cKnW80REREREpO4ZOLBXla77+uuVBAc3ruZqpLopTJBKa9vMFz8vl8tOddi4J4HgBu74eCpUEBERERGpbzp37nresZKSEg4ePABA+/YdcHI6f4tNZ2fnGqmnQYOGNG/eAg8Pz2prMyioEc2bt8DV1bXa2nQUBpvNplXyajGLxUpGRr69yzjPud0cLsfNxcxtg1oxpHsTjEZtnyMiVWM2G/Hz8yAzM5/SUqu9yxGROkj9jFRUSUkx6elJNGgQjJNTzXzorcuSkhIZN+4WoP6NQDCbjXbpXyr7O+vv74HJdPkVEbRmglRJz3aBPD62E35e5Ucd+Hu58PjYTkyf0IsWjbwoLCrl858OM+uTHRxLzLFTtSIiIiIiIlKdNM1Bqqxnu0C6twkgNjGbEpsBJ4ON1o19ykYgTH+gF7/uTuCbX48Rn5zL7EU7GNytMbcNbo2n2/nDmURERERE5OKsVhuHT2aRlV+Er4cLbZv51qnRv78ftfDbbzv49dcNfP31F8TGHiU3N4ePP/6cNm3akZ6exi+//MzWrb9x4kQ8aWlpmM1mWrRowdCh13P77XdecKrE7NkvsXr190ybNoMbb7y57PiqVd8xZ87LdOvWg3ffnceKFUtZsWIZJ07E4+zsQrdu3Zk8+TFatWp9XptPPPEIu3fv5J13PqBHj/+tEfHRR/P4+OMFjBp1E8899wJffPEpP/74A0lJibi7e9C3bz8eeeRxgoIaXfC1SE1N4cMPP2Dbti3k5uYQGBjEsGHX88ADE3njjVcv+DyuNoUJckWMRgNhLf0vOCzQaDQwpEdTerQL5OsNR9my7zS/7E5kx6FU7hwSyjWdG2E01J3OT0RERESkpkQdSmHxuiPl1i3z83Lh3uFt6Nku0I6V1YzPP/+E//73XXx9/WjatCkpKcllj3333bd8+OEHODu70KBBQ1q3bk12djaHDx8iJuYAGzdu4J13PrjgegyX88orM1izZhXBwY1p3rwF8fHxbNr0K7t2RfHhh5/StGmzSrVXWlrKX//6JFFRkTRv3oKmTZtx4kQ8a9asZteunYSHL8bb26fcNSdOxPP445PJzMzAbDbTqlVrioqK+OSTj9ixI6LWTA1RmCA1zsfDmYdv6sC1XYL5dO1hEtPyWbgqhk3Ridx/fTuaBlbfAigiIiIiInXNxdYry8wt4r3l+3h8bKc6Fyh8+OEHPPvsc4wZcztGoxGr1YrFYgGge/devPXWe3Tv3hOz+X8faVNSknnrrdfZtOkXvvzyM+6//6FK3XPfvmji4+P4z3/m061bDwBycrKZOnUKe/bs4qOP5jFjxiuVanPDhnU0atSYTz75knbt2lJaauX06dNMmfIkcXHH+eKLz3j00cfLzrfZbMycOZ3MzAw6d+7CrFn/pGHDAAAOHz7I3//+DIcOxVSqhpqiNRPkqmnX3I+XHurNuCGtcXYycuRUNi99HMmX649QWFRq7/JERERERKqFzWajqNhSLX8Kz5xdg+xSFq87QuGZ0mq5X21Zn//mm8dw223jMBrPfmQ1Go1lIw26du1G7959ywUJAIGBQcyY8Qpms5kff/yh0vcsLS3l6aenlAUJAN7ePjz11F8B2Lp1c5XafOGFl2ndOrTsWKNGjZg8+bELtrlz5w4OHjyAq6srs2b9qyxIAGjbtj3PPz+D0tLa8dlJIxPkqjKbjIzq24K+YUF8sf4IUYdSWRt5koiYZO4Z3pZe7QIwaOqDiIiIiDgom83Gq5/t5GhC9lW7Z2ZuEY+/vbFa2gpt6sPU8T3s/p78cmsBFBWdYcOG9ezZs4vk5GTOnCksC0KMRiMnTsRTVHQGF5eKb9no6enFsGHXn3e8bdv2ODs7k5eXS3Z2Fj4+vhVuMzS0LZ06dT7veMeOZ48lJJwqd3z79i0A9Os3gIYNG553Xe/e/WjUKJjTp5MqXENNUZggduHv7crjYzsTHZvO4p8Ok5JVyH+/3UfHln6Mv74djfzd7V2iiIiIiEjV6LuxK9aiRchFHzt2LJbnnnuGpKTES7aRk5NDQEDFw4RLrYfg6+tHSkoyhYWFlQoTLtamv78/AIWFBeWOnzx5AoDQ0DYXbTM0tI3CBJEurRsQ1qIPP2yNZ9W2E+yPy+TFj7Yzqm8LRvdvgbOTyd4lioiIiIhUmMFgYOr4HhSXWC9/cgUcPpnFW1/vuex5z4zrSttmvld8P2cno91HJQC4ubld8LjFYmH69OdISkqkZ88+3HffBEJD2+Dl5V027eG220aTkpJc6ekArq4XDx7OTbeo7DSQiz2Pc+39UUFBIQDu7h4XbfNSj11NChPE7pzMJsZc24r+nRrx+drD7DuewXdb4ti6/zT3Xd+WLq3PH94jIiIiIlJbGQwGXJyr50uxjiH++Hm5lNvF4Y/8vVzoGOJfp7aJvJiYmAPEx8cRGBjEv/717/OmMdhsNnJzc+1U3ZVzdz8bPhQU5F/0nEs9djVpAUapNYL83Hnmzq48NqYTfl4upGWf4e2vo3l3aTTp2WfsXZ6IiIiIyFVnNBq4d/jFh7wD3DO8Tb0IEgCSkhIACAvrcMH1EI4diz1v6oAjadasOQCxsUcves6lHruaFCZIrWIwGOjVPpDZk/sysm9zTEYDu46k8fyH21i1LZ5SS/UMFxMRERERcRQ92wXy+NizX7j9nr+XS53cFvJSzk1FSE9Pv+DjixcvuprlVLu+fa8BYNu2zWRknP8cd+yIuOxaEVeLpjlIreTqbObOIaFc06kRn605xOFT2XzzSyyb9yZx//XtaN/Cz94lioiIiIhcNT3bBdK9TQCHT2aRlV+Er4cLbZv51psRCed07NgZs9nMvn3RrFixjFtvvQ2AkpISwsM/ZO3a1Tg5OVFSUmLnSqumR49ehIV1ICbmAC+88BwzZ75WtqvDkSOHmDPnZcxmc63YHlJhgtRqTQM8eW58D7bsO82SDUdJSi/gX1/sol/HIO4aEoqPp8vlGxERERERqQOMRkO9/1LN378B99xzP59++jGvvz6Hjz9eQMOGAZw6dYK8vDwmTXqUH35YWSt2O6gKg8HA9OmzePzxyURH7+aOO26iVavWFBeXEBd3jA4dOtGlSzfWrVtz0UUcrxZNc5Baz2AwMKBzMHMe6ceQ7k0wANv2JzNtwTbWR53Caq3ciqoiIiIiIuK4Hn30caZMmUrr1qFkZ2dx6tRJQkPbMmvWazz00GR7l3fFmjdvwUcffcro0bfg4+NDXNxxiouLuO++B3nnnQ/KRiV4eNh3VweDrbJ7W8hVZbFYycioHat1XozZbMTPz4PMzHxKS2t+TYPjSTl8uuYQcafPrtLaIsiL+25oS+vGPjV+bxGxj6vdz4hI/aN+RiqqpKSY9PQkGjQIxsnJ2d7liAMxm43V0r/cf/+dHD9+jI8/XkybNm0ve35lf2f9/T0wmS4/7kAjE8ThhAR788IDvbj/+ra4uZiJT85lzqIoFv14kLxCx5wbJSIiIiIicjn79+/j+PFjeHv7EBLSyq61KEwQh2Q0GhjSoylzHunHNZ0aYQN+2Z3ItPnb2BSdiFUDbkRERERExAGdPHmCr7/+ktzc3HLHo6N38+KL/wDgllvGYjbbdwlELcAoDs3Hw5mHb+rAtV2C+WztYRLS8vl41UE2RZ/d9aFZoKe9SxQREREREamw/Pw85s59g//85y2aNWuOu7sHaWmppKQkA9C5cxceeuhhO1epkQlSR7Rr7seMh3pz55BQXJxMHD2VzcsfR/Ll+iMUFtl/2xQREREREZGKaNy4KQ88MJG2bduRnZ3N4cMHyc/Po2PHzvzlL39l7twPcHFxtXeZWoCxttMCjJWXkXOGL9YfIepQKgC+ns7cPawNvdsHYjDUr314ReqK2tbPiEjdo35GKkoLMEpVVdcCjJWlBRhFKsjf25XHx3bmmTu7EujrRlZeMR+s2M+/v9rN6YwCe5cnIiIiIiLi8BQmSJ3VuVUDZj3ch1sHhmA2Gdkfl8mLH21n+cZjFJdY7F2eiIiIiIiIw1KYIHWak9nErQNDmPVwHzq18qfUYuO7LXG88OF29hxNs3d5IiIiIiIiDklhgtQLQX7uPDOuK4+P7YSflwtp2WeY+0007y6NJj37jL3LExERERGHo6XnxFHUzO+qtoaUesNgMNCzXSAdQ/xZuTmOnyJPsutIGvvjMrhlQAjX926GuQILjYiIiIhI/WUwnH2/aLFYcXKyczEiFWC1nl30sboXo9cnJ6l3XJ3N3DkklBkP9aZtUx+KS6x880ssMxZGEBOfae/yRERERKQWM5lMGI1miooK7V2KSIUUFxdhMBgxmap3LIHCBKm3mgZ48tz4Hjx8Uxhe7k4kpRfw+he7mP/dfrLziuxdnoiIiIjUQgaDAVdXd86cyaekRO8ZpXazWq2cOZOPi4trtY9M0DQHqdcMBgPXdAqma2hDlm08xi87E9i2P5k9R9O4bVBrhnRvgtFYvf/oRERERMSxeXr6UFJSREZGCq6uHri4uGEyGQG9b5SLs1oNWCxXZ60Nm82GxVJCfn4uVqsVT0/far+HwWazaeWQWsxisZKRkW/vMi7JbDbi5+dBZmY+paVWe5dzRY4n5fDpmkPEnc4FoHmQJ/ff0I7WjX3sXJlI/VaX+hkRqZ3Uz0hlWa1W8vKyOXOmAKu11N7liAMwGo1l6xdcLc7Ornh6+uLs7FLha/z9Pf4/HLs0hQm1nMKEq89qtfHrnkSW/hJLQVEpBmBQt8bcPrg1nm5aZUfEHupaPyMitY/6Gamqs98AW7DZ9HsjF2cyGfDxcSc7u+CqjU4wGk2YTKZKX1fRMEHTHET+wGg0MKR7E3q2DeDrDUfZvO80v+5OJOpQKuOGtGZA52CM1TzfSEREREQck8FgwGzWxyq5NLPZiKurK4WFljoTWGoBRpGL8PZwZtJNHXju3u40aehBXmEJH686yGuf7+RkSp69yxMREREREbEbhQkil9GuuR8zHurNnUNCcXEycfRUNi9/HMmX649QWKT5cSIiIiIiUv8oTBCpALPJyMi+zZk9uS+92gVgtdlYG3mS5xdsIyImGS09IiIiIiIi9YnCBJFK8Pd25bGxnXnmzq4E+rqRlVfMByv28++vdnM6o8De5YmIiIiIiFwVDrObw+rVq9myZQv79+8nJSWFrKwsnJycaNmyJYMHD2bChAn4+fld8X0+//xzZs6cCUCfPn349NNPL3pueno6//3vf9mwYQMpKSl4e3vTu3dvHn30UcLCwq64FtBuDrVZSamFVdtO8MPWeEotVswmAyP7tuCm/i1wdqr8qqkicnH1tZ8RkatH/YyI1CRH6mPq3NaQt956KwcPHsTZ2ZmAgAD8/PzIyMggMTERgAYNGrBw4ULat29f5XskJydz4403kpd3dnG9S4UJ8fHx3HvvvaSlpeHu7k5ISAinT58mPT0dJycn5s6dy7Bhw6pcyzkKE2q/lMwCPvvpMPuOZQDQ0MeV8SPa0jW0oZ0rE6k76ns/IyI1T/2MiNQkR+pjKhomOMw0h/Hjx/PZZ5+xc+dOfv75Z5YuXcqGDRtYuXIlbdu2JT09nb/+9a9XdI+XXnqJwsJChgwZcsnzbDYbTz31FGlpaVx77bVs3LiRZcuWsXHjRh577DFKSkqYMmUKKSkpV1SPOIZAP3eeGdeVx8d2ws/LhbTsM8z9Jpp3l0aTll1o7/JERERERESqncOECXfeeSe9e/fGycmp3PF27doxe/ZsAI4ePUpsbGyV2l+1ahU///wz48ePp2PHjpc8d/369cTExODl5cWbb76Jl5cXAGazmaeeeorevXtTUFDAwoULq1SLOB6DwUDPdoHMntyXkX2bYzIa2HUkjRcWbOeHrXGUWmp3+igiIiIiIlIZDhMmXEqrVq3K/l5YWPlvgrOzs5k9ezaNGjXi6aefvuz5q1evBmDkyJH4+Pic9/idd95Z7jypP1ydzdw5JJSXHupN22a+FJdaWfrrMWYsjCAmPtPe5YmIiIiIiFSLOhEmREVFAZStXVBZr732GmlpaUyfPh0PD4/Lnr9nzx4AevXqdcHHzx0/ffo0ycnJla5HHF+TAE+eu7c7D98Uhre7E0npBbz+xS7mf7ef7Lwie5cnIiIiIiJyRcz2LqCqrFYrqampbN68mTfeeAOAKVOmVCgM+L2tW7eybNkyhg4dyvDhwy97fnFxMQkJCQA0b978gucEBwfj5ORESUkJx44dIygoqFI1Sd1gMBi4plMwXUMbsmzjMX7ZmcC2/cnsOZrG2GtbMaRHE0zGOpHniYiIiIhIPeNwYUJ4eDivvvpquWNdunThtddeY9CgQZVq68yZM7z44ou4u7vz4osvVuiavLw8rNaz898vNMUBzn6I9Pb2Jj09nZycnErVdCFmc+3+wHlupc+KrPhZH/l4uvDQjWEM7taET1Yf5HhSDovXHWHzvtM8OKo9rZtc+PdIRP5H/YyI1DT1MyJSk+piH+NwYUJQUBA9evTAYrGQmJhIWloaMTExrFixgm7duuHt7V3htt555x1OnDjB1KlTCQ4OrtA1RUX/G6Lu7Ox80fPOPXbmzJkK13MhRqMBP7/KjbawF29vN3uXUKv19POgW1gj1m6L45NVMcSfzmVmeCTX923BAzd2wNvj4r9PInKW+hkRqWnqZ0SkJtWlPsbhwoRRo0YxatSosp8PHjzIrFmz+P7774mNjWXp0qWYTKbLtnPgwAE++eQTOnTowP3331/h+7u4uJT9vbi4+KLnnXvM1dW1wm1fiNVqIyen4IraqGkmkxFvbzdycgqxaNeCy+oXFkiH5r58uf4Iv0UnsWZbPFuiE7lraBsGdg3GaDDYu0SRWkf9jIjUNPUzIlKTHKmP8fZ2q9AICocLE/6offv2zJs3j+HDhxMTE8MPP/zALbfcctnrnn/+eaxWKzNnzqxQ+HCOp6cnRqMRq9VKdnb2Bc+x2Wxl0xsqM1LiYkpLa/cv2zkWi9VharU3dxczE28MY2DnYD5de4iE1Hw+/P4Av+xK4P4b2tEs0NPeJYrUSupnRKSmqZ8RkZpUl/qYOjFhw9PTkz59+gCwf//+Cl1z4MABDAYDf/rTnxgwYEC5PwsXLgRg165dZceSkpKAs9MXGjduDMCJEycu2HZSUhIlJSUAVdpdQuqPts18mfFgb+4cEoqLk4mjCdm8/HEkX64/QmFRqb3LExERERERuaA6ESYAlJae/eBlsVgqfI3FYiEtLe28PwUFZ6cVlJSUlB37fbvdunUDYMeOHRds99zxRo0a0ahRo6o8HalHzCYjI/s2Z/bkvvRqF4DVZmNt5EmeX7CNiJhkbDabvUsUEREREREpp06ECVlZWURERAAQFhZWoWsOHTp00T9PPPEEAH369Ck71rRp07Jrb7jhBgB+/PHHC051WLJkCQAjR468oucl9Yu/tyuPje3Ms3d2JdDPjay8Yj5YsZ83v9rN6YzavW6GiIiIiIjULw4RJkRERPD+++9z6tSp8x7bv38/kyZNIjc3l6CgoPM+wA8dOpShQ4fy448/Vls9w4cPp127duTm5jJlyhRyc3OBsyMd5s6dS2RkJG5ubkycOLHa7in1R6dWDZg1qQ9jBoZgNhk5EJfJix9tZ9nGYxSXVHzkjYiIiIiISE1xiAUYc3JymDt3LnPnziUgIIDAwEBMJhNJSUmkpqYCZ7eMnDdvHh4e5bdRTEhIACibulAdjEYjc+fOZfz48WzcuJFBgwYREhLC6dOnSU9Px8nJiddff52goKBqu6fUL05mE7cMDKFfxyA+++kw+45l8P2WOLbtP829I9rSLbShvUsUEREREZF6zCFGJnTv3p2pU6cydOhQ3NzciIuLIyYmBqvVSt++fZk6dSqrVq2q8BSH6hASEsLKlSu577778PPz4/Dhw8DZKRBLlixhxIgRV60WqbsC/dx5ZlxXHh/bCT8vF9Kyz/DON9G8uzSatOxCe5cnIiIiIiL1lMGm1d1qNYvFSkZGvr3LuCSz2YifnweZmfl1ZpuT2uhMcSnfbY5jbeRJLFYbzmYjNw9oyQ19mmOuwD6wIo5M/YyI1DT1MyJSkxypj/H398BUgc8X+gQi4iBcnc2MGxLKSw/1pm0zX4pLrSz99RgzFkYQE59p7/JERERERKQeUZgg4mCaBHjy3L3defimMLzdnUhKL+D1L3Yxf+V+svKK7F2eiIiIiIjUAw6xAKOIlGcwGLimUzDdQhuybOMxNuxMYNuBZPbEpjH22lYM6dEEk1FZoYiIiIiI1Ax92hBxYO6uTtx3fTumP9iLkGAvCossLF53hFmf7CA2Idve5YmIiIiISB2lMEGkDmjZyJvn7+/F/Te0w93FzInkPGZ/GkX46oPkFZbYuzwREREREaljFCaI1BFGo4Eh3Zsw55F+DOjcCICNexKZNn8bm/YkYtXGLSIiIiIiUk0UJojUMd4ezkwa3YF/jO9BkwAP8gpL+Hj1QV77bCcnknPtXZ6IiIiIiNQBChNE6qi2zXyZ8WBv7hwSiouziaMJ2cwM38EX645QWFRq7/JERERERMSBKUwQqcPMJiMj+zZn9sN96dU+EKvNxk87TjJtwTYiYpKxaeqDiIiIiIhUgcIEkXrA39uVx8Z04tk7uxLo50Z2XjEfrNjPm1/tJik9397liYiIiIiIg1GYIFKPdGrVgFmT+jBmYAhmk5EDcZm8+FEEyzbGUlRisXd5IiIiIiLiIBQmiNQzTmYTtwwM4ZWH+9C5VQMsVhvfb4ln+ofb2X00zd7liYiIiIiIA1CYIFJPBfq58/S4Ljw+tjP+3i6kZZ/hnW+ieXdpNGnZhfYuT0REREREajGzvQsQEfsxGAz0bBdAxxA/vtscx9rIk+w6ksb+4xncPKAlN/RpjtmkzFFERERERMrTpwQRwdXZzLghobz0UG/aNfOluNTK0l+PMWNhBDFxGfYuT0REREREahmFCSJSpkmAJ3+/tzuTb+qAt7sTSekFvP7lbuav3E9WXpG9yxMRERERkVpC0xxEpByDwUD/To3oGtqAZRuPsWFnAtsOJLMnNo0x17ZiaI8mmIzKIUVERERE6jN9IhCRC3J3deK+69sx/cFehAR7UVhk4Yt1R5gVvoPYhGx7lyciIiIiInakMEFELqllI2+ev78XD9zQDg9XMydS8pj9aRThqw+SV1hi7/JERERERMQOFCaIyGUZjQau696E2ZP7MaBzIwA27klk2vxtbNyTiNVms3OFIiIiIiJyNSlMEJEK8/ZwZtLoDvxjfA+aBHiQV1hC+OqDvPpZFCeSc+1dnoiIiIiIXCUKE0Sk0to282XGg725a2goLs4mYhNymBm+gy/WHaGwqNTe5YmIiIiISA1TmCAiVWI2GbmhT3NmP9yXXu0Dsdps/LTjJNMWbGP7gWRsmvogIiIiIlJnKUwQkSvi7+3KY2M68exdXQn0cyM7r5h5K/fzxpe7SUrPt3d5IiIiIiJSAxQmiEi16BTSgFmT+jDm2hDMJiMx8Zm8+FEEyzbGUlRisXd5IiIiIiJSjRQmiEi1cTKbuGVACK883IcurRtgsdr4fks80z/czu6jafYuT0REREREqonCBBGpdoF+7jx1RxceH9sZf28X0rLP8M430by7NJq07EJ7lyciIiIiIlfIbO8CRKRuMhgM9GwXQKcQf1ZuOc7aiJPsOpLG/uMZ3DygJTf0aY7ZpDxTRERERMQR6Z28iNQoF2cT464L5aWJfWjXzJfiUitLfz3GjIURxMRl2Ls8ERERERGpAoUJInJVNGnowd/v7c7kmzvg7eFMUnoBr3+5m/kr95OVV2Tv8kREREREpBI0zUFErhqDwUD/jo3o2roByzce5+ddp9h2IJk9sWmMubYVQ3s0wWRUxikiIiIiUtvpXbuIXHXurk6Mv74t0yf0IiTYm8IiC1+sO8Ks8B3EJmTbuzwREREREbkMhQkiYjctG3nz/AM9eeCGdni4mjmRksfsT6MIXx1DXmGJvcsTEREREZGLUJggInZlNBi4rnsTZj/Sj4GdgwHYuCeJafO3sXFPIlabzc4VioiIiIjIHylMEJFawdvdmYmjw/jH+B40DfAgr7CE8NUHefWzKE4k59q7PBERERER+R2FCSJSq7Rt5suLD/bmrqGhuDibiE3I4eXwSBavO0xhUam9yxMRERERERQmiEgtZDYZuaFPc+ZM7kfv9oHYbLBuxymmLdjG9gPJ2DT1QURERETErhQmiEit5eflwp/HdOLZu7oS5OdGdl4x81bu540vd5OUnm/v8kRERERE6i2FCSJS63UKacDMSX0Ze20ITmYjMfGZvPhRBEt/jaWoxGLv8kRERERE6h2FCSLiEJzMRm4eEMKsh/vSpXUDLFYbP2yN54UF29l9JM3e5YmIiIiI1CsKE0TEoQT6uvHUHV144rbO+Hu7kJ5zhneWRvPON9GkZRXauzwRERERkXrBbO8CREQqy2Aw0KNtAB1b+vPdljjWRJxg99E0DsRlcPOAltzQpzlmk7JSEREREZGaonfbIuKwXJxN3HFda16a2If2zX0pLrWy9NdjzFgYQUxchr3LExERERGpsxQmiIjDa9LQg7/d053JN3fA28OZpPQCXv9yN/NW7icrr8je5YmIiIiI1Dma5iAidYLBYKB/x0Z0bd2A5RuP8/OuU2w/kMyeo2mMHdSKoT2aYDIqPxURERERqQ56Zy0idYq7qxPjr2/LixN6ExLszZliC1+sO8Ks8B3EJmTbuzwRERERkTpBYYKI1EktGnnx/AM9eWBkOzxczZxIyWP2p1GEr44hr7DE3uWJiIiIiDg0hQkiUmcZDQau69aE2Y/0Y2DnYAA27kli2vxtbNyTiNVms3OFIiIiIiKOSWGCiNR53u7OTBwdxtT7etA0wIO8whLCVx/k1c+iOJGca+/yREREREQcjsIEEak32jT1ZcZDvbl7aCguziZiE3J4OTySxesOU1hUau/yREREREQchsIEEalXTEYj1/dpzpzJ/ejdPhCbDdbtOMW0BdvYfiAZm6Y+iIiIiIhclsIEEamX/Lxc+POYTvz1rm4E+bmRnVfMvJX7eePL3SSl59u7PBERERGRWk1hgojUax1D/Jk5qS9jrw3ByWwkJj6TFz+KYOmvsRSVWOxdnoiIiIhIraQwQUTqPSezkZsHhDDr4b50ad0Ai9XGD1vjeWHBdnYfSbN3eSIiIiIitY7CBBGR/xfo68ZTd3Thyds608DbhfScM7yzNJp3vokmLavQ3uWJiIiIiNQaZnsXICJSmxgMBrq3DaBDS3++2xLHmogT7D6axoG4DG66piU39GmOk1k5rIiIiIjUb3pHLCJyAS7OJu64rjUvTexD++a+FJdaWbbxGDMWRnAgLsPe5YmIiIiI2JXCBBGRS2jS0IO/3dOdR27ugLeHM6czCnjjy93MW7mfrLwie5cnIiIiImIXmuYgInIZBoOBfh0b0aV1A5ZvOs7PO0+x/UAye46mMfbaVgzt2QSTUdmsiIiIiNQfevcrIlJB7q5OjB/Rlhcn9CYk2JszxRa+WH+EmeE7OJqQbe/yRERERESuGoUJIiKV1KKRF88/0JMHRrbDw9XMyZQ85nwaRfjqGPIKS+xdnoiIiIhIjVOYICJSBUaDgeu6NWH2I/0Y2CUYgI17kpg2fxsb9yRitdnsXKGIiIiISM1RmCAicgW83Z2ZeGMYU+/rQdMAD/IKSwhffZBXP43iRHKuvcsTEREREakRChNERKpBm6a+zHioN3cPDcXF2URsYg4vh0eyeN1hCotK7V2eiIiIiEi1UpggIlJNTEYj1/dpzpzJ/egTFojNBut2nGLa/G1sO3Aam6Y+iIiIiEgdoTBBRKSa+Xm58KdbO/HXu7oR5OdGdn4x81ce4I0vd5OUnm/v8kRERERErpjCBBGRGtIxxJ+Zk/oy9toQnMxGYuIzefGjCJb+GktRicXe5YmIiIiIVJnCBBGRGuRkNnLzgBBeebgvXVo3wGK18cPWeF5YsJ1dR1LtXZ6IiIiISJUoTBARuQoCfN146o4uPHlbZxp4u5Cec4Z3l+7lnW+iScsqtHd5IiIiIiKVYrZ3ARW1evVqtmzZwv79+0lJSSErKwsnJydatmzJ4MGDmTBhAn5+fpVq86uvvmLXrl0cOHCAtLQ0srOzcXNzo1WrVowYMYL77rsPNze38647deoUw4YNu2TbXbt2ZcmSJZWqR0TqNoPBQPe2AXRo6c93W+JYE3GC3UfTOBCXwU3XtOSGPs1xMivjFREREZHaz2BzkOXFb731Vg4ePIizszMBAQH4+fmRkZFBYmIiAA0aNGDhwoW0b9++wm326tWL3NxcXF1dCQoKwsvLi+TkZFJTzw49btmyJeHh4QQHB5e77vdhQo8ePS7Ydps2bZg5c2ZVnmo5FouVjIzavWCb2WzEz8+DzMx8Skut9i5HxGEkpuXz2dpDHDyRBUAjf3fuu74tHVr627ewWkj9jIjUNPUzIlKTHKmP8ff3wGS6/BdcDhMmLFmyhJCQELp164aTk1PZ8UOHDjFlyhQOHz5MaGgoP/zwQ4XbDA8Pp0ePHnTq1Amj8X8vVlRUFE8//TQpKSkMHjyY+fPnl7vu92HCoUOHrvCZXZrCBJG6zWazsf1AMl/+fJSc/GIA+oQFctfQNvh5udi5utpD/YyI1DT1MyJSkxypj6lzYcKlREdHM27cOABWrVpF69atr7jNVatW8cwzz2A0GomKisLd3b3sMYUJ5TnSPwyR2qrgTCnLNx3j552nsNnA1dnE2GtbMbRnE0xGTX1QPyMiNU39jIjUJEfqYyoaJtSJd6itWrUq+3thYfUsZHYukLBarRQVFVVLmyIiF+Puamb8iLa8OKE3rRp7c6bYwhfrjzAzfAdHE7LtXZ6IiIiISDkOswDjpURFRQHg7u5OSEhItbbZpEmTSy7s+Morr3Ds2DEMBgNNmjRh4MCBDB8+vNy0CRGRimrRyItp9/dk055EvvkllpMpecz5NIpruwRzx3Wt8XJ3tneJIiIiIiKOGyZYrVZSU1PZvHkzb7zxBgBTpkzBw8Ojym2WlpaSkpLCunXreOutt3BycmLatGmXvObTTz8t9/NXX31FWFgY7777Ls2aNatyLSJSfxkNBgZ3a0L3tgF880ssv0UnsSk6iZ2HUxk3JJSBXYIxGgz2LlNERERE6jGHWzMhPDycV199tdyxLl268OSTTzJo0KAqtTl79mwWLVpU7tjAgQN58skn6dat23nnnz59mjfffJPRo0cTGhpKYGAgmZmZ/Prrr7z99tukp6fTokULli1bhqenZ5VqOsdisZKTU7v3oDeZjHh7u5GTU4jFUrvn/4g4oiMnswhffZCTKXkAtG7iw4Oj2tOikZedK7t61M+ISE1TPyMiNcmR+hhvb7e6uQDj6tWrWbRoERaLhcTERNLS0jCbzdxwww3MmDEDb2/vSre5aNEiVq9eTXFxMYmJiWRkZODu7s7YsWP5xz/+gbNzxYcVx8fHc9ttt5GXl8dTTz3FY489Vul6fs9ms2HQN5Ai9Z7FYuX7zcf5/McYCossGA0wemArxt/QHg83p8s3ICIiIiJSjRwuTPijgwcPMmvWLHbs2EFYWBhLly7FZDJdUZs7duzg5Zdf5vDhwwwZMoQPPvigUte/8cYbLFiwgA4dOrB8+fIrqkUjE0Tk9zJyzvDFuiNsP5AMgI+nM/cOb0u/jkF1OnhUPyMiNU39jIjUJEfqY+rsyIQLycvLY/jw4WRmZvL6669zyy23XHGbSUlJjBgxgpKSEhYvXkzPnj0rfO369et57LHHcHd3Z9euXVdUh7aGFJEL2R+XwWdrD5OcUQBAWAs/7ru+LcENqr5uTG2mfkZEapr6GRGpSY7Ux9SrrSE9PT3p06cPAPv376+WNoODg2nbtm2V2nRyOjvk2GKxVEstIiJ/1LGlPzMn9mHsoFY4mY3ExGfy4kcRLP01lqIS9T0iIiIiUrPqRJgAZ3digOr9AH+urcq2eeTIEQAaNWpUbbWIiPyRk9nIzde05JWH+9KldQMsVhs/bI3nhQXb2XUk1d7liYiIiEgdVifChKysLCIiIgAICwurljbj4uI4fPhwpdvMz89n8eLFAAwYMKBaahERuZQAXzeeuqMLT97WmQbeLqTnnOHdpXt555toUrNq95orIiIiIuKYHCJMiIiI4P333+fUqVPnPbZ//34mTZpEbm4uQUFBjBw5stzjQ4cOZejQofz444/ljp/bFSI19fxv77Zt28bkyZOxWq106NChbArFOdOnT2ft2rUUFxeXOx4bG8vDDz/MqVOncHd3Z9KkSVV9yiIilWIwGOjeNoBXHu7H6P4tMBkN7D6axvQPt/P9ljhKavncPBERERFxLGZ7F1AROTk5zJ07l7lz5xIQEEBgYCAmk4mkpKSyMCAoKIh58+bh4VF+8bGEhAQACgoKyh1PTk7m1VdfZfbs2QQHB9OwYUNsNhsJCQlkZmYCEBoaynvvvYfRWD5ziY6OZsmSJTg5OdG8eXM8PT3JzMzkxIkTAPj4+PD222/TtGnTGnk9REQuxsXZxO2DW9O/YyM+W3uIgyeyWLbxGJv3nea+69vSsaW/vUsUERERkTrAIcKE7t27M3XqVLZv387Ro0eJi4ujuLgYb29v+vbty9ChQ7njjjvw9PSscJvDhw+nqKiIiIgIjh8/ztGjRyktLcXPz49BgwZx/fXXc+utt+Ls7HzetY8++iibNm1i3759pKWlER8fj6urKx07dmTQoEGMHz+egICA6nwJREQqpXFDD/52T3e2H0jmq5+PkpxRwJtf7qZPWCB3DW2Dn5eLvUsUEREREQdWJ7aGrMu0NaSIXKmCM6V8u+kY63eewmYDV2cTY69txdCeTTAZHWK2m/oZEalx6mdEpCY5Uh9Tr7aGFBGRi3N3NXPviLa8OKE3rRp7c6bYwhfrjzAzfAdHT2XbuzwRERERcUAKE0RE6okWjbyYdn9PJoxsh4ermZMpecz5LIqPV8WQW1B8+QZERERERP6fwgQRkXrEaDAwuFsT5jzSj2u7BAOwKTqJafO3sXFPIlbNfBMRERGRClCYICJSD3m5O/PQjWFMu68nTQM8yT9TSvjqg7z6aRTxp3PtXZ6IiIiI1HIKE0RE6rHQpj7MeKgXdw9rg6uzidjEHGZ+Esninw5TcKbU3uWJiIiISC2lMEFEpJ4zGY1c37sZsyf3o09YIDYbrIs6xfMLtrHtwGm06Y+IiIiI/JHCBBERAcDPy4U/3dqJv97djSB/d7Lzi5m/8gCvf7GLxLTavUWtiIiIiFxdChNERKScji39mTmxD7cNaoWT2cjBE1nMWBjB0l9jKSqx2Ls8EREREakFFCaIiMh5nMxGbrqmJa883JeurRtgsdr4YWs8LyzYzq4jqfYuT0RERETsTGGCiIhcVICvG0+N68qTt3emgbcr6TlneHfpXuZ+vYfUrEJ7lyciIiIidmK2dwEiIlL7dW8TQIcW/ny/NY4ft59gT2w6B+K3c9M1LRnZpzlOZmXTIiIiIvWJ3v2JiEiFuDibuH1wa2ZO6kNYCz9KSq0s33iMFxdGsD8uw97liYiIiMhVpDBBREQqJbiBB1Pu7sYjt3TAx8OZ5IwC3vxyNx+s2EdmbpG9yxMRERGRq0DTHEREpNIMBgP9OjSiS6uGfLvpGOt3niIiJoXo2HTGXNuKYT2bYDIqrxYRERGpq/ROT0REqszd1cy9I9ry4oTetG7szZliC1+uP8LLH+/g6Klse5cnIiIiIjVEYYKIiFyxFo28mHp/Tx4c1R4PVzOnUvOY81kUC1fFkFtQbO/yRERERKSaKUwQEZFqYTQYGNS1MXMe6ce1XYIB+C06iWnzt/Hr7gSsNpudKxQRERGR6qIwQUREqpWXuzMP3RjGtPt60izQk/wzpXzy4yHmfBpF/Olce5cnIiIiItVAYYKIiNSI0KY+vPhgL+4Z1gZXZxPHEnOY+Ukki386TMGZUnuXJyIiIiJXQGGCiIjUGJPRyIjezZg9uR99wgKx2WBd1CmeX7CNbftPY9PUBxERERGHpDBBRERqnJ+XC3+6tRNT7u5GI393svOLmf/dAV7/YheJafn2Lk9EREREKklhgoiIXDUdWvrz8sQ+3DaoFU5mIwdPZDFjYQRLf42lqNhi7/JEREREpIIUJoiIyFXlZDZy0zUteeXhvnQLbYjFauOHrfG88OE2dh1O1dQHEREREQdgsOldW61msVjJyKjdQ4DNZiN+fh5kZuZTWmq1dzki4mB2HUll8U9HSM85A0DX1g24d0RbAnzdALBabcQmZlNiM+BksNG6sQ9Go8GeJYtIHaT3MyJSkxypj/H398Bkuvy4A/NVqEVEROSiurcJoENLf77fEseP20+wJzadA/HbuemalgT6urFkw1Eyc4vKzvfzcuHe4W3o2S7QjlWLiIiI1G8amVDLaWSCiNQnSen5fLb2MDHxmZc99/GxnRQoiEi10fsZEalJjtTHVHRkgtZMEBGRWiO4gQdT7u7G5Js7YLjMTIYv1h3BalUeLiIiImIPChNERKRWMRgM+Hm6cLlxcxm5RRw+mXVVahIRERGR8hQmiIhIrZOVX3T5kypxnoiIiIhUL4UJIiJS6/h6uFTovJKS2j3nUERERKSuUpggIiK1Tttmvvh5XT5Q+Hj1QT5dc4jcguKrUJWIiIiInKMwQUREah2j0cC9w9tc8pzWTbwB2LArganztvHTjpOUWjRSQURERORq0NaQtZy2hhSR+izqUAqL1x0hM/d/ayP4e7lwz/A29GwXyKETmSxed4STKXkABDdw557hbegU0sBeJYuIg9L7GRGpSY7Ux1R0a0iFCbWcwgQRqe+sVhuxidmU2Aw4GWy0buyD0Wgo9/jG6ESW/XqMvMISALqFNuSuoaEE+bvbq2wRcTB6PyMiNcmR+hiFCXWEwgQRkYr1MwVnSli5OY71UaewWG2YjAZG9G7Gzde0xM3FfJUrFhFHo/czIlKTHKmPqWiYoDUTRESkTnB3deLuYW2YOakPnVs1wGK18eP2E0ydt5VNexKxKjsXERERqTYKE0REpE4JbuDBM3d25elxXQjydyenoISPVx9k1ic7OHIqy97liYiIiNQJChNERKRO6tK6IbMm9eHuoaG4uZiIP53Lq5/tZN7K/WTknLF3eSIiIiIOTZNIRUSkzjKbjFzfpzn9OjZi+aZjbNydyPYDyew6nMqofi0Y2bc5Lk4me5cpIiIi4nA0MkFEROo8bw9nJoxsz4sP9qZtM1+KS62s+O04zy/YRkRMMlqLWERERKRyFCaIiEi90aKRF8/d250/j+lEA28XMnKK+GDFfl77fCfxp3PtXZ6IiIiIw1CYICIi9YrBYKB3+0BmT+7HmGtDcDYbOXIqm5nhkYSvjiEnv9jeJYqIiIjUegoTRESkXnJ2MnHLgBDmPNKPfh2CsAEb9yQxdf5Wftx+glJL7d4DWkRERMSeFCaIiEi95u/tyiO3dGTqfT1o0ciLwiILSzYcZfqH29lzNM3e5YmIiIjUSgZbDa86ZbFY+OKLL9i8eTNGo5HrrruOcePG1eQt6xSLxUpGRr69y7gks9mIn58HmZn5lJbqmzwRqX5Xq5+x2mxs3pvE0l+PlU136NTKn3uGtSG4gUeN3VdE7E/vZ0SkJjlSH+Pv74HJdPlxB9USJnzzzTdMnz6dG264gbfffrvcY0899RRr164FwGazYTAYGDlyJG+99daV3rZeUJggInL1+5nColK+3xrHT5EnKbXYMBkNDO3RlFsHtsTd1anG7y8iV5/ez4hITXKkPqaiYUK1THPYvHkzADfddFO549u3b2fNmjXYbDa6d+/ONddcA8CPP/7IunXrquPWIiIi1c7Nxcy460KZ9XBfurdpiMVq46cdJ/nHvG38sisBq1VbSYqIiEj9Vi1hQkxMDAA9evQod/zbb78F4M4772Tx4sUsXLiQJ598EpvNxvLly6vj1iIiIjUmyM+dJ2/vwl/v6kbjhh7kFZawaM0hXg6P5GB8pr3LExEREbGbagkTMjMzcXZ2xt/fv9zxrVu3YjAYuP/++8uOjR8/HoB9+/ZVx61FRERqXMcQf16e2JvxI9ri4WrmZEoe//piF+8v30taVqG9yxMRERG56szV0Uh+fj7u7u7ljqWkpHD69GkaNmxImzZtyo77+Pjg6elJRkZGddxaRETkqjAZjQzr2ZS+HYL4dtMxNuxKYMehVHYfTWdk3+aM7tcCF2eTvcsUERERuSqqZWSCp6cnubm5FBb+79uZyMhIALp3737Ba1xcXKrj1iIiIleVp5sT913fjpcf6kNYCz9KLVa+3xLHtAXb2Lr/NDW8SZKIiIhIrVAtYcK5kQerV68uO/btt99iMBjo3bt3uXNzc3PJy8ujYcOG1XFrERERu2ga6MmUu7vxxG2dCfB1JTO3iAXfHWDOZ1EcT8qxd3kiIiIiNapapjncdNNNREZGMnPmTPbs2UNaWhqbNm3C2dmZUaNGlTt3165dALRs2bI6bi0iImI3BoOBHm0D6NzKn7WRJ/l+SzyxCTnM+mQHAzo34vbBrfH11Eg8ERERqXuqZWTCHXfcwTXXXMOZM2dYsmQJ69evx2Aw8PTTTxMQEFDu3B9//PGCIxZEREQclZPZxOj+LZnzSD8GdGoEwOa9p5k6fxs/bI2jpNRi5wpFREREqle1jEwwmUx8+OGHfP/99+zatQtvb28GDRpEz549y51XXFxMamoqvXr1YtCgQdVxaxERkVrDz8uFSTd1YEiPpixed5hjiTks/fUYG/ckctfQNnRv0xCDwWDvMkVERESumMGmlaJqNYvFSkZGvr3LuCSz2YifnweZmfmUllrtXY6I1EGO2M9YbTa270/m61+OkpVXDEBYCz/uGd6GpgGedq5ORP7IEfsZEXEcjtTH+Pt7YDJdfhJDtUxzEBERkfKMBgP9OzViziP9uOmaFphNRmLiM5mxMILP1h4ir7DE3iWKiIiIVFm1THO4nA0bNrB582aMRiODBw9mwIABV+O2IiIidufqbOa2Qa25tktjlmw4StShVH7emcD2A8mMubYV13VvjMmobF9EREQcS7VMc1i7di3//Oc/GTBgADNnziz32KuvvsqiRYvKHXvwwQd57rnnrvS29YKmOYiI1K1+JiY+ky/WHeFUah4ATRp6cPfwNnRs6W/nykTqt7rUz4hI7eNIfcxVnebw888/k5iYSK9evcod379/P5988gk2m43g4GCaN2+OzWYjPDyc7du3V8etRUREHEpYCz9mPNSL+29oh6ebEwlp+bz55W7eXRpNcmaBvcsTERERqZBqCRP27t0LQP/+/csdX7p0KQAjRoxg3bp1rFmzhvHjx2Oz2ViyZEl13FpERMThmIxGhnRvwquP9mNEr2aYjAZ2HUlj+ofb+fqXoxQWldq7RBEREZFLqpYwISMjA5PJREBAQLnjmzdvxmAwMHnyZIz/Px/00UcfBWD37t3VcWsRERGH5eHqxD3D2/DyxD50CvGn1GJj9bYTTJu/jd+ik7BqwyURERGppaolTMjNzcXDw6PcsczMTOLj4/H29qZLly5lxwMDA3FzcyM1NbU6bi0iIuLwGjf04Jk7u/LUHV0I8nMjO7+YhatieOWTHRw9lW3v8kRERETOUy1hgru7O7m5uZSU/G+bq6ioKAC6det23vlOTk6YTKbquLWIiEidYDAY6BrakFkP9+XOIaG4uZiIO53LnM+imL9yPxk5Z+xdooiIiEiZagkTWrVqhc1m49dffy07tnr1agwGAz179ix3bmFhIbm5uedNiRAREREwm4yM7NucOY/0Z1DXYAzAtgPJTFuwjZWbj1NcYrF3iSIiIiKYq6ORESNGsHv3bl544QWOHTtGamoqq1atwmg0MmrUqHLn7t27F5vNRtOmTavj1iIiInWSj4czD44KY0j3pixed5gjp7L5dtNxNu1J4s6hofRqF4DBYLB3mSIiIlJPVUuYcN9997Fy5UoOHTrEW2+9he3/F4y67777aNasWblz165di8FgOG8bSRERETlfi0Ze/GN8DyIPprBkw1HSc87w32/30baZL/cOb0PzIC97lygiIiL1ULWECS4uLixevJhPPvmE3bt34+XlxZAhQ7jpppvKnVdcXExkZCTBwcEMHDiwOm4tIiJS5xkMBvqEBdE1tCE/bj/B6m3xHD6ZxcsfRzKoW2PGDmqFt7uzvcsUERGResRgs2nfqdrMYrGSkZFv7zIuyWw24ufnQWZmPqWlVnuXIyJ1kPqZ8tKzz/D1L0eJiEkBwM3FzK0DWjK0Z1PMpmpZDkmk3lE/IyI1yZH6GH9/D0wVeD/hMGHC6tWr2bJlC/v37yclJYWsrCycnJxo2bIlgwcPZsKECfj5+VWqza+++opdu3Zx4MAB0tLSyM7Oxs3NjVatWjFixAjuu+8+3NzcLnp9eno6//3vf9mwYQMpKSl4e3vTu3dvHn30UcLCwq70KQMKE0REQP3MxRw+mcUX644Qn5wLQCN/d+4e1oYurRvYuTIRx6N+RkRqkiP1MXYNE/Ly8jhw4ADp6ekANGjQgA4dOuDp6VnlNm+99VYOHjyIs7MzAQEB+Pn5kZGRQWJiYtk9Fi5cSPv27SvcZq9evcjNzcXV1ZWgoCC8vLxITk4mNTUVgJYtWxIeHk5wcPB518bHx3PvvfeSlpaGu7s7ISEhnD59mvT0dJycnJg7dy7Dhg2r8vM9R2GCiIj6mUuxWm1s3pvE0l9jySk4u0Vzl9YNuGtoKMENPOxcnYjjUD8jIjXJkfoYu4QJ5xZg3LRpE1Zr+RfIaDQyePBgnnrqKdq1a1fptpcsWUJISAjdunXDycmp3D2nTJnC4cOHCQ0N5Ycffqhwm+Hh4fTo0YNOnTphNP7vxYqKiuLpp58mJSWFwYMHM3/+/HLX2Ww2xo4dS0xMDNdeey1vvfUWXl5elJaW8t577/H+++/j7u7OmjVrCAwMrPRz/T2FCSIi6mcqorColO+2xPFT5EksVhsmo4FhPZtyy4CWuLs6Xb4BkXpO/YyI1CRH6mOuepiwdu1a/va3v1FcXMzFmjQYDDg7O/PGG28wYsSI6rgtANHR0YwbNw6AVatW0bp16ytuc9WqVTzzzDMYjUaioqJwd3cve2zdunU8/vjjeHl5sX79enx8fMpde9999xEZGclDDz3EP/7xjyuqQ2GCiIj6mcpIzijgq5+PsvtoGgCebk7cNrgVg7o0xmjUVpIiF6N+RkRqkiP1MRUNE6pllaaTJ08yZcoUioqKaNy4MTNmzGDt2rVER0cTHR3N2rVrmTFjBk2aNKGoqIgpU6Zw8uTJ6rg1AK1atSr7e2FhYbW0eS6QsFqtFBUVlXts9erVAIwcOfK8IAHgzjvvLHeeiIjI1RLk785f7ujCs3d2JbiBO3mFJSz68RAvh0dy6ESmvcsTERGROqJawoSPPvqI4uJiunXrxsqVK7nnnnto3rw5zs7OODs707x5c+655x5WrlxJt27dKC4u5uOPP66OWwNnpyUAZWsXVGebTZo0OW9hxz179gBn11y4kHPHT58+TXJycrXUIyIiUhmdWjXg5Yl9uGd4G9xdzJxMyeOfi3fx/rf7SMuunuBdRERE6q9qCRO2bt2KwWDg5ZdfxsPj4os9ubu78/LLL2Oz2di8efMV3dNqtZKcnMyyZcuYOnUqAFOmTLnk/S+ntLSUxMREFi1axOuvv46TkxPTpk0rd05xcTEJCQkANG/e/ILtBAcHl63rcOzYsSrXIyIiciXMJiMjejXj1Uf7MaR7EwwG2HEwhecXbGf5xmMUFVvsXaKIiIg4KHN1NHL69Gk8PDwqtLBiu3bt8PT05PTp01W6V3h4OK+++mq5Y126dOG1115j0KBBVWpz9uzZLFq0qNyxgQMH8uSTT9KtW7dyx/Py8soWl7zQFAc4uzaEt7c36enp5OTkVKmm3zOba/ee4efm01RkXo2ISFWon7kyft6uPDQ6jGG9mvL52sPExGfy3ZY4ftubxF3D2tC/YxAGg9ZTkPpN/YyI1KS62MdUS5hgNpspLS2t0Lk2m42SkhLM5qrdOigoiB49emCxWEhMTCQtLY2YmBhWrFhBt27d8Pb2rnSbzZo1o0ePHhQXF5OYmEhGRgY7d+5k5cqVdOjQAWdn57Jzf79+wu+P/9G5x86cOVPpen7PaDTg5+cYW3t5e7vZuwQRqePUz1wZPz8PurQLYuveJD76bj8pGQV88O0+ft2dyOQxnWjTzO/yjYjUcepnRKQm1aU+plrChBYtWhATE8OmTZu49tprL3nupk2bKCoqqvKOC6NGjWLUqFFlPx88eJBZs2bx/fffExsby9KlSzGZTJVq84EHHuCBBx4o+3nHjh28/PLLfP755yQmJvLBBx+UPebi4lL29+Li4ou2ee4xV1fXStXyR1arjZycgitqo6aZTEa8vd3IySnEYqndK5OKiGNSP1O9wpr5MOeRvqzZfoKVv8URE5fBs29v5NquwYwbEoqvp8vlGxGpY9TPiEhNcqQ+xtvbrUIjKKolTBg6dCgHDhxg+vTpfPTRRxcNCo4ePcqLL76IwWBg2LBh1XFr2rdvz7x58xg+fDgxMTH88MMP3HLLLVfUZq9evZg/fz4jRoxgw4YNREVF0bNnTwA8PT0xGo1YrVays7MveL3NZiub3lCVkRJ/VNu3DjnHYrE6TK0i4pjUz1QfIwZG9W1Bvw6NWPprLFv2nWbTniQiYlK45ZqWDO/VDKdaPs1OpCaonxGRmlSX+phqeZfw4IMPEhQUxOnTpxkzZgx/+9vfWLZsGZs3b2bz5s0sXbqUKVOmMHbsWE6fPk1QUBATJkyojlsDZz/g9+nTB4D9+/dXS5vBwcG0bdv2vDadnZ1p3LgxACdOnLjgtUlJSZSUlABU2+4SIiIiNcHPy4WHb+rA8w/0JCTYm6JiC1//Esv0D7ez63AqNpvN3iWKiIhILVQtIxM8PT358MMP+dOf/kRCQgLff/8933///Xnn2Ww2mjZtyn//+188PT2r49Zlzq3ZYLFU38rU59r6Y5vdunXj1KlT7NixgzFjxpx33Y4dOwBo1KgRjRo1qrZ6REREakrrxj48/0BPtu0/zde/xJKSVci7y/bSoaUf9wxrQ5OA6v3/toiIiDi2ahu/2KZNG1auXMmzzz5LWFgYRqMRm82GzWbDaDQSFhbGlClTWLFiBW3atKmu2wKQlZVFREQEAGFhYdXSZlxcHIcPH75gmzfccAMAP/744wWnOixZsgSAkSNHVkstIiIiV4PRYOCaTsHMmdyP0f1bYDYZOBCXyYyFkXy+9jB5hSX2LlFERERqCYOthsYvlpSUlH3Q9vHxwcnJCYDc3FweeOABDAYDy5Ytq1BbERER7Nixg1tuuYWmTZuWe2z//v28+OKL7Nu3j6CgIFavXo2Hx/92Pxg6dCgAf//738t9uF+9ejWpqamMGjWKgICAcm1u27aN6dOnc+LECTp06MDSpUsxGv+Xu1itVsaMGcOhQ4cYNGgQ//73v/Hy8sJisfCf//yH999/Hzc3N9asWUNQUFAlXrXzWSxWMjLyr6iNmmY2G/Hz8yAzM7/OzP8RkdpF/Yx9pGQV8vXPR4k6nAqAh6uZMde24rrujTEZtZ6C1C3qZ0SkJjlSH+Pv73H1FmC8ECcnJxo2bHje8dLSUmJiYiq1n3VOTg5z585l7ty5BAQEEBgYiMlkIikpidTUs29wgoKCmDdvXrkgASAhIQGAgoLyOyIkJyfz6quvMnv2bIKDg2nYsCE2m42EhAQyMzMBCA0N5b333isXJAAYjUbmzp3L+PHj2bhxI4MGDSIkJITTp0+Tnp6Ok5MTr7/++hUHCSIiIvYU6OvG47d1JiYug8Xrj5CQms/nPx3ml10J3DO8DR1a+tu7RBEREbGTGgsTqlP37t2ZOnUq27dv5+jRo8TFxVFcXIy3tzd9+/Zl6NCh3HHHHZVah2H48OEUFRURERHB8ePHOXr0KKWlpfj5+TFo0CCuv/56br31VpydnS94fUhICCtXruS///0vGzZs4PDhw3h7e3PDDTfwpz/9iQ4dOlTX0xcREbGrsJb+vPRQbzbuTmTZxmMkpOXzxpe76d6mIXcNDSXQz93eJYqIiMhVVmPTHC4mMzOT/v37YzAYiImJuZq3dkia5iAion6mNskrLGHlb8f5eWcCVpsNs8nA9b2bM7p/C9xcHOI7CpELUj8jIjXJkfqYik5z0IRHERERqTBPNyfuHdGWlyf1oWNLP0otNlZti2fa/G1s3puEVVtJioiI1AsKE0RERKTSmjT04Nm7uvGX27sQ6OdGdn4xH/0Qw+xFO4hNOH+nIxEREalbFCaIiIhIlRgMBrq1acisSX0ZN6Q1rs4mjiflMvvTKBZ8t5/M3CJ7lygiIiI1RGGCiIiIXBEns5FRfVvw6iP9GNglGAOwdX8yU+dv5bstcRSXWOxdooiIiFQzhQkiIiJSLXw8XZh4YxgvTOhFaBMfikusLN94jBc+3M6Ogylc5TWfRUREpAZVadnlsLCw6q5DRERE6oiQYG+m3teD7THJfL0hlrTsM7z/7T7aN/fl7mFtaB7kZe8SRURE5ApVaWSCzWa7oj8iIiJStxkMBvp1aMScyf24ZUBLnMxGDp7I4uXwSBatOUROQbG9SxQREZErUKWRCU888UR11yEiIiJ1kIuziTHXtmJgl2C++SWWiJgUftmVQMSBZG4dGMKQHk0wV2AvaxEREaldDDYNFajVLBYrGRn59i7jksxmI35+HmRm5lNaarV3OSJSB6mfqTsOn8xi8brDnEjOAyC4gTt3D2tD51YN7FyZ1HfqZ0SkJjlSH+Pv74GpAkG/vgoQERGRq6ZtM19enNCbB0e1x8vdiaT0At5asoe3v97D6YwCe5cnIiIiFVSlaQ4iIiIiVWU0GhjUtTG92gXy3ZbjrNtxiujYdPYfz2B4r6bcfE0I7q56iyIiIlKbaWSCiIiI2IW7q5m7hrZh5qQ+dGndAIvVxpqIk0ydv5WNexKxWjUTU0REpLZSmCAiIiJ2FdzAg6fHdeXpcV1p5O9ObkEJ4asPMvOTSA6fzLJ3eSIiInIBGkMoIiIitUKX1g3o0NKPn3cmsOK345xIzuO1z3fSJyyQcdeF0sDH1d4lioiIyP9TmCAiIiK1htlk5PrezejXMYhvNx7j1z2JRMSksOtIGqP6NmdUvxa4OJnsXaaIiEi9p2kOIiIiUut4uzvzwMj2zHiwN+2a+VJSamXl5jimzd/G9gPJaGdrERER+1KYICIiIrVW8yAv/n5vdx4b04kG3q5k5hYxb+V+Xv18J3Gnc+xdnoiISL2lMEFERERqNYPBQK/2gcye3Jexg1rh7GTk6KlsZoXvYOGqGLLziuxdooiISL2jMEFEREQcgrOTiZuvacmrj/Snf8cgbMBv0UlMnb+N1dvjKSm12rtEERGRekNhgoiIiDgUPy8XJt/ckWn39yQk2IszxRa+3hDL9I+2s/tImtZTEBERuQoUJoiIiIhDCm3iw/MP9GLS6DB8PJxJySzknaXR/HvJHhLS8u1dnoiISJ2mMEFEREQcltFgYEDnYOY80o8b+7XAbDKw/3gGMz6KYPFPh8k/U2LvEkVEROokhQkiIiLi8NxczNxxXWteebgv3ds0xGqzsS7qFFPnbWPDzlNYrFpPQUREpDopTBAREZE6I9DPnSdv78Jf7+5Gk4Ye5BWW8Onaw7z8cSQxcRn2Lk9ERKTOUJggIiIidU7Hlv68NLE340e0xcPVzKnUfF7/cjfvLdtLSlahvcsTERFxeGZ7FyAiIiJSE0xGI8N6NqVvhyBWbDrOhl0JRB1OZU9sOjf0acbo/i1wddZbIRERkarQyAQRERGp0zzdnBh/fVtemtibsBZ+lFqs/LA1nqnzt7F5bxJWbSUpIiJSaQoTREREpF5oGuDJlLu78eRtnQnwdSU7r5iPfohh9qIoYhOz7V2eiIiIQ1GYICIiIvWGwWCge9sAXnm4H3dc1xoXZxPHk3KYvSiKBd8dIDO3yN4lioiIOASFCSIiIlLvOJmN3NivBa8+0o+BnYMB2Lr/NNPmb+P7LXGUlFrsXKGIiEjtpjBBRERE6i1fTxcmjg5j+oRetG7iTVGJhWUbj/H8gu1EHUrBpvUURERELkhhgoiIiNR7IcHeTLuvJ4/c3AE/LxfSss/w3vJ9vP7FLk6m5Nm7PBERkVpHYYKIiIgIZ9dT6NexEXMm9+Pma1riZDZy8EQWL30cwadrDpFbUGzvEkVERGoNhQkiIiIiv+PibGLsoFbMfrgvvdoHYrPBhl0JTJ23jZ92nKTUYrV3iSIiInanMEFERETkAhr6uvHYmE48d293mgV6UlBUyhfrjjBjYQT7jqXbuzwRERG7UpggIiIicgntmvsx48HePDCyHZ5uTiSlF/DvJXuY+/UekjMK7F2eiIiIXZjtXYCIiIhIbWc0GriuWxP6tA9k5eY41kedYk9sOvuOZzCiVzNuuqYl7q56WyUiIvWHRiaIiIiIVJC7qxN3D2vDzEl96NyqARarjR8jTjBt/lY27knEatVWkiIiUj8oTBARERGppOAGHjxzZ1eeHteFIH93cgpKCF99kFmf7ODwySx7lyciIlLjNB5PREREpIq6tG5Ih5b+/Bx1ihWbjxOfnMtrn++kT1gg464LpYGPq71LFBERqREKE0RERESugNlk5Po+zenXsRHLNx1j4+5EImJS2H0kjVH9WjCyb3NcnEz2LlNERKRaaZqDiIiISDXw9nBmwsj2vPhgb9o286W41MqK347z/IJtRMQkY7NpPQUREak7FCaIiIiIVKMWjbx47t7u/HlMJxp4u5CRU8QHK/bz2uc7iT+da+/yREREqoXCBBEREZFqZjAY6N0+kNmT+zHm2hCczUaOnMpmZngk4atjyMkvtneJIiIiV0RhgoiIiEgNcXYyccuAEOY80o9+HYKwARv3JDF1/lZ+3H6CUovV3iWKiIhUicIEERERkRrm7+3KI7d0ZOp9PWjRyIvCIgtLNhxl+ofb2XM0TespiIiIw1GYICIiInKVtGnqy/QJvXjoxvZ4eziTnFnI3G+ieevrPSSm5du7PBERkQpTmCAiIiJyFRkNBq7t0phXH+nHqH7NMZsM7DuWwYyFEXyx7gj5Z0rsXaKIiMhlKUwQERERsQM3FzPjrgtl1sN96d6mIRarjZ92nGTqvG38sisBq1VTH0REpPZSmCAiIiJiR0F+7jx5exf+elc3Gjf0IK+whEVrDvHSx5EcjM+0d3kiIiIXpDBBREREpBboGOLPyxN7M35EWzxczZxKzeNfX+ziveV7Sc0qtHd5IiIi5ZjtXYCIiIiInGUyGhnWsyl9OwTx7aZjbNiVQNShVPYcTWdk32bc2K8Frs56+yYiIvankQkiIiIitYynmxP3Xd+Olx/qQ1gLP0otVr7fEs+0+dvYuu80Vm0lKSIidqYwQURERKSWahroyZS7u/HEbZ1p6ONKVl4xC74/wKufRnEsMcfe5YmISD2mMEFERESkFjMYDPRoG8DsyX25fXArXJxMxCbm8MqiHXz0/QGy8orsXaKIiNRDChNEREREHICT2cTo/i2Z80g/BnRqBMDmfaeZOn8bP2yNo6TUYucKRUSkPlGYICIiIuJA/LxcmHRTB154oBetGntTVGxh6a/HeOHD7ew8nIpN6ymIiMhVoDBBRERExAG1auzNtPt7MvmmDvh6OpOadYb/LNvLG1/u5lRqnr3LExGROk5hgoiIiIiDMhoM9O/UiDmP9OOma1pgNhmJic9kxsIIPlt7iLzCEnuXKCIidZTCBBEREREH5+ps5rZBrZk9uS892wVgs8HPOxOYOm8r63acpNRitXeJIiJSxyhMEBEREakjAnzdeHxsZ/52T3eaBniSf6aUxeuO8NLHkew/nmHv8kREpA5RmCAiIiJSx4S18GPGQ724/4Z2eLo5kZiWz5tf7eadb6JJziywd3kiIlIHmO1dgIiIiIhUP5PRyJDuTegTFsjK3+L4eecpdh9NY++xdK7v3YybrmmJm4veCoqISNVoZIKIiIhIHebh6sQ9w9vw8sQ+dArxx2K1sXr7CabO38am6ESs2kpSRESqQGGCiIiISD3QuKEHz9zZlafu6EKQnxs5+cV8vOogsz7ZwdFT2fYuT0REHIzCBBEREZF6wmAw0DW0IbMe7sudQ0JxczERfzqXOZ9FMX/lfjJyzti7RBERcRCaKCciIiJSz5hNRkb2bU7/To1YvjGWTXuS2HYgmZ1HUrmxXwtG9mmOs5PJ3mWKiEgtppEJIiIiIvWUj4czD44K48UHe9OmqQ/FJVa+3XSc5xdsI/JgCjatpyAiIhehMEFERESknmvRyIt/jO/Bn27tiL+3C+k5Rfz323388/OdxJ/OtXd5IiJSCylMEBEREREMBgN9woKYPbkftw4Mwdls5PCpbGaGRxK++iA5+cX2LlFERGoRh1kzYfXq1WzZsoX9+/eTkpJCVlYWTk5OtGzZksGDBzNhwgT8/Pwq3J7FYmHbtm388ssv7Nq1i7i4OM6cOYOvry+dO3fmrrvu4rrrrrvgtadOnWLYsGGXbL9r164sWbKkMk9RRERExO5cnEzcOjCEgZ2D+fqXo0TEpLBxTyKRB5O5ZUAIw3o2xWzS91EiIvWdweYgk+FuvfVWDh48iLOzMwEBAfj5+ZGRkUFiYiIADRo0YOHChbRv375C7X399de88MILABiNRpo3b46Hhwfx8fHk5eUBcNddd/Hyyy9jMBjKXfv7MKFHjx4XbL9NmzbMnDmzSs/19ywWKxkZ+VfcTk0ym434+XmQmZlPaanV3uWISB2kfkbEfg6fzOKLdUeITz473SHI3517hoXSpXVDO1dWvdTPiEhNcqQ+xt/fA1MFQmOHGZkwfvx4QkJC6NatG05OTmXHDx06xJQpUzh8+DB//etf+eGHHyrcZrt27bj//vsZOXIkXl5eAJSWlvLJJ5/w+uuv89VXX9G+fXvuvffei7bxxRdfVP1JiYiIiNRybZv5Mn1CLzbvTWLpr7EkZxTw9tfRdG7VgLuHhRLcwMPeJYqIiB04zMiES4mOjmbcuHEArFq1itatW1/2mqysLHx8fM4bdXDO9OnTWbJkCe3bt2fFihXlHvv9yIRDhw5dYfWXppEJIiLqZ0Rqi8KiUr7bEsdPkSexWG2YjAaG9WzKLQNa4u7qdPkGajH1MyJSkxypj6noyIQ6MeGtVatWZX8vLCys0DW+vr4XDRIABg0aBMDx48evrDgRERGROsLNxcydQ0J55eG+dAttiMVqY23kSf4xbxu/7E7AanX476hERKSCHGaaw6VERUUB4O7uTkhISLW0eebMGQDc3Nwued4rr7zCsWPHMBgMNGnShIEDBzJ8+HCMxjqR04iIiIicJ8jfnb/c0YV9x9L5Yv0RktILWPTjITbsTODe4W1o17zii2KLiIhjctgwwWq1kpqayubNm3njjTcAmDJlCh4e1TNv79zaCz179rzkeZ9++mm5n7/66ivCwsJ49913adasWbXUIiIiIlIbdWrVgJdb+LFhVwIrNh3nZEoe/1y8i17tA7nzutY09L30lzIiIuK4HC5MCA8P59VXXy13rEuXLrz22mtlUxOu1Lp169iwYQMGg4GHH374vMfNZjO33HILo0ePJjQ0lMDAQDIzM/n11195++23iYmJYdKkSSxbtgxPT88rrsdsrt2jHM7Np6nIvBoRkapQPyNSe5nNRkb1a8HALsEs/fUYG3aeYsfBFPYcTWNUvxbcfE1LXJxN9i7zstTPiEhNqot9jMMtwLh69WoWLVqExWIhMTGRtLQ0zGYzN9xwAzNmzMDb2/uK2o+NjeWuu+4iNzeXBx98kKlTp1bq+vj4eG677Tby8vJ46qmneOyxx66oHpvNdsm1HURERERqk+OJ2Xy4Yh/RR9MAaODjyoOjOzC4R1O9pxERqUMcLkz4o4MHDzJr1ix27NhBWFgYS5cuxWSqWvqdlJTEvffeS2JiIoMHD+a9994rtw1lRb3xxhssWLCADh06sHz58irVco7FYiUnp2KLStqLyWTE29uNnJxCLJbavTKpiDgm9TMijsVms7HjUCpf/HSYtOyz61CFNvXhvuvb0qqxj52ruzD1MyJSkxypj/H2dqvQCAqHm+bwR+3bt2fevHkMHz6cmJgYfvjhB2655ZZKt5OamsqDDz5IYmIiffr04d13361SkADQvXt3AOLi4qp0/R/V9q1DzrFYrA5Tq4g4JvUzIo6je2hDOrX0Y23kSb7fEs/RU9m8tDCSAZ0bcfvg1vh6uti7xAtSPyMiNaku9TF1YsKGp6cnffr0AWD//v2Vvj49PZ0JEyYQFxdH9+7d+eCDD3Bxqfr/4M6FEBaLpcptiIiIiDg6J7OJ0f1bMueRfvTv2AiAzXtPM3X+NlZti6ekjryhFhGpj+pEmABQWloKVP4DfFZWFg899BCxsbF07NiRBQsWXPGOEEeOHAGgUaNGV9SOiIiISF3g5+XC5Js78Pz9PQkJ9qao2MI3v8Qy/cPt7DqcioPPuhURqZfqRJiQlZVFREQEAGFhYRW+Li8vj4kTJ3Lo0CHatm3LRx99hJeX1xXVkp+fz+LFiwEYMGDAFbUlIiIiUpe0buLD8w/0ZNLoMHw8nUnJKuTdZXt586vdnErNs3d5IiJSCQ4RJkRERPD+++9z6tSp8x7bv38/kyZNIjc3l6CgIEaOHFnu8aFDhzJ06FB+/PHHcscLCwt55JFH2L9/P61atSI8PBw/P78K1TN9+nTWrl1LcXFxueOxsbE8/PDDnDp1Cnd3dyZNmlTJZyoiIiJStxkNBgZ0DmbO5H6M7t8Cs8nAgbhMXloYyedrD5NXWGLvEkVEpAIcYgHGnJwc5s6dy9y5cwkICCAwMBCTyURSUhKpqakABAUFMW/evPOmKCQkJABQUFBQ7viiRYuIiooq+/mJJ5646P3feecdAgICyn6Ojo5myZIlODk50bx5czw9PcnMzOTEiRMA+Pj48Pbbb9O0adMre+IiIiIidZSbi5nbB7fm2q6N+frno0QdTmX9zlNsO3CaMde24rrujTEZHeJ7LxGReskhwoTu3bszdepUtm/fztGjR4mLi6O4uBhvb2/69u3L0KFDueOOO/D09Kxwm78fVXDs2LFLnltUVFTu50cffZRNmzaxb98+0tLSiI+Px9XVlY4dOzJo0CDGjx9fLnwQERERkQsL9HXj8ds6ExOXweL1R0hIzefznw7zy64E7hnehg4t/e1dooiIXIDBphVvajWLxUpGRr69y7gks9mIn58HmZn5dWabExGpXdTPiNQPFquVjbsTWbbxGPlnzi6u3b1NQ+4aGkqgn3uN3lv9jIjUJEfqY/z9PTCZLj8yzCFGJoiIiIhI3WcyGhnSoym9w4JY+dtxft6ZwK4jaew9ls71vZszun8L3Fz09lVEpDbQRDQRERERqVU83Zy4d0RbXp7Uh44t/Si12Fi1LZ5p87exeW8SVg2sFRGxO4UJIiIiIlIrNWnowbN3deMvt3ch0M+N7PxiPvohhtmLdnA0Idve5YmI1GsKE0RERESk1jIYDHRr05BZk/oybkhrXJ1NHE/KZc6nUSz4bj+ZuUWXb0RERKqdwgQRERERqfWczEZG9W3Bq4/0Y2CXYAzA1v3JTJ2/le82H6e4xGLvEkVE6hWFCSIiIiLiMHw8XZh4YxgvTOhFaBMfikusLN90nOcXbGfHwRS0UZmIyNWhMEFEREREHE5IsDdT7+vBI7d0wM/LhfScM7z/7T7+tXgXJ5Jz7V2eiEidpzBBRERERBySwWCgX4dGzJncj1sGtMTJbOTQySxeDo9k0Y8HySkotneJIiJ1lsIEEREREXFoLs4mxlzbitmT+9InLBCbDX7ZncjUedtYG3mSUovV3iWKiNQ5ChNEREREpE5o6OPGn27txD/G96B5kCeFRaV8uf4IMxZGsPdYur3LExGpUxQmiIiIiEid0raZLy9O6M2Do9rj5e5EUnoBby3Zw9tf7+F0RoG9yxMRqRMMNi15W6tZLFYyMvLtXcYlmc1G/Pw8yMzMp7RUwwhFpPqpnxGRqio4U8p3W46zbscpLFYbJqOB4b2acvM1Ibi7mgGwWm3EJmZTYjPgZLDRurEPRqPBzpWLSF3iSO9l/P09MJkuP+5AYUItpzBBRET9jIhcuaT0fL76+SjRsWenO3i5O3HboFa4uzrx5fojZOYWlZ3r5+XCvcPb0LNdoL3KFZE6xpHeyyhMqCMUJoiIqJ8RkeoTHZvOl+uPVGi6w+NjOylQEJFq4UjvZSoaJmjNBBERERGpN7q0bsDMSX24c2joZc/9Yt0RrFZ97yYiciEKE0RERESkXjGbjLQM8rrseRm5RRw+mVXzBYmIOCCFCSIiIiJS72TlF13+JGDJhqP8Fp1ETkFxDVckIuJYzPYuQERERETkavP1cKnQeXGnc1m4KgYD0LqJD93aNKRbaEOCG7hjMGjHBxGpvxQmiIiIiEi907aZL35eLuV2cfgjb3dnBncLJjo2g/jkXI4mZHM0IZtvfokl0M+NbqFng4U2zXwwGTXgV0TqF+3mUMtpNwcREfUzIlIzog6l8N7yfRd9/Pe7OWTknGHP0TR2HU3jYHwmpZb/vYX2cDXTuXUDuoU2pFNIA9xd9X2diJTnSO9ltDVkHaEwQURE/YyI1JyoQyksXnek3AgFfy8X7hne5qLbQhYWlXIgLoPdR9LYE5tOXmFJ2WMmo4F2zX3LRi009HWr8ecgIrWfI72XUZhQRyhMEBFRPyMiNctqtRGbmE2JzYCTwUbrxj4YjRVbD+HctbuPpLH7aBpJ6QXlHm8a4PH/6ywE0DLYC6PWWRCplxzpvYzChDpCYYKIiPoZEal51dXPnM4oODti4Wgah09l8ft32j4eznT9/xELYS39cHEyVUPlIuII/q+9e4+Our7zP/6aSSaT6ySTTC4QgglJuBoygMKqFVy0RY5bpV0vP2V7sGrr9qZ7Tulu2QW7LqcHz7anLsW10h5drNb16LZVD4rpUdT2hy6BkAQIIeTCxVyAzOQecp2Z3x8DI/lxy0CGme/wfJzjOcl85vvNe4LnfWZe+VyM9F6GMCFKECYAAH0GQOiFos/0DYxoX6NblQ0u7W9ya3DYExiLizVrdn66nMUOlRZmKDV5fKdLADAmI72XGW+YwO4wAAAAQAgkJ1h00/U5uun6HI16vKo71nV6OUS73D1DqmrwL42QpGmTbf59FoodynUkcewkgIjHzIQIx8wEAKDPAAi9q9lnfD6fPj/Zp+rTYcLhtt4x447U+ECwMD0vTbHj+AshgMhmpPcyLHOIEoQJAECfARB64ewznb1Dqm50qbrepQNHOzVy1s9PsMaqZFq6nEUOlRRmKCneclVrAzAxjPRehmUOAAAAgAHYU6y6zZmr25y5Ghr26MCRDlU2uLS3waWeUyMqrz2p8tqTMptMmp6XKmdxppxFGcqyJ4a7dADXMGYmRDhmJgAAfQZA6EVin/H6fDrc2uPfW6HepRbX2PeEkx1JgeUQ0ybZxn2cJYCrLxJ7zIWwzCFKECYAAH0GQOgZoc+c7BpQdb1/n4W6Y13ynvU23pZo0dxCf7AwJz9d1jiOnQQiiRF6zBmECVGCMAEA6DMAQs9ofaZ/cET7mtyqbnBrb6NbA0OjgbHYGLNm59vlLHKotMghewrHTgLhZqQeQ5gQJQgTAIA+AyD0jNxnRj1e1X/epcrTyyFc3YNjxvNzUuQsdshZ5FBeVjLHTgJhYKQeQ5gQJQgTAIA+AyD0oqXP+Hw+tbr6A/ssNLX26Ow3++k2a2CfhRl5dlliOXYSuBqM1GMIE6IEYQIA0GcAhF609pnu/mHtbfDvs1BzuEPDZ722+LgYXV+QLmexQ3MLHUpO4NhJIFSM1GM4GhIAAAC4xqUmxenW0sm6tXSyhkc8OnC0U9Wnw4XuvmHtrmvX7rp2mUxS8ZS0wKyFnHSOnQRwccxMiHDMTAAA+gyA0LvW+ozX59PR472qrPcvh2hu7xsznpOeGNhnoSg3lWMngStkpB7DMocoQZgAAPQZAKF3rfcZV/eAqhvcqqpv18FjXfJ4v/iIkJxg0dzCDDmLHJpTkK4EK5ObgWAZqccQJkQJwgQAoM8ACD36zBcGhka1/3CHqurbtbfRrf7Bs4+dNGnmVHtg1kK6LT6MlQLGYaQeQ5gQJQgTAIA+AyD06DPn5/F61dDcraoGlyrrXTrZOTBmfGp2cmCfheuyUzh2ErgAI/UYwoQoQZgAAPQZAKFHn7k0n8+n4x2nVFXvUmWDS40t3Tr7k4Q9xarSIoecRRmadZ1dltiY8BULRBgj9RjChChBmAAA9BkAoUefCV7PqWHta3Srqt6l/Yc7NDTiCYxZLTGaU5AuZ5FDc4syZEuMC2OlQPgZqcdwNCQAAACAkLElxumWkkm6pWSSRkY9OnisS1X1/mMnO3uHtOdQu/YcapdJUmFuamCfhUkZiSyHAKIAMxMiHDMTAIA+AyD06DMTx+fz6diJPlU1+I+dPHqid8x4lj3Bv89CkUPFeamKMV/6L6CA0Rmpx7DMIUoQJgAAfQZA6NFnQqejZ1DVDS5VNbhVe7RDo54vPn4kxceq5PSxk9cXZCgxnonTiE5G6jGECVGCMAEA6DMAQo8+c3UMDI3qwJEOVdW7VN3oVt/ASGAsxmzSjKlpgVkLjrSEMFYKTCwj9RjChChBmAAA9BkAoUefufq8Xp8aW7sD+yy0uU+NGZ+SmXR6n4VM5U9KkZl9FmBgRuoxhAlRgjABAOgzAEKPPhN+JzpOBfZZONTcNebYydSkOJUWZchZlKlZ+XZZLRw7CWMxUo8hTIgShAkAQJ8BEHr0mcjSNzDiP3aywaV9TW4NDn9x7GRcrFmz89PlLHaotDBDqcnWMFYKjI+RegxHQwIAAAAwpOQEi266Pkc3XZ+jUY9XdYFjJ9vl7hnyz2BocEmSpk22+fdZKHYo15HEsZPAVcLMhAjHzAQAoM8ACD36jDH4fD41t/erqr5dVQ0uHW4be+ykIzU+ECxMz0tT7Dj+ugpcDUbqMSxziBKECQBAnwEQevQZY+rsHdLeRv8+CweOdmrkrH+7BGusSqaly1nkUElhhpLiLWGsFNc6I/UYljkAAAAAiGr2FKuWOHO1xJmroWGP/9jJBpeqG1zqOTWi8tqTKq89KbPJpOl5qXIWZ8pZlKEse2K4SwcMj5kJEY6ZCQBAnwEQevSZ6OL1+XS4tSdwOkSLa+z76cmOpMByiGmTbDKb2WcBoWWkHsMyhyhBmAAA9BkAoUefiW4nuwZUXe/ftPHQ513yeL/4CJSSaFFpoT9YmJOfLmscx05i4hmpxxAmRAnCBACgzwAIPfrMtePU4Ij2NfmXQ+xtdGtgaDQwFhtj1ux8u5xFDpUWOWRP4dhJTAwj9RjChChBmAAA9BkAoUefuTaNeryq/7xLlaeXQ7i6B8eM5+ekyFnskLPIobysZI6dxGUzUo8hTIgShAkAQJ8BEHr0Gfh8PrW6+gP7LDS19ujsD0rpNmtgn4UZeXZZYjl2EuNnpB5DmBAlCBMAgD4DIPToM/j/dfcPa2+Df5+FmsMdGj7r/wtrXIxKCtLlLHZobqFDyQkcO4mLM1KP4WhIAAAAALhMqUlxurV0sm4tnazhEY8OHO1U9elwobtvWLvr2rW7rl0mk1Q8JS0wayEnnWMncW1gZkKEY2YCANBnAIQefQbj5fX5dPR4ryrrXapucOnzk31jxnPSEwP7LBTm2hRjZjkEjNVjWOYQJQgTAIA+AyD06DO4XK7uAVU3uFXV4NLBo51jjp1MTrBobmGGnEUOzSlIV4KVieHXKiP1GMKEKEGYAAD0GQChR5/BRBgYGtX+wx2qqm/X3ka3+gfPPnbSpJlT7YFZC+m2+DBWiqvNSD2GMCFKECYAAH0GQOjRZzDRPF6vGpq7VdXgUmW9Syc7B8aMT81K9gcLxQ5dl53CsZNRzkg9hjAhShAmAAB9BkDo0WcQSj6fT8c7TgWOnWxo6dbZn8LsKVaVFmbIWezQrOvsssTGhK9YhISRegxhQpQgTAAA+gyA0KPP4GrqOTWsfY1uVdW7tP9wh4ZGPIExqyVGcwrS5SxyaG5RhmyJcWGsFBPFSD2GoyEBAAAAIALZEuN0S8kk3VIySSOjHh081qWqev+xk529Q9pzqF17DrXLJKkwNzWwz8KkjESWQyBiMDMhwjEzAQDoMwBCjz6DSODz+XTsRF9gOcTRE71jxrPSEgLBQnFeKsdOGoiRegzLHKIEYQIA0GcAhB59BpGoo2dQ1aeXQ9Qe7dCo54uPbknxsSo5fezk9QUZSoxn0nkkM1KPIUyIEoQJAECfARB69BlEuoGhUR040qGqepeqG93qGxgJjMWYTZoxNU3OIv+sBUdaQhgrxfkYqcdEXZiwbds2ffrpp6qpqdHJkyfV1dUli8Wi/Px8LVmyRKtWrZLdbh/3/Twej/73f/9XH3/8sSorK3XkyBENDg4qLS1NJSUleuCBB3Tbbbdd9B5ut1u/+tWv9NFHH+nkyZOy2Wy68cYb9fjjj2vWrFlX+IrP1EmYAAD0GQChRp+BkXi9PjW2dgf2WWhznxozPiUz6fRyiEzlT0qRmX0Wws5IPSbqwoR77rlHBw8eVFxcnDIzM2W329XR0aHW1lZJUkZGhl566SXNnDlzXPd78803tXbtWkmS2WzW1KlTlZSUpKNHj6qvr0+S9MADD+jpp58+7yYnR48e1UMPPSSXy6XExEQVFBTo+PHjcrvdslgs2rhxo26//fYrft2ECQBAnwEQevQZGNmJs46drG/ulvesj3ipSXEqLcqQsyhTs/Ltslo4djIcjNRjoi5MeOONN1RQUCCn0ymLxRJ4vK6uTqtXr9ahQ4dUVFSkd999d1z3e/PNN/XKK6/oG9/4hu68806lpKRIkkZHR/Xyyy/rZz/7mXw+n37yk5/ooYceGnOtz+fT1772NdXW1urWW2/Vs88+q5SUFI2Ojuo///M/9fzzzysxMVFlZWXKysq6otdNmAAA9BkAoUefQbToGxjRvib/Pgv7mtwaHP7i2Mm4WLNm56fLWexQaWGGUpOtYaz02mKkHhN1YcLF7N27V/fdd58k6b333lNhYeElr+nq6lJqauoFj1ZZt26d3njjDc2cOVNvv/32mLEPPvhA3/ve95SSkqIPP/xQqampY8b/7u/+Trt27dI3v/lN/fjHP77MV+VHmAAA9BkAoUefQTQa9XhVd9axk+6ewTHj0ybbAvss5GYmcexkCBmpx4w3TIiKLT+nTZsW+HpgYGBc16SlpV10fPHixXrjjTd0+PDhc8a2bdsmSbrzzjvPCRIk6f7779euXbu0bdu2Kw4TAAAAAOByxMaYNacgXXMK0vXQl4vV3N6vqvp2VTW4dLitV02tPWpq7dEf/twkR2q8P1godmh6Xppix/FhEte2qAgTKioqJCmwd8FEGBz0p3YJCefuhFpdXS1JuuGGG8577ZnHjx8/rhMnTig7O3tCagIAAACAy2EymZSXlay8rGR99ZYCdfUNqfr0PgsHjnbK1T2oDyqa9UFFsxKsMSqZ5j92sqQwQ0nxlkv/AFxzDBsmeL1etbe3a8eOHfr5z38uSVq9erWSkpIm5P5n9l5YsGDBmMeHh4fV0tIiSZo6dep5r500aZIsFotGRkbU1NREmAAAAAAgoqQlW7XEmaslzlwNDXv8x042uFTd4FLPqRGV155Uee1JmU0mTc9LDcxayLInhrt0RAjDhQlbtmzRhg0bxjw2d+5cPfPMM1q8ePGE/IwPPvhAH330kUwmkx577LExY319ffJ6/WtczrfEQfKnfjabTW63Wz09PVdcT2xsZE8xOrOeZjzragDgctBnAIQafQbXsthYs26cna0bZ2fL6/OpqaVHlYfaVVnfrub2fh081qWDx7r0+vYG5TqSNG96puYVO1SYmyqzmX0WxiMae4zhwoTs7GzNnz9fHo9Hra2tcrlcqq2t1dtvvy2n0ymbzXZF929sbAzsc7Bq1SrNnz9/zPjQ0FDg67i4uAve58zYmeUSl8tsNslun5jZFqFms527JAQAJhJ9BkCo0WcAKSM9WTeWTJYkHXf3q7zmuHbWHFdNk1strn61uPq19dMjSk2O042zcrRwTo7mTc9UvNVwHy+vumjqMYb7116+fLmWL18e+P7gwYNav369tm7dqsbGRv3+979XTMzlnZ3a1tamxx57TL29vVqyZIlWr159znOs1i+OTxkeHr7gvc6MxcfHX1YtZ3i9PvX0nLqie4RaTIxZNluCenoG5PFE9s6kAIyJPgMg1OgzwPlZzdKtJTm6tSRH/YMj2tfo1p5D7drb4FZ337A+2HVMH+w6JkuMWbML7JpXnClnsUPptiv7HBRtjNRjbLaEa+M0h5kzZ2rz5s264447VFtbq3fffVd333130Pdpb2/Xww8/rNbWVi1cuFCbNm2SxXLuRiPJyckym83yer3q7u4+7718Pl9gecOVzpSQFPFHh5zh8XgNUysAY6LPAAg1+gxwYdbYGN0wI0s3zMjSqMer+s+7VNXgVmV9u1zdg6pucKu6wS1tk/JzUgL7LORlJXPs5GnR1GMMHyZI/g/4CxcuVFlZmWpqaoIOE9xut1atWqUjR45o3rx5euGFF8bMQDhbXFycJk+erObmZh07duycZRCSf4bDyMiIJE3Y6RIAAAAAECliY8yalZ+uWfnp+j+3F6nV1a+qBpeqGlxqaunRkeO9OnK8V2/938NKt1n9wUKRQzOm2mWJ8D3hMD5RESZI0ujoqCTJ4/EEdV1XV5e++c1vqrGxUXPmzNFvfvObS54I4XQ61dzcrN27d2vFihXnjO/evVuSlJOTo5ycnKDqAQAAAAAjMZlMys1MVm5msu66KV/d/cPaezpYqDncoY6eIW3f06Lte1pkjYtRSUG6nMUOzS10KDmBYyeNKirChK6uLpWXl0uSZs2aNe7r+vr69Mgjj6iurk7Tp0/Xiy++qJSUlEtet2zZMm3dulXvv/++fvSjH51zqsMbb7whSbrzzjuDeBUAAAAAYHypSXG6tXSybi2drOERj2qPdgZmLXT3DWt3Xbt217XLZJKKc1PlPL3PQk46x04aicnn8/nCXcSllJeXa/fu3br77rs1ZcqUMWM1NTV66qmntH//fmVnZ2vbtm1jZhYsXbpUkvSP//iPYz7cDwwM6NFHH1VFRYWmTZumV199VRkZGeOqx+v1asWKFaqrq9PixYv1i1/8QikpKfJ4PHruuef0/PPPKyEhQWVlZcrOzr6i1+7xeNXR0X9F9wi12Fiz7PYkdXb2R836HwCRhT4DINToM0DoeX0+HT3eq6p6f7Dw+cm+MeM56YmBfRYKc22KMUfPcggj9Zj09KTo2YCxp6dHGzdu1MaNG5WZmamsrCzFxMSora1N7e3tkvxHRm7evPmcJQotLS2SpFOnxp6I8Nvf/lYVFRWB77///e9f8Of/8pe/VGZmZuB7s9msjRs3auXKlfrzn/+sxYsXq6CgQMePH5fb7ZbFYtHPfvazKw4SAAAAACBamE0mFUyyqWCSTV9bPE2u7gFVN7hV1eDSwaOdOt5xSu+XH9P75ceUnGDR3MIMOYscmlOQrgSOnYw4hvgXmTdvntasWaOdO3eqoaFBR44c0fDwsGw2mxYtWqSlS5fq3nvvVXJy8rjvefaxjk1NTRd97tDQ0DmPFRQU6J133tGvfvUrffTRRzp06JBsNpuWLVumv//7v9fs2bPH/wIBAAAA4BrjSE3Q7Qum6PYFUzQwNKr9hztUVd+uvY1u9Q2M6NP9x/Xp/uOKjTFp5lS7nMX+TRw5djIyGGKZw7WMZQ4AQJ8BEHr0GSByeLxeNTR3+/dZqHfpROfAmPGpWcn+YKHYoeuyUwxx7KSResx4lzkQJkQ4wgQAoM8ACD36DBC52tz9gWChoaVbZ3+CTUuOC+yzMOs6uyyxMeEr9CKM1GMIE6IEYQIA0GcAhB59BjCGnlPD2tfo32dhf1OHhkY8gTGrJUZzCtJVWpSh0kKHbElxYax0LCP1mKjagBEAAAAAAFtinG4pmaRbSiZpZNSjg8e6AqdDdPYOac+hdu051C6TpMLcVDmLHSotcmhyRqIhlkMYCTMTIhwzEwCAPgMg9OgzgLH5fD4dO9EXWA5x9ETvmPGstITABo7FealX/dhJI/UYljlECcIEAKDPAAg9+gwQXTp6BlXd6FZVvUu1Rzs06vniY29SfKxKpmXIWezQ9QUZSowP/YR9I/UYwoQoQZgAAPQZAKFHnwGi1+DwqGoOd6iqwaXqBv+xk2fEmE2aMTVNpUX+WQuZaQkhqcFIPYYwIUoQJgAAfQZA6NFngGuD1+tTY2t3YJ+FNvepMeNTMpNOL4fIVP6kFJknaJ8FI/UYwoQoQZgAAPQZAKFHnwGuTSc6TgX2Wahv7pb3rI/HqUlxKi3KkLMoU7Py7bJaLv/YSSP1GMKEKEGYAAD0GQChR58B0Dcwon1N/n0W9jW5NTj8xbGTcbFmzc5P958OUZih1GRrUPc2Uo8hTIgShAkAQJ8BEHr0GQBnG/V4VXesKzBrwd0zOGZ82mSbSoscmlfkUG5m0kWPnTyztGLEZ5LF5FPh5FSZzZF7TCVhQpQgTAAA+gyA0KPPALgQn8+n5vZ+VdW3q6rBrcNtPWPGHanxchY55Cx2aHpemmLP+iBeUXdSr31Qr87eocBj9hSrHrqjWAtmZF211xAMwoQoQZgAAPQZAKFHnwEwXl19Q6o+PWPhwNFOjZzVMxKsMf5jJ4sc8vh8enFr7QXv872vXR+RgQJhQpQgTAAA+gyA0KPPALgcQyMeHTjSoap6l6obXOo5NXLpi05LT7Hq379zc8QteRhvmBB7FWoBAAAAACDqWC0xmlecqXnFmfL6fDrc2qOqBpd2HjghV/fgRa/t6B3Soc+7NPM6+1WqdmIRJgAAAAAAcIXMJpMKc1NVmJuq3Mwk/fqdA5e8pqt/6JLPiVSXnrsAAAAAAADGLS1pfEdHjvd5kYgwAQAAAACACTQ9L032lIsHBekpVk3PS7s6BYUAYQIAAAAAABPIbDbpoTuKL/qcB+8ojrjNF4NBmAAAAAAAwARbMCNL3/va9efMUEhPsUbssZDBYANGAAAAAABCYMGMLM0rzlRja7dGfCZZTD4VTk419IyEMwgTAAAAAAAIEbPZpFn56bLbk9TZ2a/RUW+4S5oQLHMAAAAAAABBIUwAAAAAAABBIUwAAAAAAABBIUwAAAAAAABBIUwAAAAAAABBIUwAAAAAAABBIUwAAAAAAABBIUwAAAAAAABBIUwAAAAAAABBIUwAAAAAAABBIUwAAAAAAABBIUwAAAAAAABBIUwAAAAAAABBMfl8Pl+4i8CF+Xw+eb2R/08UE2OWx+MNdxkAohh9BkCo0WcAhJJReozZbJLJZLrk8wgTAAAAAABAUFjmAAAAAAAAgkKYAAAAAAAAgkKYAAAAAAAAgkKYAAAAAAAAgkKYAAAAAAAAgkKYAAAAAAAAgkKYAAAAAAAAgkKYAAAAAAAAgkKYAAAAAAAAgkKYAAAAAAAAgkKYAAAAAAAAgkKYAAAAAAAAgkKYAAAAAAAAgkKYAAAAAAAAghIb7gJgPO3t7dqxY4f279+vffv2qba2VkNDQ1q4cKFeeeWVcJcHwOB8Pp8qKyu1fft2VVRUqKmpSX19fUpJSdHs2bO1YsUKffWrX5XJZAp3qQAMbNu2bfr0009VU1OjkydPqqurSxaLRfn5+VqyZIlWrVolu90e7jIBRJFPPvlE3/72tyVJubm52r59e5grujImn8/nC3cRMJYtW7Zow4YN5zxOmABgInz22Wd6+OGHA9/n5eXJZrOppaVFXV1dkqTbbrtNmzZtUlxcXHiKBGB499xzjw4ePKi4uDhlZmbKbrero6NDra2tkqSMjAy99NJLmjlzZpgrBRAN+vv79Td/8zeBHhMNYQIzExC05ORk3XzzzSopKVFJSYkOHDig559/PtxlAYgSPp9PU6ZM0apVq3TXXXcpIyMjMPbWW29p3bp1+vjjj7Vx40b96Ec/CmOlAIxs5cqVKigokNPplMViCTxeV1en1atX69ChQ/rhD3+od999N4xVAogWzz77rFpbW3X77bfrww8/DHc5E4KZCbhir776qtavX8/MBAAToq+vT1ardcyb+7O98MILevbZZ5WWlqbPPvtMZjPb/wCYWHv37tV9990nSXrvvfdUWFgY5ooAGFlVVZUefPBB/fVf/7XuuOMOrVmzJipmJvAODAAQUZKTky8YJEjS4sWLJUldXV3q6Oi4WmUBuIZMmzYt8PXAwEAYKwFgdCMjI1q3bp3i4+P11FNPhbucCUWYAAAwlMHBwcDX8fHxYawEQLSqqKiQJCUmJqqgoCDM1QAwss2bN+vQoUN68sknlZOTE+5yJhR7JgAADOXM+uWZM2cqOTk5zNUAiBZerzdwYtXPf/5zSdLq1auVlJQU5soAGFVjY6M2b96sOXPm6Bvf+Ea4y5lwhAkAAMPYv3+/Xn/9dUkKHK0EAFfifKdUzZ07V88880xgWRUABMvn82nt2rUaHR3V008/rZiYmHCXNOFY5gAAMASXy6Uf/OAHGh0d1Ze//GXddddd4S4JQBTIzs7W/PnzVVpaqszMTJlMJtXW1urtt99WT09PuMsDYFCvvfaa9uzZo5UrV6qkpCTc5YQEMxMAABGvt7dX3/rWt9Ta2qo5c+bomWeeCXdJAKLE8uXLtXz58sD3Bw8e1Pr167V161Y1Njbq97//fVT+RRFA6Jw4cUK/+MUvlJ2drX/4h38Idzkhw8wEAEBE6+/v12OPPaYDBw6ouLhYL774InslAAiZmTNnavPmzbLb7aqtrQ3s0wIA47V+/Xr19fVp7dq1Uf2ehZkJAICINTAwoMcff1xVVVXKz8/Xf/3Xf8lut4e7LABRLjk5WQsXLlRZWZlqamp09913h7skAAZy4MABSdLTTz+tp59+eszYmVOp2tradMstt0iSNm3apPnz51/dIicAYQIAICINDQ3pO9/5jnbt2qXc3Fxt2bJFmZmZ4S4LwDVidHRUkuTxeMJcCQCjcrlcFxzzer2B8ZGRkatV0oQiTAAARJyRkRH94Ac/0Geffabs7Gy9/PLLmjRpUrjLAnCN6OrqUnl5uSRp1qxZYa4GgNFs3779gmN/+MMftGbNGuXm5l70eUbAngkAgIji8Xj0wx/+UJ988okyMzP18ssvKy8vL9xlAYgi5eXlev7559Xc3HzOWE1NjR599FH19vYqOztbd955ZxgqBIDIx8wEBK2trU0rVqwIfD88PCxJ2rNnjxYtWhR4/LHHHtO3vvWtq10eAIPbtm2bysrKJElxcXH653/+5ws+d926dZo9e/bVKg1AlOjp6dHGjRu1ceNGZWZmKisrSzExMWpra1N7e7sk/5GRmzdvVlJSUpirBYDIRJiAoHk8HnV1dZ3z+Ojo6JjHz2wuAgDBOBNQSlJLS4taWlou+Nze3t6rURKAKDNv3jytWbNGO3fuVENDg44cOaLh4WHZbDYtWrRIS5cu1b333hvVu7ADwJUy+Xw+X7iLAAAAAAAAxsGeCQAAAAAAICiECQAAAAAAICiECQAAAAAAICiECQAAAAAAICiECQAAAAAAICiECQAAAAAAICiECQAAAAAAICiECQAAAAAAICiECQAAAAAAICiECQAAAJdhxowZmjFjhnbu3BnuUgAAuOpiw10AAACIDps2bdJzzz037ufX1dWFsBoAABBKhAkAAGDCORyOcJcAAABCiDABAABMuB07doS7BAAAEELsmQAAAAAAAILCzAQAABB2S5cuVUtLizZs2KCvfOUr2rx5s/70pz+pra1NCQkJWrBggR5//HGVlpZe8B4ej0d//OMf9c4776iurk79/f2y2+2aN2+eVq5cqUWLFl20hra2Nr3yyivasWOHmpubNTIyoqysLBUXF2vZsmVavny5rFbrea/t6+vTb37zG5WVlam1tVUJCQlyOp367ne/e9GaAQAwKsIEAAAQMXp6enTvvffq8OHDslgsslqt6urq0ocffqiPPvpI69ev17333nvOdb29vfrud7+r8vJySVJMTIySkpLU3t6usrIylZWV6ZFHHtE//dM/nffnvvXWW3rqqac0NDQkSbJYLEpKSlJbW5s+//xzbd++XTNmzNCsWbPOuba9vV1f//rXdfToUVmtVpnNZnV1denjjz/Wjh079MILL+hLX/rSBP6WAAAIP5Y5AACAiPHcc8+po6ND//Ef/6GqqipVVFTovffe08KFC+X1evWTn/xENTU151z3L//yLyovL5fFYtHatWtVUVGhXbt26S9/+Yv+9m//VpL00ksv6b//+7/Pufbjjz/Wj3/8Yw0NDWn+/Pn63e9+p71792rnzp2qrKzU7373O91///2yWCznrfnf/u3fZLFY9PLLL6uqqkqVlZV68803VVBQoJGRET311FPyer0T+4sCACDMTD6fzxfuIgAAgPGdfTTkpU5zWL58udauXRv4/swyB0nasmWLbrrppjHPHxwc1D333KMjR45oyZIl+vWvfx0Yq66u1v333y/J/8H+gQceOOfnPfHEEyorK5Pdbtcnn3wSWK4wOjqqZcuWqbm5WQsWLNCWLVsUFxc3rtc7Y8YMSVJ6erq2bt2qjIyMMeN1dXW6++67JUmvvfaaFixYMK77AgBgBMxMAAAAE87lcl30v76+vvNeN3/+/HOCBEmKj4/Xo48+Kkn6y1/+ot7e3sDYe++9J0nKycnRfffdd977Pvnkk5Kkzs7OMSdN7Ny5U83NzZKkNWvWjDtIONv9999/TpAg+cOGKVOmSPIHCwAARBP2TAAAABPucj88/9Vf/dUlx7xer2pqagLf79+/X5K0aNEimc3n/ztJYWGhsrOzdeLECe3fv19Lly6VJFVWVkqSMjMzVVJSclk1X2yDxaysLDU3N6u7u/uy7g0AQKRiZgIAAIgY2dnZ4xrr6OgIfO12uy95reSfuXD28yX/5omSNHny5OCLPS0pKemCY7Gx/r/bjI6OXvb9AQCIRIQJAADgmmUymcJdAgAAhkSYAAAAIsaJEyfGNZaenh74+sx+BcePH7/ovc+Mn72/wZmNIltbW4MvFgCAaxhhAgAAiBg7d+685JjZbNbs2bMDj19//fWB8QsdwdjY2BgII87eG2H+/PmS/Msd9u3bd2XFAwBwDSFMAAAAEaOiouK8gcLQ0JBeeuklSdKXvvQl2Wy2wNhdd90lyT9z4c033zzvfX/5y19Kkux2u26++ebA44sWLVJeXp4kacOGDRoeHp6YFwIAQJQjTAAAABEjJSVFTzzxhN5///3ApoWNjY369re/raamJsXExOiJJ54Yc83cuXO1bNkySdL69ev16quvamBgQJJ/xsHatWv1/vvvS/IfEWm1WgPXxsTEaN26dTKZTKqoqNDDDz+s3bt3B2Y4DA8Pa+fOnVq9erUaGhpC/voBADAKjoYEAAAT7pZbbrnkczZt2hRYZnDG97//fb3++ut68sknFRcXJ6vVqt7eXkn+zRL/9V//9bxHOP70pz9VZ2enysvLtX79em3YsEFJSUnq6emRz+eTJD3yyCN68MEHz7l2yZIleuaZZ7Ru3TpVVFRo5cqViouLU2Jiovr6+gKhxqOPPhr07wEAgGhFmAAAACacy+W65HNGRkbOecxms+l//ud/tHnzZv3pT39SW1ub0tLSNG/ePD3++OOaN2/eee+VkpKiLVu26I9//KPefvtt1dXV6dSpU3I4HJo/f75WrlypRYsWXbCWFStW6IYbbtBvf/tb7dixQ62trRoaGtLkyZM1ffp0feUrX1FhYeH4fwEAAEQ5k+9MXA8AABAmS5cuVUtLizZs2KCvf/3r4S4HAABcAnsmAAAAAACAoBAmAAAAAACAoBAmAAAAAACAoBAmAAAAAACAoLABIwAAAAAACAozEwAAAAAAQFAIEwAAAAAAQFAIEwAAAAAAQFAIEwAAAAAAQFAIEwAAAAAAQFAIEwAAAAAAQFAIEwAAAAAAQFAIEwAAAAAAQFAIEwAAAAAAQFD+HzvlTA2s9J7PAAAAAElFTkSuQmCC\n"
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Tg42jJqqM68F"
},
"source": [
"#### Test Data Preparation\n",
"\n",
"We'll need to apply all of the same steps that we did for the training data to prepare our test data set."
]
},
{
"cell_type": "code",
"source": [
"## Preprocess test data\n",
"test_df['modified_message'] = test_df.astype(str).apply(lambda row : process_text(row['message']), axis = 1)\n",
"print(test_df.sample(10))\n",
"\n",
"test_sentences = test_df.modified_message.values\n",
"test_labels = test_df.is_toxic.values\n"
],
"metadata": {
"id": "eDuXSdtmfT-i"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "mAN0LZBOOPVh",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "b026c478-92d6-4917-821d-654bb9eee71b"
},
"source": [
"import pandas as pd\n",
"\n",
"# Load the test dataset into a pandas dataframe.\n",
"# df = pd.read_csv(\"./test_dataset.csv\", delimiter='\\t', header=None, names=['sentence', 'is_toxic'])\n",
"\n",
"# Report the number of sentences.\n",
"print('Number of test sentences: {:,}\\n'.format(test_df.shape[0]))\n",
"\n",
"# Create sentence and label lists\n",
"# sentences = test_df.sentence.values\n",
"# labels = test_df.label.values\n",
"\n",
"# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
"input_ids = []\n",
"attention_masks = []\n",
"\n",
"# For every sentence...\n",
"for sent in test_sentences:\n",
" # `encode_plus` will:\n",
" # (1) Tokenize the sentence.\n",
" # (2) Prepend the `[CLS]` token to the start.\n",
" # (3) Append the `[SEP]` token to the end.\n",
" # (4) Map tokens to their IDs.\n",
" # (5) Pad or truncate the sentence to `max_length`\n",
" # (6) Create attention masks for [PAD] tokens.\n",
" encoded_dict = tokenizer.encode_plus(\n",
" str(sent), # Sentence to encode.\n",
" add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
" max_length = 512, # Pad & truncate all sentences.\n",
" pad_to_max_length = True,\n",
" return_attention_mask = True, # Construct attn. masks.\n",
" return_tensors = 'pt', # Return pytorch tensors.\n",
" )\n",
"\n",
" # Add the encoded sentence to the list.\n",
" input_ids.append(encoded_dict['input_ids'])\n",
"\n",
" # And its attention mask (simply differentiates padding from non-padding).\n",
" attention_masks.append(encoded_dict['attention_mask'])\n",
"\n",
"# Convert the lists into tensors.\n",
"input_ids = torch.cat(input_ids, dim=0)\n",
"attention_masks = torch.cat(attention_masks, dim=0)\n",
"test_labels = torch.tensor(test_labels)\n",
"\n",
"# Set the batch size.\n",
"batch_size = 16\n",
"\n",
"# Create the DataLoader.\n",
"prediction_data = TensorDataset(input_ids, attention_masks, test_labels)\n",
"prediction_sampler = SequentialSampler(prediction_data)\n",
"prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)\n",
"\n",
"print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))\n"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Predicting labels for 15 test sentences...\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2393: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
" warnings.warn(\n"
]
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "16lctEOyNFik"
},
"source": [
"#### Evaluate on Test Set\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "rhR99IISNMg9"
},
"source": [
"\n",
"With the test set prepared, we can apply our fine-tuned model to generate predictions on the test set."
]
},
{
"cell_type": "code",
"metadata": {
"id": "Hba10sXR7Xi6",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "32cfa74f-f116-43b7-bb83-71da0fbf537f"
},
"source": [
"# Prediction on test set\n",
"print(\"Running Test set...\")\n",
"\n",
"t0 = time.time()\n",
"\n",
"# Put the model in evaluation mode--the dropout layers behave differently\n",
"# during evaluation.\n",
"model.eval()\n",
"\n",
"# Tracking variables\n",
"total_eval_accuracy = 0\n",
"total_eval_loss = 0\n",
"nb_eval_steps = 0\n",
"\n",
"\n",
"# Put model in evaluation mode\n",
"model.eval()\n",
"\n",
"# Tracking variables\n",
"predictions , true_labels = [], []\n",
"\n",
"# Predict\n",
"for batch in prediction_dataloader:\n",
" # Add batch to GPU\n",
" batch = tuple(t.to(device) for t in batch)\n",
"\n",
" # Unpack the inputs from our dataloader\n",
" b_input_ids, b_input_mask, b_labels = batch\n",
"\n",
" # Telling the model not to compute or store gradients, saving memory and\n",
" # speeding up prediction\n",
" with torch.no_grad():\n",
" # Forward pass, calculate logit predictions.\n",
" # result = model(b_input_ids,\n",
" # token_type_ids=None,\n",
" # attention_mask=b_input_mask,\n",
" # labels=b_labels,\n",
" # return_dict=True)\n",
"\n",
" result = model(b_input_ids,\n",
" token_type_ids=None,\n",
" attention_mask=b_input_mask,\n",
" return_dict=True)\n",
"\n",
" # Get the loss and \"logits\" output by the model. The \"logits\" are the\n",
" # output values prior to applying an activation function like the\n",
" # softmax.\n",
" # loss = result.loss\n",
" logits = result.logits\n",
"\n",
" # Accumulate the validation loss.\n",
" # total_eval_loss += loss.item()\n",
"\n",
" # Move logits and labels to CPU\n",
" logits = logits.detach().cpu().numpy()\n",
" label_ids = b_labels.to('cpu').numpy()\n",
"\n",
" # Store predictions and true labels\n",
" predictions.append(logits)\n",
" true_labels.append(label_ids)\n",
"\n",
" # Calculate the accuracy for this batch of test sentences, and\n",
" # accumulate it over all batches.\n",
" total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
"\n",
"\n",
"print(' DONE.')\n",
"# Report the final accuracy for this run.\n",
"avg_val_accuracy = total_eval_accuracy / len(prediction_dataloader) * 100.0\n",
"print(\" Accuracy: {0:.2f}\".format(avg_val_accuracy))\n"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Running Test set...\n",
" DONE.\n",
" Accuracy: 73.33\n"
]
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "cRaZQ4XC7kLs",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "00271fac-c8bd-4c5d-a185-a51a738d5d0a"
},
"source": [
"from sklearn.metrics import matthews_corrcoef\n",
"\n",
"matthews_set = []\n",
"\n",
"# Evaluate each test batch using Matthew's correlation coefficient\n",
"print('Calculating Matthews Corr. Coef. for each batch...')\n",
"final_labels = []\n",
"# For each input batch...\n",
"for i in range(len(true_labels)):\n",
"\n",
" # The predictions for this batch are a 2-column ndarray (one column for \"0\"\n",
" # and one column for \"1\"). Pick the label with the highest value and turn this\n",
" # in to a list of 0s and 1s.\n",
" pred_labels_i = np.argmax(predictions[i], axis=1).flatten()\n",
" final_labels.append(pred_labels_i)\n",
" # Calculate and store the coef for this batch.\n",
" matthews = matthews_corrcoef(true_labels[i], pred_labels_i)\n",
" matthews_set.append(matthews)\n",
"\n",
"print(final_labels)"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Calculating Matthews Corr. Coef. for each batch...\n",
"[array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])]\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"#### Calculate Accuracy & F1 Score"
],
"metadata": {
"id": "ORnn8VPz9uLC"
}
},
{
"cell_type": "code",
"source": [
"import numpy as np\n",
"from sklearn.metrics import f1_score\n",
"\n",
"count = 0\n",
"# Calculate accuracy\n",
"for i in range(len(true_labels[0])):\n",
" if true_labels[0][i] == final_labels[0][i]:\n",
" count = count + 1\n",
"\n",
"\n",
"print(\" Accuracy: {0:.2f}\".format(count/len(true_labels[0]) * 100.0))\n",
"\n",
"\n",
"def f1(actual, predicted, label):\n",
"\n",
" \"\"\" A helper function to calculate f1-score for the given `label` \"\"\"\n",
" # F1 = 2 * (precision * recall) / (precision + recall)\n",
" tp = np.sum((actual==label) & (predicted==label))\n",
" fp = np.sum((actual!=label) & (predicted==label))\n",
" fn = np.sum((predicted!=label) & (actual==label))\n",
"\n",
" precision = tp/(tp+fp)\n",
" recall = tp/(tp+fn)\n",
" f1 = 2 * (precision * recall) / (precision + recall)\n",
" return f1\n",
"\n",
"def f1_macro(actual, predicted):\n",
" # `macro` f1- unweighted mean of f1 per label\n",
" return np.mean([f1(actual, predicted, label)\n",
" for label in np.unique(actual)])\n",
"\n",
"\n",
"print(\" F1 Score: {0:.2f}\".format(f1_macro(true_labels[0], final_labels[0])*100.0))\n",
"\n",
"#define array of actual classes\n",
"# actual = np.repeat(true_labels, repeats=[160, 240])\n",
"\n",
"#define array of predicted classes\n",
"# pred = np.repeat(final_labels, repeats=[120, 40, 70, 170])\n",
"\n",
"#calculate F1 score\n",
"# f1_score(actual, pred)"
],
"metadata": {
"id": "_0dxa1JW9roc",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "2cfa6e26-69a9-44be-a39e-f9bd25d06b37"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
" Accuracy: 73.33\n",
" F1 Score: nan\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"<ipython-input-36-5caed28a9aa0>:22: RuntimeWarning: invalid value encountered in long_scalars\n",
" precision = tp/(tp+fp)\n"
]
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "oCYZa1lQ8Jn8",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "d4b4b8ec-3fb9-45a4-d4cd-cd7fc579daab"
},
"source": [
"# Combine the results across all batches.\n",
"flat_predictions = np.concatenate(predictions, axis=0)\n",
"\n",
"# For each sample, pick the label (0 or 1) with the higher score.\n",
"flat_predictions = np.argmax(flat_predictions, axis=1).flatten()\n",
"\n",
"# Combine the correct labels for each batch into a single list.\n",
"flat_true_labels = np.concatenate(true_labels, axis=0)\n",
"\n",
"# Calculate the MCC\n",
"mcc = matthews_corrcoef(flat_true_labels, flat_predictions)\n",
"\n",
"print('Total MCC: %.3f' % mcc)"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Total MCC: 0.000\n"
]
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "xlQG7qgkmf4n"
},
"source": [
"This post demonstrates that with a pre-trained BERT model you can quickly and effectively create a high quality model with minimal effort and training time using the pytorch interface, regardless of the specific NLP task you are interested in."
]
},
{
"cell_type": "markdown",
"source": [
"#### Saving New model"
],
"metadata": {
"id": "eggfbrc6-WQg"
}
},
{
"cell_type": "code",
"source": [
"import os\n",
"\n",
"# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()\n",
"\n",
"output_dir = './preprocess_contrastive_model-2/'\n",
"\n",
"# Create output directory if needed\n",
"if not os.path.exists(output_dir):\n",
" os.makedirs(output_dir)\n",
"\n",
"print(\"Saving model to %s\" % output_dir)\n",
"\n",
"# Save a trained model, configuration and tokenizer using `save_pretrained()`.\n",
"# They can then be reloaded using `from_pretrained()`\n",
"model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training\n",
"model_to_save.save_pretrained(output_dir)\n",
"tokenizer.save_pretrained(output_dir)\n",
"\n",
"# Good practice: save your training arguments together with the trained model\n",
"# torch.save(args, os.path.join(output_dir, 'training_args.bin'))\n"
],
"metadata": {
"id": "wE8Sc7N1-ZP5",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "a475a6ce-f694-4571-9671-8cf9660bb2d9"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Saving model to ./contrastive_model/\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"('./contrastive_model/tokenizer_config.json',\n",
" './contrastive_model/special_tokens_map.json',\n",
" './contrastive_model/vocab.txt',\n",
" './contrastive_model/added_tokens.json')"
]
},
"metadata": {},
"execution_count": 38
}
]
},
{
"cell_type": "code",
"source": [
"# Mount Google Drive to this Notebook instance.\n",
"from google.colab import drive\n",
"\n",
"drive.mount('/content/drive')"
],
"metadata": {
"id": "apZeu_nr-oys",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "26841f4b-924c-4442-9347-fa84fceaeaa3"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Mounted at /content/drive\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# Copy the model files to a directory in your Google Drive.\n",
"!cp -r ./preprocess_contrastive_model-2/ ./drive/MyDrive"
],
"metadata": {
"id": "lF10V-Dw-p9s"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "NIWouvDrGVAi"
},
"source": [
"## Cros-Entropy Loss Fine tuning\n",
"\n"
]
},
{
"cell_type": "code",
"source": [
"# Load model\n",
"# Mount Google Drive to this Notebook instance.\n",
"from google.colab import drive\n",
"\n",
"drive.mount('/content/drive')\n",
"\n",
"!cp -r ./drive/MyDrive/preprocess_contrastive_model-2/ ."
],
"metadata": {
"id": "9kCmjivJIvay",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "3a988a43-65dd-4e07-d497-f48a8feaf816"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"#### Model Initialize and Tokenize"
],
"metadata": {
"id": "hD8eBR1axG9X"
}
},
{
"cell_type": "code",
"source": [
"output_dir = './preprocess_contrastive_model-2/'\n",
"# Load a trained model and vocabulary that you have fine-tuned\n",
"from transformers import BertTokenizer\n",
"from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
"\n",
"# Load the BERT tokenizer.\n",
"print('Loading BERT tokenizer...')\n",
"# tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
"tokenizer = BertTokenizer.from_pretrained(output_dir, num_labels = 2)\n",
"\n",
"# Load BertForSequenceClassification, the pretrained BERT model with a single\n",
"# linear classification layer on top.\n",
"model = BertForSequenceClassification.from_pretrained(output_dir)\n",
"\n",
"\n",
"# model = model_class.from_pretrained(output_dir)\n",
"# tokenizer = tokenizer_class.from_pretrained(output_dir)\n",
"\n",
"# Copy the model to the GPU.\n",
"model.to(device)\n",
"\n",
"# Get all of the model's parameters as a list of tuples.\n",
"params = list(model.named_parameters())\n",
"\n",
"print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
"\n",
"print('==== Embedding Layer ====\\n')\n",
"\n",
"for p in params[0:5]:\n",
" print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
"\n",
"print('\\n==== First Transformer ====\\n')\n",
"\n",
"for p in params[5:21]:\n",
" print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
"\n",
"print('\\n==== Output Layer ====\\n')\n",
"\n",
"for p in params[-4:]:\n",
" print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
],
"metadata": {
"id": "d6_nrom8I7AE",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "512a7e36-a588-409a-832d-3a76b87a7318"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Loading BERT tokenizer...\n",
"The BERT model has 201 different named parameters.\n",
"\n",
"==== Embedding Layer ====\n",
"\n",
"bert.embeddings.word_embeddings.weight (30522, 768)\n",
"bert.embeddings.position_embeddings.weight (512, 768)\n",
"bert.embeddings.token_type_embeddings.weight (2, 768)\n",
"bert.embeddings.LayerNorm.weight (768,)\n",
"bert.embeddings.LayerNorm.bias (768,)\n",
"\n",
"==== First Transformer ====\n",
"\n",
"bert.encoder.layer.0.attention.self.query.weight (768, 768)\n",
"bert.encoder.layer.0.attention.self.query.bias (768,)\n",
"bert.encoder.layer.0.attention.self.key.weight (768, 768)\n",
"bert.encoder.layer.0.attention.self.key.bias (768,)\n",
"bert.encoder.layer.0.attention.self.value.weight (768, 768)\n",
"bert.encoder.layer.0.attention.self.value.bias (768,)\n",
"bert.encoder.layer.0.attention.output.dense.weight (768, 768)\n",
"bert.encoder.layer.0.attention.output.dense.bias (768,)\n",
"bert.encoder.layer.0.attention.output.LayerNorm.weight (768,)\n",
"bert.encoder.layer.0.attention.output.LayerNorm.bias (768,)\n",
"bert.encoder.layer.0.intermediate.dense.weight (3072, 768)\n",
"bert.encoder.layer.0.intermediate.dense.bias (3072,)\n",
"bert.encoder.layer.0.output.dense.weight (768, 3072)\n",
"bert.encoder.layer.0.output.dense.bias (768,)\n",
"bert.encoder.layer.0.output.LayerNorm.weight (768,)\n",
"bert.encoder.layer.0.output.LayerNorm.bias (768,)\n",
"\n",
"==== Output Layer ====\n",
"\n",
"bert.pooler.dense.weight (768, 768)\n",
"bert.pooler.dense.bias (768,)\n",
"classifier.weight (2, 768)\n",
"classifier.bias (2,)\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# This code is taken from:\n",
"# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L102\n",
"\n",
"# Don't apply weight decay to any parameters whose names include these tokens.\n",
"# (Here, the BERT doesn't have `gamma` or `beta` parameters, only `bias` terms)\n",
"# no_decay = ['bias', 'LayerNorm.weight']\n",
"\n",
"# # Separate the `weight` parameters from the `bias` parameters.\n",
"# # - For the `weight` parameters, this specifies a 'weight_decay_rate' of 0.01.\n",
"# # - For the `bias` parameters, the 'weight_decay_rate' is 0.0.\n",
"# optimizer_grouped_parameters = [\n",
"# # Filter for all parameters which *don't* include 'bias', 'gamma', 'beta'.\n",
"# {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
"# 'weight_decay_rate': 0.1},\n",
"\n",
"# # Filter for parameters which *do* include those.\n",
"# {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n",
"# 'weight_decay_rate': 0.0}\n",
"# ]\n",
"\n",
"# Note - `optimizer_grouped_parameters` only includes the parameter values, not\n",
"# the names.\n",
"\n",
"# Note: AdamW is a class from the huggingface library (as opposed to pytorch)\n",
"# I believe the 'W' stands for 'Weight Decay fix\"\n",
"from transformers import get_linear_schedule_with_warmup\n",
"\n",
"optimizer = AdamW(model.parameters(),\n",
" lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
" eps = 1e-8 # args.adam_epsilon - default is 1e-8.\n",
" )\n"
],
"metadata": {
"id": "X-M2UrjUf3_9"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
"input_ids = []\n",
"attention_masks = []\n",
"i = 0\n",
"# For every sentence...\n",
"for sent in sentences:\n",
" # `encode_plus` will:\n",
" # (1) Tokenize the sentence.\n",
" # (2) Prepend the `[CLS]` token to the start.\n",
" # (3) Append the `[SEP]` token to the end.\n",
" # (4) Map tokens to their IDs.\n",
" # (5) Pad or truncate the sentence to `max_length`\n",
" # (6) Create attention masks for [PAD] tokens.\n",
" encoded_dict = tokenizer.encode_plus(\n",
" str(sent), # Sentence to encode.\n",
" add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
" max_length = 512, # Pad & truncate all sentences.\n",
" truncation=True,\n",
" pad_to_max_length = True,\n",
" return_attention_mask = True, # Construct attn. masks.\n",
" return_tensors = 'pt', # Return pytorch tensors.\n",
" )\n",
"\n",
" # Add the encoded sentence to the list.\n",
" input_ids.append(encoded_dict['input_ids'])\n",
"\n",
" # And its attention mask (simply differentiates padding from non-padding).\n",
" attention_masks.append(encoded_dict['attention_mask'])\n",
"\n",
"\n",
"# Convert the lists into tensors.\n",
"input_ids = torch.cat(input_ids, dim=0)\n",
"attention_masks = torch.cat(attention_masks, dim=0)\n",
"labels = torch.tensor(labels)\n",
"\n",
"# Print sentence 0, now as a list of IDs.\n",
"print('Original: ', sentences[0])\n",
"print('Token IDs:', input_ids[0])"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "fQvnQxusf2TJ",
"outputId": "169a4d41-a9cc-4103-cc74-3c45184e93c4"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2393: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
" warnings.warn(\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Original: this and below assignments also should be removed\n",
"Token IDs: tensor([ 101, 2023, 1998, 2917, 14799, 2036, 2323, 2022, 3718, 102,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
" 0, 0])\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"from torch.utils.data import TensorDataset, random_split\n",
"\n",
"# Combine the training inputs into a TensorDataset.\n",
"dataset = TensorDataset(input_ids, attention_masks, labels)\n",
"\n",
"# Create a 90-10 train-validation split.\n",
"\n",
"# Calculate the number of samples to include in each set.\n",
"train_size = int(0.98 * len(dataset))\n",
"val_size = len(dataset) - train_size\n",
"\n",
"# Divide the dataset by randomly selecting samples.\n",
"train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n",
"\n",
"print('{:>5,} training samples'.format(train_size))\n",
"print('{:>5,} validation samples'.format(val_size))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "GgXH77OkgMLi",
"outputId": "4a6735aa-6ef9-4bc3-b643-ee540f9106de"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"19,257 training samples\n",
" 394 validation samples\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
"\n",
"# The DataLoader needs to know our batch size for training, so we specify it\n",
"# here. For fine-tuning BERT on a specific task, the authors recommend a batch\n",
"# size of 16 or 32.\n",
"\n",
"# Create the DataLoaders for our training and validation sets.\n",
"# We'll take training samples in random order.\n",
"train_dataloader = DataLoader(\n",
" train_dataset, # The training samples.\n",
" sampler = RandomSampler(train_dataset), # Select batches randomly\n",
" batch_size = BATCH_SIZE # Trains with this batch size.\n",
" )\n",
"\n",
"# For validation the order doesn't matter, so we'll just read them sequentially.\n",
"validation_dataloader = DataLoader(\n",
" val_dataset, # The validation samples.\n",
" sampler = SequentialSampler(val_dataset), # Pull out batches sequentially.\n",
" batch_size = BATCH_SIZE # Evaluate with this batch size.\n",
" )"
],
"metadata": {
"id": "yXzRGzUogaDb"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "QxSMw0FrptiL"
},
"source": [
"# Number of training epochs. The BERT authors recommend between 2 and 4.\n",
"# We chose to run for 4, but we'll see later that this may be over-fitting the\n",
"# training data.\n",
"epochs = 4\n",
"\n",
"# Total number of training steps is [number of batches] x [number of epochs].\n",
"# (Note that this is not the same as the number of training samples).\n",
"total_steps = len(train_dataloader) * epochs\n",
"\n",
"# Create the learning rate scheduler.\n",
"scheduler = get_linear_schedule_with_warmup(optimizer,\n",
" num_warmup_steps = 0, # Default value in run_glue.py\n",
" num_training_steps = total_steps)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"#### Cross-entropy training loop"
],
"metadata": {
"id": "O5Eeo6YKLMmX"
}
},
{
"cell_type": "code",
"source": [
"import numpy as np\n",
"import time\n",
"import datetime\n",
"\n",
"# Function to calculate the accuracy of our predictions vs labels\n",
"def flat_accuracy(preds, labels):\n",
" pred_flat = np.argmax(preds, axis=1).flatten()\n",
" labels_flat = labels.flatten()\n",
" return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
"\n",
"\n",
"\n",
"def format_time(elapsed):\n",
" '''\n",
" Takes a time in seconds and returns a string hh:mm:ss\n",
" '''\n",
" # Round to the nearest second.\n",
" elapsed_rounded = int(round((elapsed)))\n",
"\n",
" # Format as hh:mm:ss\n",
" return str(datetime.timedelta(seconds=elapsed_rounded))\n"
],
"metadata": {
"id": "owrPlWvnLQxP"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import random\n",
"import numpy as np\n",
"\n",
"# This training code is based on the `run_glue.py` script here:\n",
"# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
"\n",
"# Set the seed value all over the place to make this reproducible.\n",
"seed_val = 42\n",
"\n",
"random.seed(seed_val)\n",
"np.random.seed(seed_val)\n",
"torch.manual_seed(seed_val)\n",
"torch.cuda.manual_seed_all(seed_val)\n",
"\n",
"# We'll store a number of quantities such as training and validation loss,\n",
"# validation accuracy, and timings.\n",
"training_stats = []\n",
"\n",
"# Measure the total training time for the whole run.\n",
"total_t0 = time.time()\n",
"\n",
"# For each epoch...\n",
"for epoch_i in range(0, epochs):\n",
"\n",
" # ========================================\n",
" # Training\n",
" # ========================================\n",
"\n",
" # Perform one full pass over the training set.\n",
"\n",
" print(\"\")\n",
" print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
" print('Training...')\n",
"\n",
" # Measure how long the training epoch takes.\n",
" t0 = time.time()\n",
"\n",
" # Reset the total loss for this epoch.\n",
" total_train_loss = 0\n",
"\n",
" # Put the model into training mode. Don't be mislead--the call to\n",
" # `train` just changes the *mode*, it doesn't *perform* the training.\n",
" # `dropout` and `batchnorm` layers behave differently during training\n",
" # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
" model.train()\n",
"\n",
" # For each batch of training data...\n",
" for step, batch in enumerate(train_dataloader):\n",
"\n",
" # Progress update every 40 batches.\n",
" if step % 40 == 0 and not step == 0:\n",
" # Calculate elapsed time in minutes.\n",
" elapsed = format_time(time.time() - t0)\n",
"\n",
" # Report progress.\n",
" print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
"\n",
" # Unpack this training batch from our dataloader.\n",
" #\n",
" # As we unpack the batch, we'll also copy each tensor to the GPU using the\n",
" # `to` method.\n",
" #\n",
" # `batch` contains three pytorch tensors:\n",
" # [0]: input ids\n",
" # [1]: attention masks\n",
" # [2]: labels\n",
" b_input_ids = batch[0].to(device)\n",
" b_input_mask = batch[1].to(device)\n",
" b_labels = batch[2].to(device)\n",
"\n",
" # Always clear any previously calculated gradients before performing a\n",
" # backward pass. PyTorch doesn't do this automatically because\n",
" # accumulating the gradients is \"convenient while training RNNs\".\n",
" # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
" model.zero_grad()\n",
"\n",
" # Perform a forward pass (evaluate the model on this training batch).\n",
" # In PyTorch, calling `model` will in turn call the model's `forward`\n",
" # function and pass down the arguments. The `forward` function is\n",
" # documented here:\n",
" # https://huggingface.co/transformers/model_doc/bert.html#bertforsequenceclassification\n",
" # The results are returned in a results object, documented here:\n",
" # https://huggingface.co/transformers/main_classes/output.html#transformers.modeling_outputs.SequenceClassifierOutput\n",
" # Specifically, we'll get the loss (because we provided labels) and the\n",
" # \"logits\"--the model outputs prior to activation.\n",
" result = model(b_input_ids,\n",
" token_type_ids=None,\n",
" attention_mask=b_input_mask,\n",
" labels=b_labels,\n",
" return_dict=True)\n",
"\n",
" loss = result.loss\n",
" logits = result.logits\n",
"\n",
" # Accumulate the training loss over all of the batches so that we can\n",
" # calculate the average loss at the end. `loss` is a Tensor containing a\n",
" # single value; the `.item()` function just returns the Python value\n",
" # from the tensor.\n",
" total_train_loss += loss.item()\n",
"\n",
" # Perform a backward pass to calculate the gradients.\n",
" loss.backward()\n",
"\n",
" # Clip the norm of the gradients to 1.0.\n",
" # This is to help prevent the \"exploding gradients\" problem.\n",
" torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
"\n",
" # Update parameters and take a step using the computed gradient.\n",
" # The optimizer dictates the \"update rule\"--how the parameters are\n",
" # modified based on their gradients, the learning rate, etc.\n",
" optimizer.step()\n",
"\n",
" # Update the learning rate.\n",
" scheduler.step()\n",
"\n",
" # Calculate the average loss over all of the batches.\n",
" avg_train_loss = total_train_loss / len(train_dataloader)\n",
"\n",
" # Measure how long this epoch took.\n",
" training_time = format_time(time.time() - t0)\n",
"\n",
" print(\"\")\n",
" print(\" Average training loss: {0:.2f}\".format(avg_train_loss))\n",
" print(\" Training epcoh took: {:}\".format(training_time))\n",
"\n",
" # ========================================\n",
" # Validation\n",
" # ========================================\n",
" # After the completion of each training epoch, measure our performance on\n",
" # our validation set.\n",
"\n",
" print(\"\")\n",
" print(\"Running Validation...\")\n",
"\n",
" t0 = time.time()\n",
"\n",
" # Put the model in evaluation mode--the dropout layers behave differently\n",
" # during evaluation.\n",
" model.eval()\n",
"\n",
" # Tracking variables\n",
" total_eval_accuracy = 0\n",
" total_eval_loss = 0\n",
" nb_eval_steps = 0\n",
"\n",
" # Evaluate data for one epoch\n",
" for batch in validation_dataloader:\n",
"\n",
" # Unpack this training batch from our dataloader.\n",
" #\n",
" # As we unpack the batch, we'll also copy each tensor to the GPU using\n",
" # the `to` method.\n",
" #\n",
" # `batch` contains three pytorch tensors:\n",
" # [0]: input ids\n",
" # [1]: attention masks\n",
" # [2]: labels\n",
" b_input_ids = batch[0].to(device)\n",
" b_input_mask = batch[1].to(device)\n",
" b_labels = batch[2].to(device)\n",
"\n",
" # Tell pytorch not to bother with constructing the compute graph during\n",
" # the forward pass, since this is only needed for backprop (training).\n",
" with torch.no_grad():\n",
"\n",
" # Forward pass, calculate logit predictions.\n",
" # token_type_ids is the same as the \"segment ids\", which\n",
" # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
" result = model(b_input_ids,\n",
" token_type_ids=None,\n",
" attention_mask=b_input_mask,\n",
" labels=b_labels,\n",
" return_dict=True)\n",
"\n",
" # Get the loss and \"logits\" output by the model. The \"logits\" are the\n",
" # output values prior to applying an activation function like the\n",
" # softmax.\n",
" loss = result.loss\n",
" logits = result.logits\n",
"\n",
" # Accumulate the validation loss.\n",
" total_eval_loss += loss.item()\n",
"\n",
" # Move logits and labels to CPU\n",
" logits = logits.detach().cpu().numpy()\n",
" label_ids = b_labels.to('cpu').numpy()\n",
"\n",
" # Calculate the accuracy for this batch of test sentences, and\n",
" # accumulate it over all batches.\n",
" total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
"\n",
"\n",
" # Report the final accuracy for this validation run.\n",
" avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
" print(\" Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
"\n",
" # Calculate the average loss over all of the batches.\n",
" avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
"\n",
" # Measure how long the validation run took.\n",
" validation_time = format_time(time.time() - t0)\n",
"\n",
" print(\" Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
" print(\" Validation took: {:}\".format(validation_time))\n",
"\n",
" # Record all statistics from this epoch.\n",
" training_stats.append(\n",
" {\n",
" 'epoch': epoch_i + 1,\n",
" 'Training Loss': avg_train_loss,\n",
" 'Valid. Loss': avg_val_loss,\n",
" 'Valid. Accur.': avg_val_accuracy,\n",
" 'Training Time': training_time,\n",
" 'Validation Time': validation_time\n",
" }\n",
" )\n",
"\n",
"print(\"\")\n",
"print(\"Training complete!\")\n",
"\n",
"print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
],
"metadata": {
"id": "coCQazMoLZXx",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "f5f181e1-40fa-46a0-c883-8ef21900ac01"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"======== Epoch 1 / 4 ========\n",
"Training...\n",
" Batch 40 of 1,204. Elapsed: 0:00:54.\n",
" Batch 80 of 1,204. Elapsed: 0:01:48.\n",
" Batch 120 of 1,204. Elapsed: 0:02:43.\n",
" Batch 160 of 1,204. Elapsed: 0:03:37.\n",
" Batch 200 of 1,204. Elapsed: 0:04:31.\n",
" Batch 240 of 1,204. Elapsed: 0:05:26.\n",
" Batch 280 of 1,204. Elapsed: 0:06:20.\n",
" Batch 320 of 1,204. Elapsed: 0:07:14.\n",
" Batch 360 of 1,204. Elapsed: 0:08:08.\n",
" Batch 400 of 1,204. Elapsed: 0:09:03.\n",
" Batch 440 of 1,204. Elapsed: 0:09:57.\n",
" Batch 480 of 1,204. Elapsed: 0:10:51.\n",
" Batch 520 of 1,204. Elapsed: 0:11:46.\n",
" Batch 560 of 1,204. Elapsed: 0:12:40.\n",
" Batch 600 of 1,204. Elapsed: 0:13:34.\n",
" Batch 640 of 1,204. Elapsed: 0:14:28.\n",
" Batch 680 of 1,204. Elapsed: 0:15:23.\n",
" Batch 720 of 1,204. Elapsed: 0:16:17.\n",
" Batch 760 of 1,204. Elapsed: 0:17:11.\n",
" Batch 800 of 1,204. Elapsed: 0:18:06.\n",
" Batch 840 of 1,204. Elapsed: 0:19:00.\n",
" Batch 880 of 1,204. Elapsed: 0:19:54.\n",
" Batch 920 of 1,204. Elapsed: 0:20:49.\n",
" Batch 960 of 1,204. Elapsed: 0:21:43.\n",
" Batch 1,000 of 1,204. Elapsed: 0:22:37.\n",
" Batch 1,040 of 1,204. Elapsed: 0:23:32.\n",
" Batch 1,080 of 1,204. Elapsed: 0:24:26.\n",
" Batch 1,120 of 1,204. Elapsed: 0:25:20.\n",
" Batch 1,160 of 1,204. Elapsed: 0:26:15.\n",
" Batch 1,200 of 1,204. Elapsed: 0:27:09.\n",
"\n",
" Average training loss: 0.20\n",
" Training epcoh took: 0:27:14\n",
"\n",
"Running Validation...\n",
" Accuracy: 0.93\n",
" Validation Loss: 0.15\n",
" Validation took: 0:00:12\n",
"\n",
"======== Epoch 2 / 4 ========\n",
"Training...\n",
" Batch 40 of 1,204. Elapsed: 0:00:54.\n",
" Batch 80 of 1,204. Elapsed: 0:01:49.\n",
" Batch 120 of 1,204. Elapsed: 0:02:43.\n",
" Batch 160 of 1,204. Elapsed: 0:03:37.\n",
" Batch 200 of 1,204. Elapsed: 0:04:32.\n",
" Batch 240 of 1,204. Elapsed: 0:05:26.\n",
" Batch 280 of 1,204. Elapsed: 0:06:20.\n",
" Batch 320 of 1,204. Elapsed: 0:07:15.\n",
" Batch 360 of 1,204. Elapsed: 0:08:09.\n",
" Batch 400 of 1,204. Elapsed: 0:09:03.\n",
" Batch 440 of 1,204. Elapsed: 0:09:58.\n",
" Batch 480 of 1,204. Elapsed: 0:10:52.\n",
" Batch 520 of 1,204. Elapsed: 0:11:46.\n",
" Batch 560 of 1,204. Elapsed: 0:12:41.\n",
" Batch 600 of 1,204. Elapsed: 0:13:35.\n",
" Batch 640 of 1,204. Elapsed: 0:14:29.\n",
" Batch 680 of 1,204. Elapsed: 0:15:24.\n",
" Batch 720 of 1,204. Elapsed: 0:16:18.\n",
" Batch 760 of 1,204. Elapsed: 0:17:12.\n",
" Batch 800 of 1,204. Elapsed: 0:18:07.\n",
" Batch 840 of 1,204. Elapsed: 0:19:01.\n",
" Batch 880 of 1,204. Elapsed: 0:19:55.\n",
" Batch 920 of 1,204. Elapsed: 0:20:50.\n",
" Batch 960 of 1,204. Elapsed: 0:21:44.\n",
" Batch 1,000 of 1,204. Elapsed: 0:22:38.\n",
" Batch 1,040 of 1,204. Elapsed: 0:23:33.\n",
" Batch 1,080 of 1,204. Elapsed: 0:24:27.\n",
" Batch 1,120 of 1,204. Elapsed: 0:25:21.\n",
" Batch 1,160 of 1,204. Elapsed: 0:26:15.\n",
" Batch 1,200 of 1,204. Elapsed: 0:27:10.\n",
"\n",
" Average training loss: 0.12\n",
" Training epcoh took: 0:27:15\n",
"\n",
"Running Validation...\n",
" Accuracy: 0.95\n",
" Validation Loss: 0.17\n",
" Validation took: 0:00:12\n",
"\n",
"======== Epoch 3 / 4 ========\n",
"Training...\n",
" Batch 40 of 1,204. Elapsed: 0:00:54.\n",
" Batch 80 of 1,204. Elapsed: 0:01:49.\n",
" Batch 120 of 1,204. Elapsed: 0:02:43.\n",
" Batch 160 of 1,204. Elapsed: 0:03:37.\n",
" Batch 200 of 1,204. Elapsed: 0:04:32.\n",
" Batch 240 of 1,204. Elapsed: 0:05:26.\n",
" Batch 280 of 1,204. Elapsed: 0:06:20.\n",
" Batch 320 of 1,204. Elapsed: 0:07:15.\n",
" Batch 360 of 1,204. Elapsed: 0:08:09.\n",
" Batch 400 of 1,204. Elapsed: 0:09:03.\n",
" Batch 440 of 1,204. Elapsed: 0:09:57.\n",
" Batch 480 of 1,204. Elapsed: 0:10:52.\n",
" Batch 520 of 1,204. Elapsed: 0:11:46.\n",
" Batch 560 of 1,204. Elapsed: 0:12:40.\n",
" Batch 600 of 1,204. Elapsed: 0:13:35.\n",
" Batch 640 of 1,204. Elapsed: 0:14:29.\n",
" Batch 680 of 1,204. Elapsed: 0:15:23.\n",
" Batch 720 of 1,204. Elapsed: 0:16:18.\n",
" Batch 760 of 1,204. Elapsed: 0:17:12.\n",
" Batch 800 of 1,204. Elapsed: 0:18:06.\n",
" Batch 840 of 1,204. Elapsed: 0:19:01.\n",
" Batch 880 of 1,204. Elapsed: 0:19:55.\n",
" Batch 920 of 1,204. Elapsed: 0:20:49.\n",
" Batch 960 of 1,204. Elapsed: 0:21:44.\n",
" Batch 1,000 of 1,204. Elapsed: 0:22:38.\n",
" Batch 1,040 of 1,204. Elapsed: 0:23:32.\n",
" Batch 1,080 of 1,204. Elapsed: 0:24:27.\n",
" Batch 1,120 of 1,204. Elapsed: 0:25:21.\n",
" Batch 1,160 of 1,204. Elapsed: 0:26:15.\n",
" Batch 1,200 of 1,204. Elapsed: 0:27:10.\n",
"\n",
" Average training loss: 0.06\n",
" Training epcoh took: 0:27:14\n",
"\n",
"Running Validation...\n",
" Accuracy: 0.96\n",
" Validation Loss: 0.16\n",
" Validation took: 0:00:12\n",
"\n",
"======== Epoch 4 / 4 ========\n",
"Training...\n",
" Batch 40 of 1,204. Elapsed: 0:00:54.\n",
" Batch 80 of 1,204. Elapsed: 0:01:49.\n",
" Batch 120 of 1,204. Elapsed: 0:02:43.\n",
" Batch 160 of 1,204. Elapsed: 0:03:37.\n",
" Batch 200 of 1,204. Elapsed: 0:04:32.\n",
" Batch 240 of 1,204. Elapsed: 0:05:26.\n",
" Batch 280 of 1,204. Elapsed: 0:06:20.\n",
" Batch 320 of 1,204. Elapsed: 0:07:15.\n",
" Batch 360 of 1,204. Elapsed: 0:08:09.\n",
" Batch 400 of 1,204. Elapsed: 0:09:03.\n",
" Batch 440 of 1,204. Elapsed: 0:09:58.\n",
" Batch 480 of 1,204. Elapsed: 0:10:52.\n",
" Batch 520 of 1,204. Elapsed: 0:11:46.\n",
" Batch 560 of 1,204. Elapsed: 0:12:40.\n",
" Batch 600 of 1,204. Elapsed: 0:13:35.\n",
" Batch 640 of 1,204. Elapsed: 0:14:29.\n",
" Batch 680 of 1,204. Elapsed: 0:15:23.\n",
" Batch 720 of 1,204. Elapsed: 0:16:18.\n",
" Batch 760 of 1,204. Elapsed: 0:17:12.\n",
" Batch 800 of 1,204. Elapsed: 0:18:06.\n",
" Batch 840 of 1,204. Elapsed: 0:19:01.\n",
" Batch 880 of 1,204. Elapsed: 0:19:55.\n",
" Batch 920 of 1,204. Elapsed: 0:20:49.\n",
" Batch 960 of 1,204. Elapsed: 0:21:44.\n",
" Batch 1,000 of 1,204. Elapsed: 0:22:38.\n",
" Batch 1,040 of 1,204. Elapsed: 0:23:32.\n",
" Batch 1,080 of 1,204. Elapsed: 0:24:27.\n",
" Batch 1,120 of 1,204. Elapsed: 0:25:21.\n",
" Batch 1,160 of 1,204. Elapsed: 0:26:15.\n",
" Batch 1,200 of 1,204. Elapsed: 0:27:09.\n",
"\n",
" Average training loss: 0.03\n",
" Training epcoh took: 0:27:14\n",
"\n",
"Running Validation...\n",
" Accuracy: 0.96\n",
" Validation Loss: 0.19\n",
" Validation took: 0:00:12\n",
"\n",
"Training complete!\n",
"Total training took 1:49:46 (h:mm:ss)\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"\n",
"# Display floats with two decimal places.\n",
"# pd.set_option('precision', 2)\n",
"pd.options.display.max_rows = 5\n",
"# pd.options.precision = 2\n",
"print(training_stats)\n",
"# Create a DataFrame from our training statistics.\n",
"df_stats = pd.DataFrame(data=training_stats)\n",
"\n",
"# Use the 'epoch' as the row index.\n",
"df_stats = df_stats.set_index('epoch')\n",
"\n",
"# A hack to force the column headers to wrap.\n",
"#df = df.style.set_table_styles([dict(selector=\"th\",props=[('max-width', '70px')])])\n",
"\n",
"# Display the table.\n",
"df_stats"
],
"metadata": {
"id": "Hyz17eQ0Leez",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 244
},
"outputId": "5c7b9dce-7c27-4d61-c2de-57dd15092410"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[{'epoch': 1, 'Training Loss': 0.20411855078583113, 'Valid. Loss': 0.14585627406835555, 'Valid. Accur.': 0.932, 'Training Time': '0:27:14', 'Validation Time': '0:00:12'}, {'epoch': 2, 'Training Loss': 0.1152435676600344, 'Valid. Loss': 0.1749358731787652, 'Valid. Accur.': 0.9484999999999999, 'Training Time': '0:27:15', 'Validation Time': '0:00:12'}, {'epoch': 3, 'Training Loss': 0.06414671170705484, 'Valid. Loss': 0.15540768602164462, 'Valid. Accur.': 0.9570000000000001, 'Training Time': '0:27:14', 'Validation Time': '0:00:12'}, {'epoch': 4, 'Training Loss': 0.031750682121859414, 'Valid. Loss': 0.1890620315662818, 'Valid. Accur.': 0.9620000000000001, 'Training Time': '0:27:14', 'Validation Time': '0:00:12'}]\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
" Training Loss Valid. Loss Valid. Accur. Training Time Validation Time\n",
"epoch \n",
"1 0.204119 0.145856 0.9320 0:27:14 0:00:12\n",
"2 0.115244 0.174936 0.9485 0:27:15 0:00:12\n",
"3 0.064147 0.155408 0.9570 0:27:14 0:00:12\n",
"4 0.031751 0.189062 0.9620 0:27:14 0:00:12"
],
"text/html": [
"\n",
"\n",
" <div id=\"df-3eeb6233-d59e-42b6-a258-cca41fb9dfcb\">\n",
" <div class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Training Loss</th>\n",
" <th>Valid. Loss</th>\n",
" <th>Valid. Accur.</th>\n",
" <th>Training Time</th>\n",
" <th>Validation Time</th>\n",
" </tr>\n",
" <tr>\n",
" <th>epoch</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>0.204119</td>\n",
" <td>0.145856</td>\n",
" <td>0.9320</td>\n",
" <td>0:27:14</td>\n",
" <td>0:00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>0.115244</td>\n",
" <td>0.174936</td>\n",
" <td>0.9485</td>\n",
" <td>0:27:15</td>\n",
" <td>0:00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>0.064147</td>\n",
" <td>0.155408</td>\n",
" <td>0.9570</td>\n",
" <td>0:27:14</td>\n",
" <td>0:00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>0.031751</td>\n",
" <td>0.189062</td>\n",
" <td>0.9620</td>\n",
" <td>0:27:14</td>\n",
" <td>0:00:12</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3eeb6233-d59e-42b6-a258-cca41fb9dfcb')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
"\n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
" </svg>\n",
" </button>\n",
"\n",
"\n",
"\n",
" <div id=\"df-0e0addba-6b7a-492f-bc65-42df9cc9f206\">\n",
" <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0e0addba-6b7a-492f-bc65-42df9cc9f206')\"\n",
" title=\"Suggest charts.\"\n",
" style=\"display:none;\">\n",
"\n",
"<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <g>\n",
" <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
" </g>\n",
"</svg>\n",
" </button>\n",
" </div>\n",
"\n",
"<style>\n",
" .colab-df-quickchart {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-quickchart:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-quickchart {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-quickchart:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
"</style>\n",
"\n",
" <script>\n",
" async function quickchart(key) {\n",
" const containerElement = document.querySelector('#' + key);\n",
" const charts = await google.colab.kernel.invokeFunction(\n",
" 'suggestCharts', [key], {});\n",
" }\n",
" </script>\n",
"\n",
" <script>\n",
"\n",
"function displayQuickchartButton(domScope) {\n",
" let quickchartButtonEl =\n",
" domScope.querySelector('#df-0e0addba-6b7a-492f-bc65-42df9cc9f206 button.colab-df-quickchart');\n",
" quickchartButtonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"}\n",
"\n",
" displayQuickchartButton(document);\n",
" </script>\n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" flex-wrap:wrap;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-3eeb6233-d59e-42b6-a258-cca41fb9dfcb button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-3eeb6233-d59e-42b6-a258-cca41fb9dfcb');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
" </div>\n"
]
},
"metadata": {},
"execution_count": 62
}
]
},
{
"cell_type": "code",
"source": [
"%matplotlib inline"
],
"metadata": {
"id": "QKrfLhYvLh5Q"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import seaborn as sns\n",
"\n",
"# Use plot styling from seaborn.\n",
"sns.set(style='darkgrid')\n",
"\n",
"# Increase the plot size and font size.\n",
"sns.set(font_scale=1.5)\n",
"plt.rcParams[\"figure.figsize\"] = (12,6)\n",
"\n",
"# Plot the learning curve.\n",
"plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n",
"plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n",
"\n",
"# Label the plot.\n",
"plt.title(\"Training & Validation Loss\")\n",
"plt.xlabel(\"Epoch\")\n",
"plt.ylabel(\"Loss\")\n",
"plt.legend()\n",
"# plt.xticks([1, 2, 3, 4])\n",
"plt.xticks(list(range(1, epochs)))\n",
"\n",
"plt.show()"
],
"metadata": {
"id": "V1rjGrcQLm7U",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 592
},
"outputId": "6d0e2ed3-ab3f-4b5f-adf7-94b92da7457e"
},
"execution_count": null,
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"<Figure size 1200x600 with 1 Axes>"
],
"image/png": "iVBORw0KGgoAAAANSUhEUgAABCIAAAI/CAYAAACiSXgsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAADhZUlEQVR4nOzdd3iUVdoG8Pudlp7MpPcekpCAVAUbiKAgKGABERVkFTWoq66rrooddb9vdT91xbpUF1dRmoUiTRApIdSEhPSQ3idlUqa93x+TDAxJIAlJJpPcv+viumTeMs9EOMzcc855BFEURRARERERERER9QGJtQsgIiIiIiIiosGDQQQRERERERER9RkGEURERERERETUZxhEEBEREREREVGfYRBBRERERERERH2GQQQRERERERER9RkGEURERERERETUZxhEEBEREREREVGfYRBBRERERERERH2GQQQREVEXHD58GNHR0YiOju7xe2/YsAHR0dGYNGlSj9+besYDDzyA6OhofPzxx106dqX37guTJk1CdHQ0NmzYYJXnJyKiwUNm7QKIiIgudiUf8t99913ceeedPVgNddXJkyexYsUKJCUlQa1Ww83NDX5+frjuuuswbdo0xMTEdOu+xcXFmDRpEoxGI55//nn86U9/6tR1mzZtwgsvvADAFPbExcV16/lt1YYNG1BYWIirr74a11xzjbXL6XEvvvgiNm7ciICAAOzevdva5RARUScwiCAion7H09Oz3ccbGhrQ0NBwyXPs7e17rS4AcHBwQFhYWK/c28XFBWFhYfDx8emV+/eF77//HkuXLoXRaARg+nk1NDTg9OnTOH36NI4dO4a1a9d2695+fn649tpr8fvvv2PDhg2dDiJ++OEHAEBsbGyvhhB+fn4ICwuDSqXqtefojo0bN+LIkSN44oknLhlEBAUFQaFQwMXFpQ+rIyKiwYhBBBER9TsHDhxo9/GPP/4Y//rXvy55Tm8bPnw4tm3b1iv3njJlCqZMmdIr9+4LVVVVePPNN2E0GhEbG4u3334b8fHxAID8/Hzs3r0b2dnZV/Qcd999N37//XdkZmbi5MmTuOqqqy55fn5+PhITEwEAd9111xU99+X8z//8T6/ev7etXr3a2iUQEdEgwSCCiIiIesTRo0fR3NwMAPjf//1fREVFmY8FBQVhwYIFV/wcN998M5RKJdRqNX744YfLBhEbNmyAKIpQKBS4/fbbr/j5iYiI6MoxiCAiogGjdW+JNWvWIDIyEl988QX27t2LkpISNDU14ezZswCAxsZG7Nq1C/v27cPZs2dRWlqK+vp6KJVKDB8+HHPnzsWECRPafY7Dhw/jwQcfBADz/Vpt2LABf/vb38xr1ZOTk/Hll1+a90rw8fHB5MmTkZCQADc3tzb3vvj6C7XOBrn66quxdu1aHDx4ECtXrsSpU6eg0WgQGBiI6dOn45FHHoGdnV2HP6OdO3dizZo1OHPmDAwGA4KCgnD77bdj4cKF+Oyzzyyeo6ukUqn5v3treYlCocDMmTOxevVq/Pzzz3jppZc6XI5jNBqxadMmAKbZJkqlEgCQnp6O7du3IzExEUVFRSgrK4NMJkNwcDAmTJiABQsWwN3dvcu1PfDAA+YlEE8++WSb4waDAevWrcOGDRuQk5MDhUKB6OhozJ8/H1OnTr3kvfPz87F161YcPnwYBQUFKC0thSAI5r03HnroIfj7+1tc0/rnqdW//vUv84yiVrt27UJgYCAA02aVhYWFHe6zYjAYsHHjRmzZsgVnz56FRqOBSqXCyJEjMX/+/A6XfVz4c3niiSewfv16rF+/HllZWRBFEUOGDMF9992HmTNnXvJn0BvKy8uxYsUK7Nu3D4WFhQCAgIAATJgwAYsWLepwCVhNTQ1WrVqFvXv3Ii8vD1qtFm5ubnB3d8fIkSMxbdo0jB8/3uKapqYm/Oc//8GOHTuQnZ2NhoYGuLi4wN3dHcOGDcOkSZNw66239vprJiLqDxhEEBHRgHPu3Dk8++yzqKiogJ2dHWQyy3/utm7dav6AJggCnJ2dIZPJUF5ejl27dmHXrl1YtGiReYPD7vjxxx/xt7/9DTqdDi4uLjAYDCgoKMCqVatw4MABfPvtt3BycurWvb/66iv84x//AGDaV0Kn0yE7Oxsff/wxjhw5gpUrV1qEAq3+/ve/Y8WKFebfu7q6IisrC//4xz/w22+/YfTo0d17sS3Gjx8Pd3d3VFVVYc2aNXjiiSeu6H4dufvuu7F69WrU19dj+/btHX6APXjwIIqKigBYLst47LHHzB867ezs4ODggJqaGqSmpiI1NRUbN27EqlWrEB4e3mM1a7VaPP744/j9998BABKJBHK5HImJiThy5AgeeeSRS17/0ksv4ciRIwAAuVwOJycn1NbWIisrC1lZWdi4cSM+++wzjBkzxnyNvb09PD09UVNTA51OB0dHRzg6Olrct70/J+2pq6tDQkKCuQapVAonJyeUl5dj+/bt2L59+2X/zhgMBixZsgS7du2CTCaDvb09NBoNTpw4gRMnTiAvLw9PPfVUp+rpCUeOHMGSJUtQW1sLAOafTWZmJjIzM/H9999j+fLlFj9TACgpKcG8efPMf7YkEglcXFxQXV2NiooKpKenIycnxyKIqK+vx/z585GWlgbANO64uLigrq4O1dXVyMrKQmJiIoMIIho02L6TiIgGnHfeeQcuLi5YtWoVTpw4gWPHjlns6+Dq6opFixZh3bp1OH78OI4ePYoTJ05g//79ePLJJyGXy7FixQrs2rWrW89fVVWFl156CbNmzcLevXtx9OhRHDt2DK+++irkcjkyMjLw1VdfdeveaWlpeP/997F48WL88ccfSExMxNGjR7FkyRIAphkbGzdubHPdzz//bA4hZsyYgX379iExMRHHjh3DW2+9hVOnTuGbb77pVk2tHB0dzR9EP/nkE2zZsuWK7teRIUOGYPjw4QDOb0TZntZjAQEBFh8Kx44di/feew979uzBqVOncPjwYZw6dQqrVq3C8OHDUVpaiueee65Ha37//ffx+++/QxAEPP3000hMTERiYiIOHDiAefPm4csvv0RqamqH18fExODVV1/F9u3bzTWfPn0a69evxw033IC6ujo888wzaGpqMl9z22234cCBAxg5ciQAYNGiRThw4IDFLz8/v07V//LLL+PIkSOQy+V45ZVXkJSUhMTEROzfv98c8qxYseKSf4bWrVuHI0eO4L333kNSUhKSkpLw22+/4aabbgIAfPrpp8jNze1UPVequLjYHEJERkaax4Ljx4/jP//5D8LCwlBTU4MlS5agtLTU4tqPP/4YRUVFCAgIwKpVq5CcnIwjR47g9OnT2L17N15//fU2S4bWrFmDtLQ0KJVKfPzxxzh16hQSExNx+vRp7Nu3D3//+99x3XXX9clrJyLqDxhEEBHRgCORSLBq1SqMHz8eEonpn7oLO11MnjwZL7zwAkaPHg0HBwfz497e3njiiSfwzDPPAEC3uzs0NjZi+vTpePvtt80f9BwcHDB//nzcf//9AEzBQHfU1tYiISEBzz77rHn5gLOzM5566inccsst7d5bFEV8+OGHAIDrrrsO//jHP8xLJ+zs7DBnzhy8/vrrqKmp6VZNrQoLC80Bi9FoxIsvvnjJoOBK3H333QBM32rn5+e3OV5TU4OdO3cCAO68807znwPANDNk9uzZFksZFAoFxo8fj1WrVsHT0xMpKSk4evRoj9RaWlqKr7/+GgDw+OOP4/HHH4ezszMAwMPDA6+//jpmzJiBurq6Du/x8ssvY/78+QgNDTW/FplMhuHDh+Pzzz9HdHQ0ysrKsH379h6p+UInT54033fp0qV44IEHzH9vvLy88M4775i/yf/www/N+4RcrKamBv/6178we/Zs83IaX19ffPTRR/D29obRaMTWrVt7vP72fPbZZ6itrYWbmxtWrVplMRtozJgxWLVqFZydnaFWq/H5559bXHv8+HEAwLPPPovx48ebZ5VIpVIEBARg3rx5bYKs1msWLVqEW265BQqFAoBprPLx8cGsWbPw1ltv9drrJSLqbxhEEBHRgDNz5kz4+vp2+/qJEycCAE6cOAGDwdCtezz++OPtPn7zzTcDAPLy8tDY2Njl+yoUCixatOiS975474rU1FTk5eUBAB599FEIgtDm2os/mHdVTU0NFixYgIyMDMybNw8ffvghBEHAyy+/3GGg85///AfR0dHdmo4+ffp0ODg4QBTFdmeA/PTTT2huboZEIsHs2bM7fV8nJyeMHTsWAHDs2LEu19We7du3Q6/Xw97evsOWo1eyjEUqleKGG24AACQlJXX7Ph355ZdfAJhCg3vuuafdc/785z8DAKqrqzvsaDNq1CiMGzeuzeMKhQLXX389gLZ/dnuDKIrmGVL33nsvvLy82pzj6+uLe++9F0DbYM/V1RWAaX+JzurONUREAxn3iCAiogFn1KhRlz2noqIC69atw4EDB5Cbm4u6uro2oUNjYyNqamq6vHGhUqlESEhIu8e8vb3N/11bW2sxI6MzoqKiOtxbovXeF89sSElJAWDaW6B1mv7FBEHA2LFjsXnz5i7V0+rtt99Gfn4+RowYgaVLl0IqlcJgMOCvf/0r3n77bTQ0NODRRx+1uKZ1yntsbGyXn8/Z2Rm33norNm3ahE2bNuGJJ56wmPXQOhNj/PjxCAgIaHP9nj17sHnzZpw+fRqVlZXthkIlJSVdrqs9ycnJAID4+HjzTIiLhYWFwcfHp80ygAsdPXoU33//PU6cOIHS0lI0NDS0OedS13dXa/3XXHONxc/4QhEREeb6k5OTMWnSpDbnXKrDSUd/dntDQUEB1Go1ALTZUPJC1113Hb766iuo1Wrk5+cjKCgIgCmoPH78ON5//31kZ2djypQpGDVqVIf/b1uv+emnn/D111+jqqoKt912G0aNGtWtTVGJiAYCBhFERDTgeHh4XPL48ePHsXjxYvMmdYBpfwMHBwcIggCDwYDq6moA6NashUttQnnh5oA6na5X7q3X6y0eb30tSqXSPCW8Pd3tdFFeXm7+1jwhIcFcx/Tp06HT6fC3v/0NH3zwATQaDZ599lnzdYmJiQBg3iOgq+6++25s2rQJhYWFOHjwoHmNfVpamjl8aV3C0cpoNOKvf/0rfvrpJ/NjMpkMbm5ukMvlAEwbMzY3N3fr/317KisrAVz+5+vr69thkPC///u/FvuKSKVSi5obGhrMv3paV+tvPf9il/qz27qh7MV/dnvDhfVd6jVdeKyqqsocRPzpT39CWloatm7diu+++w7fffcdBEFAVFQUrr/+etxzzz1tNjq9/fbbcerUKXz99df4+eefzbMsQkJCcN111+Guu+5CfHx8T75MIqJ+jUsziIhowOnoW1vA9EHnL3/5C2praxEbG4svvvgCSUlJOH78OP744w8cOHAA3333nfl8URT7omSbdubMGfMHyIs7b8yaNQtvv/02BEHA559/jrfffhuiKCI7OxvHjx+Hm5sbJk+e3K3nHTt2LEJDQwGYWlW2av1vpVLZ5t7ff/89fvrpJ0ilUixZsgQ7duzA6dOnceTIEfMGjq1LRfrL//sDBw6YQ4j77rsPP/74Y5uaFyxYYOUqBw+5XI7/+7//w+bNm7FkyRKMGzcODg4OSE9Px4oVKzBjxgyL7jStXn75ZWzbtg3PPvssbrzxRri6uiIvLw/r1q3DXXfdhWXLllnh1RARWQdnRBAR0aBy4sQJFBYWQiqV4vPPP2/3G9GBto5bpVIBANRqNbRabYezIro7rV+j0Vzy+F133QW9Xo/XXnsNa9euhUajQW1tLURRxIIFC7rdxrT13u+//z5+/fVX81KX1m4dt99+e5vX2vpN9N13391hq8iKiopu19Oe1hk6l/v5dnS8tebrr78er732Wrvn9HTNF/Lw8EBOTs5ll6q0Hr/cjCRru7C+0tLSDtu0Xvj/o70lFDExMYiJiQFgCjgTExPxySefIDExEf/zP/+Da6+91ny8VUhICB599FE8+uijMBqNOHXqFL788kvs3LkTa9aswbhx48x7vRARDWScEUFERINKcXExANMHi46mZR88eLAvS+p1cXFxAExLQVp377+YKIrd7hLROmUdAA4dOtTuOXPnzsXSpUsBmGYs7Ny5E2FhYXj44Ye79ZytZs2aBalUiubmZvz444/YvXu3eSnKxcsygPMflocOHdru/TQaDU6ePHlFNV2sdcp9cnJyh6FNbm5uhx/0L1ezKIod/twBmDcn7e4Mj9b6Dx8+DKPR2O45WVlZ5g/uw4YN69bz9JXAwEAolUoAl/67/scffwAwzay58M94e2QyGcaPH4/PP/8cCoUCoiiar++IRCLBiBEj8NFHH5k3ir3cNUREAwWDCCIiGlRcXFwAmL5Bbu9b5JKSkm637eyvYmNjzZtnfvHFF+1+IN28eTMKCwu7df/4+HgEBwcDMO1l0BoEXGz+/PmYNm2a+fcxMTGws7Pr1nO28vb2xo033gjAFHC0LsuIi4tr8200APOGgmlpae3eb/ny5Zed4dFVt956K6RSKZqamtqdsg8An3zySYfXX67mb775pt0Wphdff+GeKF0xffp0AKYZAuvXr2/3nI8++giAafbNtdde263n6SuCIJj/HH777bftzoAqLS3Ft99+CwCYMWOGxTGtVtvhvRUKhXmPlAuXiF3qGqlUat7ro72ONkREAxGDCCIiGlRGjx4NR0dHiKKIp59+Gjk5OQAAg8GA/fv344EHHrByhT1PEAQ8+eSTAIDff/8dL7zwgvnb6+bmZqxfvx6vvfYa3Nzcun3/V199FVKpFLm5ubjnnnuwfft2NDc3AzD9bI8dO4annnoKW7duNX/Y2rp1K/75z39e8etrnfmQnJyMffv2ATAt2WhPa5vL9evX49tvvzV/QCwvL8c777yDr776yvxteU/x8fHBfffdB8AUdHz++eeor68HYNoE8c0338SWLVvMIVlHNe/btw+ffPKJeUPK2tpafPbZZ3j77bcvWXNUVJT5+u4svxk+fLh534y33noLX3/9tXkjz/Lycrzyyivmdph//vOfrzhc6i6j0YiqqqpL/mr9uT/22GNwdXWFWq3GQw89ZNGqNSkpCQ899BBqa2uhVCqxePFii+e56aab8P777+PEiRMWAUNeXh6ee+45NDY2QiKRmFuSAsA999yDt99+G4cPH7bYULS0tBRvvfWWub3uhAkTeuVnQ0TU33CPCCIiGlRcXFzw/PPP4/XXX0diYiKmTp0KR0dHGAwGNDc3Q6VS4d1338Xjjz9u7VJ71O23347Tp09j9erV2Lx5M7Zs2QJXV1c0NDRAp9Nh3LhxuOqqq8xTy7vqhhtuwAcffICXX34Z+fn5eOqppyCTyeDs7AyNRmPuEOLv74933nkH+/btw4oVK/DZZ5/By8sL999/f7df28SJE+Hp6YmKigoYjUbY2dnh9ttvb/fcRYsWYfv27cjOzsarr76K119/Hc7Ozqirq4Moipg7dy60Wi02btzY7Xra89e//hVZWVn4448/8MEHH+DDDz+Es7Ozea+MRx55BCdPnsSRI0faXDtr1ixs2rQJR48exUcffYSPP/4Yrq6uqKurg9FoxMSJExEbG4tPP/203eeePXs2Vq5ciby8PEycOBHu7u7msGDdunXw9fW9bP3Lli1DdXU1jhw5grfeegvvvvsunJyczPUDpp/tvHnzruCndGWKi4sv2Y4TAG6++WYsX74cvr6++OSTT5CQkICMjAzMmzcPjo6OAGAOClxdXfHJJ5+0WcJVUVGBL774Al988QUkEglcXFzQ1NRkDt4EQcALL7yAyMhI8zV1dXVYu3Yt1q5dC0EQ4OLiAr1ebxFKLFy40Bw6ERENdAwiiIho0Jk3bx78/f3x1VdfITk5GQaDAT4+PpgwYQIeeeSRbrXVtAUvvfQSxo4dizVr1uDMmTPQarUIDw/HzJkzsWDBArz33nsATB/AumPq1KkYNWoU1q1bh3379iEvLw8ajQZKpRJxcXGYMmUK7rjjDigUClxzzTXIzc3F7t27sWzZMnh4eFgs2+gKmUyGWbNmmTtLTJkypcPX4Orqiv/+97/45JNPsHPnTpSVlUEqleLqq6/G3LlzMX36dLz44ovdquNS7Ozs8OWXX2LdunXYsGEDcnJyIIoixowZY16y0tFsHLlcjhUrVuCLL77ATz/9hMLCQoiiiOHDh2PWrFmYO3fuJZd2hIaGYs2aNfj8889x6tQpqNVqc5eTzrbLdHFxwapVq7Bx40Zs3rwZZ8+eRUNDAzw9PTFq1CjMnz8f11xzTdd/MFZ09dVX45dffsHKlSvx22+/obCwEIIgICIiAhMmTMCiRYvg5eXV5roVK1bg8OHDSEpKQnFxsXmJV0hICEaPHo358+e3acX5wQcf4Pfff8fRo0dRUFCAiooK6PV6BAQE4KqrrsKcOXMuG6IQEQ0kgthfelMRERGRVd177704fvw4nnrqKSxZssTa5RAREdEAxT0iiIiICEeOHDF31OD0cCIiIupNDCKIiIgGiTfeeAMbNmxAeXm5eV1/bW0t/vvf/yIhIQEAMG7cOAwfPtyaZRIREdEAx6UZREREg8TMmTPNLSAVCgUcHBwsNhuMjIzEihUr2mzOR0RERNSTGEQQERENErt27cLOnTtx6tQpVFRUoL6+Hs7OzoiMjMSUKVMwd+5cODg4WLtMIiIiGuAYRBARERERERFRn+EeEURERERERETUZxhEEBEREREREVGfkVm7AOo9oijCaOz/K28kEsEm6iQi28Vxhoh6E8cYIupttjLOSCQCBEG47HkMIgYwo1FEVZXG2mVckkwmgUrlhNraBuj1RmuXQ0QDEMcZIupNHGOIqLfZ0jjj7u4EqfTyQQSXZhARERERERFRn2EQQURERERERER9hkEEEREREREREfUZBhFERERERERE1GcYRBARERERERFRn2EQQURERERERER9hkEEEREREREREfUZBhFERERERERE1GcYRBARERERERFRn2EQQURERERERER9hkEEEREREREREfUZBhFERERERERE1GcYRBARERERERFRn2EQQURERERERER9RmbtAmjwMhpFpOZWQZdTDbkgIsLfDRKJYO2yiIiIiIiIqBcxiCCrSDpbhnU7M1Bd12x+TOVih/smR2F0tLcVKyMiIiIiIqLexKUZ1OeSzpbhk43JFiEEAFTXNeOTjclIOltmpcqIiIiIiIiotzGIoD5lNIpYtzPjkud8szMDRqPYRxURERERERFRX2IQQX0qPV/dZibExarqmpGer+6bgoiIiIiIiKhPMYigPqXWXDqE6Op5REREREREZFsYRFCfUjrZ9eh5REREREREZFtsrmvGoUOHsHLlSpw8eRINDQ3w9/fH1KlTsXjxYjg6Onb6PgaDAYcOHcLevXtx/Phx5ObmoqmpCUqlEsOGDcPcuXMxceLES96jsrISn376Kfbs2YOysjK4urpi7NixePTRRxEbG3vJa7dv346vv/4aaWlp0Ol0CAkJwR133IEHH3wQcrm806/D1gwJUkLlYnfJ5RmOdjJEBbr1YVVERERERETUVwRRFG1mV8C1a9di2bJlEEURvr6+cHd3R2ZmJrRaLSIiIrBu3ToolcpO3Wv9+vV45ZVXAAASiQTBwcFwcnJCXl4e6uvrAQBz587FG2+8AUEQ2lyfl5eH++67DxUVFXB0dERYWBhKSkpQWVkJuVyODz/8EDfffHO7z/33v/8dK1asAAAEBwfDwcEBmZmZMBgMGDt2LFasWAGFQtGNn5Alg8GIqirNFd+np7V2zbiUsTHeWDgtBg52NpeVEVE/I5NJoFI5obpaA73eaO1yiGiA4RhDRL3NlsYZd3cnSKWXX3hhM0szkpOT8c477wAA3nzzTezduxcbN27Ezp07ERcXh6ysLCxdurRL94yOjsbbb7+NI0eOYPv27diwYQMOHz6M559/HoIg4Ntvv8U333zT5jpRFPHnP/8ZFRUVuOGGG7Bv3z5s2LAB+/btQ0JCAnQ6HZ577jmUlbVtQ/nrr7+ag4bly5fj119/xZYtW/Djjz8iMDAQiYmJ+OCDD7r3Q7IRo6O9sWR2PFQulssv3F3scP1wP0glAhLTyvDW6qMoLK+3UpVERERERETUG2xmRkRCQgJ27dqFWbNm4e9//7vFsdzcXEybNg1GoxGbN29GTEzMZe+nVqvh5ubW7mwHAFi6dCm+++47xMTEYPPmzRbHdu7ciSVLlsDFxQW7du2Cm5vlMoL7778fiYmJeOihh/Diiy9aHJs5cybS0tKwZMkSPPXUUxbHDh48iIULF0KhUOC3336Du7v7ZV/HpfTXGRGtjEYRWUU10IkC5IKICH83SCQCMgtr8OmmZFTXNUMhl2DBrTEYH+9r7XKJyEbZ0rcIRGR7OMYQUW+zpXFmQM2I0Gg02L9/PwBgzpw5bY6HhoZi3LhxAIBt27Z16p5KpbLDEAIAbrzxRgBATk5Om2Nbt24FAEydOrVNCHFhja3ntcrNzUVaWhoA07KPi40fPx4hISHQarXYtWtXp16HLZNIBMSGumPCqEDEhrpDIjH9/4gMcMNrD41FXKgKWp0RX/50Bmu2n4Wun/+lIyIiIiIiosuziSAiNTUVWq0WCoUCw4cPb/ec0aNHAwBOnjzZI8/Z1NQEAHBwcGhzrPU5xowZ0+61rY+XlJSgtLTU/PiJEycAAEFBQfDx8Wn32p5+HbbK1VGBZ+aMwB3XhUIAsPd4Id79OgkV6kZrl0ZERERERERXwCaCiNZZCf7+/h12lAgODrY490r9/PPPAM4HA620Wi0KCwstnvNifn5+5jqzs7PNj+fm5l7yuguP9dTrsGUSiYBZN4Tj6TlXwclehtySOryxKhGnsiqsXRoRERERERF1k020JKipqQGAdpdBtGo91nruldi5cyf27NkDQRDw8MMPWxyrr6+H0Wi8ZD2CIMDV1RWVlZWora01P96V13HhdVdCJuvfWVPr+qFLrSMaOcQLbz0yDv/64RSyi2rxf+tP4Y7rQnHnhAjzcg4ioo50ZpwhIuoujjFE1NsG4jhjE0FEc3MzAHQ4GwKAud1l67ndlZWVZd5gcsGCBRg1alS7tVz4nJeqp3WJx4XXduZ1XHhdd0kkAlQqpyu+T19wdW27BOZCKpUT/vHnG/HvLSn4+UAOthzIRV5ZPZ6bPwbKi7pvEBG153LjDBHRleAYQ0S9bSCNMzYRRNjZmT5o6nS6Ds/RarUW53ZHcXExHn74YdTV1WHChAl47rnnOqzlwue8VD329vZtru3M67jwuu4yGkXU1jZc8X16k1QqgaurA2prG2EwXH4zyrk3RSDYywkrfk7FyYwKPPX+Hiy5cxiGBCl7v1giskldHWeIiLqCYwwR9TZbGmdcXR06NXPDJoKIziy76Myyh0spLy/HwoULUVRUhKuvvhoff/xxuzMXnJ2dIZFIYDQaO6xHFEXz0gpXV1fz463/3ZnXceF1V6K/t3dpZTAYO13r2Bhv+Hs6YfnG0yiubMC7a5Nwz8QITBkbdMlOKEQ0uHVlnCEi6iqOMUTU2wbSOGMTi0xCQ0MBAEVFRR3OJjh37pzFuV1RWVmJBQsWIDc3FyNHjsRnn33W4cwKhUIBf39/i+e8WHFxsbnOsLAw8+Ot/52Xl9dhLVfyOgaTAE8nLF0wBtcM9YHBKOK/uzOxfFMyGpv11i6NiIiIiIiILsEmgojY2FjI5XJotVqcOnWq3XOSkpIAACNGjOjSvdVqNR566CFkZWUhLi4OX375JZycLr2vQutzHD16tN3jrY/7+vrC19fX/PhVV10FACgoKLBo63mh7r6OwcheIcPi24di/pQhkEoEJJ0tx5urElFQVm/t0oiIiIiIiKgDNhFEODs74/rrrwcAfPfdd22O5+bm4tChQwCAqVOndvq+9fX1WLRoEc6ePYshQ4bg3//+N1xcXC573a233goA2LZtW7vLLFprvLiWsLAwDBkyBADw7bfftrnu4MGDyMvLg1wux80339zp1zGYCYKAm0cH4sX7R8Hd1Q6l1Y14e81RHDhdbO3SiIiIiIiIqB02EUQAQEJCAgRBwObNm/Htt99CFEUAQFlZGZ599lkYjUZMnjwZMTExFtdNmjQJkyZNwrZt2yweb2xsxOLFi5GSkoLw8HCsWrUKKpWqU7VMnjwZ0dHRqKurw3PPPYe6ujoAgMFgwIcffojExEQ4ODhg0aJFba594oknAABffvkldu/ebX48Ozsbr7zyCgDgvvvug7u7eyd/MgQAEf5ueP2hqxEf7g6t3oh//5yK1dvSoNMbrF0aERERERERXUAQWz/R24BVq1bhvffegyiK8PPzg0qlQmZmJrRaLcLCwrBu3bo2H+Cjo6MBAO+++y7uvPNO8+Off/45PvjgAwBAeHg4lEplh8/70UcfwcvLy+KxnJwczJ8/H5WVlXB0dERYWBhKSkpQWVkJuVyOf/7zn5gyZUq793vnnXewevVqAEBwcDAcHR2RkZEBg8GA0aNHY+XKlVfU/aOVwWBEVZXmiu/Tm2QyCVQqJ1RXa3pk4xWjKOKnA7nY/HsORAAhPi5ImB0PL+XAaXVDRF3T0+MMEdGFOMYQUW+zpXHG3d1p4HTNaLVw4UJER0djxYoVOHXqFCorK+Hv74+pU6di8eLFl93b4UIXtt7Mzs6+5LnNzc1tHgsLC8OWLVvw6aefYs+ePUhPT4erqytuvfVWPPbYYxg6dGiH93vppZcwcuRIrFu3DqmpqSgrK0NERATuuOMOLFy4sN1uHdQ5EkHAHdeHITzAFV9sOYO80jq8sTIRD88YihFRntYuj4iIiIiIaNCzqRkR1DWDcUbEhapqm/DppmRkFZlaqd42LgSzbwyDVGIzK5KIqAfY0rcIRGR7OMYQUW+zpXGmszMi+ImMBix3V3u8MH8UJo8JBAD8cigP7//3BGo02stcSURERERERL2FQQQNaDKpBPdNHoLHZsbBTiFF2jk1Xl95BOn5amuXRkRERERENCgxiKBB4epYH7y6YAwCPJ1QU6/F/6w7jm2Hz4Erk4iIiIiIiPoWgwgaNPw8nPDKg2MwLs4HRlHEd3sy8cnGZDQ06a1dGhERERER0aDBIIIGFTuFFI/MGIoHbo2GTCrgWHo53lydiHOlddYujYiIiIiIaFBgEEGDjiAIuGlkAP52/2h4uNqjrLoRy9Ym4fdTxdYujYiIiIiIaMBjEEGDVpifK157aCyGR3hApzdixS+pWPlLKrQ6g7VLIyIiIiIiGrAYRNCg5uwgx1N3D8fsG8MhCMD+U8V4Z20SyqobrF0aERERERHRgMQgggY9iSDg9mtD8Ze5I+DiKMe5snq8seoojqeXW7s0IiIiIiKiAYdBBFGLoaHueP2hqxEZ4IbGZj0+3nAa6/dkwmA0Wrs0IiIiIiKiAYNBBNEFVC52eP6+kbhlbBAAYOvhc/jHNydQU99s5cqIiIiIiIgGBgYRRBeRSSW49+YoJMyKh71CirP5ary+MhFnz1VbuzQiIiIiIiKbxyCCqANjYrzx6sKxCPRyQo1Gi//55jh+OZQHURStXRoREREREQ0CRtGIs1WZ+D0vEWerMmEUB8aycUHkp6oBy2AwoqpKY+0yLkkmk0ClckJ1tQZ6ff/8S9WsM2Dt9rP4I7kEADAi0hMPz4iFo73cypURUWfYwjhDRLaLYwwR9ZYTZaexPmML1M015seUdm64J+oOjPAeZsXKOubu7gSp9PLzHTgjgugy7ORS/Gl6LBZMjYZMKsGJzAq8sSoReSV11i6NiIiIiIgGoBNlp/Fl8lqLEAIA1M01+DJ5LU6UnbZSZT2DQQRRJwiCgAkjAvDSA6Pg6WaPcnUTlq1Nwr6TRdYujYiIiIiIBhCjaMT6jC2XPOf7jC02vUyDQQRRF4T6uuK1h8biqggP6A1GrNqahhU/p6JZZ7B2aURERERENABkqnPazIS4WHVzDTLVOX1UUc9jEEHURU72cjx593DcNSEcggD8froYy9YkobS6wdqlERERERGRDRJFEfl1hdiaswvr0n7o1DW1zbW9XFXvkVm7ACJbJBEETB8finB/N3y+ORkF5fV4c1UiFt02FKOjvaxdHhERERER9XNN+iakVWUgpTINKZVpqNF2bQ86VzvXXqqs9zGIILoCsSEqvPbQ1fhsczIyCmrwycbTuPXqINw1IQKyTuwWS0REREREg4MoiihtKEdKZRqSK9OQpc6BQTy/xFshVSBGFYVY9yHYmrsTtZcIJlR2bohUhvVF2b2CQQTRFVK52OGv80Ziw2/Z2HbkHLYfyUd2US0emxkPlYudtcsjIiIiIiIr0Rp0yFBnI6UyFSkVaahoqrI47u3giTjPGMR5xCBSGQ65xPQR3VXhjC+T13Z437uj7oBEsN0vPgVRFEVrF0G9w2AwoqpKY+0yLmmg9d5OOluOFb+cQWOzAa6Ocjw6Mx6xISprl0U0qA20cYaI+heOMUR0scrGavNyi7PVmdAZdeZjMkGKKFUE4jxiEOcRDW/Hjpd1nyg7jfUZWyw2rlTZueHuqDswwntYr76G7nJ3d4K0EzPDGUQMYAwirKO0qgGfbDTtGyEIwJ03hmPauBBIBMHapRENSgNxnCGi/oNjDBEZjAZk1+QipfIskitTUawptTiutHNDnEcM4j1iMEQVCXtZ52dNG0UjcupyoZdpIdMrEOYS2q9nQjCIIAYRVqTVGfD1jnT8froYAHBVhAcevn0onOzlVq6MaPAZqOMMEfUPHGOIBqdabR1SKs8ipTINaVXpaNQ3mY8JEBDuFoJ4j1jEecbA38kXwhV8KWlL40xngwjuEUHUCxRyKRZNj0VkoBu+3pGOk1mVeGNlIpbMHoYQXxdrl0dERERERF1gFI04V1eAlArTRpPn6gosjjvLnTDUIxpxHjGIdR8CJ7mjlSq1DQwiiHrRjVf5I8THBcs3nUa5ugnL1ibhvilRmHCV/xWlokRERERE1LsadI1IrUo37/dQr7OcbR7sEtCy10MsQlwD+/WSif6GQQRRLwvxdcFrC8fi3z+n4nhGBdZsO4uM/Bo8ODUadnKptcsjIiIiIiKY2msWa0qRXJmKlMo0ZNfkwSieXwphL7VDjPsQxHvEYKhHNNzsXK1YrW1jEEHUBxzt5XjizmHYdvgcfvgtGwdTSnCurA5LZg+DrzunbRERERERWUOzQYv06kwkV6YhpSIN1c1qi+O+jt6I8zRtNBnuFgqZhB+hewJ/ikR9RBAETBsXgnB/V3y2OQWF5Rq8uSoRi26LxZgYb2uXR0REREQ0KJQ3VCKlMg3JlanIUGdDb9Sbj8klMgxRRbYsuYiBp4O7FSsduNg1YwBj14z+S13fjM82pyA9Xw0AuGVsEO6eGAFZJ3aYJaKuGazjDBH1DY4xRP2f3qhHpjrHvNdDaUO5xXF3exXiW4KHIaoIKKQKK1XaPlsaZ9g1g6gfUzrb4a/zRmDDb9nYevgcdiTmI7uoFo/PiofKpfN9hYmIiIiIqC11c01L8HAWaVXpaDZozcckggQRbqGI94xFnEcMfB29uZF8H2MQQWQlUokE99wUicgAN3z1cyoyC2vw+sojePSOOAwN5RQwIiIiIqLOMopG5NaeM7fXLKgvsjjuonA2L7eIdY+Cg8zBSpUSwCCCyOpGDvHCa15OWL4xGefK6vH+tycw64ZwTB8fAgmTWSIiIiKidtXrNEitTEdyZSpSK9Oh0TeYjwkQEOIahDiPaMR7xCLQxZ/tNfsRBhFE/YC3yhEvPTAa63amY9/JYmzcl42swho8PGMonB3k1i6PiIiIiMjqRFFEQX0xUlraa+bUnIOI81seOsgcMNR9COJa2mu6KJytWC1dCoMIon5CIZdi4bRYRAYosXbHWZzKqsQbKxORMDseYX7sUUxEREREg0+Tvglp1ZlIqTBtNFmjrbU4HuDsZ15yEeYaDKlEaqVKqSsYRBD1M9cP90OwjzOWb0pGWXUj3v06CfNujsLEkQHcRIeIiIiIBjRRFFHWUN7SXjMNmeocGESD+bhCIke0e5S5y4XKXmm9YqnbGEQQ9UPBPi54dcFYrPglFcfSy7F2RzoyCmuw4NYY2CmY8hIRERHRwKEz6JChzkZyS3vNisZKi+OeDh6I94hBvEcsIpVhkEu5dNnWMYgg6qcc7WVYMjse24/k4/u9WTiUUor80nokzI6Hn4eTtcsjIiIiIuq2qqbqlvaaaThblQmtUWc+JhWkiFKGI87TNOvBx9HLipVSb2AQQdSPCYKAqdcEI9zfFZ9uTkZhhQZvrj6Kh6bF4OpYH2uXR0RERETUKQajAdk1eebwoUhTYnFcaeeGOI9oxHnEIloVCXuZnZUqpb7AIILIBgwJUuL1hWPx+ZYUpJ1T47PNKcgsqMGcSZGQSdmGiIiIiIj6nzptPc5UnjW116xKR6O+yXxMgIAwtxDzXg8Bzn7cD20QYRBBZCPcnO3wl3tHYNP+HPx8MA87kwqQU1yLx2fFw93V3trlEREREdEgZxSNyK8rNO/1cK62wKK9ppPcEUPdYxDvEY1Yj2g4yR2tWC1ZE4MIIhsilUhw14QIRAS44asfzyCrqBavr0zEo3fEIS7M3drlEREREdEg06BrRFp1BpIrUnGm8izqdPUWx4NcAhDnEYN4jxiEuAZBInA2LwGCKIri5U8jW2QwGFFVpbF2GZckk0mgUjmhuloDvd5o7XJsSrm6Ecs3JiOvtA4CgJnXh2HGdaGQcEobkQWOM0TUmzjG0GAjiiKKNaXmvR6yanJhFM//2beX2iHGPQpxHrEY6jEESjs3K1Y7MNjSOOPu7gRpJ5aO29yMiEOHDmHlypU4efIkGhoa4O/vj6lTp2Lx4sVwdOza1J6CggIcPHgQp0+fRnJyMtLT06HT6TB79my89957HV734osvYuPGjZ16jt27dyMgIMD8+w0bNuBvf/vbJa955JFH8Nxzz3XuRdCg5aV0wEsPjMK6nRn47UQRNv2eg8zCGjxy+1C4OCqsXR4RERERDRBagxZnqzORUnkWKZVpqGqqtjju4+ht3ushQhkKmcTmPmZSH7OpPyFr167FsmXLIIoifH194efnh8zMTHz66afYsWMH1q1bB6VS2en7rV69GmvWrOlyHaGhoRg1alSHx3Nzc1FVVQU/Pz/4+fm1e46zszOGDBnS7rELgwuiS5HLpFgwNQaRAW5Yu/0sknOq8MaqRCTMGoZwf1drl0dERERENqqisdK010NFGtLVWdAb9eZjcokMUaoI85ILTwcPK1ZKtshmgojk5GS88847AIA333wTc+bMgSAIKC0txeOPP46UlBQsXboUH3/8cafvqVKpMHHiRAwbNgzDhg3Djh078P3331/2usceewyPPfZYu8dEUcSUKVNQVVWFmTNnQiJpf1rK0KFDsXbt2k7XSnQp1w3zQ4iPCz7ZeBql1Y149+sk3HtzFCaNCuDuw0RERER0WXqjHlnqXCRXpiKl8ixKG8osjqvslIj3jEW8RwyGqCKgkHIGLnWfzQQRy5cvh9FoxKxZszB37lzz4z4+Pvjggw8wbdo07NixA2lpaYiJienUPRMSEix+f+jQoSuuMzExEfn5+QCAO++884rvR9RZgd7OeHXhWKz8JRVHz5bjP7+mI6NAjYXTYmCvsJm/6kQ9yigacbYqG/paLWR6BcJcQrlJFhERUYua5lrzXg9pVRloMjSbj0kECSLcQhHXsuTCz8mHX3BRj7GJTycajQb79+8HAMyZM6fN8dDQUIwbNw5//PEHtm3b1ukgoje07h0xatQohISEWK0OGpwc7GR4fFY8fj1agPV7MnEktQz5ZfVYMnsY/D2drF0eUZ86UXYa6zO2QN1cY35MaeeGe6LuwAjvYVasjIiIyDqMohG5tfmm8KEiFfn1RRbHXeTOpuDBMwYxqig4yh2sVCkNdDYRRKSmpkKr1UKhUGD48OHtnjN69Gj88ccfOHnyZB9Xd15DQwO2bdsG4PKzIYqKivDiiy+iuLgY9vb2CA8Px6233ooRI0b0QaU0kAmCgFvGBiHMzwWfbkpGcWUD3lp9FAumRWPcUF9rl0fUJ06UncaXyW2Xv6mba/Bl8lo8Ev8AwwgiIhoUNLoGpFaeRXJlGs5UnYVG12A+JkBAsGugea+HIJcAzhykPmETQUROTg4AwN/fH3K5vN1zgoODLc61hu3bt6OhoQEODg6YNm3aJc8tKChAQUGB+fd79+7FihUrMH36dCxbtgwODkwf6cpEBSrx+kNX4/MtKUjNq8YXW84gs6AGcydFQS7jPzA0cBlFI9ZnbLnkOd9nbMFwrzi+2SIiogFHFEUU1hebNpqsTENOTR5EiObjDjIHDHUfgjiPGAz1iIaLwtmK1dJgZRNBRE2NaVqtm1vHPWhbj7Weaw0bNmwAAEyZMgXOzu3/hXZ1dcXDDz+Mm266CSEhIXBzc0NhYSE2bdqEr776Cj///DMMBgM+/PDDHqlJ1s8/cLb2mO1Mr1nqOnc3e7wwfxQ27MvGlt9zsPtYIXJL6vDEXcPh6WZv7fKIesXZqmyL5RjtqW6uQU5dLqLdI/uoKiIaqPhehvqDJn0z0qoycLo8FckVaW3+HfR39sUwz1jEe5raa0olUitVSt0xEMcZmwgimptNm6Z0NBsCABQKhcW5fS0/Px+JiYkALr0sY/LkyZg8ebLFY2FhYXjmmWcQHR2NZ555Btu2bcPRo0cxZsyYK6pJIhGgUtnGvgCurpwB0psemT0cI2N88P5/kpBdVIvX/n0Ez80fjVEx3tYujahHGEUjzqkLkVyWjv25hzt1zYbMnzDcdyiC3fwR6OqHQFdfKGTcAZyIuofvZaivFdWV4nhRMo4VJyO1PNOivaadVIF4n2iM9IvHKL94eDq5W7FS6ikDaZyxiSDCzs4OAKDT6To8R6vVWpzb1zZt2gRRFBEQEIBx48Z16x633XYbVq1ahZMnT+LXX3+94iDCaBRRW9tw+ROtSCqVwNXVAbW1jTAYjNYuZ0CL8HXGG3+6Gv/64TRyimvx+pcHMfOGMMy6IRwSCXdAJttiFI0orC9BelUW0quzkF6VhQZ9Y5fukasuQK76/BI5AQI8Hdzh7+zb8ssHfs6+8HX0glzacRBORIMb38tQX9EZdMiozsbpCtOsh7KGCovjXg4eGOYVi3jPWAxRhZ//t0sLVGs1VqiYeootjTOurg6dmrlhE0FEZ5ZddGb5Rm8RRRGbNm0CAMycOfOK2tqMHDkSJ0+eRF5eXo/Uptf37z+orQwGo83UastUznZ4cf4o/HdXBvYcL8Sm/TnIyFfjkTvi4OrIb4Kp/zKKRhRrSpFenYUMdTYyq7Oh0VsGrXZSBSKUYYhyC8fu/P2o09V3eD9nuROmhU5GaUMZijWlKNKUQKNrQHljJcobK3GyPMV8rgABXo4e8HfyhZ+TT8svX3g7ekImsYl/RomoD/C9DPWG6ia1ea+Hs1UZ0BrPfzErFaSIUoYjziMacZ6x8HbwPP85RLSdzwHUeQNpnLGJd1ChoaEATJ0mdDpdu0s0zp07Z3FuXzpy5AgKCgogCMJlu2VcTutr0+v1lzmTqHvkMgkeuDUakYFuWL0tDSm51XhjZSIenxWPyIC+D/KI2mMUjSjRlCFdnYWM6mxkqrNRr7P8NkchVSDCLRRDVBGIUkYg2CXAvObV29Gz3a4ZreZF32nRNUMURdTp6lFcX4piTSmKNSUtAUUpGvWNKGuoQFlDBU6UJ5uvkQgSeDt6mcOJ1qDCy8GDa2+JiKhbDEYDcmrPIaUyDckVqSjSlFgcd1O4mjpceMYgWhUJexn3/CLbZBNBRGxsLORyObRaLU6dOoXRo0e3OScpKQkArNL+cuPGjQCAMWPGICgo6IrulZGRAQDw9WWbRepd4+N8EeztjE82JqOkqgF//88xzJkUicmjA69oVg9Rd4iiiJKGMtOMh5ZZD22CB4kcEcowDFFGIEoVjmCXwA4/8I/wHoZH4h/A+owtFht2qezccHfUHW1adwqCAFeFC1zdXSw2sBRFETXa2pZworQlqDCFFE2GZpRoSlGiKcXxC+4lE6TmgMLf+fwsCk8HD3bpICKiNuq09ThTeRYplWk4U5WOxguWGgoQEOYWjDiPWMR5xCDQ2Y/v02hAsIkgwtnZGddffz327NmD7777rk0QkZubi0OHDgEApk6d2qe1aTQabN++HQAwe/bsK7pXWloa9u/fDwC47rrrrrg2ossJ8HLG0gVjsGprGhLTyvDNzgxkFtRg4bQYONjZxPBANkoURZQ2lCG9Ortl1kNWm+BBLpEjwi0UUaoIDFGFI8QlqEszDUZ4D8Nwrzjk1OVCL9NCplcgzCW0S2GAIAhQ2rlBaeeGWPchFvWrm2tQ1Dp7onUmRUMptAYtijQlKNKUIKns5AWvRwZfR2/4OvnC38kHfs6mgMLdXsWAgohoEDGKRhTUFSG5MhXJlWk4V1tg0V7TSeaIoR7RiPOIQazHEDjLbWPzeaKusJlPGgkJCdi7dy82b96MUaNGYc6cORAEAWVlZXj22WdhNBoxefJkxMTEWFw3adIkAMDzzz/fKyHF9u3b0dDQAEdHx8vev76+Hq+88goefPBBjBw50iLN3L9/P1566SUYDAbExMTglltu6fFaidrjYCfDYzPjEBnohu92ZyIxrQz5ZfVYMjseAV7sK009QxRFlDWUm5dapKuzUKe13MNBLpEh/IKlFiGugVe8B4NEkCDaPRIqlROqqzU9tq5SEASo7JVQ2SsR5xFtftwoGlHdpEZRy6yJ1l8lmlLojHrk1xchv77I4l4KiRy+5r0nzs+iUNkp+a0XEdEA0ahvRGpVhmnWQ+VZ1GrrLI4HOfsjziMGcZ6xCHUNYkBNA54giqJ4+dP6h1WrVuG9996DKIrw8/ODSqVCZmYmtFotwsLCsG7dOri7W7amiY42vUF899132+zfkJSUhISEBPPvm5qa0NTUBIVCAUdHR/Pjr776KqZPn95uTQ888ACOHDmC2bNn47333rtk/bW1tRg7diwAwMnJCUFBQVAoFCgqKkJFhWnX26ioKHzxxRfw9/fv5E+lYwaDEVVV/XuHXJlM0uMfEKj7Mgtr8OmmZFTXNUMhl2DB1BiMj+MyIeo6URRR1liBjOos8waTF7/pkktkCHMLxRBlOKJUEQhxDYK8FzZ/7A/jjFE0oqKx6oJwwhRUlGrKoBcN7V5jL7UzBxT+LRtk+jn7wE3hyoCCqB/pD2MM9T+tSw6TK1KRUpmGrJpcGMXzfz7spArEuA9BvEcMhnpEQ2nHfbqoY7Y0zri7Ow2crhmtFi5ciOjoaKxYsQKnTp1CZWUl/P39MXXqVCxevBhOTl2btqTX66FWq9s8rtVqze1AAaC5ubnd6/Pz85GYmAigc8syHBwc8Pzzz+PEiRNIT09HUVERGhoa4OzsjGuuuQa33nor7r77bqu1ICWKDHDDaw+NxZdbUpCSW40vfzyDzIIa3HtzFOQyJvPUMVEUUd5YYZ7tkFGdjRptrcU5MokMYa7B5hkPoa5Bg6YtpmljS094O3riKq848+MGowEVjZXmzh2tQUVpQzmaDM3IrT2H3NpzFvdykDlc0L3jfBcPV4UzAwoiIivSGrRIr85CSkuXi8qmaovjPo5eplkPHjGIVIax8xINajY1I4K6hjMiqLuMRhFbDuTgxwO5EAGE+rogYVY8PJUO1i6N+glRFFHRWIV0dSYyqrORoc622BQSMG3aGOYWgqiWGQ9hrsFWCR5scZzRG/Uoa6iwWN5RrClFeWOFxTdqF3KSO5pDiQtnUTgruLaYqDfZ4hhDPaeiscrU4aIyFRnVWdAZz3e+k0lkGKKMMIcPXo4eVqyUbJktjTOdnRHBIGIAYxBBV+p0diW+2JICTZMeTvYyPHL7UAyP8LR2WWQFoiiisqkK6dXZyFCblltcHDxIBSlCW2Y8DFGFI9Q1BIp+MONhII0zOqMeZQ3lKK4/3160WFOCisYqi43OLuQidzYFFM6WIYWj3LHd84moawbSGEOXpzfqkV2Ti+QK06yHkoYyi+MqOyXiPGMQ7xGDIapI2EkVVqqUBhJbGmcYRBCDCOoRlTVNWL4pGTnFpmn2M64NwazrwyGRcAr4QFfZWIV0dbZ5n4fqZrXFcVPwEGTqaqGMQJhbMBT98A3XYBhntAYdShvKTOFE/fklHpVNVR1e46ZwMe87ceFMCgf2pCfqksEwxgx2Nc21SGlpr5lWlY4mw/ll2xJBgnC3EMS3tNf0c/LhMjnqcbY0zjCIIAYR1GN0eiO+3Z2B3ccKAQCxISo8ekccXJ3634dO6r7KxmpkXNDVouqita0SQYJQ1yAMUUYgShWBcLeQfhk8XGwwjzNN+maUNpSdbzOqKUVxfWmbUOlCKjulxf4T/s6+8HH0hr2M+xcRtWcwjzEDlVE0Iq82v2XJRRry6wotjrvInc+313QfAkc5l65S77KlcYZBBDGIoB536EwJVm89i2adASoXOzw+Mx6Rgdzl2VZVN6mRXp1l3lzy4m/PJYIEIS5BiFKFY4gqAuFuoTY5xZTjTFuN+iaUXLD3ROssios3GL2Qh73KYuaEn7MPfB19+sXyGyJr4hgzMGh0DUitSkdyRRpSq86iXmf5HjrEJQhxHtGI94xFkEsA22tSn7KlcYZBBDGIoF5RWKHB8o2nUVzZAKlEwD0TIzBlbBCnIdqA6iY1MtTZpnaa1VmoaCd4CHYJbOlqEY5wt9AB8S04x5nOa9A1oFhTZtHBo1hTgjptfbvnCxDg4eAO/9ZwouWXj5N3r7RiJeqPOMbYJlEUUaQpMbfXzK7Js9hrx0Fmb9Fe01XhYsVqabCzpXGGQQQxiKBe06TVY9XWNBxJNW3QNDraC4tui4WDHT949Cfq5pqW0MG0wWR5Y6XFcQECgl0DzUstItxCYD8A9wfgOHPl6rWa80s7WmdRaEqg0TW0e75EkMDLwaNNi1FvR0+2q6MBh2OM7WjSN+Nsdaa5vebFmy77Ofm07PUQjXC3UEglUitVSmTJlsYZBhHEIIJ6lSiK2H2sEP/dlQGDUYSPygFLZg9DoLeztUsbtGqaa00bS7YstShrrLA4LkBAsEsgolThiFKGI0IZNig2JuQ40ztEUUSdrh7F9ednThS1hBSN+sZ2r5EIEng7ep3ff6JlJoWXgwff8JPN4hjTv5U1lCOl8iySK1KRqc6GXjSYj8klckSrIhHvGYOh7jHwcFBZsVKijtnSOMMgghhEUJ/IKqrBp5uSUVXbDIVMggdujcZ1w/ysXdagUNNc17K5ZBYy1NkobSi3OC5AQJCLP6KUERiiikCEMhQOssG3oRbHmb4liiJqtLUtG2NazqK4cKf5C8kEKbwdveDvbLnEw9PBg+uwqd/jGNO/6Ix6ZKqzkdLSXvPiUN7T3h1xnqYOF0OU4ZBznxuyAbY0zjCIIAYR1GfqG3X44scUJGeb9hyYMMIf902OglzGbzh7Uq22ztzRIqM6G6UX9S4XICDQxR9RStPmkhFuYdzJGxxn+gtRFKFurjm//8QFMym0Rl2718glMvg6esPXyRf+LRtk+jn5wt1eyYCC+g2OMdZX3aRuWW5xFmnVGdAatOZjUkGKCGUY4j1iEO8RA29HL+5rRTbHlsYZBhHEIIL6lFEU8dOBXGz+PQcigBAfFyTMjoeXkh+Eu6tOW48MdbZpuUV1FkraCR4CnP1MXS2UEYhUhsFR7milavsvjjP9m1E0oqpJbd6Doqi+FCWaEpQ0lEFn1Ld7jUKqgK+jt7m9aOsMCpWdkh8wqM9xjOl7BqMBObXnzHs9FNYXWxx3U7ggziMGcZ6xiFZFDopliDSw2dI4wyCCGESQVSTnVOKLLWdQ36iDo50MD98+FCMiPa1dlk2o09YjU51j2mBSnYViTWmbcwKc/Vo2lwxHpDIcTgweLovjjG0yikZUNFaZZ020Lu8o1ZRZrPG+kL3UDr5OPqbZE62tRp194KZwZUBBvYZjTN+o09a3tNdMRWpVOhou2ItGgIBQ12DEe8YgziMGgc7+/DtPA4otjTMMIohBBFlNVW0TPt2UjKyiWgDA9PEhmHVDGKQSTqW+UL1Wg0x1NtJbZj0UaUranOPv5IsolWmPh0hlGJzlTlao1LZxnBlYDEYDKhorWzbGvCCgaCiHUWz//6+DzOGiDh6mmRQucmd+WKErxjGmdxhFIwrqi8x7PeTW5lu013SSOSLWYwjiPGIw1D0azgr++0gDly2NMwwiiEEEWZXeYMR3ezKx82gBACAmWIlHZ8bDzUlh5cqsR6NrMAUPLZtLXjyVFGgNHsIRpYxAlDKcb6x6AMeZwUFv1KOsocJic8xiTQnKGys7DCic5I7mmRMXzqLg3zvqCo4xPadR34S0qgwkV6biTOVZ1GrrLI4HOvsjziMG8Z4xCHUN5l4xNGjY0jjDIIIYRFC/cCS1FCu3pqFZa4CbswKPz4zHkCCltcvqEw26BmSoc5ChNu3xUFRfYvFtDgD4OvmYl1pEKcPhomD7057GcWZw0xn1KGsoR3H9+faixZoSVDRWtfn72MpF7mwKJS7Yf8LfyYd7sFC7OMZ0nyiKKG0oQ3JlGlIq0pBZk2MRHCqkCsSqohDXsuRCaedmxWqJrMeWxhkGEcQggvqN4koNlm9MRmGFBhJBwN0TI3Dr1UEDbkp0g64Rmeps8waTBfXFbYMHR2+LpRauChcrVTt4cJyh9mgNWpQ0lF3QvcP0q7KpqsNr3BSuLQFFazjhC18nH26EN8hxjOkarUGH9OpMpFSeRUplKiqbqi2Oezt6It7D1F4zQhkGuURmpUqJ+g9bGmcYRBCDCOpXmrUGrN6ehkMppg0YRw3xwqLbYuFob7tvMBr1jRdsLpmNgrqiNsGDj6OXKXhQhiNSGQE3OwYPfY3jDHVFk74ZpQ1llntQ1Jeiulnd4TUqO+X5/SecTcs8fBy9YS+z67vCyWo4xlxeZWMVUirTkFyZhvTqTIuOODKJDFHKcFOXC48YeDtyg2uii9nSOMMgghhEUL8jiiL2nijCNzvToTeI8FY5IGFWPIJ9bOPDeaO+CVnqHKSrs5BRnY38usI2wYO3oyeilKYZD1HKcLjZuVqpWmrFcYZ6QqO+CSUtsyaKNCXmmRQ12toOr/GwV53v3tEyk8LX0QcKqbwPK6fexjGmLYPRgKyaXCRXpiKl8ixKLuoCpbJTIs4jGvGesRiiioSddPDuH0XUGbY0zjCIIAYR1G/lFNdi+cZkVNY2QS6T4IFbonH9cD9rl9VGk74JWTW5yKg2bTB5rq6gTfDg5eDREjqY9nng+tX+h+MM9aYGXQOKNWWmcOKCPSjqtPXtni9AgKeD+/lwoqWDh7ejF6eg2yiOMSY1zXU4U2nqcJFalYEmQ5P5mESQINwtxDzrwd/Jd8AtzyTqTbY0zjCIIAYR1K/VN+rw5Y9ncDq7EgBw41V+uG/yECjkUqvV1KRvbgkeTEstztUVtNlt39PBA0OU4YhqmfGgsldap1jqNI4zZA31Wo1Fe9HWoEKja2j3fIkggZeDxwUtRk1BhbejJ2QMKPq1wTrGGEUj8moLkFKZhpTKVJyrK7Q47ix3agkeohHrPoSbvRJdAVsaZxhEEIMI6veMooifD+Zh0/5siCIQ7O2MhNnx8Fb1zZuVJn0zcmryWpZaZCGvveDB3t0cOgxRRTB4sEEcZ6i/EEURdbp687KOC2dRNOob271GIkjg7ehl0V7Uz8kHXg4ekEqsF9zSeYNpjGnQNSC1Kh3JlWk4U3kW9TrL95nBLoHm9prBLoFsr0nUQ2xpnGEQQQwiyGacya3C51tSUNegg4OdDA9Pj8XIIV49/jzNBi2yL1hqkVeX3yZ48LBXmfd4iFSGw8NB1eN1UN/iOEP9nSiKqNHWtgQUJRZdPJoMze1eIxOk8HHyvmAGhemXp4MHP/z1sYE8xoiiiCJNCVIqTBtN5tTmWfy7aS+1R6x7FOI8YzHUPZobMhP1ElsaZxhEEIMIsinVdc34dFMyMgtrAADTrgnGnRPCIZV0/w211qBFdk0eMqqzkK7ORl5tPgyiweIclZ3StMdDS2cLDwf3K3od1P9wnCFbJYoiqpvV54OJetMsihJNKbRGXbvXyCUy+Dp6w9fJ1L3D1GrUF+72SgYUvWSgjTHNBi3OVmW0LLk426ZjjJ+Tj2nWg0cMwt1COTOHqA/Y0jjDIIIYRJDN0RuM+H5vFnYk5gMAooOUeGxmHNycO9cCT2vQIacmDxnqLKRXZyG3neBBaeeGIaoIDFGawgcPexU3zBrgOM7QQGMUjahqUptnTxTVl6JEU4KShjKLtogXUkgV8HX0hr+Tb0s4YfqlslNyDLxCA2GMKWuoaAke0pBRnQX9Bf92yiVyRKsiEOcRiziPaAb2RFZgS+MMgwhiEEE262haGVb8koomrQFuTgo8NjMO0cFtl0joDDrk1J5DenUWMtRZyK05Z/HmCTAFD6alFqY9Hjzs3fmme5DhOEODhVE0oqKxqs3yjlJNWZuxsZW91O6i5R2moMJN4cqxspNscYzRGfXIUue0tNdMQ1lDhcVxD3sV4jxiEe8ZgyhlBFvOElmZLY0zDCKIQQTZtJKqBnyy8TQKyzWQCALumhCOm8f4Ia8uH+nqbGRUZyGn9hz0F33756ZwbVlqEY4oZQS8HDz4ZnqQ4zhDg53BaEB5Y6W5tahpo8xSlDWUt9knp5WDzMEioGidSeEid+aYehFbGWPUzTVIqTDNekirzkCzQWs+JhEkiFSGI84jGvEesfBx9OL/Z6J+xFbGGYBBBIFBBNk+TVMzvth1AKmVmZC4VkHmUgNRsPxWz1XhYgoeWrpaeDl48s0TWeA4Q9Q+vVGPsoYKi4CiWFOK8sbKDgMKJ7nj+WDiglkUzgqnPq6+/+ivY4xRNCKn5hxSKtOQXJmKwvpii+OuChfzXg/R7lFwkNlbqVIiupz+Os60p7NBBBtTE1G/oTfqkVubb+pqoc5CTk0udA56yANNx0UAgt4O0e6RGOEXjSHKcHjzWxsiom6RSWTwd/aFv7MvgKvMj+uMepQ1lKOo/sIlHiWoaKyCRteATHUOMtU5FvdykTvDz9n3olkUPnCU9007ZjKp12pwpuosUirTkFqZDo2+wXxMgIBQ1yDTXg+e0Qh09ucGpkRkNQwiiMhq9EY98moLkKHOQkZ1NrJqcqG7aCd4F7kzolThcJcE4MBBLarKZTgtleKqW4LhE+BtpcqJiAYuuUSGAGc/BDj7WTyuNWhR0lDW0mb0fEBR2VSNOl096qozkV6daXGNm8LVFEw4n1/i4evkw2/fe4goiiioL0JyRRpSKlORW5sPEecnOzvKHDDUIxpxHjEY6h49qGeuEFH/wqUZAxiXZlB/YzAakFdXYGqnWZ2F7JrcNi3onOVO5laaUaoI+Dp6m2c8aJp0+OrHMziZVQkAuH6YH+bfMgR2crYOo45xnCHqXU36ZpQ2lKFIU4riC2ZRXNz28UIqO+UFAYWp1aivkw/spIq+K7yH9PUY06RvQpq5vWYaarR1FscDnP1allzEItQ1iO01iQYAW3ovwz0iiEEEWZ3BaMC5ugLzUousmlxoL9gcCzCtN45SmjaXHKKMgJ+TzyWXWhhFEVsP5WHDvmyIIhDo5Ywld8bDR8Xpv9Q+jjNE1tGob0KJphRFrftPtMykqNHWdniNh727ZRcPZx/4Ovr0664NvT3GiKKI0obylr0e0pClzrFoTa2QKhCjikK8RwyGekRDZa/s8RqIyLps6b0MgwhiEEF9zmA0IL++0NROszobWTU5FrtyA4CTzNHc0SJKFQ4/J59urVFNzavG55uTUdugg4OdFItuG4rR0V499VJoAOE4Q9S/NOgaTLMnWn+1zKKo09W3e74AAZ4O7qbWoq37Tzj7wtvRC3KJ9VcZ98YYozXokKHOMs16qEhDRVOVxXFvB0/EecYgziMGkcrwfvFzIKLeY0vvZRhEEIMI6nUGowEF9UWm4EGdjSx1DpoMzRbnOMocENWyzGKIKqLbwUN7quua8dnmZGQU1AAApl4djDsnhEPWicGPBg+OM0S2oV6rsWgv2vrfGl1Du+dLBAm8HDzMnTtaQwofR68+XY7QU2NMZWN1y3KLVJytzrLYM0kmSBGlikCcRwziPKLh7cjgnWgwsaX3MgwiiEEE9TijaERBXRHS1VnIqM5CpjoXTYYmi3MczMGDaamFv7Nvr+7KrTcY8cNvWdh+JB8AEBXohsdmxkPlYtdrz0m2heMMke0SRRF1unoU11+wxKPlV6O+sd1rJIIEPo5eFu1F/Zx84OXg0SsBRXfHGIPRgOyaXCS37PVQrCm1OK60czO31xyiioS9jP+uEQ1WtvRehkEEMYigK2YUjSioL0JGdTYy1FnIVOegUX9x8GCPSGUYhigjEKWKQICzn1XagSWdLcOKX1LR2GyAq6Mcj86MR2yIqs/roP6H4wzRwCOKImq0tS37TlgGFBfPzGslE6TwcfJuE1B4Orh3+98to2hETl0u9DItZHoFwlxCL3mvWm0dUipN7TXTqtIt/k0VICDcLQTxHrGI84yBv5Mv21MTEQDbei/DIIIYRFCXGUUjCutLkFGdiXR1dkvwYPmNk73UFDxEqcIxRBXRr/qQl1Y14JONySgor4cgAHfeGI5p40Ig4Ru5QY3jDNHgIYoiqpvV5lCiqGX/iRJNaZsuTa3kEhl8Hb3h6+QLf+fzIYW7vfKS/76dKDuN9RlboG6uMT+mtHPDPVF3YIT3MACmf1fP1RUgpcK00eS5ugKLezjLnS5orzkEjnJuvExEbdnSexkGEcQggi7LKBpRVF+CDHU20quzkKnORkOb4MEOEcowDFFFIEoZjkBn/37dCkyrM+DrHen4/XQxAOCqCA88fPtQONn33x3XqXdxnCEio2hEVZPaNHui3rQHRYmmBCUNZdAZ9e1eo5Aq4OvoDX8n35Y2o6ZfKjslTpYn48vktR0+36SgG6DRNSClMg31Osv3YsEuAYjziEWcRwxCXAP7TZhPRP2XLb2XYRBBDCKoDaNoRLGm1Ly5ZGZ1NjR6y03A7KQKU/CgjDDPeOjPwUNH9p0swtc70qE3GOHpZo8ls4chxNfF2mWRFXCcIaKOGEUjKhorLZZ2FGtKUaopg/6CFpkXspPYQS/qLVpoXoq91A4x7kNa2mvGwM2O/xYRUdfY0nsZBhHEIIJgFI0o0ZSZN5fMUGe32X1cIVUgwi20ZcZDBIJdAmwyeGhPXkkdlm86jXJ1E2RSCe6bEoUJV/lzze0gw3GGiLrKYDSg3BxQlJjbjZY1lMModm4cGeU9HDcEjEO4WyhkbK9JRFfAlt7LMIggBhGDkCiKKGkoM814aAkeLp4SqpDIEaEMQ5TStMdDsEvggAke2tPQpMNXP6XiRGYFAODaeF88cGs07OQD9zWTJY4zRNRT9EY99uT/jk1Zv1z23IeGzsMY35F9UBURDXS29F6ms0EE41kiGyaKIkobypBenW2e9XBx8CCXyBHhFoooVQSGqMIR7BI4qL6ZcbSX48m7hmHb4XP4/rcs/JFcgrzSOiyZPQy+7twUjIiIOk8mkSHENahT57raufZyNUREtmvwfBohGgBMwUM5MtRZyGgJH+q09RbnyCUyhLuFIqplj4cQ18EVPLRHEARMGxeCcH9XfLo5BYXlGry5KhGLbovFmBhva5dHREQ2JFIZBqWdm0W3jIup7NwQqQzrw6qIiGwLl2YMYFyaYftEUURZY4XFUotabZ3FOTKJDOGuIaY9HlQRCHENgnyQBw+Xoq5vxmebU5CerwYA3DI2CHdPjICsE1PIyDZxnCGinnai7PQlu2Y8Ev+AuYUnEdGVsqX3MtwjghhE2CBRFFHeWGGe7ZBRnY0aba3FOTKJDGGuwaalFsoIhLoGQS5la8quMBiN2PBbNrYePgcAiAxww+Oz4qFysbNyZdQbOM4QUW84UXYa6zO2WMyMUNm54e6oOxhCEFGPsqX3MgM2iDh06BBWrlyJkydPoqGhAf7+/pg6dSoWL14MR8eurfcuKCjAwYMHcfr0aSQnJyM9PR06nQ6zZ8/Ge++9d8lro6OjL3nc09MTBw4c6PD4mTNn8MUXXyAxMRG1tbXw9vbGTTfdhISEBLi7u3fpdXSEQUT/J4oiKhqrkK7OREZ1NjLU2W2mesoEKcLcQhClDEeUKgJhrsEMHnrI8fRyfPVzKhqb9XBxlOPRO+IwNLRn/v5R/zHYxxki6j1G0YiculzoZVrI9AqEuYRCInCGHRH1LFt6LzMgN6tcu3Ytli1bBlEU4evrCz8/P2RmZuLTTz/Fjh07sG7dOiiVyk7fb/Xq1VizZs0V1RQfHw+FQtHm8UvVsWPHDjz77LPQ6XTw8PBAVFQUcnJysHbtWmzbtg3ffPMNgoI6txES2RZRFFHZVIX06mxkqLOQXp3VJniQClKEugZjSMvmkqGuIVAweOgVI4d44TUvJyzfmIxzZfV4/9sTmHVDOKaPD4GELT6JiOgyJIIE0e6RNvMBgYiov7CZICI5ORnvvPMOAODNN9/EnDlzIAgCSktL8fjjjyMlJQVLly7Fxx9/3Ol7qlQqTJw4EcOGDcOwYcOwY8cOfP/9912q68MPP0RgYGCnzy8tLcXzzz8PnU6HhIQELFmyBDKZDHV1dXjmmWewf/9+PP300/j+++8h8IPQgFDZWGXa40GdjfTqLFQ3qy2Om4KHIESpIhClDEe4WwgU0rbhFvUOb5UjXnpgNP7zazr2nyrGxn3ZyCqswcMzhsLZgQEQEREREVFPs5kgYvny5TAajZg1axbmzp1rftzHxwcffPABpk2bhh07diAtLQ0xMTGdumdCQoLF7w8dOtSjNbfnq6++QmNjI8aOHYs///nP5sddXFzw/vvv4+abb0ZycjL27NmDSZMm9Xo91PMqG6vNsx0y1Nmoaqq2OC4RJAh1DcIQpWlzyTC3ENgxeLAqhVyKh26LRWSgG77ekY5TWZV4Y2UiEmbHI8yP7deIiIiIiHqSTQQRGo0G+/fvBwDMmTOnzfHQ0FCMGzcOf/zxB7Zt29bpIMIatm/fDqD91+Hm5oapU6di/fr12Lp1K4MIG1HdpEZ6dZZ5c8nKpiqL4xJBghCXIESpwjFEGYFwZSiDh37qhuH+CPFxwfJNySirbsS7Xydh3s1RmDgygDOUiIiIiIh6iE0EEampqdBqtVAoFBg+fHi754wePRp//PEHTp482ae1LV++HGVlZTAYDPDx8cG4ceNw2223tbtvRHFxMUpLSwEAY8eObfd+Y8aMwfr16/v8dVDnVTepzcssMqqzUNFO8BDsEmhqp6kMR7hbKOxl7MZgK4J9XPDqgrFY8UsqjqWXY+2OdGQU1mDBrTGwU0itXR4RERERkc2ziSAiJycHAODv7w+5vP0128HBwRbn9pUffvjB4vcbN27ERx99hI8//hhxcXEWx3JzcwEAcrkcvr6+7d6vdZPK/Px86HS6Dl8v9R11c01L6GDaYLK8sdLiuAABwa6B5qUWEW4hsJfZW6la6gmO9jIsmR2P7Ufy8f3eLBxKKUV+aT0SZsfDz8PJ2uUREREREdk0mwgiampMXQXc3Nw6PKf1WOu5ve3mm2/GzJkzERMTA19fX2g0Ghw8eBD//Oc/kZ+fj0WLFmHTpk3w8/MzX6NWq821djTNu7XbhtFoRH19PVQq1RXVKZP17xZSra1dOtPipa+om2qQXp2Ns1WZSK/OQllDhcVxU/AQgGj3SAxRRSBSFQYHBg8D0ozrQhEV5IZPNpxGYYUGb60+ij/NGIprhvpYuzTqgv44zhDRwMExhoh620AcZ2wiiGhubgaAS84OaF0K0Xpub1u+fLnF7+3s7DB9+nSMHz8ed911F4qKivCvf/0Ly5YtM5/Tlddx4fndJZEIUKls49tbV1cHqz23urEGKeXpSCnLQErZWRTXlVkcFwQBYcogDPUegnjvIYjxjISjwnr1Ut8ap3JCdJgn/vfrJJzOqsAnG07jXLkGD82Ig7yfB31kyZrjDBENfBxjiKi3DaRxxiaCCDs70/p6nU7X4TlardbiXGtxd3fH4sWL8frrr2Pnzp14++23zbMfuvI6Ljy/u4xGEbW1DVd0j94mlUrg6uqA2tpGGAx903u7trkO6dVZOFtl6mxRorkoeICAQBd/RLtHmPZ5UIXDUX7+L32zxohmjaZPaqX+49m5w7FhbzZ+/CMXP+7PxpnsSjxx5zB4uHE2TH9njXGGiAYPjjFE1NtsaZxxdXXo1MwNmwgiOrPsojPLN/rKyJEjAZiWYqjVavPyigtfhyiK7S7PaF2+IZFI4OzsfMW16PX9+w9qK4PB2Gu11mnrkaHORkZ1S/DQ0DZ4CHD2M3e1iFSGwVHuaHGOrfwcqXfNvjEcYX6u+OqnM8gqrMHSrw7j0TviEBfmbu3SqBN6c5whIuIYQ0S9bSCNMzYRRISGhgIAioqKOtzA8dy5cxbnWtOF9RkMBvN/t9am0+lQXFwMf3//Ntfm5+cDAAIDA7lRZTedDx5Mm0sWa0rbnBPg7NeyuWQ4IpXhcLooeCDqyIgoT7z20Fgs35iMvNI6fPDtCcy8PgwzrguFhC0+iYiIiIguyyaCiNjYWMjlcmi1Wpw6dQqjR49uc05SUhIAYMSIEX1cXVsZGRkATEsrWjefBExdP7y9vVFWVoajR4/ijjvuaHPt0aNHAfSP12Er6rUaZKqzka42dbYo0pS0OcffyRdRKtNSi0hlGJzltrF3BvVPXkoHvPTAKKzbmYHfThRh0+85yCyswSO3D4WLY9vWvUREREREdJ5NBBHOzs64/vrrsWfPHnz33Xdtgojc3FwcOnQIADB16lRrlGim1+uxcuVKAMC4ceMgk1n+iG+99VasXbsW3333XZsgoqamBtu2bQNg/dfRF4yiEWersqGv1UKmVyDMJRQS4fLriTS6BvNSiwx1Ngrri9ucYwoewhGljECUMhzOCgYP1LPkMikWTI1BZIAb1m4/i+ScKryxKhEJs4Yh3N/V2uUREREREfVbNhFEAEBCQgL27t2LzZs3Y9SoUZgzZw4EQUBZWRmeffZZGI1GTJ48GTExMRbXTZo0CQDw/PPP99iH+3/84x+IiIjAlClTLPZxKC4uxltvvYUTJ05AJpNhyZIlba7905/+hPXr1yMxMREffvghnnjiCUilUtTV1eEvf/kL6urqMHToUHPdA9WJstNYn7EF6ubz+34o7dxwT9QdGOE9zOLcBl0DMtQ5pj0e1Fkoqi+BCNHiHF8nHwxRhiNKZQoeXBRXvr8GUWdcN8wPIT4u+GTjaZRWN+Ldr5Nw781RmDQqoMM2vUREREREg5kgiqJ4+dP6h1WrVuG9996DKIrw8/ODSqVCZmYmtFotwsLCsG7dOri7W24aFx0dDQB49913ceedd1ocS0pKQkJCgvn3TU1NaGpqgkKhgKPj+T0DXn31VUyfPt38+4SEBOzatQtSqRRBQUFwc3NDXV0dcnJyIIoi7Ozs8Pbbb7e79AIAtm3bhr/85S/Q6/Xw8PCAr68vcnJy0NDQAE9PT6xbtw4hISFX/PMyGIyoqup/3R1OlJ3Gl8lrOzz+YOxcOMjszbMeCuqL2wYPjt7m0CFKFQ5XhUtvl010SY3Neqz4JRVJZ8sBANcM9cGCqdGwV9hM3jtgyWQSqFROqK7WDJgNnoio/+AYQ0S9zZbGGXd3p4HTNaPVwoULER0djRUrVuDUqVOorKyEv78/pk6disWLF8PJqWvT7/V6vblLxYW0Wq1FG83m5maL4/PmzYOnpyeSk5NRVlaGwsJCyOVyREVFYfz48bj//vsRHBzc4fNOnToVQUFB+Pzzz3H06FGkp6fD29sbd955JxISEuDh4dGl12FLjKIR6zO2XPKcNanftnnMx9ELUcrwlj0eIuBmx+CB+hcHOxkSZsXj18R8rN+bhcNnSnGutA5LZg+DvyeXBhERERERtbKpGRHUNf1xRkR6dRY+PP75Zc9T2rkiziPWvNzCzY5r7sl2ZBSo8emmZKjrtbCTS7FgWjTGDfW1dlmDli19i0BEtodjDBH1NlsaZzo7I+LyZxD1oNrm2k6dNztiOu6LuQtjfEcyhCCbExWoxOsPXY3YEBWadQZ8seUMvt5xFrp+/g8HEREREVFfYBBBfcq1k6FCZ88j6q9cnRT4y9wRmHFtKABg97FCvPefY6isabJuYUREREREVsYggvpUpDIMSju3S56jsnNDpDKsjyoi6j0SiYA7bwzH0/cMh5O9DDnFtXhjVSKSsyutXRoRERERkdUwiKA+JREkuCeq/W4ire6OugMSgX80aeAYHuGJ1xaORaivC+obdfjndyexaX82jEZu0UNEREREgw8/7VGfG+E9DI/EP9BmZoTKzg2PxD+AEd7DrFQZUe/xVDrgb/ePxsSRARABbDmQi3+uP4naBu1lryUiIiIiGkjYNWMA649dMy5kFI3IqcuFXqaFTK9AmEsoZ0LQoHAwuQSrt6dBqzNC5WKHx2fFIzLg0kuWqPtsaadpIrI9HGOIqLfZ0jjDrhnU70kECaLdI3F9yFhEu0cyhKBBY3y8L155cAx83R1RXdeMv//nGH49mg/mwkREREQ0GPCTHxGRFQR6OWPpgjEYG+MNg1HENzsz8NnmFDQ2661dGhERERFRr2IQQURkJQ52Mjw2Mw7zJkdBKhGQmFaGt1YfRWF5vbVLIyIiIiLqNQwiiIisSBAETBkThBfmj4LKxQ4lVQ14a81RHEwpsXZpRERERES9gkEEEVE/EBnghtceGou4UBW0OiO+/PEM1m4/C10/35CIiIiIiKirGEQQEfUTro4KPDNnBO64LhQCgD3HC/Hu10moUDdauzQiIiIioh7DIIKIqB+RSATMuiEcT8+5Ck72MuSW1OGNVYk4lVVh7dKIiIiIiHoEgwgion5oWLgHXn/oaoT5uULTpMf/rT+FDfuyYDSyxScRERER2TYGEURE/ZSHmz1enD8Kk0YFAAB++iMP7397ArUarZUrIyIiIiLqPgYRRET9mFwmwf23RGPxHUNhJ5ciNa8ab6xKRGZBjbVLIyIiIiLqFgYRREQ2YNxQX7yyYAz8PBxRXdeMv687hh2J+RBFLtUgIiIiItvCIIKIyEYEeDph6YIxuDrWGwajiP/uysCnm5LR2Ky3dmlERERERJ3GIIKIyIbYK2R49I44zJ8yBFKJgKNny/Hm6qMoKKu3dmlERERERJ3CIIKIyMYIgoCbRwfixftHwd3VDqVVDXh7zVEcOF1s7dKIiIiIiC6LQQQRkY2K8HfD6w9djfhwd2j1Rvz751Ss3pYGnd5g7dKIiIiIiDrEIIKIyIY5O8jx9D1XYdb1YRAA/HaiCO+sPYZydaO1SyMiIiIiaheDCCIiGycRBNxxfRiemXsVnB3kyCutwxsrE3Eis8LapRERERERtcEggohogIgP88DrD41FhL8rGpr1+Oj7U/jhtywYjEZrl0ZEREREZMYggohoAHF3tccL80dh8uhAAMDPB/Pw/n9PoEajtXJlREREREQmDCKIiAYYmVSC+6YMwWMz42CnkCLtnBqvrzyC9Hy1tUsjIiIiImIQQUQ0UF0d64NXF4xBgKcTauq1+J91x7Ht8DmIomjt0oiIiIhoEGMQQUQ0gPl5OOGVB8dgXJwPjKKI7/Zk4pONyWho0lu7NCIiIiIapBhEEBENcHYKKR6ZMRQP3BoNmVTAsfRyvLk6EedK66xdGhERERENQgwiiIgGAUEQcNPIAPzt/tHwcLVHWXUjlq1Nwu+niq1dGhERERENMgwiiIgGkTA/V7z20FgMC/eATm/Eil9SsWprKrQ6g7VLIyIiIqJBgkEEEdEg4+wgx5/vGY7ZN4RBALDvZDHe+ToJZdUN1i6NiIiIiAYBBhFERIOQRBBw+3VhePbeEXBxlONcaT3eWHUUx9PLrV0aEREREQ1wDCKIiAaxuFB3vLZwLCID3NDYrMfHG05j/Z5MGIxGa5dGRERERAMUgwgiokHO3dUez983EreMDQIAbD18Dv/45gRq6putXBkRERERDUQMIoiICDKpBPfeHIWEWfGwV0hxNl+N11cm4uy5amuXRkREREQDDIMIIiIyGxPjjVcXjkWAlxNqNFr87zcnsPVQHkRRtHZpRERERDRAMIggIiILvu6OeOWBMRgf5wujKGL93iz8a8NpNDTprF0aEREREQ0ADCKIiKgNO4UUD8+IxYNToyGTCjieUYE3ViUir6TO2qURERERkY1jEEFERO0SBAETRwTgpQdGw9PNHuXqJixbm4R9J4usXRoRERER2TAGEUREdEmhvq547aGxuCrCA3qDEau2pmHFz6lo1hmsXRoRERER2SAGEUREdFlO9nI8efdw3DUhHIIA/H66GMvWJKG0usHapRERERGRjWEQQUREnSIRBEwfH4rn7h0JV0c5Csrr8eaqRCSdLbd2aURERERkQwTRxnqyHTp0CCtXrsTJkyfR0NAAf39/TJ06FYsXL4ajo2OX7lVQUICDBw/i9OnTSE5ORnp6OnQ6HWbPno333nuvw+uqq6uxc+dO/PHHH0hJSUFxcTEkEgn8/Pxw/fXXY+HChQgMDGz32o8//hj/+te/LlnX66+/jnnz5nXptbTHYDCiqkpzxffpTTKZBCqVE6qrNdDrjdYuh4g6qbquGZ9tTkZGQQ0AYOrVwbhzQjhk0v6Xb3OcIaLexDGGiHqbLY0z7u5OkHbi/aCsD2rpMWvXrsWyZcsgiiJ8fX3h5+eHzMxMfPrpp9ixYwfWrVsHpVLZ6futXr0aa9as6XIdS5YsQVJSEgDAyckJERERaGpqQn5+PtauXYsffvgBH374IW688cYO7+Hh4YGQkJB2j3l5eXW5JiKivqRyscNf543ED79lYfuRfGw7cg7ZRTV4dGY8VC521i6PiIiIiPoxmwkikpOT8c477wAA3nzzTcyZMweCIKC0tBSPP/44UlJSsHTpUnz88cedvqdKpcLEiRMxbNgwDBs2DDt27MD3339/2eukUilmzJiBe++9F6NGjYJUKgUAlJSU4KWXXsKBAwfwzDPPYPv27fD09Gz3HjfeeOMlZ10QEfV3MqkEcydFITLADSt+SUV6QQ3eWJWIR++IQ2yIytrlEREREVE/1etBhMFgwDfffIMDBw5AIpFg4sSJuOeee7p8n+XLl8NoNGLWrFmYO3eu+XEfHx988MEHmDZtGnbs2IG0tDTExMR06p4JCQkWvz906FCnrvvoo4+gUrV9k+3r64sPP/wQt9xyC6qqqvDTTz9h4cKFnbonEZGtGh3tjUAvZ3yyMRkF5fX4x3+P484bwzFtXAgkgmDt8oiIiIion+mRxbzff/89YmNj8fTTT7c59uyzz2LZsmXYu3cvdu3ahVdffRXPPPNMl+6v0Wiwf/9+AMCcOXPaHA8NDcW4ceMAANu2bev6C+ii9kKIVi4uLhgxYgQAICcnp9drISLqD3zcHfHKg6Nx/TA/iCLww2/Z+Pj7U9A06axdGhERERH1Mz0SRBw4cAAAMGPGDIvHDx8+jO3bt0MURYwcORLXXnstAFNYsHPnzk7fPzU1FVqtFgqFAsOHD2/3nNGjRwMATp482Z2X0KOam5sBAA4ODh2ek5aWhr/85S948MEH8fjjj+P//u//kJGR0VclEhH1OIVcikXTY7FwWgxkUglOZlXijZWJyCups3ZpRERERNSP9EgQkZqaCgAYNWqUxeObNm0CYJrFsG7dOqxYsQJPPvkkRFHExo0bO33/1pkF/v7+kMvl7Z4THBxsca61lJaW4siRIwCAMWPGdHheamoqfvrpJxw+fBi7d+/Gp59+ittvvx3vvPMODAZDX5VLRNTjbrzKHy8/MBpeSntU1DRh2dok7D1RCBtr0kREREREvaRH9oiorq6GQqGAu7u7xeMHDx6EIAh44IEHzI/Nnz8fH330EZKTkzt9/5oaU3s4Nze3Ds9pPdZ6rrW89dZb0Ol0iIyMxE033dTmuLe3N5566inccMMNCAwMhLOzM3JycrBu3Tr897//xerVqyGTyfD888/3SD0yWf9rpXeh1tYunWnxQkS2IyLQDW8+fA2+2JKC4+kVWLPtLLIKa7HwthjYyaV9WgvHGSLqTRxjiKi3DcRxpkeCCI1GA0dHR4vHysrKUFJSAk9PT0RFRZkfd3Nzg7OzM6qqqjp9/9alDh3NhgAAhUJhca41fPHFF/j1118hl8vx3nvvmbtpXOjCjTZbRUdH44033kBgYCD+8Y9/YPXq1bjvvvsQGBh4RfVIJAJUKqcrukdfcXXteBkLEdkmFYA3Fl+LDXsyseaXMzhwuhiFFRq8uGAsAryc+7wejjNE1Js4xhBRbxtI40yPBBHOzs6oqalBY2OjeV+ExMREAMDIkSPbvcbOrvN95lvP1ek63vRMq9V2+b49aePGjfjggw8gCAKWLVuGYcOGdfkeixYtwpo1a1BWVobdu3fjwQcfvKKajEYRtbUNV3SP3iaVSuDq6oDa2kYYDEZrl0NEvWDSSH/4uzvgkw2nkVtci6c/2IuHbx+Kq2N9+uT5Oc4QUW/iGENEvc2WxhlXV4dOzdzokSAiKioKR48exdatW3HnnXcCMO0PIQgCxo4da3FuXV0d6uvrERoa2un7d2bZRWeWb/SWX375BS+//DJEUcTrr7+OmTNndus+UqkUV111FX799Vfk5eX1SG16ff/+g9rKYDDaTK1E1HWRAW547aGx+GxzCtLz1fjXD6dxy1g17p4YAVkfTTPkOENEvYljDBH1toE0zvRIEDFjxgwkJibizTffxMmTJ1FRUYH9+/dDoVBg2rRpFuceP34cALoURLSeW1RUBJ1O1+4SjXPnznX5vj3h119/xV//+lcYDAa88MILmDdv3hXdr/W16fX6niiPiKjfUDrb4a/zRmDDb9nYevgcdiTmI7uoFo/PiofKxTqz2YiIiIio7/XI11B33303rr32WjQ1NeG7777Drl27IAgCnn76aXh5eVmcu23btnZnSlxKbGws5HI5tFotTp061e45SUlJAIARI0Z0+3V01W+//YZnnnkGer0eTz31FBYtWnTF92xt4enr63vF9yIi6m+kEgnuuSkST945DA52MmQW1uD1lUdwJrfz+wYRERERkW3rkRkRUqkUX331FX766SccP34crq6uuPHGGzF69GiL87RaLcrLyzFmzBjceOONnb6/s7Mzrr/+euzZswffffddm/vm5ubi0KFDAICpU6de+QvqhIMHD+LJJ5+ETqfDo48+iiVLllzxPffu3WsOIq677rorvh8RUX81cogXXvNywvKNyThXVo/3vz2BWTeEY/r4EEgEwdrlEREREVEv6pEgAgAkEgnuuOMO3HHHHR2eo1Ao8OWXX3br/gkJCdi7dy82b96MUaNGYc6cORAEAWVlZXj22WdhNBoxefJkxMTEWFw3adIkAMDzzz/fYyHF8ePHkZCQgObmZixcuBDPPvtsp67LyMjA2rVrcd9991nUaTQa8csvv+C1114DANx0000YPnx4j9RKRNRfeasc8dIDo/GfX9Ox/1QxNu7LRlZhDR6eMRTODh13SSIiIiIi2yaIoihau4jOWrVqFd577z2Iogg/Pz+oVCpkZmZCq9UiLCwM69atg7u7u8U10dHRAIB3333XvJFmq6SkJCQkJJh/39TUhKamJigUCot2pK+++iqmT59u/v2tt96K3Nxc8+aSHRk6dCiWLl1q/n1qaipmzZoFAFAqlfD394dUKsW5c+fMm22OGTMGn376KVxdXbv402nLYDCiqkpzxffpTTKZBCqVE6qrNQNm4xUi6rr9p4rw9Y506PRGeLjaI2F2PML8rnwcBDjOEFHv4hhDRL3NlsYZd3envuuacTl79uzBgQMHIJFIMGHChG4vO1i4cCGio6OxYsUKnDp1CpWVlfD398fUqVOxePFiODk5del+er0earW6zeNardbcDhQAmpubLY63thE1GAw4duxYh/eXySx/vAEBAXj66adx4sQJZGVlIS8vD1qtFm5ubrjxxhsxY8YMzJgxA1KptEuvg4jI1t0w3B8hPi5YvjEZZepGvPt1EubdHIWJIwMgcKkGERER0YDSIzMiduzYgb///e+47rrr8Oabb1oce/fdd7FmzRqLxxYuXIgXXnjhSp+WLoMzIojI1jQ06bHil1QcSy8HAIyL88GCW2Ngp+h+QMtxhoh6E8cYIupttjTOdHZGRI90zdi9ezeKioowZswYi8dTUlKwevVq81KK4OBgiKKIVatW4fDhwz3x1ERENIA42suwZHY85twUCYkg4FBKKd5ecxTFlf07VCUiIiKizuuRIOL06dMAgPHjx1s8/sMPPwAApkyZgp07d2L79u2YP38+RFHEd9991xNPTUREA4wgCJh6TTCev28k3JwVKKzQ4M3VR3EktdTapRERERFRD+iRIKKqqgpSqRReXl4Wjx84cACCIOCRRx6BRGJ6qkcffRQAcOLEiZ54aiIiGqCGBCnx+sKxiAlWollrwGebU7Du13ToDf17SiIRERERXVqPBBF1dXVtNoqsrq5GXl4eXF1dLVpRent7w8HBAeXl5T3x1ERENIC5OdvhL/eOwPTxIQCAnUkF+Pu6Y6iqbbJyZURERETUXT0SRDg6OqKurs7cTQIwtcYEgBEjRrQ5Xy6XszMEERF1ilQiwV0TIvDUXcPhaCdDVmEtXl+ZiJScKmuXRkRERETd0CNBRHh4OERRxG+//WZ+bOvWrRAEAaNHj7Y4t7GxEXV1dW2WcRAREV3KiChPvPrQWIT4uKC+UYcPvj2BLb/nwHjlzZ+IiIiIqA/JeuImU6ZMwYkTJ/DKK68gOzsb5eXl+OWXXyCRSDBt2jSLc0+fPg1RFBEYGNgTT01ERIOIt9IBLz0wCut2ZuC3E0XY9HsOMgtr8MjtQ+HiqLB2eURERETUCT0yI+L+++9HdHQ01Go1/vnPf2Lt2rUQRRH3338/goKCLM7dsWMHBEFo0+qTiIioM+QyKRZMjcGfpsdCIZMgOacKb6xKRHZRrbVLIyIiIqJO6JEZEXZ2dli3bh1Wr16NEydOwMXFBTfddBNmzJhhcZ5Wq0ViYiL8/Pxw/fXX98RTExHRIHXdMD+E+Ljgk42nUVrdiHe/TsK9N0dh0qgACIJg7fKIiIiIqAOCKHJx7UBlMBhRVaWxdhmXJJNJoFI5obpaA72eLfmIqOsam/VY8Usqks6aujFdM9QHC6ZGw14hg9EoIquoBjpRgFwQEeHvBomEIQUR9Ry+lyGi3mZL44y7uxOk0ssvvOiRGRFERETW4mAnQ8KsePyamI/1e7Nw+EwpzpXWYeKIAGw7cg7Vdc3mc1UudrhvchRGR3tbsWIiIiKiwa1XZkTU19fjzJkzqKysBAB4eHhg6NChcHZ27umnokvgjAgiGmwyCtT4dFMy1PXaS563ZHY8wwgi6hF8L0NEvc2WxhmrzIg4e/Ys/vnPf2L//v0wGi1/QBKJBBMmTMCf//xnREdH9+TTEhERAQCiApV4dcFYPP/ZH9AbOs7Zv9mZgZFRXlymQURERGQFPdI1AzB1w5gzZw5+++03GAwGiKJo8ctgMGDPnj2YM2cOfv311556WiIiIgslVQ2XDCEAoKquGen56r4piIiIiIgs9MiMiPz8fDz33HPQarUICAjAww8/jOuuuw6+vr4AgJKSEhw4cAD//ve/UVBQgOeeew4//fRTm9aeREREV0qtab78SV04j4iIiIh6Vo/MiPj3v/8NrVaLESNGYMuWLZg3bx6Cg4OhUCigUCgQHByMefPmYcuWLRgxYgS0Wi1WrlzZE09NRERkQelk16nzdhzJx+nsSrB5FBEREVHf6pEg4uDBgxAEAW+88QacnJw6PM/R0RFvvPEGRFHEgQMHeuKpiYiILAwJUkLlcvkwIrekDv/87iRe+eow9p4ohFZn6IPqiIiIiKhHgoiSkhI4OTl1ahPK6OhoODs7o6SkpCeemoiIyIJEIuC+yVGXPOeBW4Zgypgg2CukKK5swJptZ/Hc8j+wYV821PVcskFERETUm3pkjwiZTAa9Xt+pc0VRhE6ng0zWow07iIiIzEZHe2PJ7His25mB6rrzwYK7ix3mTY4yt+6cdUMY9p8sws6kAlTUNOGnP3Kx9VAerhnqg1vGBiHYx8VaL4GIiIhowOqRNCAkJASpqanYv38/brjhhkueu3//fjQ3NyMiIqInnpqIiKhdo6O9MTLKC1lFNdCJAuSCiAh/N4uWnQ52MtxydTBuHhOI4+kV2HE0H5kFNfgjuQR/JJcgJliJKWODcFWkJyQCW30SERER9YQeCSImTZqEM2fOYOnSpfj3v//dYciQmZmJV199FYIg4Oabb+6JpyYiIuqQRCIgNtQdKpUTqqs10OuN7Z4nlUgwJsYbY2K8kV1Uix2J53A0rRxp59RIO6eGt8oBU8YE4bphvrBXcEYfERER0ZUQxB7YLry+vh7Tp09HaWkp5HI5pk6divHjx8PHxweAaQ+JgwcPYvv27dDpdPD19cVPP/0EZ2fnK34B1DGDwYiqKo21y7gkmUxy2Q8IRERXorvjTFVtE3YlFeC3E0VoaDYtP3S0k+HGEf6YPDoQ7q72vVUyEdkQvpchot5mS+OMu7sTpNLLb0XZI0EEAGRkZOCxxx5DYWEhhA6mr4qiiMDAQHz66aeIirr0RmJ05RhEEBFd+TjTpNXjwOkS/Ho0H2XVjQAAiSBgTIwXbhkbjHB/154umYhsCN/LEFFvs6Vxps+DCADQaDT4z3/+g23btuHs2bMwGEyt0KRSKaKjo3Hbbbdh3rx5l2zxST2HQQQRUc+NM0ZRxKnMSuxIPIe0c2rz45EBbrhlbBBGDvGEVNIjzaiIyIbwvQwR9TZbGmesEkRcSKfToaamBgDg5uYGuVwOAKirq8ODDz4IQRCwYcOG3nhqasEggoiod8aZc6V1+DUxH4fOlMJgNP0z6uFqj8ljAnHDcH842nMfCaLBgu9liKi32dI4Y/UgoiPV1dUYP348BEFAampqXz71oMMggoiod8cZdX0zdh8rxN7jhahv1AEA7BVSXD/cD5PHBMFb6dCjz0dE/Q/fyxBRb7OlcaazQQS/siEiIuompbMd7rwxHDPGh+BgSgl+PVqAogoNdh4twK6kAoyK8sKUsUGICnTrcP8kIiIiosGGQQQREdEVUsilmDAiADde5Y+UnCrsSMxHck4VktLLkZRejlBfF9wyNghjYrwh68S3BEREREQDGYMIIiKiHiIIAuLDPRAf7oHCCg1+TczHwZQS5JbU4Ysfz2D93ixMGhWACSMC4Owgt3a5RERERFbBIIKIiKgXBHg6YeG0GNw5IRx7jxdi97FCVNc144ffsvHjH7m4Lt4PU8YGwdfd0dqlEhEREfUpBhFERES9yNVRgTuuC8O0a0JwJLUUOxLzkV9Wjz3HC7HneCGGR3jg1rFBiAlRcR8JIiIiGhQYRBAREfUBuUyC64b54dp4X6SdU+PXxHyczKzAqaxKnMqqRKCXM24ZG4RrhvpALuM+EkRERDRwMYggIiLqQ4IgIDZEhdgQFUqrGvDr0Xz8froYBeX1WPFLKr7/LQuTRgZg4qgAuDoqrF0uERERUY8TRFEUu3pRbGzsFT2pKIoQBAGpqalXdB+6NIPBiKoqjbXLuCRb6olLRLbJFsYZTZMO+04UYWdSAarrmgEAMqkE4+N8MGVsEAK9nK1cIRF1xBbGGCKybbY0zri7O0HaiQ5h3ZoR0Y3sgoiIiDrgZC/HtHEhmDI2CEfPluHXxHzkFNdh/6li7D9VjLhQFaaMDUZ8uDsk3EeCiIiIbFy3gognnniip+sgIiIa9GRSCcYN9cU1sT7ILKzBjsR8HEsvR0puNVJyq+Hn4YgpY4IwPt4XdnKptcslIiIi6pZuLc0g28ClGUREtj/OlKsbsSupAPtOFqFJawAAODvIMXGkPyaNCoTS2c7KFRINbrY+xhBR/2dL40xnl2YwiBjAGEQQEQ2ccaaxWY/9p4qx82g+KmqaAABSiYCrY31wy9gghPi6WLlCosFpoIwxRNR/2dI4wyCCGEQQEWHgjTNGo4hj6eXYcTQfmQU15sejg5S4ZWwQror0hETCfSSI+spAG2OIqP+xpXGmVzerJCIiIuuQSASMifHGmBhv5BTXYkdiPo6mleFsvhpn89XwVjpg8phAXD/cD/YK/jNPRERE/Q9nRAxgnBFBRDQ4xpmq2ibsOlaAfSeKoGnSAwAc7GSYMMIfk0cHwt3V3soVEg1cg2GMISLrsqVxhksziEEEEREG1zjTrDXgQHIxfk3MR2l1IwBAIggYE+OFKWODEOHvZuUKiQaewTTGEJF12NI4w6UZREREg4ydQopJowIxcWQATmVVYseRc0g7p8aR1DIcSS1DRIArbhkbjFFDPCGVXP5NAhEREVFvsLkg4tChQ1i5ciVOnjyJhoYG+Pv7Y+rUqVi8eDEcHR27dK+CggIcPHgQp0+fRnJyMtLT06HT6TB79my89957l71eo9Hgiy++wPbt21FUVARHR0dcddVVWLRoEa655po+ex1EREQXkggCRkR6YkSkJ86V1uHXxHwcTi1FVmEtPi1MhoerPW4eHYgbr/KHo73NvRUgIiIiG2dTSzPWrl2LZcuWQRRF+Pr6wt3dHZmZmdBqtYiIiMC6deugVCo7fb9ly5ZhzZo1bR7vTBBRVVWF++67Dzk5OVAoFIiMjERVVRVKSkogCAKWLl2K+fPn98nr6AiXZhARcZxpVVPfjN3HCrHneCHqG3UATDMobhjuh8ljguCtdLByhUS2iWMMEfU2WxpnOrs0w2bmZSYnJ+Odd94BALz55pvYu3cvNm7ciJ07dyIuLg5ZWVlYunRpl+6pUqkwceJEPPnkk/jiiy9w9913d/ral19+GTk5OYiLi8POnTuxceNG7N27F2+++SZEUcSyZcuQmpraJ6+DiIjoctyc7TD7xnD8I+FaLJwWA39PJzRrDdh5tAB/++wg/rXhNNLz1bCh7yeIiIjIRtlMELF8+XIYjUbMnDkTc+fOhSCYeqT7+Pjggw8+gEQiwY4dO5CWltbpeyYkJODzzz/HE088gQkTJsDV1bVT1505cwa7d++GRCLBP//5T/j4+AAABEHA3LlzMXPmTBgMBixfvrxPXgcREVFnKeRS3HiVP97609V4du5ViA93hwjgWHo53vvPMby5+igOppRAb+jf37gQERGR7bKJIEKj0WD//v0AgDlz5rQ5HhoainHjxgEAtm3b1uv1bN++HQAwbtw4hPx/e3ceHmV59n38O5N9ISvZF0hYwpKwBQQVUQERquKOVau1tWJFrX3U56m2ouJe62tVrEutLEWpaKvSWoEgoCIIhLAGAoTsGwkkIfs+8/4REolJYIBMJjP8PsfBIbnve64578icmTlzXdc5YECn87fccgsA33zzDbW1te3H+9p9iIjI+ctgMBAfE8jDc8bw7K8mMmV0OC7ORnKOVPHef/bzf29v5r/fZ7cv4xARERHpKXZRiEhLS6OxsRFXV1dGjRrV5TWJiYkA7N692+rx7Nq1C4Dx48d3eX7UqFG4urrS0NDQYXlGX7sPERERgIj+Xtw1axivzLuI6y+JwdfLlePVjfzrm0we/csm/r7mIEWlfXvPIREREbEfdlGIyMrKAiA8PBwXF5cur4mOju5wrTVlZ2d3eM4fc3FxISwsrFM8fe0+RERETtbP05VrLo7h5fsu4u6rhhMd7E1js4mvdxbwh/e28tonu9mfXaZ9JEREROSc2EXProqKCgB8fX27vabtXNu1fSWeysrKs3pcT92Hs3PfrjW17ahqyc6qIiJnQ3nmzDk7G7l0bARTxoRzIKec1dty2XXoGHsyStmTUUpUsDdXXhDNpPgQXJ2dbB2uiE0px4iItTlinrGLQkRDQwNAt7MIAFxdXTtc21fiqa+vP6vH9cR9GI0G/P29znmc3uDjo7ZxImJdyjNn56IAby4aG0Xh0Wr+szGTr5JzySup5m9f7OefX2fwk4sGMuuiGPz6udk6VBGbUo4REWtzpDxjF4UIN7fWNzdNTd1vmNXY2NjhWmvHU1dXZ1E87u7uHR4HvXcfJpOZysra019oQ05ORnx8PKisrKNFO7SLiBUoz/QMD2cDcy4fxFWTovl6ZwFrk/Moq2pgedJBPlmXzoXxocycGE1ksLetQxXpVcoxImJt9pRnfHw8LJq5YReFCEuWK1iy7KGn+Pj4UFdXZ1E8J7cEtcV9NDf37X+obVpaTHYTq4jYJ+WZnuHm4sSVF0QzLTGSlINHSUrOI6uokm93F/Lt7kJGDPRnxoRo4mMDMJ5oUS1yPlCOERFrc6Q8YxeFiIEDBwJQWFhIU1NTl0sbcnNzO1xr7XiKi4vJycnp8nxTUxOFhYWd4ulr9yEiInK2nJ2MTBwRwgXDg8koqCQpOZeUQ0fZn13O/uxywgI9uWJ8FBfGh+Lmon0kRERE5Ad2sdvF8OHDcXFxobGxkT179nR5TUpKCgBjxoyxejxtz9H2nD+2Z88empqacHNzY/jw4e3H+9p9iIiInCuDwcDgSF/mXZ/AH++9kBkTonB3daKotJa/rznIo3/ZxL++yaC8yvp7OImIiIh9sItChLe3N5MnTwbg448/7nQ+OzubLVu2ADBz5kyrx3PllVcCsHXr1i5nRaxYsQKAKVOm4OX1w2aRfe0+REREelJ/Pw9+Om0I/+/+i/nptCH093Wnpr6Z/36fw/+9vZn3/rOPnCNVtg5TREREbMwuChEA8+bNw2AwsHLlSlasWNHew7ykpISHH34Yk8nE9OnTGTZsWIfHTZ06lalTp7J69eoei2XkyJFcfvnltLS08D//8z+UlJQAYDabWbFiBStXrsRoNHLffff12H2IiIjYCw83Z2ZMiOKley/k/uvjGRLpS4vJzPf7ilmwJJmXPtzBjkNHMZnMtg5VREREbMBgbvskbAeWLFnCSy+9hNlsJiwsDH9/fw4fPkxjYyMxMTEsX76cgICADo+Ji4sD4MUXX+SGG27ocC4lJYV58+a1f11fX099fT2urq54enq2H3/yySe56qqrOjy2rKyMW2+9lezsbFxdXRk8eDDl5eUUFRVhMBj4wx/+wB133NFj93E2WlpMlJXVnPM41uTsbMTf34vy8hqH2XhFRPoW5Zm+IauokrXJeSQfKKHlRAEi2M+DaeMjmZwQhoebXWxbJdKJcoyIWJs95ZmAAC/H6ZrR5q677iIuLo5FixaxZ88eSktLCQ8PZ+bMmcydO7fDMghLNDc3c/z48U7HGxsb29toAjQ0dF7XGhAQwL/+9S/ee+89Vq9ezeHDh/H09GTKlCncfffdTJo0qdfuQ0REpK+LCfNh7uyR3HTZINbvKOCbXQWUHK/jH1+l8/nGLC4dHc60xEgCfd1PP5iIiIjYNbuaESFnRjMiRESUZ/qqhsYWNqcWkbQ9n+KyWgCMBgOJcUHMmBDFoAjrt+MW6QnKMSJibfaUZxxyRoSIiIg4BjdXJy4fF8mlYyPYk1HK2uQ80nLKST5QQvKBEgaF+3DFhCgS44JwMtrNllYiIiJiARUiRERExGaMBgNjBvdnzOD+5BZXsXZ7Hlv3F5NRWEnGyn0E+rgxLTGKKaPD8XTX2xYRERFHoKUZDkxLM0RElGfsUUV1Axt2FrBhZwFVtU1A6wyKSxLCmD4+kmB/z9OMINJ7lGNExNrsKc9YujRDhQgHpkKEiIjyjD1ram7h+33FrE3Oo+BY688zAzBmSH9mTIhiaJQfBoPBtkHKeU85RkSszZ7yjPaIEBEREbvm4uzElNHhXDIqjP3Z5axJziU1s4yd6cfYmX6MASH9mDEhignDg3G24E2PiIiI9A2aEeHANCNCRER5xtEUHqth7fY8NqceoenE/08/b1emjovksrEReHu42DhCOd8ox4iItdlTntHSDFEhQkQE5RlHVVXbyNe7Clm/I5+K6kYAXJ2NXBQfyhUToggL9LJxhHK+UI4REWuzpzyjQoSoECEigvKMo2tuMbEtrZik5Dxyi6vbj48aFMgVE6IYMcBf+0iIVSnHiIi12VOe0R4RIiIi4vCcnYxcFB/GhSNDOZh7nKTkPHYfPsaejFL2ZJQSGeTFFeOjmDQyBBdnJ1uHKyIiImhGhEPTjAgREeWZ81FxeS1fJefz3d4iGppaAPDxdOGysRFcPi4SXy9XG0cojkQ5RkSszZ7yjJZmiAoRIiIoz5zPauqb+HZ3IetS8imrbADA2cnApBGhzJgQRWSwt40jFEegHCMi1mZPeUaFCFEhQkQE5Rlp3Udix6GjJCXnkVlY2X58xEB/ZkyIIj42EKP2kZCzpBwjItZmT3lGe0SIiIiI0LqPxAXDQ7hgeAiHCypISs4j5WAJ+7PL2Z9dTmiAJ1dMiOKi+FDcXLSPhIiIiLWpECEiIiLnjcERvgyO8OXY8Tq+Ssln455CjpTVsmzNQT79JoPLxkYwdVwk/v3cbB2qiIiIw9LSDAempRkiIsozcmp1Dc18t6eItdvzOFZRD4CT0cCE4cHMmBDFwFAfG0cofZ1yjIhYmz3lGe0RISpEiIigPCOWMZnM7Ew/xtrkXA7lV7QfHxrpy4wLohkzuD9Go/aRkM6UY0TE2uwpz2iPCBERERELGY0GEuOCSIwLIvtIJUnJeSSnlXAov4JD+XsJ8nNn+vgoJieE4eGmt08iIiLnQjMiHJhmRIiIKM/I2SuvamD9jny+3llATX0zAB5uTkwZHc60xEj6+3rYOELpC5RjRMTa7CnPaGmGqBAhIoLyjJy7hsYWNqcWkbQ9n+KyWgCMBgPj4oKYMSGKwRG+No5QbEk5RkSszZ7yjJZmiIiIiPQAN1cnLh8XyaVjI9ibUUpSch5pOeVsP1DC9gMlxIb7MGNCFIlxQTgZT//mS0RE5HynQoSIiIiIBYwGA6MH92f04P7klVSzNjmPLfuPkFlYyTsr9xHo48a0xCimjA7D093F1uGKiIj0WVqa4cC0NENERHlGrKuippENO/LZsLOAqtomANxcnJg8KowrxkcS7O9p4wjF2pRjRMTa7CnPaI8IUSFCRATlGekdTc0tfL+vmLXJeRQca/3ZawDGDOnPjAlRDI3yw2BQ+09HpBwjItZmT3lGe0SIiIiI9BIX59ZuGpeMCmN/djlJyXnszSxlZ/oxdqYfIzrEmxkTorhgeAjOFrxBExERcWSaEeHANCNCRER5Rmyn8FgNX23PY3PqERpP/Nvz9XZl2rhILhsbgbeH9pFwBMoxImJt9pRntDRDVIgQEUF5Rmyvuq6Jr3cWsG5HPhXVjQC4Ohu5KD6UKyZEERboZeMI5Vwox4iItdlTnlEhQlSIEBFBeUb6juYWE8lpJaxJziW3uLr9eEJsIDMmRDFioL/2kbBDyjEiYm32lGe0R4SIiIhIH+LsZOTC+FAmjQzhUN5xkpLz2JV+jL2ZpezNLCUiyIsrxkdx4cgQXJydbB2uiIiI1WhGhAPTjAgREeUZ6duKy2v5ans+3+0poqGpBYB+ni5cPjaCy8dF4uvlauMI5XSUY0TE2uwpz2hphqgQISKC8ozYh9r6Jr7dXcS6lDxKKxsAcHYyMHFECDMmRBMV7G3jCKU7yjEiYm32lGdUiBAVIkREUJ4R+9JiMpFy8Chrk/PIKKxsPz58gD8zJkSRMCgQo/aR6FOUY0TE2uwpz2iPCBERERE742Q0csHwEC4YHsLhggqSkvNIOVhCWk45aTnlhAZ4csX4SC6KD8PNVftIiIiIfVIhQkRERKQPGhzhy+AIX45V1LEuJZ9vdxdypKyWZUmH+PTbTC4dE8G0xEj8+7nZOlQREZEzoqUZDkxLM0RElGfEcdQ1NPPd3iK+2p7H0eP1ADgZDUwYFswVE6KICfOxcYTnJ+UYEbE2e8ozWpohIiIi4kA83Jy5YnwU08ZFsuvwMZKS8ziUd5wt+4vZsr+YIZG+zJgQzdgh/TEatY+EiIj0XSpEiIiIiNgRo9HAuKFBjBsaRPaRStYm57EtrYT0/ArS8/cS5OfO9MQoJo8Kw8NNb/VERKTv0dIMB6alGSIiyjNyfiivamD9jny+3llATX0zAB5uTlwyKpzpiZH09/OwcYSOSzlGRKzNnvKM2neKChEiIijPyPmloamFzalHWJucx5GyWgAMBkgcGsSMC6IZHOFr4wgdj3KMiFibPeUZ7REhIiIicp5xc3Hi8rERXDomnNTMUpKS89ifXc72g0fZfvAoseE+XDE+isS4IJwteKMoIiJiDSpEiIiIiDgYo8HAqEH9GTWoP/kl1SRtz2PLviNkFlby7r/3EeDjxrTESC4dHY6nu4utwxURkfOMlmY4MC3NEBFRnhFpU1HTyIYd+WzYWUBVbRPQOoNickIY0ydEEuLvaeMI7ZNyjIhYmz3lGe0RISpEiIigPCPyY03NLWzZV0zS9jwKjra+TzAAowf3Z8aEKOKi/TAY1P7TUsoxImJt9pRntEeEiIiIiHTi4uzEJaPDmTwqjP055axNzmNPRim7Dh9j1+FjRId4c8X4KCaOCNE+EiIiYhV2NyNiy5YtLF68mN27d1NbW0t4eDgzZ85k7ty5eHqe3ZTCNWvW8MEHH3DgwAGampoYMGAAs2fP5s4778TFpfO6yTvuuINt27ZZNPbBgwc7fL1w4ULefPPNUz7m6aef5tZbb7X8BrqhGREiIsozIpYoKq1h7fZ8Nu8tovHE68TXy5WpiZFcNiacfp6uNo6w71KOERFrs6c845AzIpYtW8bzzz+P2WwmNDSUsLAwDh8+zNtvv01SUhLLly/Hz8/vjMb84x//yKJFiwCIjo7Gw8OD9PR0Xn75ZTZs2MCiRYtwde34w3fo0KE0Nzd3O+ahQ4eorq5m7Nix3V4TGBjIgAEDujwXFBR0RvcgIiIici7CAr2488o4bpgSyze7CvgqJZ+K6kY++zaTLzZnc1F8KFeMjyK8v5etQxUREQdgN4WI1NRUXnjhBQCeeeYZ5syZg8FgoLi4mPvuu499+/Yxf/58Fi5caPGYa9eubS80vPbaa0ybNg2AjIwM5s6dS3JyMq+++iqPPfZYh8fNnz+/2zFra2u5+OKLAbjxxhu7vW7KlCm89NJLFscqIiIiYm3eHi5cdeFArrwgmuS0EpKS88gpruKbXYV8s6uQ+NgAZkyIYuTAAO0jISIiZ81uFv699dZbmEwmrr32Wm655Zb2H34hISG8+uqrGI1GkpKSOHDggMVjti2RuOeee9qLEACDBg3iueeeA+DDDz+krKzM4jHXrFlDbW0tHh4ezJo1y+LHiYiIiPQVzk5GLowP5cm7xvO728Yydkh/DEBqZhmvrtjNk+9v49vdhTQ2tdg6VBERsUN2UYioqalh48aNAMyZM6fT+YEDBzJp0iQAVq9ebdGY2dnZ7UWLW265pdP5Cy+8kAEDBtDY2Mi6dessjvXTTz8FYPr06Xh7e1v8OBEREZG+xmAwEBftz4M3juLFeycxPTESN1cnCo7VsGTVAR59azOffZtJRXWDrUMVERE7YhdLM9LS0mhsbMTV1ZVRo0Z1eU1iYiKbN29m9+7dFo25a9cuAKKioggJCel2zJycHHbv3s3NN9982jHz8/NJTk4GTr0sA+DAgQM88sgjHD16FC8vL+Li4rjqqqsYMmSIRfGLiIiI9KZgf09uu2Io110Sw7e7i1iXkkdpZQP/2ZzNqq05TBwRwhXjo4gO6WfrUEVEpI+zi0JEVlYWAOHh4V12sYDWjSZPvvZ0srOzOzyuJ8b8/PPPMZvNhIeHt8/Q6E5aWhppaWntX69fv5533nmHO++8k9/97nc4OTlZ9JwiIiIivcnT3YWZE6O5YkIkKQePsjY5j4zCSjbtPcKmvUcYPsCfKyZEMWpQIEbtIyEiIl2wi0JERUUFAL6+vt1e03au7dqeHLOysvK045nNZj777DMArr322m43cAoODuY3v/kNl1xyCZGRkXh7e5OVlcXy5cv56KOPWLp0Kc7Ozvzf//2fRfdxOs7OfXv1TVtrF0tavIiInA3lGRHrcMbIRQlhXJQQxuH8ClZvy2V7WglpOeWk5ZQTGuDJjAuiuGRUOG6ujvsLFuUYEbE2R8wzdlGIaGhoXXfY3WwIoL3FZtu1PTlmfX39acfbtm0b+fn5ANxwww3dXtfVfhRxcXEsWLCAyMhIXnnlFZYuXcptt91GZGTkaZ/3VIxGA/7+9tFmy8fHw9YhiIiDU54RsZ4J/l5MSAinpLyWL77LImlLNkfKavn76oN8+k0mV04awNWTY+nv57ivQ+UYEbE2R8ozdlGIcHNzA6CpqanbaxobGztc25Njuru7n3a8ttkQ48ePP+Vyj1P55S9/yd///ndKSkpYv349d95551mN08ZkMlNZWXtOY1ibk5MRHx8PKivraGkx2TocEXFAyjMivccFuH7yQGZOiGTjnkKStuVRUl7HvzYc5vNvMpgwPJiZE6OJDe9+Rqq9UY4REWuzpzzj4+Nh0cwNuyhEWLLswpKlFifz8fGxeMy2a7tTU1PDmjVrALj++ustev6uODk5MXr0aNauXUtOTs5Zj3Oy5ua+/Q+1TUuLyW5iFRH7pDwj0ntcnIxMHRvJZaMj2H34GEnJeRzMO86WfcVs2VfMkEhfZkyIYuyQIIxGx9hHQjlGRKzNkfKMXRQiBg4cCEBhYSFNTU1dLqfIzc3tcO3pxMTEAJzyA7+lY65Zs4ba2lo8PT2ZNWuWRc/fnbZ7a25uPqdxRERERGzNaDQwdmgQY4cGkXOkiqTkXLallZCeX0F6fgX9fd2ZPj6KS0aF4eFmF29LRUSkB9jFbhfDhw/HxcWFxsZG9uzZ0+U1KSkpAIwZM8aiMUePHg20ttwsLi4+pzHblmXMmDEDL69z25MhPT0dgNDQ0HMaR0RERKQvGRDaj3uuGcnL913EVRcOwMvdmWMV9Xy0Lp1H39rER+vSOXa8ztZhiohIL7CLQoS3tzeTJ08G4OOPP+50Pjs7my1btgAwc+ZMi8aMiYlh6NChAKxYsaLT+e+//56cnBxcXFyYNm1at+Pk5eWRnJwMnNuyDICvv/66vRBx8cUXn9NYIiIiIn2Rfz83brx0EK/cfzF3XhlHWKAndQ0tJCXn8bt3v+cvn+3lcH4FZrPZ1qGKiIiV2EUhAmDevHkYDAZWrlzJihUr2n84lZSU8PDDD2MymZg+fTrDhg3r8LipU6cydepUVq9e3WnMBx54AID33nuP9evXtx/PzMzkiSeeAOC2224jICCg27g+//xzzGYzERERTJw48ZT3kJ6ezpNPPsmBAwc6HDeZTHzxxRc88sgjAFx++eWMGjXqlGOJiIiI2DM3FycuGxvBs7+ayG9vHs3Igf6YzZBy8CgvfJDCc39PYev+Ypr7+MZsIiJy5gxmOyo3L1myhJdeegmz2UxYWBj+/v4cPnyYxsZGYmJiWL58eaeiQVxcHAAvvvhil201X3jhBZYuXQpAdHQ0np6epKen09LSQmJiIosXL+62E4fZbGb69Onk5+fzwAMP8OCDD54y/rS0NK677joA/Pz8CA8Px8nJidzc3PaNMcePH8/bb7992g0yLdHSYqKsrOacx7EmZ2cj/v5elJfXOMzGKyLStyjPiNiP/JJqkrbnsWXfDwUI/35uTE+MZMqYcLzcu2+7bivKMSJibfaUZwICvCzqmmFXhQhoXTKxaNEi9uzZQ21tLeHh4cycOZO5c+d2uT/D6QoRAKtWrWL58uWkpaXR1NREdHQ0s2fP5q677upyY8w2W7du5c4778RgMLB27VqioqJOGXtlZSUffvghu3btIiMjg7KyMhobG/H19WXEiBFcffXVXH311Tg5OZ3Bd6R7KkSIiCjPiNijyppGNuwsYMOOfCprW1utu7k4cXFCKFeMjyIkwNPGEf5AOUZErM2e8ozDFiLEcipEiIgoz4jYs6bmFrbsL2Ztch75R1vf0xiA0YP7c8WEKIZF+2Ew2Lb9p3KMiFibPeUZSwsR6pMkIiIiIn2Si7MTl4wKZ3JCGGk55SQl57Eno5Rdh4+x6/AxooO9uWJCFBcMD8HF2W62PhMROe9pRoQD04wIERHlGRFHU1Raw9rt+WzeW0Tjide0r5crU8dFcNnYCPp5uvZqPMoxImJt9pRntDRDVIgQEUF5RsRRVdc18c2uAtal5HO8uhEAF2cjF44M5YoJUUT077x3mDUox4iItdlTnlEhQlSIEBFBeUbE0TW3mEg+UEJSch45R6raj8fHBDBjQhQjYwKsuo+EcoyIWJs95RntESEiIiIiDs/ZqXUWxKQRIaTnV5CUnMfOQ0dJzSojNauM8P5eXDE+kgtHhuLq0jOdyURE5NxoRoQD04wIERHlGZHzUcnxOr7ansfGPUU0NLYA4O3hwuVjI5g6LgJfb7ceey7lGBGxNnvKM1qaISpEiIigPCNyPqutb2bjnkK+2p5HaWUDAM5OBiYOD+GKCVFEh/Q75+dQjhERa7OnPKNChKgQISKC8oyIQIvJxI5Dx0hKziWjoLL9+LBoP2ZMiGbU4ECMZ7mPhHKMiFibPeUZ7REhIiIiIgI4GY1MGBbMhGHBZBRWsDY5j+0HjnIg9zgHco8T4u/B9PFRTE4Iw81V+0iIiFibZkQ4MM2IEBFRnhGRrpVW1LNuRz7f7CqkrqEZAE83Zy4dE860xEgCfNwtGkc5RkSszZ7yjJZmiAoRIiIoz4jIqdU3NvPdniK+2p5PyfE6AJyMBsYPC2bGhChiwnxO+XjlGBGxNnvKMypEiAoRIiIoz4iIZUwmM7sPHyMpOY+Decfbjw+O9GXG+CjGDQ3CaDR0ekxGYQVNZgMuBjODwn07XSMicq7s6b2M9ogQEREREbGQ0Whg7NAgxg4NIudIFUnJeWxLK+ZwfgWH8yvo7+vO9MRILhkdjoebMykHS1j+VTrlVQ3tY/j3c+O26UNIjAu24Z2IiPR9mhHhwDQjQkREeUZEzl55VQMbdubz9c5CquuaAHB3dWJolB97Mkq7fdz918erGCEiPcae3stYOiPi9FeIiIiIiJyH/Pu5ccOUQfxp3kXcOTOOsEBP6htbTlmEAPjHV+mYTPpdn4hId1SIEBERERE5BTcXJy4bE8Gzv5rITZcNOu31ZVUNHDppnwkREelIhQgREREREQsYDQYCfNwsunb7wRKqahutHJGIiH3SZpUiIiIiIhby87KsELF+RwEbdhQwMMyH+JgAEmIDiQnvh5NRvwcUEVEhQkRERETEQkOj/PDv59ahW8aPubs6EejjRsGxWrKKKskqquQ/m7PxdHNmREwA8Sf+BPi492LkIiJ9hwoRIiIiIiIWMhoN3DZ9CH/5LLXba+6+ajiJccGUVzWQmlVKamYZ+7PLqKlvZvuBErYfKAEgIsirtSgRG8jQSD9cnDVbQkTOD2rf6cDUvlNERHlGRKwj5WAJy79K7zAzIqCfG7dOH9Jl684Wk4msoipSM0tJzSojq7CSk9+Eu7oYGRbt376MI9jfA4PB0At3IiJ9nT29l7G0facKEQ5MhQgREeUZEbEek8lMRmEFTWYDLgYzg8J9MRotKx5U1zWxP7uMvZmtMyYqajpubBnk5058TCDxsQEMi/bHw00TmUXOV/b0XkaFCFEhQkQE5RkRsa6eyDFms5n8ozWkZpayN7OU9PwKWkw/vEV3MhoYEulLfGwg8TEBRAV7a7aEyHnEnt7LqBAhKkSIiKA8IyLWZY0cU9/YzIGc4+zNKiU1s5Sjx+s7nPf1cm3fW2JkTADeHi498rwi0jfZ03sZSwsRmuMlIiIiItKHuLs6M2ZIf8YM6Q9AcXktqZmtyzgO5JZTUdPIptQjbEo9ggEYGOZDQmxrYSImTC1CRaTv04wIB6YZESIiyjMiYl29nWOamk2k5x9vLUxklVJwtON7vbYWoQknZkz493OzekwiYl329F5GMyJERERERByMi7OREQMDGDEwgDkMpqyynn1ZZezNKmN/Vhm1DZ1bhCac2PRyiFqEikgfoRkRDkwzIkRElGdExLr6Uo6xtEVoQmxrYSLE39NmsYqI5fpSnjkdbVYpZ12IMJvNtLS0YDZb/x+5k5MBX19PKipqaWnRP0WxDaPRCScnJ1uHIVZiTz+8RcT+9OUcU13XxL6ssvbCRJctQmMDSYgJZNgAP9xdNVlapC/qy3nmx1SIkDMuRJhMJqqrK6ivr8VkarZiZB0ZjUZMpr79ghLH5+rqjre3H66uWkvraOzph7eI2B97yTFms5m8kmpSTxQmumsRmnCiE4dahIr0HfaSZ0CFCOHMChEmk4ny8hKam5twd/fCzc3jxD8g6/8AcnIyaDaE2JCZ5uYmamqqaGlpJjAwFGdntUFzJPb0w1tE7I+95pi6hmYO5JaTmlXG3oxSjlX8qEWod2uL0ITYQEYMVItQEVuypzyjQoScUSGisrKcurpqAgKCcXHp3d8IOzsb+/wLShyfyWSitLQIFxdX/PyCbB2O9CB7+uEtIvbHEXKM2WympLyOvSeWcBzILaex6Yd7MQAx4T7En+jEERvmg9Go2RIivcWe8oy6ZojFzGYz9fW1uLt79XoRQqSvMBqNuLt7UVtbhdls1nRUERE5bxgMBkICPAkJ8GT6+Ciamls4lF/BvpNahGYWVpJZWMm/N2Xj5e7MiIEB7YUJtQgVkTOlQoTQ0tKCydSMm5uHrUMRsSlXVzdqaipoaWnW8gwRETlvuTg7MXJgACNPahGamlVG6okWoTX1zSQfKCH5RIvQyCAv4mMDiY9Ri1ARsYwKEdLeHcOSKTQijsxobH0NaMWaiIjIDwJ83JkyOpwpo8NbW4QWVrUv48guqiT/aA35R2tYvTUXVxcjw6P9WwsTahEqIt1QIUJOoqnocr7Ta0BERORUnIxGBkf6MjjSl+unxFJV28i+7LITyzjKqKxpZHdGKbszSgEI9vNgZGyAWoSKSAfKBCIiIiIiclb6eboyaUQok0aEYjKbyS+pZm9mKfuyykjPr6DkeB0lOwrYsKMAJ6OBoVF+7XtLRAZ5aU8mkfOUChEiIiIiInLOjAYD0SH9iA7px1UXDmxtEZpzokVoZmuL0LScctJyyvnk6wz8vF2Jj2ldwqEWoSLnFxUiROzU888/zapVX/D73z/FT35yTY+M+cADc9m1awdvvPEO48aN75ExRURE5Pzk4ebM2KFBjB0ahNlspri8jtS2FqE55RyvbuS7vUV8t7cIgwFiw3wYGRNAQmwgMWoRKuLQVIgQsdDkyWf3wfyTT/5NWFh4D0cjIiIiYj8MBgOhAZ6EntwiNK+C1KxSUjPLKDhWQ0ZhJRkntQgdGRPAyJgA4mPUIlTE0agQIWKhhITRnY41NTVx4MB+AIYNG4GLS+cpha6urlaJJzCwP9HRA/Dy8u6xMUNCQomOHoC7u3uPjSkiIiLyYy7OTu2Fhlum8kOL0MxS9mWXU1PfzLa0EraltbUI9SY+NoCEmAAGq0WoiN0zmNWnzmG1tJgoK6s57XVNTY2UlhYRGBiGi4t1PjSfirOzkeZmU68/b08oKirk5ptnA5r54Ahs/VoQ63B2NuLv70V5eY3d5hoR6buUY3pexxahpWQXVXHyBxY3FyeGRfsRHxtIQmwAwWoRKg7OnvJMQIAXTk6nLxRqRoSIiIiIiPQZ3bUITc0sI7WrFqH+Hu2dOIZFq0WoiD2wu1fpli1bWLx4Mbt376a2tpbw8HBmzpzJ3Llz8fQ8u2romjVr+OCDDzhw4ABNTU0MGDCA2bNnc+edd3Y51T4/P59p06adcszRo0fz8ccf9+p92COTyUxadhmllfX4ebkxNMrPYTYmOnm2xHffbeebbzbwySf/ICPjMFVVlSxe/CFDhsRRWnqMr79ez/fff0dubg7Hjh3D2dmZAQMGMHXqDG68cU6Xyzu626zyyy//wwsvLGDMmHEsXPguK1f+i5UrPyU3NwdXVzfGjBnLPffMIzZ2UKcxu9us8v3332Xx4veYNetqfve7J/jHP5axevV/KSoqxNPTi4kTJzF37v2EhIR2+b04erSEv/3tHbZs2UxVVSXBwSFMmzaDO+/8Ja+88mKPb7opIiIijqO7FqGpmWUcLqigpLyO9eUFrN9RgLOTgSGRfieWcQQSoRahIn2SXRUili1bxvPPP4/ZbCY0NJSwsDAOHz7M22+/TVJSEsuXL8fPz++MxvzjH//IokWLAIiOjsbDw4P09HRefvllNmzYwKJFi065xn/cuHFdHh8yZEiv3oc9SjlYwvKv0imvamg/5t/PjdumDyExLtiGkfW8Dz9cyttvL8TPz5/IyEhKSorbz/3nP5/zt7+9g6urG4GB/Rk0aBAVFRUcOnSQtLT9fPvtBt54450ui2Kn89xzT7FmzZeEhYUTHT2AnJwcNm78hp07U/jb35YRGRl1RuM1NzfzyCMPkpKSTFRUNJGRUeTm5rBmzSp27tzBkiXL8fHx7fCY3Nwc7r//HsrLy3B2diY2dhANDQ0sXfo+27dv03IWERERsVh3LUL3nthfokOL0A1qESrSV9lNISI1NZUXXngBgGeeeYY5c+ZgMBgoLi7mvvvuY9++fcyfP5+FCxdaPObatWvbCw2vvfZa+yyHjIwM5s6dS3JyMq+++iqPPfZYt2P84x//sPl92KOUgyX85bPUTsfLqxr4y2ep3H99vEMVI/72t3d4+OHfcd11N2I0GjGZTLS0tAAwdux4/vznvzB2bCLOzj+8JEtKivnzn//Exo1f89FHH3DHHb84o+dMTd1DTk42b775V8aMaS2YVVZW8Pjjj7J7907ef/9dnnrquTMac8OGrwgNDWfp0o8YNGgwAEeOHOHRRx8kOzuLf/zjA+699/72681mM888M5/y8jISEkbx7LN/pH//IAAOHTrA//3f/3DwYNoZxSAiIiLSpqsWoW2zJQ7mdt0iND62tTARE6oWoSK2Yjfbzb711luYTCauvfZabrnllvYpViEhIbz66qsYjUaSkpI4cOCAxWO++eabANxzzz0dlloMGjSI555r/YD24YcfUlZW1qfvozeYzWYaGlt65E9dfTMfrj10yudb/lU6dfXNPfJ8fWE/1muuuY4bbrgZo7H1JWc0GttnOIwePYYJEyZ2KEIABAeH8NRTz+Hs7Mzq1f894+dsbm7mt799tL0IAeDj48tDDz0CwPffbzqrMZ94YkF7EQIgNDSUe+6Z1+WYO3Zs58CB/bi7u/Pssy+3FyEAhg4dxh/+8BTNzc1nHIeIiIjIj7W1CL1ifBT/M2c0C397CY/cMoYZE6II7++F2QwZhZWs/C6L5/+ewkNvbOSdlal8t6eI49UNp38CEekxdjEjoqamho0bNwIwZ86cTucHDhzIpEmT2Lx5M6tXr2bYsGGnHTM7O7v9w/4tt9zS6fyFF17IgAGtU9nXrVvHzTfffI53YZ376A1ms5kXP9jB4YKKXnvO8qoG7n/t2x4Za3CkL4/fPs6m6wNPt/dBQ0M9GzasY/funRQXF1NfX9deQDEajeTm5tDQUI+bm+VtNb29+zFt2oxOx4cOHYarqyvV1VVUVBzH19fP4jEHDx5KfHxCp+MjR7YeKyjI73B869bNAEyadDH9+/fv9LgJEyYRGhrGkSNFFscgIiIiYomTW4TCDy1C92aWsr+bFqEJsa2bXg6J9MXZgp3/ReTs2EUhIi0tjcbGRlxdXRk1alSX1yQmJrJ582Z2795t0Zi7du0CICoqipCQkG7HzMnJYffu3d0WIp577jkyMzMxGAxEREQwefJkpk+f3v6bb2vfR6/RrLVzMmBATLfnMjMz+N3v/oeiosJTjlFZWUlQkOWFiFPt/+Dn509JSTF1dXVnVIjobsyAgNYf8HV1tR2O5+XlAjB4cPd7pgwePESFCBEREbG6AB93powOZ8rocFpMJjILK9mbWca+Ey1C849Wk3+0mlVbc3FzcWL4AH/iYwOIj1GLUJGeZheFiKysLADCw8O73bAvOjq6w7Wnk52d3eFxZzvmsmXLOny9YsUKhg8fzsKFC4mK6vihzRr30RsMBgOP3z6Oxqae6Vl7KO84f/7k9IWW/7l5NEOj/M75+VxdjDbfLdnDw6PL4y0tLcyf/zuKigpJTLyAn/3s5wwePIR+/Xzal2rccMNVlJQUn/ESBnf37osWbYWyM1220t19dFV4A6itrQPA09Or2zFPdU5ERETEGpyMRoZE+jEk0o8bpsRSWdvI/qyy9sJEZW0Tuw4fY9fhY0Bri9CEE5teDov2x83VycZ3IGLf7KIQUVHRuiTA19e322vazrVd25NjVlZWdjju7OzM7Nmzueqqqxg8eDDBwcGUl5fzzTff8Nprr5GWlsbdd9/Np59+ire3t1Xv43ScnU8/pcxkOv2HdIPB0GMJd2RMAP793Dp0y/ixgH5ujIwJcPgNhNLS9pOTk01wcAgvv/xqp6UXZrOZqqoqG0V37jw9WwsXtbU13V5zqnO24uRksOi1I/bB6cTUWidNsRURK1COcQwBPu5MHh3O5NHhmMxmcour2JtRyt6MUtLzW1uErivPZ92OfJydDAyN8mfUoEASBgUSqRahYmWOmGfsohDR0ND6gfVU7QvbWmy2XduTY9bX13c4Hhoayp/+9KcOx0JCQpgzZw4TJ07khhtuICcnh7///e/MmzfPqvdxKkajAX//0/+2ub7eiWPHjL364etnV8ax8J97uj1/+5VxuNpBpfnkZODkZOzw/Tv5XHff15KS1iUJI0aMxMur85S/w4fT25c7/Hj8th94RmPH/29txRuD4fT/P7sb88fHz2TMk89HRw8AICsro9vHZWQc7vI+bMFkMmA0GvH19TzljBKxTz4+Xc/oERHpCcoxjiUwwJuxw8MAqK1vYnf6MXYeLCHlQDEl5XXszy5jf3YZH61LJ8DHncRhwYyNC2bM0CD6ebraOHpxVI6UZ+yiEOHm5gZAU1NTt9c0NjZ2uLYnxzyTDyQDBgzg1ltv5b333mPt2rUdChHWuI9TMZnMVFbWnva6xsaGE+0kzTQ398zyi9MZO7g/918fz/Kv0jvMjAjo58at04cwdnD/XovlXLS0mDr8/eSYTz7X3b24uLT+oDp27FiX1yxbtrTb8duWVZhMHf+/mUzm9vOn+x52N+aPj5/JmCefv+CCC/nww7+zefN3lJQcJSAgsMO127dva98b48f3YQstLWZMJhMVFbXU1bXYNBbpOU5ORnx8PKisrOvwuhQR6QnKMeeHYZE+DIv04adTB3GkrJY9J2ZLpOWUU1ZZz9ptuazdlovBAIMifBkV2zpbIiZMLULl3NlTnvHx8bBo5oZdFCIsWa5gybKHk/n4+Fg8Ztu1lho7dizwwz4UbaxxH6djyQe7lhbbtLdMjAtm7JAgMgorKK2sx8/LjaFRfudVsh45MgFnZ2dSU/ewcuWnXHvtDUBrsWrJkr+RlLQKFxeXUxav+rJx48YzfPgI0tL288QTv+OZZ15q756Rnn6QF15YgLOzc59r4dmbRTnpPT8usImI9CTlmPNHkK8H08ZFMm1cJI1NLRzKP05qZhmpWWUUHqvhcH4Fh/Mr+PTbTLzcnRkZE0D8if0l/LzP/ZeNcv5ypDxjF4WIgQMHAlBYWEhTU1OXSxtyc3M7XHs6MTGtXQxycnK6veZMx2zTFl9LS8ffqFrjPuyd0Whg+MAAh3lBnamAgEBuvfUOli1bzJ/+9AKLF79H//5B5OfnUl1dzd1338t///tvu+0qYTAYmD//We6//x727NnFTTddTWzsIBobm8jOzmTEiHhGjRrDV1+t6XbDSxEREZG+ytXFqbXIENM667O0op7UrFJSs1qXbvy4RWhUsPeJThxqESrnN7soRAwfPhwXFxcaGxvZs2cPiYmJna5JSUkBYMyYMRaNOXr0aADy8/MpLi7usoXnmY7ZJj09HWjdS+Jk1rgPsX/33ns/ISGhfPbZJ+Tm5lBfX8/gwUO58cY5XH75dP7733/bOsRzEh09gPffX8b777/Lli2byM7Oon//IH72s7u4665f8dxzTwHg5aXuGSIiImLfAn3duXRMBJeOiaC5pbVFaGpWKamZZWQfqSKvpJq8kmpWbcnFzdWJ4dEnWoTGBhLs5zjr/0VOx2A+0/59NvLrX/+aDRs2cN111/HHP/6xw7ns7GxmzZqFyWRi5cqVDBs2zKIxr7nmGg4dOsT999/Pb37zmw7nvv/+e+666y5cXFz49ttvCQgIsGjMmpoaZs+eTX5+PrfddhtPPfWU1e+jOy0tJsrKTt+RoKmpkdLSIgIDw9r3LOhNzs7G83ZGhMAdd8whKyuTxYuXM2TIUJvGYuvXgliHs7MRf38vystrlGtEpMcpx4ilKmsa2ZddRupJLUJPFuLvQXxsIPExahEqHdlTngkI8LJojwi7mQs0b948DAYDK1euZMWKFe2b6pWUlPDwww9jMpmYPn16pw/vU6dOZerUqaxevbrTmA888AAA7733HuvXr28/npmZyRNPPAHAbbfd1qkIMX/+fJKSkto3lmyTkZHBr371K/Lz8/H09OTuu+/usfsQcUT79qWSlZWJj48vMTGxtg5HRERExGp8vFy5cGQo91wzglcfnMxTd03gxktjGRrlh5PRQHF5HetS8nn9n3t48PVveeWjnazemkv+0Wrs5HfHIhazmxkRAEuWLOGll17CbDYTFhaGv78/hw8fprGxkZiYGJYvX96paBAXFwfAiy++yA033NBpzBdeeIGlS1s7E0RHR+Pp6Ul6ejotLS0kJiayePHiTh0srr32Wg4cOICLiwvR0dF4e3tTXl7evr+Dr68vr732GhdddFGP3cfZ0IwI6Qvy8nLZsmUzM2deRb9+/dqP79mziwULnqC4+Ag/+9ld/PrXD9gwyla2fi2IddjTbxFExP4ox0hPqK1vJi2nnH1ZpezNLKO0sr7Def9+bsTHtC7hGDHQHy/3znvNieOypzxj6YwIuypEQOuSiUWLFrFnzx5qa2sJDw9n5syZzJ07t8s15qcrRACsWrWK5cuXk5aWRlNTE9HR0cyePbt9acaPffnll2zcuJHU1FSOHTtGZWUl7u7uDBgwgClTpnD77bcTFBTUo/dxNlSIkL7gwIH9/OpXd+Lk5ERUVDSenl4cO3aUkpJiABISRvHaa2/h5mZ5m1xrsfVrQazDnn54i4j9UY6RnmY2mzlSVktqZhl7s0o5mHucppP+bRkMMCjct70wMTC033nVde58ZE95xmELEWI5FSKkL6isrOSjjz4gOXkLR44coaqqEjc3NwYOjGXatBlcd92NuLr2jQ/9tn4tiHXY0w9vEbE/yjFibY1NLRzKO05qVhl7M0spKq3tcN7bw+VEi9DWP75qEepw7CnPqBAhKkSInCFbvxbEOuzph7eI2B/lGOlt7S1CM8vYn1NGXUNLh/PRwd6MjA0gISaQwWoR6hDsKc9YWoiwi/adIiIiIiIi0n2L0L2ZZeQcqSK3pJrcH7UITTjRIjRILUKlj1AhQkRERERExA45OxkZGuXH0Cg/bpgy6KQWoaWkZpVRVdvErsPH2HX4GAAhAZ7ExwSQEBtAXLQ/bi5qESq2oUKEiIiIiIiIA2hrEXrhyFBMZjN5xdXszSwlNbOUwwWVFJfVUlxWy7qU/BNFDF/iYwJJiA0gvL8XBoM2vZTeoUKEiIiIiIiIgzEaDAwI7ceA0H5cfdHA9hahrftLlFJa2cD+7HL2Z5fz8YYfWoQmnGgR6qkWoWJFKkSIiIiIiIg4OE93ZxLjgkiMC2pvEbo3s3UZx8G845RXNbBxTxEb9xRhNBiIDfchPra1MDEgtB9GzZaQHqRChIiIiIiIyHnEYDAQFuhFWKAXMyZEtbcI3ZtZRmpWa4vQwwUVHC6o4PONWWoRKj1OhQgREREREZHzmKuLE/GxgcTHBgJDOFZRR2pWGamZZaTllFFd18TW/cVs3V8MtLYIjY9t3VtiUIRahMqZUyFCRERERERE2vX39eCyMRFcdlKL0L0nOnGc3CL0yy05uLk6MWKAf2shIyZALULFIipEiIiIiIiISJdObhF646UnWoRmlbE3q5R9J1qE7kw/xs70H1qEJsQEEB8bSFy0n1qESpdUiBARERERERGL+Hi5cmF8KBfGt7YIzS2uYm9mGft+1CL0qxMtQuOifNtnS6hFqLRRIUJERERERETOmNFgYGCoDwNDfbimvUVo2Yn9JVpbhO7LLmdfdjkraG0RmhAbQHyMWoSe71SIEOnDJk8eD8B3323vcPyBB+aya9cO3njjHcaNG2/xeDt2bOc3v/k1Y8aM4803/9qjsXanqKiQm2+eTWhoGP/853965TlFREREpPe1tggNJjEuGLPZTFFpLakn9pY4kNvaIvTb3UV8u/tEi9AIn/ZlHGoRen5RIULEQs8//zSrVn3BxIkX8f/+3xunvb6srJTrr/8JLS0t/PnPf2HChIm9EGXve//9dwGYM+c2+vXrZ+NoRERERKQvMBgMhPf3Iry/FzMuiKbhRIvQ1JNbhOZXcDi/gs9OtAiNjwlobxOqFqGOTYUIEQvNmnU1q1Z9wfbtWyktPUZgYP9TXp+UtIqWlhaCg0NITJzQo7GEhIQSHT0Ad3f3Hh33bCxe/B4AP/nJNV0WIpydnYmOHkBQUHBvhyYiIiIifYSbixMJsYEktLUIPX6iRWhWGfuzW1uEbtlfzJa2FqEh3sTHqEWoo1IhQsRCY8cmEhYWTlFRIUlJq7n11p+d8vpVq/4LwMyZV2E09mzinD//mR4dz5qCgoJZvvxftg5DRERERPqQ/n4eXDY2gsvGtrYIzSioOLG3RBk5xVXkFleTW9zaItTd1YnhJ1qEJsQE0F8tQu2eChEiFjIYDMyceRWLF7/H6tX/PWUhIj39IBkZ6UDrTAoREREREemas5ORuGh/4qL9ufHSQVTUNLIvq7S9MFFd17FFaGiAJ/EnNr1Ui1D7pEKE2JTJbOJgWSbltRX4uPkw2C8Go6HvTruaOfMqliz5GxkZ6aSnH2TIkLgur1u9unU2RELCKKKiotm3L5Vvv93Ajh3JlJQUU1FRgY+PLyNGjOTmm28946Ubp9qs0mQy8dln/+Tf//6MvLxcPD09GTVqDL/4xT2nHPNMY3z//Xfbl2UA3Hzz7A7n22I73WaVNTXVrFixnG++2UBBQR4Gg4GIiCguvfRybrnlNjw9vTo95qabruHIkSLeeOMdgoNDeP/9d0lJSaa6uoqwsHCuumo2P/3pz3p8JoqIiIiIWJ+vlysXxYdxUXwYJrOZnCNV7Z04MgoqOVJWy5GyWr7afqJFaLQf8Sc2vQwP9FSLUDugQoTYzK6SvXyS/m+ON1S0H/Nz8+XmIbMZE5xgw8i6FxERyahRY9i9eyerVn3RZSGiubmZpKTVAMyc2Tob4plnnqCgIJ9+/XwIDOxPYGAQR4+W8N1337Jp00Z++9tHufHGW845PrPZzIIFT7BuXRIAoaFh+Pr6sXXrZrZs2cwvfvGrbh97pjGGhISSkDCavXt3AzBs2AhcXH5oweTt7X3aeI8cOcJvfzuP/PxcjEYjMTGxAGRmHubw4UOsXbua1157i+DgkC4fn55+kMcff4Tm5mYGDozF2dmZnJxs3nrrDY4cKeLhh393+m+aiIiIiPRZRoOBmDAfYsLaWoQ2sT+7/MT+EqWUVTawL6uMfVllrFh/mAAfN+JjAomPCVCL0D5MhQixiV0le3kvdVmn48cbKngvdRn3xN/RZ4sRs2Zdze7dO1m7dg3z5j2Es3PHl9HWrd9TXl6Gq6sb06bNAOCuu37FyJEJREcP6HBtSkoyTz/9BxYu/DMXX3wpoaGh5xTbv//9GevWJeHq6saCBc9zySWXAVBdXc3zzz/d3uGiK2ca49VXX8vVV1/b3mL02WdfIiws/IziXbDgD+Tn5zJ48FCef/5lIiIiAcjLy+X3v3+UrKxMnnlmfretRt9+eyGzZl3Ngw8+jKenJwDr1q3l6ad/z2ef/ZObbvppp/sREREREfvl6e7C+GHBjB/W2iK0sLSWfZml7M0q42DuccoqG/h2dyHf7i7EaDAwKMKnfbaEWoT2HZq3LBYxm800tDT2yJ+65no+PrTylM/3Sfq/qWuu75HnM5vNPfq9mDp1Ou7u7pSXl7F16/edzq9a9QUAl1xyafusgFmzru7yA3Fi4gTmzp1Hc3MzX321+pziMpvNfPDBUgBuv/3O9iIEtM5OePLJZ/Hy6rzMoU1vxHiynTtT2Lt3N0ajkQULXmgvQgBERUXz9NMvYDAY2LVrB7t27ehyjKioaB599PH2IgTAtGlXcPHFl2A2m9myZVOPxSsiIiIifYvBYCDiRHvQR24Zw8LfXsL/zBnN9PGRhAZ4YjKbST/RHvTZpdv57Rvf8df/7GNzahEVNY22Dv+8phkRclpms5lXd7xFZkVOrz3n8YYKHv32yR4ZK9Z3IA+Pu6/H1op5enpx6aVTWbPmS1av/i8XX3xJ+7nKyko2b94ItLazPFlhYQFffbWG9PRDVFQcp6mpCWjdIwFalxmci9zcHIqKCgC6XObh4eHBVVddy/Llf+92DGvHeLItWzYDcMEFkxgwYGCn84MGDWbChIls27aFrVu/Z8yYcZ2uueaa63By6rw50ciRCXz33bcUFOT3WLwiIiIi0rd1bBFKe4vQvZmlpOWUt7YI3VfMln2tLUIHhPQ7semlWoT2NhUixEKawnSyWbOuZs2aL9m06Vuqqqro168fAOvXJ9HY2Ej//kGMH39B+/Uff7yct956g+bm5m7HrKio6PacJXJysgHw9w/Az8+vy2va9mDoSm/EeLLc3NbCVmzsoG6viY0dzLZtW9rv7cciI6O7PO7vHwBAXV3duQUpIiIiInaruxahezNLyS2uJqe4ipziKv77/Q8tQhNiW/eXUItQ61IhQk7LYDDw8Lj7aDQ19ch4h49n8tbuRae9bt7oXzLYr/sPzpZyNbr0+M65iYkTCAkJpbj4COvWJXHddTcCsGpVa7eMK6/8Sftv6vfu3c0bb7yK0WjkF7+4h0svnUp4eDju7h4YjUZSUpJ56KH7TlkAsERdXS0A/v7+3V7T9gH9x3orxpPV1rbFG9jtNQEBgSeurenyvLu7e5fH27pl9PSyHBERERGxT922CM0sIzWrc4vQsEBPRsYEkBAbSFyUH65qEdqjVIgQixgMBtycXHtkrOEBQ/Fz8+3QLePH/N18GR4wtM+28jQYDMyceRVLl77P6tX/5brrbiQ3N4d9+/YCrTMm2rS18rzlltu5++57O43VU7MMPDxa90koLy/v9pry8rIuj/dWjCdr29ehvLy022vKykpPXNv93hYiIiIiImeqyxahJza9zCyopKi0lqLS1hahLs5Ghkb5kXBi08swtQg9ZypESK8zGozcPGR2l10z2tw0ZHafLUK0mTXrapYufZ/U1D3k5eW2f5gfPnwkAwfGtF9XVFQIwOjRY7scp614ca7a9lk4fryc48ePd7k8Iysrs8vH9laMJ2vbGDMzM6Pba9rOdbWHhIiIiIhIT+jQIvTimJNahJayN7OM8qofWoRyUovQhNgAhg8IwNNdH6vPlL5jYhNjghO4J/4OPkn/d4eZEf5uvtw0ZHafbd15ssjIKBISRrN3725WrfqCNWu+BDrOhgBwc2tdPlBaeqzTGOXl5e1dNs5VdPQAwsIiKCoq4LPPPuEXv7inw/n6+nq+/PLfXT72XGJ0c3OjoaGBhoaGM4p30qSL+PDDpe17QPy42JCZmUFy8pb2a0VEREREekNXLUJTM0tJzSzlYF5F1y1CY1sLE9EhahFqCRUixGbGBCcwKmgkWVXZlNdW4OPmw2C/mD4/E+JkP/nJNezdu5sVKz6koaEBV1dXpk+/ssM1Y8aMZePGr1m2bDFjxya2zwQoLCzgqad+T319fY/EYjAYuO22O/h//+8lPvxwKUOGxDF58hSgtevFCy8soLq6usvHnkuMERGRZGZmsGtXSoeZIKczdmwio0aNYc+eXTz99O957rmX21t4FhTks2DBHzCbzYwZM67bmRoiIiIiItbU1iI0or8XV14QTUNTCwdzj7cWJrLKOFJWS3p+RWub0G8z6efp0rq3REwgI2MC8PHqmeXtjkaFCLEpo8FIXMBgmn1Mtg7lrEydOp3XX3+l/YP6RRddgo+PT4drrrnmelau/JTc3BzuuGMOUVEDcHIykpWViYeHB/PmPchrr73SI/Fcd92N7NixnQ0bvuKxxx4mLCwcX18/srMzMZnM3H33vbz77l86Pe5cYpw+/Ur++te3eOWVl/j000/w8fEF4KGHHmHIkLhTxvvkk8/x29/eR3r6IW699QZiYgYBZrKyMjGZTERFRfPkk8+e8/dFRERERKQnuLk4MWpQIKMGtW6qfvREi9DUzFL255RTVdt1i9CE2EBiw33UIvQEFSJEzoGXlzdTplxOUtIqoHWGxI95enryl7/8jffee4tNm74lPz8Xf/8AZsyYxS9+cQ/FxUd6LB6DwcDTTz/P6NFj+M9/PicvL5e6ulomTJjEL385l6qqyi4fdy4x3nbbnZhMJr76ag35+fk0Nrbu61BVVXXaeENDQ3n//WV89NGHfPPNegoK8oDWNqOXXTaNW265TRtVioiIiEifFeTnweVjI7j8pBahezPLSM3q3CLUw82J4QMCiI8NaG0R6nv6FqEmk5m07DKasspxMZgZFO6L0Wj/Sz8MZvW3c1gtLSbKyrpue3iypqZGSkuLCAwMw8Wl96cOOTsbaW62zxkR4lhs/VoQ63B2NuLv70V5eY1yjYj0OOUYEelORXUDqSc2uWxrEXqysEBP4mMCiY8N6LJFaMrBEpZ/lU551Q97sfn3c+O26UNIjAvulXs4UwEBXjhZMOtDhQgHpkKEyJmx9WtBrEMfEkTEmpRjRMQSJpOZnOIq9p7YWyKjoIKTP4m7OBuJi/IjPjaQ+JgACo/V8Nbnqd2Od//18X2yGGFpIUJLM0RERERERESsyGj8oUXo7ItjqKlvIu1HLUJTT8ycADjd6ot/fJXO2CFBdrtMQ4UIERERERERkV7k9eMWocdq2JtZxr6sUtJyyzGdZoJVWVUDh/KOM2yAf+8E3MNUiBARERERERGxEYPBQESQNxFB3sycGM13e4pY9GXaaR93vKbhtNf0VeodIiIiIiIiItJH9Pd1t+g6Py83K0diPSpEiIiIiIiIiPQRQ6P88O936iJDQD83hkb59U5AVqBChIiIiIiIiEgfYTQauG36kFNec+v0IXa7USWoECEiIiIiIiLSpyTGBXP/9fGdZkYE9HPrs607z4Q2q5STmE9/iYhD02tARERERPqGxLhgxg4JIqOwgiazAReDmUHhvnY9E6KNChGCwdA6MaalxYSLi42DEbEh04k+SQaD/Sd3EREREbF/RqOB4QMD8Pf3ory8hubm0/T1tBNamiE4OTlhNDrT0FBn61BEbKqxsQGDwYiTk2q0IiIiIiLWokKEYDAYcHf3pL6+hqYm++1FK3IuTCYT9fU1uLm5a0aEiIiIiIgV2d2v/bZs2cLixYvZvXs3tbW1hIeHM3PmTObOnYunp+dZjblmzRo++OADDhw4QFNTEwMGDGD27NnceeeduHSxVqG4uJikpCS+//570tLSOHr0KC4uLkRFRXH55Zfz85//nICAgC6f67HHHuOzzz47ZTzvvfceU6ZMOat7OVve3r40NTVQVlaCu7sXbm4eODkZAet/IDOZDLS0aG2+2IbZbKalpYmamipMJhPe3n62DklERERExKHZVSFi2bJlPP/885jNZkJDQwkLC+Pw4cO8/fbbJCUlsXz5cvz8/M5ozD/+8Y8sWrQIgOjoaDw8PEhPT+fll19mw4YNLFq0CFdX1w6PmTNnDkeOHAHAz8+PoUOHUlFRQXp6OgcPHuSTTz7hb3/7GyNGjOj2ecPCwggLC+vynK+v7xndQ08wGo34+wdTXV1BfX0tdXVVvfrcbWvzRWzF1dUdH59gnJ21UYqIiIiIiDXZTSEiNTWVF154AYBnnnmGOXPmYDAYKC4u5r777mPfvn3Mnz+fhQsXWjzm2rVr2wsNr732GtOmTQMgIyODuXPnkpyczKuvvspjjz3W4XGurq7ceuut3HTTTYwcObJ9GndGRgb/+7//y759+3jggQdYtWoVbm5unZ4X4MYbb+TBBx88m2+F1RiNRnx8/OnXz4+WlhbMZusXB5ycDPj6elJRUatZEWIzRqMTTk5Otg5DREREROS8YDeFiLfeeguTycR1113HLbfc0n48JCSEV199lVmzZpGUlMSBAwcYNmyYRWO++eabANxzzz3tRQiAQYMG8dxzz3HXXXfx4YcfMnfu3A5LLT7++GP8/f07jTdo0CAWLlzIlVdeSUFBARs3bmT69Olne8s2YzAYcHbunX8azs5G3N3dqatrcZgdYEVERERERKR7drFZZU1NDRs3bgRal0X82MCBA5k0aRIAq1evtmjM7OxsDhw4ANChsNHmwgsvZMCAATQ2NrJu3boO57oqQrSJiIggNjYWgMzMTItiERERERERETlf2MWMiLS0NBobG3F1dWXUqFFdXpOYmMjmzZvZvXu3RWPu2rULgKioKEJCQrodMycnh927d3PzzTdbHG9DQ2vnCQ8Pj26v2bp1K+np6Rw/fhwfHx9GjhzJ7NmziYiIsPh5REREREREROyNXRQisrKyAAgPD++yiwW0bjR58rWnk52d3eFxPTEmtO5l0Tb2+PHju70uOTm5w9dr167lL3/5Cw899BD33HOPxc8nIiIiIiIiYk/sohBRUVEBnLqbRNu5tmt7cszKykqLxmxqamLBggUATJ48meHDh3e6ZsCAATz22GNMmjSJiIgIXF1dOXjwIIsWLWL16tW88soreHp6cvvtt1v0nKfj7Ny3V9+0tgj94b8iIj1NeUZErEk5RkSszRHzjF0UItqWOnQ3GwJob7HZdm1PjllfX2/RmM8++yx79uzBx8eHZ555pstr7rvvvk7HRo8ezeuvv86CBQtYvnw5r732Gtdddx1eXl4WPW93jEYD/v7nNkZv8fHpfhmLiEhPUJ4REWtSjhERa3OkPGMXhYi2FphNTU3dXtPY2Njh2p4c093d/bTjvfnmm6xYsQJXV1feeOONs9rr4eGHH+aTTz6hsrKSLVu2dOjkcTZMJjOVlbXnNIa1OTkZ8fHxoLKyjpYWdc0QkZ6nPCMi1qQcIyLWZk95xsfHw6KZG3ZRiLBk2YUlSy1O5uPjY/GYbdd2Z9GiRSxcuBAXFxdef/11LrzwQoti+LF+/foxZMgQ9u/fT05OzlmN8WP20hKzpcVkN7GKiH1SnhERa1KOERFrc6Q8YxeFiIEDBwJQWFhIU1NTl8spcnNzO1x7OjExMQCn/MBvyZgffPABf/zjH3FycuLll19m6tSpFj1/d9rurbm5+ZzGgdalGQEBWpohIgLKMyJiXcoxImJt9pBnjEaDRdfZRSFi+PDhuLi40NjYyJ49e0hMTOx0TUpKCgBjxoyxaMzRo0cDkJ+fT3FxcZctPE835scff8xzzz2HwWDg+eef5yc/+YlFz92d5uZmMjMzAQgNDT2nsQAMBgNOTpb9Q7A1R9p4RUT6JuUZEbEm5RgRsTZHyjN2cSfe3t5MnjwZaP3w/2PZ2dls2bIFgJkzZ1o0ZkxMDEOHDgVgxYoVnc5///335OTk4OLi0uVeDStXruSpp57CbDbz9NNPc/3111t8P91ZsWIFVVVVODs7M2nSpHMeT0RERERERKSvsYtCBMC8efMwGAysXLmSFStWYDabASgpKeHhhx/GZDIxffp0hg0b1uFxU6dOZerUqaxevbrTmA888AAA7733HuvXr28/npmZyRNPPAHAbbfdRkBAQIfHJSUl8fjjj2MymfjDH/7AT3/6U4vuYdOmTfzpT38iOzu7w/HGxkaWLVvGiy++CMBPf/pTgoODLRpTRERERERExJ4YzG2f6O3AkiVLeOmllzCbzYSFheHv78/hw4dpbGwkJiaG5cuXdyoaxMXFAfDiiy9yww03dBrzhRdeYOnSpQBER0fj6elJeno6LS0tJCYmsnjx4k6dOOLj42lqasLDw4Phw4d3G++ll17Kr3/96/avv/rqK+6//34A+vfv374cJCsri9ra1u4WV155Ja+88kp761ARERERERERR2IXe0S0ueuuu4iLi2PRokXs2bOH0tJSwsPDmTlzJnPnzsXL68w3Zvz973/P2LFjWb58OWlpaZSUlDBo0CBmz57NXXfd1eXGmG0tP+vq6tixY0e3Yw8YMKDD1yNHjmTevHns2rWLnJwcsrKyaGpqIiAggMmTJ3P99def82aXIiIiIiIiIn2ZXc2IEBERERERERH7Zjd7RIiIiIiIiIiI/VMhQkRERERERER6jQoRIiIiIiIiItJrVIgQERERERERkV6jQoSIiIiIiIiI9BoVIkRERERERESk16gQISIiIiIiIiK9RoUIEREREREREek1KkSIiIiIiIiISK9xtnUAcn45evQomzZtIjU1lb1795KWlkZDQwMXXHABy5Yts3V4ImLnzGYzO3fuZP369aSkpJCZmUl1dTX9+vVjxIgRXHfddVxzzTUYDAZbhyoidmzVqlVs3ryZffv2UVJSwvHjx3FxcWHgwIFceuml/PznP8ff39/WYYqIA/nmm2+YO3cuABEREaxfv97GEZ0bg9lsNts6CDl/LFmyhBdffLHTcRUiRKQnfP/999x1113tX0dFReHj40NBQQHHjx8H4LLLLmPhwoW4urraJkgRsXvXXnstBw4cwNXVlaCgIPz9/SkrK6OwsBCAwMBAFi1axLBhw2wcqYg4gpqaGq6++ur2HOMIhQjNiJBe5e3tzUUXXURCQgIJCQns37+ft956y9ZhiYiDMJvNREZG8vOf/5yrrrqKwMDA9nOff/458+fP5+uvv+b111/nf//3f20YqYjYs9tvv52YmBjGjBmDi4tL+/GDBw/y6KOPcujQIR555BH++9//2jBKEXEUf/7znyksLGTatGmsW7fO1uH0CM2IEJv64IMPePbZZzUjQkR6RHV1NW5ubh0+GJzsnXfe4c9//jN+fn58//33GI3aKklEetaePXu4+eabAfjyyy8ZNGiQjSMSEXu2a9cubr31Vi6//HKmT5/O448/7hAzIvQOTEREHIa3t3e3RQiAKVOmAHD8+HHKysp6KywROY/Exsa2/72urs6GkYiIvWtqamL+/Pm4u7vz5JNP2jqcHqVChIiInDfq6+vb/+7u7m7DSETEUaWkpADg6elJTEyMjaMREXv27rvvcujQIR566CFCQ0NtHU6P0h4RIiJy3mhbrz1s2DC8vb1tHI2IOAqTydTeGeyVV14B4NFHH8XLy8vGkYmIvcrIyODdd99l5MiR3HHHHbYOp8epECEiIueF1NRUPvroI4D29lciIueiq25go0aN4qWXXmpfCiYicqbMZjNPPPEEzc3NLFiwACcnJ1uH1OO0NENERBzesWPHePDBB2lubuaKK67gqquusnVIIuIAQkJCGDduHKNHjyYoKAiDwUBaWhorV66ksrLS1uGJiJ1avnw5O3bs4PbbbychIcHW4ViFZkSIiIhDq6qq4p577qGwsJCRI0fy0ksv2TokEXEQs2bNYtasWe1fHzhwgGeffZYvvviCjIwM/vWvfznkbzJFxHqKi4t59dVXCQkJ4be//a2tw7EazYgQERGHVVNTw69+9Sv279/PkCFDeP/997U3hIhYzbBhw3j33Xfx9/cnLS2tfV8aERFLPfvss1RXV/PEE0849HsWzYgQERGHVFdXx7333suuXbsYOHAgixcvxt/f39ZhiYiD8/b25oILLmDNmjXs27eP2bNn2zokEbEj+/fvB2DBggUsWLCgw7m27l9FRUVcfPHFACxcuJBx48b1bpA9QIUIERFxOA0NDdx3330kJycTERHBkiVLCAoKsnVYInKeaG5uBqClpcXGkYiIvTp27Fi350wmU/v5pqam3gqpR6kQISIiDqWpqYkHH3yQ77//npCQEJYuXUpYWJitwxKR88Tx48fZtm0bAMOHD7dxNCJib9avX9/tuU8//ZTHH3+ciIiIU15nD7RHhIiIOIyWlhYeeeQRvvnmG4KCgli6dClRUVG2DktEHMi2bdt46623yM/P73Ru37593H333VRVVRESEsLMmTNtEKGISN+nGRHSq4qKirjuuuvav25sbARgx44dTJw4sf34r371K+65557eDk9E7NyqVatYs2YNAK6urvz+97/v9tr58+czYsSI3gpNRBxEZWUlr7/+Oq+//jpBQUEEBwfj5OREUVERR48eBVrber777rt4eXnZOFoRkb5JhQjpVS0tLRw/frzT8ebm5g7H2zZiERE5E23FTYCCggIKCgq6vbaqqqo3QhIRBzN27Fgef/xxtm7dyuHDh8nOzqaxsREfHx8mTpzI1KlTuemmmxx6t3sRkXNlMJvNZlsHISIiIiIiIiLnB+0RISIiIiIiIiK9RoUIEREREREREek1KkSIiIiIiIiISK9RIUJEREREREREeo0KESIiIiIiIiLSa1SIEBEREREREZFeo0KEiIiIiIiIiPQaFSJEREREREREpNeoECEiIiIiIiIivUaFCBEREZFeFhcXR1xcHFu3brV1KCIiIr3O2dYBiIiIiCxcuJA333zT4usPHjxoxWhERETEmlSIEBERkT6lf//+tg5BRERErEiFCBEREelTNm3aZOsQRERExIq0R4SIiIiIiIiI9BrNiBARERG7NnXqVAoKCnjxxReZMWMG7777LklJSRQVFeHh4UFiYiL33nsvo0eP7naMlpYWPvvsM/79739z8OBBampq8Pf3Z+zYsdx+++1MnDjxlDEUFRWxbNkyNm3aRH5+Pk1NTQQHBzNkyBCuvPJKZs2ahZubW5ePra6u5r333mPNmjUUFhbi4eHBmDFjmDdv3iljFhERsVcqRIiIiIhDqKys5KabbiIrKwsXFxfc3Nw4fvw469atY8OGDTz77LPcdNNNnR5XVVXFvHnz2LZtGwBOTk54eXlx9OhR1qxZw5o1a/jlL3/J7373uy6f9/PPP+fJJ5+koaEBABcXF7y8vCgqKiIvL4/169cTFxfH8OHDOz326NGj3HDDDeTk5ODm5obRaOT48eN8/fXXbNq0iXfeeYfJkyf34HdJRETE9rQ0Q0RERBzCm2++SVlZGa+99hq7du0iJSWFL7/8kgsuuACTycRTTz3Fvn37Oj3uD3/4A9u2bcPFxYUnnniClJQUkpOT2bhxIzfeeCMAixYt4h//+Eenx3799dc89thjNDQ0MG7cOD788EP27NnD1q1b2blzJx9++CFz5szBxcWly5ifeeYZXFxcWLp0Kbt27WLnzp188sknxMTE0NTUxJNPPonJZOrZb5SIiIiNGcxms9nWQYiIiMj57eT2nafrmjFr1iyeeOKJ9q/blmYALFmyhAsvvLDD9fX19Vx77bVkZ2dz6aWX8te//rX93O7du5kzZw7QWhS45ZZbOj3fb37zG9asWYO/vz/ffPNN+xKL5uZmrrzySvLz80lMTGTJkiW4urpadL9xcXEABAQE8MUXXxAYGNjh/MGDB5k9ezYAy5cvJzEx0aJxRURE7IFmRIiIiEifcuzYsVP+qa6u7vJx48aN61SEAHB3d+fuu+8GYOPGjVRVVbWf+/LLLwEIDQ3l5ptv7nLchx56CIDy8vIOHT22bt1Kfn4+AI8//rjFRYiTzZkzp1MRAloLFZGRkUBrUUJERMSRaI8IERER6VPO9oP3pEmTTnvOZDKxb9++9q9TU1MBmDhxIkZj17+fGTRoECEhIRQXF5OamsrUqVMB2LlzJwBBQUEkJCScVcyn2owyODiY/Px8KioqzmpsERGRvkozIkRERMQhhISEWHSurKys/e+lpaWnfSy0zpg4+Xpo3WgSIDw8/MyDPcHLy6vbc87Orb8vam5uPuvxRURE+iIVIkRERETOgsFgsHUIIiIidkmFCBEREXEIxcXFFp0LCAho/3vb/gxHjhw55dht50/ez6FtU83CwsIzD1ZEROQ8pkKEiIiIOIStW7ee9pzRaGTEiBHtx+Pj49vPd9cmMyMjo72QcfJeEOPGjQNal2js3bv33IIXERE5j6gQISIiIg4hJSWly2JEQ0MDixYtAmDy5Mn4+Pi0n7vqqquA1hkTn3zySZfjvvHGGwD4+/tz0UUXtR+fOHEiUVFRALz44os0Njb2zI2IiIg4OBUiRERExCH069eP3/zmN6xevbp9g8eMjAzmzp1LZmYmTk5O/OY3v+nwmFGjRnHllVcC8Oyzz/LBBx9QV1cHtM50eOKJJ1i9ejXQ2sbTzc2t/bFOTk7Mnz8fg8FASkoKd911F9u3b2+fWdHY2MjWrVt59NFHOXz4sNXvX0RExF6ofaeIiIj0KRdffPFpr1m4cGH70og2DzzwAB999BEPPfQQrq6uuLm5UVVVBbRuLPn000932Wbz+eefp7y8nG3btvHss8/y4osv4uXlRWVlJWazGYBf/vKX3HrrrZ0ee+mll/LSSy8xf/58UlJSuP3223F1dcXT05Pq6ur2gsjdd999xt8HERERR6VChIiIiPQpx44dO+01TU1NnY75+Pjwz3/+k3fffZekpCSKiorw8/Nj7Nix3HvvvYwdO7bLsfr168eSJUv47LPPWLlyJQcPHqS2tpb+/fszbtw4br/9diZOnNhtLNdddx3jx4/n73//O5s2baKwsJCGhgbCw8MZOnQoM2bMYNCgQZZ/A0RERBycwdxW6hcRERGxQ1OnTqWgoIAXX3yRG264wdbhiIiIyGlojwgRERERERER6TUqRIiIiIiIiIhIr1EhQkRERERERER6jQoRIiIiIiIiItJrtFmliIiIiIiIiPQazYgQERERERERkV6jQoSIiIiIiIiI9BoVIkRERERERESk16gQISIiIiIiIiK9RoUIEREREREREek1KkSIiIiIiIiISK9RIUJEREREREREeo0KESIiIiIiIiLSa1SIEBEREREREZFe8/8Bn92e07qX3XEAAAAASUVORK5CYII=\n"
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"source": [
"Saving the model"
],
"metadata": {
"id": "rJ6ylBCnMGBN"
}
},
{
"cell_type": "code",
"source": [
"import os\n",
"\n",
"# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()\n",
"\n",
"output_dir = './cross_entropy_model/'\n",
"\n",
"# Create output directory if needed\n",
"if not os.path.exists(output_dir):\n",
" os.makedirs(output_dir)\n",
"\n",
"print(\"Saving model to %s\" % output_dir)\n",
"\n",
"# Save a trained model, configuration and tokenizer using `save_pretrained()`.\n",
"# They can then be reloaded using `from_pretrained()`\n",
"model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training\n",
"model_to_save.save_pretrained(output_dir)\n",
"tokenizer.save_pretrained(output_dir)\n",
"\n",
"# Good practice: save your training arguments together with the trained model\n",
"# torch.save(args, os.path.join(output_dir, 'training_args.bin'))\n",
"\n",
"\n",
"# Copy the model files to a directory in your Google Drive.\n",
"!cp -r ./cross_entropy_model/ ./drive/MyDrive"
],
"metadata": {
"id": "OdCk0ZDvMIUL",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "46abf954-6e03-4677-879f-420b74c9e4d1"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Saving model to ./cross_entropy_model/\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"#### Testing After Cross-entropy training"
],
"metadata": {
"id": "so-0GYfcLuaf"
}
},
{
"cell_type": "code",
"source": [
"import pandas as pd\n",
"\n",
"# Load the test dataset into a pandas dataframe.\n",
"# df = pd.read_csv(\"./cola_public/raw/out_of_domain_dev.tsv\", delimiter='\\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])\n",
"\n",
"# Report the number of sentences.\n",
"print('Number of test sentences: {:,}\\n'.format(test_df.shape[0]))\n",
"\n",
"# Create sentence and label lists\n",
"sentences = test_df.sentence.values\n",
"labels = test_df.label.values\n",
"# sentences=[\"go fuck yourself\",\n",
"# \"this is crap\",\n",
"# \"thank you for the information\",\n",
"# \"yeah that sucked, fixed, Done.\",\n",
"# \"Crap, this is an artifact of a previous revision. It's simply the last time a change was made to Tuskar's cloud.\",\n",
"# \"Ah damn I misread the bug -_-\",\n",
"# \"wtf...\",\n",
"# \"I appreciate your help.\",\n",
"# \"fuuuuck\",\n",
"# \"what the f*ck\",\n",
"# \"absolute shit\",\n",
"# \"Get the hell outta here\",\n",
"# \"shi*tty code\",\n",
"# \"you are an absolute b!tch\",\n",
"# \"Nothing particular to worry about\"]\n",
"\n",
"# labels = [1,1,0,1,1,1,0,0,1,1,1,1,1,1,0]\n",
"# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
"input_ids = []\n",
"attention_masks = []\n",
"\n",
"# For every sentence...\n",
"for sent in sentences:\n",
" # `encode_plus` will:\n",
" # (1) Tokenize the sentence.\n",
" # (2) Prepend the `[CLS]` token to the start.\n",
" # (3) Append the `[SEP]` token to the end.\n",
" # (4) Map tokens to their IDs.\n",
" # (5) Pad or truncate the sentence to `max_length`\n",
" # (6) Create attention masks for [PAD] tokens.\n",
" encoded_dict = tokenizer.encode_plus(\n",
" str(sent), # Sentence to encode.\n",
" add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
" max_length = 512, # Pad & truncate all sentences.\n",
" pad_to_max_length = True,\n",
" return_attention_mask = True, # Construct attn. masks.\n",
" return_tensors = 'pt', # Return pytorch tensors.\n",
" )\n",
"\n",
" # Add the encoded sentence to the list.\n",
" input_ids.append(encoded_dict['input_ids'])\n",
"\n",
" # And its attention mask (simply differentiates padding from non-padding).\n",
" attention_masks.append(encoded_dict['attention_mask'])\n",
"\n",
"# Convert the lists into tensors.\n",
"input_ids = torch.cat(input_ids, dim=0)\n",
"attention_masks = torch.cat(attention_masks, dim=0)\n",
"labels = torch.tensor(labels)\n",
"\n",
"# Set the batch size.\n",
"batch_size = 16\n",
"\n",
"# Create the DataLoader.\n",
"prediction_data = TensorDataset(input_ids, attention_masks, labels)\n",
"prediction_sampler = SequentialSampler(prediction_data)\n",
"prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)\n",
"\n",
"print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))\n"
],
"metadata": {
"id": "Tx1rw9UuLyre",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "b504e8af-3403-496a-e9b9-dbfd9145923f"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Predicting labels for 15 test sentences...\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2393: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
" warnings.warn(\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# Prediction on test set\n",
"print(\"Running Test set...\")\n",
"\n",
"t0 = time.time()\n",
"\n",
"# Put the model in evaluation mode--the dropout layers behave differently\n",
"# during evaluation.\n",
"model.eval()\n",
"\n",
"# Tracking variables\n",
"total_eval_accuracy = 0\n",
"total_eval_loss = 0\n",
"nb_eval_steps = 0\n",
"\n",
"\n",
"# Put model in evaluation mode\n",
"model.eval()\n",
"\n",
"# Tracking variables\n",
"predictions , true_labels = [], []\n",
"\n",
"# Predict\n",
"for batch in prediction_dataloader:\n",
" # Add batch to GPU\n",
" batch = tuple(t.to(device) for t in batch)\n",
"\n",
" # Unpack the inputs from our dataloader\n",
" b_input_ids, b_input_mask, b_labels = batch\n",
"\n",
" # Telling the model not to compute or store gradients, saving memory and\n",
" # speeding up prediction\n",
" with torch.no_grad():\n",
" # Forward pass, calculate logit predictions.\n",
" # result = model(b_input_ids,\n",
" # token_type_ids=None,\n",
" # attention_mask=b_input_mask,\n",
" # labels=b_labels,\n",
" # return_dict=True)\n",
"\n",
" result = model(b_input_ids,\n",
" token_type_ids=None,\n",
" attention_mask=b_input_mask,\n",
" return_dict=True)\n",
"\n",
" # Get the loss and \"logits\" output by the model. The \"logits\" are the\n",
" # output values prior to applying an activation function like the\n",
" # softmax.\n",
" # loss = result.loss\n",
" logits = result.logits\n",
"\n",
" # Accumulate the validation loss.\n",
" # total_eval_loss += loss.item()\n",
"\n",
" # Move logits and labels to CPU\n",
" logits = logits.detach().cpu().numpy()\n",
" label_ids = b_labels.to('cpu').numpy()\n",
"\n",
" # Store predictions and true labels\n",
" predictions.append(logits)\n",
" true_labels.append(label_ids)\n",
"\n",
" # Calculate the accuracy for this batch of test sentences, and\n",
" # accumulate it over all batches.\n",
" total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
"\n",
"\n",
"print(' DONE.')\n",
"# Report the final accuracy for this run.\n",
"avg_val_accuracy = total_eval_accuracy / len(prediction_dataloader) * 100.0\n",
"print(\" Accuracy: {0:.2f}\".format(avg_val_accuracy))\n"
],
"metadata": {
"id": "4IanskVZL4Qn",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "e3707fd9-74f4-432b-ba42-fd15391f50db"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Running Test set...\n",
" DONE.\n",
" Accuracy: 86.67\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"from sklearn.metrics import matthews_corrcoef\n",
"\n",
"matthews_set = []\n",
"\n",
"# Evaluate each test batch using Matthew's correlation coefficient\n",
"print('Calculating Matthews Corr. Coef. for each batch...')\n",
"final_labels = []\n",
"# For each input batch...\n",
"for i in range(len(true_labels)):\n",
"\n",
" # The predictions for this batch are a 2-column ndarray (one column for \"0\"\n",
" # and one column for \"1\"). Pick the label with the highest value and turn this\n",
" # in to a list of 0s and 1s.\n",
" pred_labels_i = np.argmax(predictions[i], axis=1).flatten()\n",
" final_labels.append(pred_labels_i)\n",
" # Calculate and store the coef for this batch.\n",
" matthews = matthews_corrcoef(true_labels[i], pred_labels_i)\n",
" matthews_set.append(matthews)\n",
"\n",
"print(final_labels)"
],
"metadata": {
"id": "CHmFFznvL5L4",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "8dac4d8f-642c-40ac-c305-e419dc176596"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Calculating Matthews Corr. Coef. for each batch...\n",
"[array([1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0])]\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import numpy as np\n",
"from sklearn.metrics import f1_score\n",
"\n",
"count = 0\n",
"# Calculate accuracy\n",
"for i in range(len(true_labels[0])):\n",
" if true_labels[0][i] == final_labels[0][i]:\n",
" count = count + 1\n",
"\n",
"\n",
"print(\" Accuracy: {0:.2f}\".format(count/len(true_labels[0]) * 100.0))\n",
"\n",
"\n",
"def f1(actual, predicted, label):\n",
"\n",
" \"\"\" A helper function to calculate f1-score for the given `label` \"\"\"\n",
" # F1 = 2 * (precision * recall) / (precision + recall)\n",
" tp = np.sum((actual==label) & (predicted==label))\n",
" fp = np.sum((actual!=label) & (predicted==label))\n",
" fn = np.sum((predicted!=label) & (actual==label))\n",
"\n",
" precision = tp/(tp+fp)\n",
" recall = tp/(tp+fn)\n",
" f1 = 2 * (precision * recall) / (precision + recall)\n",
" return f1\n",
"\n",
"def f1_macro(actual, predicted):\n",
" # `macro` f1- unweighted mean of f1 per label\n",
" return np.mean([f1(actual, predicted, label)\n",
" for label in np.unique(actual)])\n",
"\n",
"\n",
"print(\" F1 Score: {0:.2f}\".format(f1_macro(true_labels[0], final_labels[0])))\n",
"\n",
"#define array of actual classes\n",
"# actual = np.repeat(true_labels, repeats=[160, 240])\n",
"\n",
"#define array of predicted classes\n",
"# pred = np.repeat(final_labels, repeats=[120, 40, 70, 170])\n",
"\n",
"#calculate F1 score\n",
"# f1_score(actual, pred)"
],
"metadata": {
"id": "6QmtX2DCL75J",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "7ad9dd90-8ed8-4e73-d28a-d3519373cfec"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
" Accuracy: 86.67\n",
" F1 Score: 0.85\n"
]
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment