Skip to content

Instantly share code, notes, and snippets.

@morganmcg1

morganmcg1/_admin_init.ipynb

Created February 21, 2021 22:08
Show Gist options
  • Save morganmcg1/9ad869d73c7dcbe15a27d7e261aa3fb5 to your computer and use it in GitHub Desktop.
Save morganmcg1/9ad869d73c7dcbe15a27d7e261aa3fb5 to your computer and use it in GitHub Desktop.
Download ZIP
Testing out ADMIN with BTE, trained SentencePieceUnigram and PreTrained T5 Tokenizers
Display the source blob
Display the rendered blob
Raw
_admin_init.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"#default_exp xtransformer"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"#hide\n",
"from nbdev.showdoc import *"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"#hide\n",
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"#!pip install datasets"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"#export\n",
"from fastai.basics import *\n",
"from fastai.text.all import *\n",
"from fastai.callback import *\n",
"from fastai.callback.wandb import *\n",
"\n",
"from transformers_sandbox.core import *\n",
"from transformers_sandbox.layers import *\n",
"from transformers_sandbox.attention import *\n",
"from transformers_sandbox.tokenizers import ByteTextTokenizer, SPUniTokenizer\n",
"from transformers_sandbox.metrics import bpc\n",
"from transformers_sandbox.transformer import LMMixin, EncDecMixin, TransformerLM\n",
"from transformers_sandbox.config import ConfigBase, update_sig\n",
"from transformers_sandbox.xtransformer import *\n",
"\n",
"import wandb\n",
"from datasets import load_dataset"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# ADMIN Init\n",
"\n",
"> ADMIN Initialisation"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"📕 **Paper**: https://arxiv.org/abs/2004.08249\n",
"\n",
"Transformers have proved effective in many NLP tasks. However, their training requires non-trivial efforts regarding designing cutting-edge optimizers and learning rate schedulers carefully (e.g., conventional SGD fails to train Transformers effectively). Our objective here is to understand what complicates Transformer training from both empirical and theoretical perspectives. \n",
"\n",
"Our analysis reveals that unbalanced gradients are not the root cause of the instability of training. Instead, we identify an amplification effect that influences training substantially -- for each layer in a multi-layer Transformer model, heavy dependency on its residual branch makes training unstable, since it amplifies small parameter perturbations (e.g., parameter updates) and results in significant disturbances in the model output. \n",
"\n",
"Yet we observe that a light dependency limits the model potential and leads to inferior trained models. Inspired by our analysis, we propose Admin (**Ad**aptive **m**odel **in**itialization) to stabilize stabilize the early stage's training and unleash its full potential in the late stage. Extensive experiments show that Admin is more stable, converges faster, and leads to better performance. Implementations are released at: [this https URL](https://github.com/LiyuanLucasLiu/Transforemr-Clinic)."
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"#export\n",
"class AdminResidual(Module):\n",
" def __init__(self, sublayer, d_model):\n",
" self.sublayer = sublayer\n",
" self.w = torch.nn.Parameter(torch.ones(d_model))\n",
" def forward(self, x, *args, **kwargs):\n",
" return x*self.w + self.sublayer(x, *args, **kwargs)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"#export\n",
"class TransformerEncoderBlockAdmin(Module):\n",
" \"\"\"\n",
" Bacis transformer encoder block. Consists of multi-head attention and positional \n",
" feedforward layers\n",
" \"\"\"\n",
" def __init__(self,\n",
" d_model:int, \n",
" n_heads:int = 8, \n",
" d_ff:int = None, \n",
" attn_dropout:float = 0.1,\n",
" ff_dropout:float = 0.1,\n",
" causal:bool = False, \n",
" attn_bias:bool = False, \n",
" prenorm:bool=False,\n",
" shared_qk:bool=False):\n",
" store_attr('attn_dropout') # mb separate argument attn_post_dropout\n",
" \n",
" self.attn = PostNorm(d_model, AdminResidual(Attention(d_model, n_heads=n_heads, causal=causal, dropout=attn_dropout, bias=attn_bias, shared_qk=shared_qk), d_model))\n",
" self.ff = PostNorm(d_model, AdminResidual(FeedForward(d_model, d_ff=d_ff, dropout=ff_dropout), d_model))\n",
" \n",
" def forward(self, x, mask=None):\n",
" out = self.attn(x, mask=mask)\n",
" return self.ff(out)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"# bs = 4\n",
"# sl = 128\n",
"# d = 64\n",
"# x = torch.randn(bs, sl, d)\n",
"# m = TransformerEncoderBlockAdmin(d)\n",
"# out = m(x)\n",
"# assert (out.size() == (bs, sl, d))\n",
"# out.shape"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"#export\n",
"class TransformerEncoderAdmin(Module):\n",
" \"\"\"Stack of TransformerEncoderBlocks\"\"\"\n",
" def __init__(self, \n",
" d_model, \n",
" n_layers=6, \n",
" n_heads=8, \n",
" d_ff=None,\n",
" ff_dropout=0.1, \n",
" attn_dropout=0.1,\n",
" attn_bias=False,\n",
" causal=False, \n",
" prenorm=False,\n",
" shared_qk:bool=False,\n",
" final_norm=None):\n",
" store_attr('d_model')\n",
" self.layers = nn.ModuleList([]) \n",
" for _ in range(n_layers):\n",
" self.layers.append(TransformerEncoderBlockAdmin(d_model, n_heads, causal=causal, \n",
" d_ff=d_ff, attn_dropout=attn_dropout, ff_dropout=ff_dropout, \n",
" prenorm=prenorm, attn_bias=attn_bias, shared_qk=shared_qk))\n",
" self.norm = None if final_norm is None else final_norm(d_model)\n",
" \n",
" def forward(self, x, mask=None):\n",
" for layer in self.layers: x = layer(x, mask=mask)\n",
" if self.norm is not None: x = self.norm(x)\n",
" return x"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"#export\n",
"class TransformerLMAdmin(Module, LMMixin):\n",
" \"\"\"\n",
" tmp\n",
" Basic Transformer for language modelling\n",
" \n",
" Parameters:\n",
" * vocab_sz: int\n",
" * d_model: int - inner dimension of the model\n",
" * n_layers: int (default: 6) \n",
" * n_heads: int (default: 8)\n",
" * d_ff: int - inner dimension of the pointwise FeedForward net, if None defaults to 4*d_model\n",
" * attn_dropout: float - attention dropout\n",
" * ff_dropout: float - feed-forward dropout\n",
" * emb_dropout: float - embedding dropout\n",
" * causal: bool (default: True) - if True does causal masking automatically\n",
" * max_seq_len: int (default: 512)\n",
" * tie_weights: bool - if True target embedding weights are used for computation output projection\n",
" * prenorm: bool - wether to use PreNorm or PostNorm\n",
" * attn_bias: bool - wether to allow biases in attention projection layers\n",
" * pad_idx: int - padding token id, required for autogeneration of padding mask\n",
" * pos_enc: str from {'absolute', 'fixed', 'axial'} - type of positional encoding to use\n",
" * axial_shape: tuple - [optional] should be factors of max_seq_len\n",
" * axial_emb_dims: tuple - [optional] axial embedding components, should sum to d_model\n",
" Inputs:\n",
" * x - input ids, shape [bs, sl]\n",
" * mask - optional boolean mask, shape [bs, sl]\n",
" Returns:\n",
" * logits - target token logits, shape [bs, sl, vocab_sz]\n",
" \"\"\"\n",
" def __init__(self, \n",
" vocab_sz:int, \n",
" d_model:int, \n",
" n_layers:int=6,\n",
" n_heads:int=8,\n",
" d_ff:int=None,\n",
" attn_dropout:float=0.1,\n",
" ff_dropout:float=0.1,\n",
" emb_dropout:float=0.1,\n",
" tie_weights:bool=True,\n",
" causal:bool=True,\n",
" pos_enc:str='absolute',\n",
" max_seq_len:int=512,\n",
" axial_shape:tuple=None,\n",
" axial_emb_dims:tuple=None,\n",
" pad_idx:int=None,\n",
" prenorm:bool=False,\n",
" attn_bias:bool=False,\n",
" shared_qk:bool=False):\n",
" store_attr()\n",
" self.emb = TransformerEmbedding(vocab_sz, d_model, max_seq_len, dropout=emb_dropout, \n",
" pos_enc=pos_enc, axial_shape=axial_shape, \n",
" axial_emb_dims=axial_emb_dims)\n",
" final_norm = None\n",
" self.encoder = TransformerEncoderAdmin(d_model, n_layers, n_heads, causal=causal, d_ff=d_ff,\n",
" attn_dropout=attn_dropout, ff_dropout=ff_dropout,\n",
" prenorm=prenorm, attn_bias=attn_bias,\n",
" shared_qk=shared_qk, final_norm=final_norm)\n",
" self.proj = nn.Linear(d_model, vocab_sz)\n",
" if tie_weights: self.proj.weight = self.emb.emb.weight\n",
" \n",
" def forward(self, x, mask=None):\n",
" x = self.emb(x)\n",
" x = self.encoder(x, mask=mask)\n",
" return self.proj(x)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"@patch(cls_method=True)\n",
"def from_config(cls:TransformerLMAdmin, config):\n",
" return cls(**config.dict())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Data"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Reusing dataset wikitext (/home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91)\n",
"Loading cached processed dataset at /home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91/cache-0b8034a134b9567b.arrow\n",
"Reusing dataset wikitext (/home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91)\n",
"Loading cached processed dataset at /home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91/cache-d23112255b6f8a24.arrow\n"
]
}
],
"source": [
"train_ds = load_dataset('wikitext', name='wikitext-2-raw-v1', split='train')\n",
"\n",
"train_ds = train_ds.filter(lambda x: x['text'] != '')\n",
"train_df = train_ds.data.to_pandas()\n",
"\n",
"valid_ds = load_dataset('wikitext', name='wikitext-2-raw-v1', split='validation')\n",
"\n",
"valid_ds = valid_ds.filter(lambda x: x['text'] != '')\n",
"valid_df = valid_ds.data.to_pandas()\n",
"\n",
"df = pd.concat([train_df, valid_df])\n",
"df['is_valid'] = [False]*len(train_df) + [True]*len(valid_df)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Tokenizer + DataLoaders"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"ByteTextTokenizer"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"tok = ByteTextTokenizer(is_lm=True, add_bos=False, add_eos=False)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"#Run setup\n",
"n_epochs = 10\n",
"bs = 16\n",
"sl = 512\n",
"n_layers = 6\n",
"pad_id = tok.pad_token_id\n",
"vocab_size = tok.vocab_size"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"df['toks'] = df['text'].apply(tok)\n",
"df['lens'] = df['toks'].apply(len)\n",
"splits = ColSplitter()(df)\n",
"tfms = [attrgetter(\"text\"), tok]\n",
"dsets = Datasets(df, [tfms], splits=splits, dl_type=LMDataLoader)\n",
"dl_kwargs = [{'lens':df['lens'].values[splits[0]]},\n",
" {'val_lens':df['lens'].values[splits[1]]}]\n",
"dls = dsets.dataloaders(bs=bs, seq_len=sl, dl_kwargs=dl_kwargs, shuffle_train=True, n_workers=2)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>text</th>\n",
" <th>text_</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>The film is set in 2034 , two years after the events of 2nd GIG . Togusa is now the team leader for Public Security Section 9 , which has increased considerably in size . Section 9 deals with a series of complicated incidents , including the assassination of Ka Rum , a former dictator of the Siak Republic , which leads to a terrorist plot using children as vectors for a cybernetic virus . Investigations reveal that a hacker nicknamed the \" The Puppeteer \" is behind the entire series of events . \\n Mozambiqu</td>\n",
" <td>The film is set in 2034 , two years after the events of 2nd GIG . Togusa is now the team leader for Public Security Section 9 , which has increased considerably in size . Section 9 deals with a series of complicated incidents , including the assassination of Ka Rum , a former dictator of the Siak Republic , which leads to a terrorist plot using children as vectors for a cybernetic virus . Investigations reveal that a hacker nicknamed the \" The Puppeteer \" is behind the entire series of events . \\n Mozambique</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>združenje ) its association with Yugoslavia on 25 June 1991 . The European Economic Community and the Conference on Security and Cooperation in Europe urged Croatian authorities to place a three @-@ month moratorium on the decision . Croatia agreed to freeze its independence declaration for three months , initially easing tensions . Nonetheless , the Croatian War of Independence escalated further . On 7 October , the eve of expiration of the moratorium , the Yugoslav Air Force attacked Banski dvori , the m</td>\n",
" <td>druženje ) its association with Yugoslavia on 25 June 1991 . The European Economic Community and the Conference on Security and Cooperation in Europe urged Croatian authorities to place a three @-@ month moratorium on the decision . Croatia agreed to freeze its independence declaration for three months , initially easing tensions . Nonetheless , the Croatian War of Independence escalated further . On 7 October , the eve of expiration of the moratorium , the Yugoslav Air Force attacked Banski dvori , the ma</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"(16, None)"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#collapse_output\n",
"dls.bs, dls.show_batch(max_n=2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Setup"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"#config = XConfig(vocab_sz=vocab_size, d_model=512, n_layers=n_layers, max_seq_len=512, pad_idx=pad_id)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"class CharLMConfig(ConfigBase):\n",
" _model = TransformerLM\n",
" _d = {\n",
" 'vocab_sz':256,\n",
" 'd_model':512,\n",
" 'n_layers':6,\n",
" 'n_heads':8,\n",
" 'd_ff':4096,\n",
" 'attn_dropout':0.1,\n",
" 'ff_dropout':0.1,\n",
" 'emb_dropout':0.1,\n",
" 'tie_weights':True,\n",
" 'causal':True,\n",
" 'pos_enc':'absolute',\n",
" 'max_seq_len':512,\n",
" 'axial_shape':None,\n",
" 'axial_emb_dims':None,\n",
" 'pad_idx':None,\n",
" 'prenorm':False,\n",
" 'attn_bias':False,\n",
" 'shared_qk':False,\n",
" }\n",
" @update_sig(_d)\n",
" def __init__(self, **kwargs):\n",
" super().__init__(**kwargs)\n",
"\n",
"config = CharLMConfig(vocab_sz=vocab_size, d_model=512, n_layers=n_layers, \n",
" max_seq_len=512, pad_idx=pad_id)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Learner"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"learn = Learner(dls, TransformerLMAdmin.from_config(config),\n",
" loss_func=CrossEntropyLossFlat(ignore_index=pad_id), #opt_func=Adam,\n",
" cbs = [\n",
" #GradientClip(1.0),\n",
" SaveModelCallback(with_opt=True)],\n",
" metrics=[accuracy, perplexity, bpc]).to_fp16()"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"class BreakFitCallback(Callback):\n",
" order=-1\n",
" \"Cancels fit after one batch before weight update\"\n",
" def before_step(self):\n",
" self.model.zero_grad(set_to_none=True)\n",
" raise CancelStepException\n",
" def after_step(self):\n",
" raise CancelBatchException\n",
" def after_batch(self):\n",
" print('Fit canceled')\n",
" raise CancelFitException"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"def res_submodules(model):\n",
" return [m.sublayer for m in learn.model.modules() if isinstance(m, AdminResidual)]\n",
"\n",
"def res_modules(model):\n",
" return [m for m in learn.model.modules() if isinstance(m, AdminResidual)]"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"learn.add_cb(ActivationStats(modules=res_submodules(learn.model)));"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"12"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(learn.activation_stats.modules)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"train_eval -10\n",
"recorder 50\n",
"progress 60\n",
"save_model 60\n",
"mixed_precision 10\n",
"activation_stats -20\n"
]
}
],
"source": [
"for cb in learn.cbs:\n",
" print(cb.name, cb.order)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>perplexity</th>\n",
" <th>bpc</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Fit canceled\n"
]
}
],
"source": [
"with learn.added_cbs(BreakFitCallback()), learn.removed_cbs(SaveModelCallback):\n",
" learn.fit(1, 1e-3)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(#12) [{'mean': 0.00481663690879941, 'std': 0.3442692756652832, 'near_zero': 0.6015446980794271},{'mean': -0.0025428251828998327, 'std': 0.3987179100513458, 'near_zero': 0.5950142542521158},{'mean': 0.004401894751936197, 'std': 0.19933182001113892, 'near_zero': 0.6423584620157877},{'mean': 0.003396979533135891, 'std': 0.4013838768005371, 'near_zero': 0.5894660949707031},{'mean': 0.009360421448946, 'std': 0.26095426082611084, 'near_zero': 0.6073331832885742},{'mean': -0.0018820768455043435, 'std': 0.3994253873825073, 'near_zero': 0.5938955942789713},{'mean': -0.02019776962697506, 'std': 0.3356610834598541, 'near_zero': 0.6348047256469727},{'mean': 0.013347254134714603, 'std': 0.40092015266418457, 'near_zero': 0.5801426569620768},{'mean': -0.024423303082585335, 'std': 0.3910963237285614, 'near_zero': 0.621729850769043},{'mean': 0.005235261749476194, 'std': 0.3965536057949066, 'near_zero': 0.587321917215983}...]"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"learn.activation_stats.stats[0]"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([0.11852133, 0.15897597, 0.03973317, 0.16110902, 0.06809713,\n",
" 0.15954064, 0.11266836, 0.16073697, 0.15295633, 0.15725476,\n",
" 0.24887388, 0.15945325])"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def variances(learn):\n",
" return np.array([stat['std']**2 for stat in learn.activation_stats.stats[0]])\n",
"\n",
"variances(learn)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([0.34426928, 0.52678013, 0.56323217, 0.69162092, 0.73921352,\n",
" 0.84022453, 0.90479038, 0.98963761, 1.06411415, 1.13560279,\n",
" 1.24034978, 1.30304291])"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def _init_scales(vars):\n",
" return np.sqrt(np.cumsum(vars))\n",
"scales = _init_scales(variances(learn))\n",
"scales"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Define ADMIN Initialisation"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"def admin_init(model, scales):\n",
" ms = res_modules(model)\n",
" for m, s in zip(ms, scales):\n",
" m.w.data *= s"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## LR Find"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"SuggestedLRs(lr_min=0.03019951581954956, lr_steep=0.25118863582611084)"
]
},
"execution_count": 40,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAr2UlEQVR4nO3dd5xU5b3H8c9vZ3uFZQtlka6CdFZR7DEaexeN/V4VjRpNzDXRmKvJTdEUNVFjRZPYNYiGqBhji70sHQQElra0LbTt9bl/7CxZyQLLsmfOlO/79ZoXM3POzPnuEfztc85TzDmHiIjErji/A4iIiL9UCEREYpwKgYhIjFMhEBGJcSoEIiIxToVARCTGxfsdYG/l5OS4gQMH+h1DRCSizJo1q9w5l9vRtogrBAMHDqSoqMjvGCIiEcXMVu9qmy4NiYjEOBUCEZEYp0IgIhLjVAhERGKcCoGISIxTIRARiXExUwi21zXy6vz1NDS1+B1FRCSsRNw4gq6auWADP3ppATnpSZx/cAEXHLwf/bNTAXDOUVnfRF1D8479HeAcOBzOgRn0zkzGzHz6CUREvBEzheDcCf3Jy0zmmU/X8NB7K3jwvRUckJ9BZV0TZVX1nWop9M5M5rjheXxzRD6HDe5FckIgBMlFRLxlkbZCWWFhodvXkcXrt9by/BdrWVCyley0JHLSE8lJTyIlMUD7X/gNwwziDOqbWvh4eQXvLyujpqGZxEAc+VlJ5KYnkZuRRM/URAJxRnycERdnpCYG6JmaSK/0RLLTkijomcKA7FTiAzFzNU5EwoiZzXLOFXa4LRYLwb6oa2zmk+IKPl1RwabtdZRV1VO6vZ6ttY20tDiaWhwtLY6axmaaW75+bhMCxsBeaQzJTSczJZ7E+DiS4gOkJcUzpiCLgwdlk5mc4NNPJiLRbHeFIGYuDXWX5IQAxx6Qx7EH5O12v5YWx/a6RiqqG9hc3cDqihqWl1axvLSKZaWVVNc309DcQn1jM7WNzbS41pbHiL6ZFA7IZlh+OsPyMhial052WmKIfjoRiUUqBB6JizN6pCbSIzWRIblw8MDsXe5b19jM7DVb+Kx4M58WV/DCF2upbfz3jeteaYkMy09n//wM9s/PYHifTIb3ySA1Uf/5RGTf6f8kYSA5IcCkITlMGpIDtLYm1m+r3dGCWF5axVebKpk+ex1V9U1Aa+thcG46I/tmMnFwL44YmrOjF5SIyN5QIQhDcXFGQc9UCnqmcky7S1DOOdZvq2Px+u0sXL+Nheu28/GKCl6Zux6AQTlpHDUsh28d1JtDBmXrxrSIdIpuFkc45xwryqr4YFk5Hywr55MVFdQ2NpOTnsi3DurNqaP7MnFQNnFxGv8gEsvUayiG1DY0897SUl5dsIF3FpdS29hM/+wUJk/oz7mFBfTJSvE7ooj4QIUgRtU0NPHmok288MVaPimuIM7gmAPyuOSwARw9LFetBJEYokIgrK6o5q9FJbxQtJayynoG9ErlkkMHcF5hf7JSNHZBJNqpEMgODU0tvLFoI09+vIqi1VvITI7n6qOHcPmkgaQlqe+ASLRSIZAOLVy3jd+/9RVvLS6lV1oi3zlmCBcfOkBzKIlEIRUC2a3Za7Zwz5tf8eHycvpkJXPjccM4d0KBup+KRJHdFQL9SxfG79eTp6+cyLNXTSQ/M5lbpi/g+Hvf5+/z1hNpvyiIyN5TIZAdJg3J4eVrJ/HYpYUkBuL47nNzuGjqZ6wqr/Y7moh4SIVAvsbMOH5EPq/feCS/OmsUC9Zt41u/f58H31tOY7NWdxOJRioE0qFAnHHhxP1466ajOfaAPH7zxlLOeOAjisuq/I4mIt1MhUB2Kz8zmYcvmcDDF09gw7ZaTn/gI15fsMHvWCLSjVQIpFNOHNmbV284kqF56Vz7zGx+/uqXulQkEiVUCKTT+vVI4cWrD+Oywwbw+IcrufCxT6moqvc7lojsIxUC2SuJ8XH87IyR/OGCscwr2cZZD37M8lLdNxDx2herNlO6vc6T71YhkC45Y2w/np9yKDUNTZz94Ed8tLzc70giUau5xXHR1M+Y+uFKT75fhUC6bPx+PXn52sPpnZXMZU98zl+L1vodSSQqrd9aS0NTC4Ny0jz5fhUC2Sf9s1OZ9p1JHDq4FzdPm88zn632O5JI1CkODuocHImFwMx6mNk0M1tiZovN7LCdtpuZ3Wdmy81svpmN9zKPeCMzOYGplxXyjQPzuO3lhfzl41V+RxKJKiuD43cG5UZgIQD+ALzhnDsQGAMs3mn7ScCw4GMK8JDHecQjyQkBHr54AieMyOeOGYuY+kGx35FEokZxeTUZSfHkpid58v2eFQIzywKOAh4HcM41OOe27rTbGcCTrtWnQA8z6+NVJvFWYnwcf7xoPCeP6s0vXlusYiDSTYrLqhmUm4aZN6sKetkiGASUAX8yszlmNtXMdm7X9APa32EsCb4nESohEMd9F4zbUQxe/EI3kEX21cryas/uD4C3hSAeGA885JwbB1QDt3Tli8xsipkVmVlRWVlZd2YUD8QH4rj3/LEcOSyHW6bPZ6ampBDpstqGZtZtrWVwbrpnx/CyEJQAJc65z4Kvp9FaGNpbB/Rv97og+N7XOOcedc4VOucKc3NzPQkr3SspPsAjl0xg3H49ufH5uXywTAVcpCtWVbT2GPKq6yh4WAiccxuBtWZ2QPCt44Avd9ptBnBpsPfQocA255x+fYwSqYnxPHHZwQzOTWPKk7OYX7LV70giEae4LNh11KMeQ+B9r6HvAs+Y2XxgLPArM7vGzK4Jbn8dKAaWA48B13qcR0IsKzWBJ684hOy0RKY8OcuzIfIi0WplebDraCS2CACcc3ODl3RGO+fOdM5tcc497Jx7OLjdOeeuc84Ncc6Ncs5pMeIolJeRzGOXFrKttpGrn55FXWOz35FEIkZxWTV9spJJTYz37BgaWSwhMaJvJvdMHsOcNVv5ySsLtRaySCcVl1d72hoAFQIJoZNG9eF73xzGtFklPO7R5Fki0cQ5R3FZlaf3B0CFQELshm8M46SRvfnV64v5WDOWiuxWRXUD2+uaGJzjXddRUCGQEIuLM3533hgG56Zzw/Nz2LhNN49FdmVlcLI5r+YYaqNCICGXlhTPwxePp6ahmeufna0lL0V2oTg42dwQtQgkGg3Ny+Cuc0ZTtHoLv565xO84ImGpuLyaxEAc/XqmeHocFQLxzelj+nLZYQOY+uFKTUMh0oHismoG9EolEOfNZHNtVAjEVz8+ZThj+/fg5mnzWVNR43cckbCyMgRdR0GFQHyWFB/g/m+PwwxueH6O7heIBDU1t7C6otrTyebaqBCI7/pnp3LX2aOZu3Yrd7/5ld9xRMLCuq21NDY7z8cQgAqBhIlTRvfh24f05+F/rdBMpSK0m2xOl4Ykltx+6kEMy0vn+y/Mo7yq3u84Ir5aEew6qktDElNSEgPcf+E4Kusa+Z+/ztN8RBLTVpZXk5WSQM/UBM+PpUIgYeXA3pn8+OThvLe0jKc/Xe13HBHfFJe19hjyap3i9lQIJOxcetgAjto/l1++vnhH81gk1qyu8Had4vZUCCTsmBm/PXc0yQkBvv/CXHUplZhT19jMhu117NcrNSTHUyGQsJSfmcydZ41ifsk27nt7md9xREKqZEsNzsHAXmoRSIw7aVQfzhlfwB/fXc6s1Vv8jiMSMqvKW0fZD1CLQAR+evoI+mSlcPO0eVriUmLG6s1thUAtAhEykhO465xRFJdV8wddIpIYsbqimozk+JB0HQUVAokARw7LZXJhAY++X8yCkm1+xxHx3OqKGgb2Ck3XUVAhkAhx2ykj6JWWyM3T5tHQpF5EEt1WV1SHrMcQqBBIhMhKSeCXZ41iycZKHnpvhd9xRDzT1NxCyZZaBqoQiPyn40fkc/qYvjzw7jKWbNzudxwRT6zfWkdTi2NAdmhuFIMKgUSYO04bQUZyAj+aNp8mDTSTKLSqonXW0VB1HQUVAokwvdKT+OnpBzGvZBtPfLTS7zgi3a6t6+jAEE0vASoEEoFOG92Hbw7P5+43v2JlebXfcUS61eryapIT4sjLSArZMVUIJOKYGb88aySJ8XHc8tJ8Wlo0XbVEj1UVNQzIDl3XUVAhkAiVn5nMT04ZzmcrN/Ps52v8jiPSbdZsrg7p/QFQIZAINrmwP0cMzeGumUtYv7XW7zgi+6ylxbG6okaFQKSzzIw7zx5Fc4vjf19ZqBXNJOJtqqyjvqklZHMMtVEhkIjWPzuVH5ywP28vKeW1BRv8jiOyT1ZXhHbW0TYqBBLxLp80kNEFWfx0xiK21jT4HUeky1YHxxCEah2CNioEEvHiA3HcdfZottQ08qvXF/sdR6TLVlfUkBAw+mQlh/S4nhYCM1tlZgvMbK6ZFXWw/Rgz2xbcPtfMbvcyj0SvEX0zmXLUYF4sKuHj5eV+xxHpktUVNRT0TCU+ENrf0UNxtGOdc2Odc4W72P5BcPtY59z/hSCPRKkbjxvGwF6p3PryAi1iIxFpVUXou46CLg1JFElOCPCrs0axuqKGB95Z7ncckb3inGNNcB2CUPO6EDjgTTObZWZTdrHPYWY2z8xmmtlBHueRKDdpaA5nj+/HI++v4KtNlX7HEem0zdUNVNY3sV929LUIjnDOjQdOAq4zs6N22j4bGOCcGwPcD7zS0ZeY2RQzKzKzorKyMk8DS+S77eThpCfFc9vLCzT9hESMf082F2WFwDm3LvhnKfAycMhO27c756qCz18HEswsp4PvedQ5V+icK8zNzfUyskSBXulJ/Pjk4XyxagsvFq31O45Ip6zeMf10FF0aMrM0M8toew6cACzcaZ/eFpxZycwOCeap8CqTxI5zJxQwcVA2v3p9MWWV9X7HEdmjVeU1mEFBz5SQH9vLFkE+8KGZzQM+B15zzr1hZteY2TXBfc4FFgb3uQ+4wGmeAOkGrTOUjqKusYWfv/ql33FE9mjt5hr6ZqWQFB8I+bHjvfpi51wxMKaD9x9u9/wB4AGvMkhsG5qXznXHDuXet77ijLF9OW54vt+RRHZpzeYa+meHvjUA6j4qUe47xwzhgPwMbnt5IZV1jX7HEdmlNZtrfOkxBCoEEuUS4+P49bmjKa2s466ZS/yOI9KhusZmSivr6d9ThUDEE2P79+CKIwbxzGdr+LRYfREk/JRsae06up8Po4pBhUBixE3HH8B+2anc8tJ8TT8hYWdNcAxBf10aEvFOSmKAu84ZxaqKGu596yu/44h8zdrNrSvs6dKQiMcmDcnh/ML+TP1gJYvWb/M7jsgOazbXkJIQICc90ZfjqxBITLn15APpmZrArdMX0KzpJyRMtPUYCo6vDTkVAokpPVITueO0g5hfso0/f7zK7zgiQOtgMr/GEIAKgcSgU0f34dgDcrn7zaU7emuI+MU5FywE/twfABUCiUFmxs/PHAnA/76yEM1qIn7aXN1AdUOzb4PJQIVAYlRBz1R+cMIBvLu0jBnz1vsdR2LYjq6jPvUYAhUCiWGXTxrI2P49+OmMRZqhVHyzdktr11G/BpOBCoHEsECc8bvzRlPd0KxLROKbtZHSIgiuLRAXfL6/mZ1uZgneRhPx3tC8DL73zWG8sWgjry3Y4HcciUFrKmrISU8iJTH000+36WyL4H0g2cz6AW8ClwB/9iqUSChNOXIwowuyuP1vi6io0iUiCa21W2rYz8euo9D5QmDOuRrgbOBB59x5gBaal6gQH4jjt+eOobKukdtnLPI7jsQYP6efbtPpQmBmhwEXAa8F3/OvHSPSzQ7oncGNxw3jtfkbeHW+ehFJaDQ2t7B+a62vYwig84Xge8CtwMvOuUVmNhh417NUIj645ughjCnI4ievLKR0e53fcSQGbNhaR4vzb9bRNp0qBM65fznnTnfO/Tp407jcOXeDx9lEQio+EMfdk8dS29DMLdMXqBeReK5tDEFEXBoys2fNLNPM0oCFwJdmdrO30URCb2heOj888UDeWVLKi0Vr/Y4jUc7vdQjadPbS0Ajn3HbgTGAmMIjWnkMiUee/Jg3k0MHZ/N/fv9zRx1vEC2u31JAQMHpnJvuao7OFICE4buBMYIZzrhFQu1miUlyc8dtzx2Bm/M9f59Gi6arFI2s211DQM5VAnD/TT7fpbCF4BFgFpAHvm9kAYLtXoUT81j87ldtPHcFnKzfzxEcr/Y4jUWrt5hoKevo7hgA6f7P4PudcP+fcya7VauBYj7OJ+Oq8wgK+OTyP3/xjKcs2VfodR6LQ2jAYQwCdv1mcZWb3mFlR8HE3ra0DkahlZtx59mjSk+L5/otzaWxu8TuSRJHtdY1sqWmMnEIAPAFUApODj+3An7wKJRIucjOS+NVZI1m4bjv3v7Pc7zgSRdaGSY8h6HwhGOKcu8M5Vxx8/AwY7GUwkXBx4sg+nD2+H398dzlz1271O45EibVhMoYAOl8Ias3siLYXZnY4UOtNJJHwc8dpB5GfkcRNL8yltqHZ7zgSBVaWBwuBj+sQtOlsIbgG+KOZrTKzVcADwNWepRIJM1kpCfz2vDEUl1fz6zeW+B1HosCKsiryMpLITPZ/Rv/O9hqa55wbA4wGRjvnxgHf8DSZSJg5fGgOl08ayJ8/XsWHy8r9jiMRbkVZFUNy0/2OAezlCmXOue3BEcYAN3mQRySs/ejEAxmcm8bN0+axrbbR7zgSoZxzrCitYkheeHS+3JelKv0dCifig5TEAPdMHktpZT0/09oF0kXlVQ1sr2uKzBbBTjTuXmLS2P49uO7YoUyfs46ZWt5SumBFWRVAZBQCM6s0s+0dPCqBviHKKBJ2vvuNoYzql8Ut0xewYZs60Mne2VEI8iKgEDjnMpxzmR08Mpxz8aEKKRJuEgJx/OGCsTQ0tXDTC/No1sR0shdWlFaTkhCgj8+zjrbZl0tDexTsbrrAzOaaWVEH283M7jOz5WY238zGe5lHpDsNzk3nZ6cfxCfFFTz6frHfcSSCrCirYnBuGnE+zzraxtNCEHSsc26sc66wg20nAcOCjynAQyHII9Jtziss4ORRvbn7zaXML9nqdxyJEOHUdRRCUwh25wzgyeCMpp8CPcysj8+ZRDrNzLjzrNHkZSRxw3NzqK5v8juShLnahmbWba2NqULggDfNbJaZTelgez+g/XqAJcH3vsbMprTNfFpWVuZRVJGuyUpN4N7zx7Jmcw23aq1j2YOV5dU4R9iMIQDvC8ERzrnxtF4Cus7MjurKlzjnHnXOFTrnCnNzc7s3oUg3mDi4Fz844QBmzFvPU5+u9juOhLFw6zoKHhcC59y64J+lwMvAITvtsg7o3+51QfA9kYjznaOHcNyBefz81S+ZvWaL33EkTK0oq8IMBuXEQIvAzNLMLKPtOXACsHCn3WYAlwZ7Dx0KbHPOaYSORKS4OOOeyWPpnZXMdc/MpqKq3u9IEoZWlFVT0DOF5ISA31F28LJFkA98aGbzgM+B15xzb5jZNWZ2TXCf14FiYDnwGHCth3lEPJeVmsBDF02gorqBG5+fq/EF8h9WlIZXjyEAzwaFOeeKgTEdvP9wu+cOuM6rDCJ+GNkvi1+cMZIfvjSfX7+xhB+fPNzvSBImWlocxeVVHDakl99Rvkajg0U8MPng/ixYt41H3y/mwN4ZnD2+wO9IEgbWb6ulrrEl7FoEfo8jEIlat582gkMHZ3PL9AVa4lKA1vsDAEPDZI6hNioEIh5JCMTx4EUTyMtI4uqniijdXud3JPHZitK2rqPh02MIVAhEPJWdlshjlxZSWdfElKdmUdeo9Y5j2YqyKnqkJpCdluh3lK9RIRDx2PA+mdwzeSzzSrby/Rfm0qKeRDGrbY4hs/CYbK6NCoFICJw4sje3nTycmQs3ctcbS/yOIz5ZUVYddpeFQL2GRELmiiMGsXZzDY++X0z/nilccthAvyNJCG2rbaSssj7segyBCoFIyJgZt592EOu21nLHjEX07ZHCccPz/Y4lIbK8NPzmGGqjS0MiIRSIM+779jgO6pvF9c/O0RoGMWTpxkoADuid4XOS/6RCIBJiqYnxPH55Ib3SE/nvP3/BmooavyNJCCzduJ20xAD9eqT4HeU/qBCI+CAvI5m//PchNLU4LvvT52yubvA7knhsycZKDuidETbLU7anQiDikyG56Uy9tJD1W2u58i9fUNugMQbRyjkXLASZfkfpkAqBiI8KB2bzhwvGMmftVq5/djaNzS1+RxIPbNpez7baRg4Mw/sDoEIg4rsTR/bh52eM5O0lpdz04jxNXR2FlmzcDoTnjWJQ91GRsHDxoQOoqm/irplLSEsMcOfZo8Ju9Kl0XVuPoXBtEagQiISJa44eQlVdEw+8u5y0pHh+cspwFYMosXRjJfmZSfRIDa85htqoEIiEkR+csD9V9U08/uFK0pLiuen4/f2OJN0gnG8UgwqBSFgxM24/dQTV9U3c9/YyUhMDXHP0EL9jyT5oam5heVkVRwzL8TvKLqkQiISZuDjjrnNGU9fUwl0zl5CSEOCySQP9jiVdtKqimoamFg7ID8/7A6BCIBKWAnHGPZPHUNfYzB0zFpGSEGDywf39jiVdsKTtRnGf8C0E6j4qEqYSAnE8cOE4jhyWw4+mz+dvc9f5HUm6YOnGSgJxFnbLU7anQiASxpLiAzx6SSETB2Vz04vzeH3BBr8jyV5asrGSQTlpJMUH/I6ySyoEImEuJTHA45cdzNj+PbjhuTm89eUmvyPJXlganGMonKkQiESAtKR4/vRfB3NQ30yufWY27y0t9TuSdEJVfRNrNtdwYBjfKAYVApGIkZmcwJP/PZGheelMeWoW//qqzO9IsgdfbQrfNQjaUyEQiSBZqQk8c+VEhuSmc9WTRWoZhLl/Ty0RvoPJQIVAJOL0TEvk2SsnMiwvnSlPzuLdJSoG4WrpxkpSEwMU9Ay/xWjaUyEQiUA90xJ55sqJ7N87naufmsU7S3QDORwt2bid/fPDczGa9lQIRCJUj9REnrniUA7oncHVT81iprqWhhXnHEs3VjI8jAeStVEhEIlgWakJPH3lREb1y+L65+bwyhwNOgsXs9dsYUtNI4UDsv2OskcqBCIRLislgaeumMghA7P5/otzee7zNX5HEuCVOetJTojjhIPy/Y6yRyoEIlGgbZzB0fvncuv0Bfzpo5V+R4ppjc0tvLZgA98cnk9GcoLfcfZIhUAkSiQnBHjkkgl866B8fvb3L3nwveV+R4pZHywrY3N1A2eO7ed3lE5RIRCJIknxAf544XjOGNuX37yxlHveXIpzWgM51F6es54eqQkctX+u31E6xfNCYGYBM5tjZq92sO1yMyszs7nBx5Ve5xGJdvGBOO6ZPJbzC/tz3zvL+eVri1UMQqiqvol/frmRU0f3ITE+Mn7XDsV6BDcCi4FdDa17wTl3fQhyiMSMQJxx59mjSEkMMPXDlVQ3NPGLM0cRCPP+7NHgzUUbqWtsiZjLQuBxi8DMCoBTgKleHkdE/lNcnHHHaSO47tghPPf5Wq5/djb1Tc1+x4p6r8xdT0HPFCYM6Ol3lE7zut3ye+CHQMtu9jnHzOab2TQz63AJJjObYmZFZlZUVqaJtkQ6y8y4+VsH8pNThjNz4Ub+609fUFXf5HesqFVaWceHy8o4Y2xfzCKn9eVZITCzU4FS59ys3ez2d2Cgc2408E/gLx3t5Jx71DlX6JwrzM2NjJsvIuHkyiMHc/d5Y/hs5WYufOxTyqvq/Y4UlV6dt4EWR0RdFgJvWwSHA6eb2SrgeeAbZvZ0+x2ccxXOuba/kVOBCR7mEYlp50wo4NFLJvDVpkrOfvBjVpRV+R0p6rwydx0j+mQyLMzXH9iZZ4XAOXerc67AOTcQuAB4xzl3cft9zKxPu5en03pTWUQ8ctzwfJ6fchjV9U2c89DHfLFqs9+RosayTZXML9nG2eMjqzUAPowjMLP/M7PTgy9vMLNFZjYPuAG4PNR5RGLN2P49ePnaw8lOTeSiqZ/x6vz1fkeKCi/NXkcgzjgjwi4LAVik9S8uLCx0RUVFfscQiXhbqhu46skiilZv4QfH78/13xgaUTc4w0lzi2PSXW8zsm8Wj19+sN9xOmRms5xzhR1ti4zRDiLS7XqmJfL0lRM5a1w/7v7nV9zw/FzqGtW9tCs+Wl7Opu31nD2+wO8oXRKKAWUiEqaSEwLcM3kM++dn8Jt/LGFNRTWPXlpIfmay39EiykuzS8hMjue44Xl+R+kStQhEYpyZ8Z1jhvDoJYUsK63itPs/ZPaaLX7HihiVdY38Y9FGThvTl+SEgN9xukSFQEQAOH5EPtOvnURyQoALHvmU57WuQafMXNA6pcQ5EyLzshCoEIhIOwf2zmTG9YczcXA2t0xfwG0vL6ChaXcTA8i02SUMykljXP8efkfpMhUCEfmaHqmJ/Pm/DuGao4fwzGdrmPzIJ6zbWut3rLC0dnMNn6/czDnj+0V0jysVAhH5D4E445aTDuTBi8azvLSKk//wAW8v3uR3rLDz0uwSzOCsCO0t1EaFQER26eRRfXj1u0fQr0cKV/yliDtfX0xjsy4VATjnmD57HYcO6kW/Hil+x9knKgQislsDc9KYfu0kLpy4H4+8X8z5j3xCyZYav2P5btbqLazZXBPRN4nbqBCIyB4lJwT41VmjuO/b4/hqU+ulopkLNvgdy1cvzS4hJSHAiSN7+x1ln6kQiEinnT6mL6/fcCSDctL4zjOz+ckrC6htiL3RyHWNzbw6fwMnjexNelLkj8tVIRCRvbJfr1T+es0krj5qME9/uoZT7v+AuWu3+h0rpN5avInKuqaInVJiZyoEIrLXEuPjuPXk4Txz5UTqGpo556GPuefNpTFzI/mlWSX0yUrmsCG9/I7SLVQIRKTLDh+awxvfP4ozx/bjvneWc9aDH7FsU6XfsTxVWlnH+8vKOXNcPwJxkTt2oD0VAhHZJ5nJCdw9eQwPXzyB9VvrOOX+D5n6QTEtLZE1xX1nzZi7nuYWxzkRuADNrqgQiEi3OHFkb/7xvaM4algOv3htMRdN/Swqu5m+NHsdYwqyGJoXWctR7o4KgYh0m9yMJB67tJBfnzOK+SVb+da97zP1g2KaouTewZfrt7N4w/aouUncRoVARLqVmXH+wfvxxveO4pBB2fzitcWc9sBHET+1dVNzC3fMWEhaYoDTxvT1O063UiEQEU/0z07licsP5uGLx7O1poFzHvqYH06bR+n2Or+jdckf3l7GF6u28MuzRpGdluh3nG6lQiAinjEzThzZh3/edDRXHTmYl+es45jfvcd9by+LqIFoHy0v54F3l3PehALOHBc9N4nbqBCIiOfSk+L58cnDeeumozl6/1zu+edXHPu793hlzjqcC+/eRWWV9XzvhbkMyU3nZ2cc5HccT6gQiEjIDOiVxkMXT+DFqw8jNyOJ770wl3Mf/oQFJdv8jtahlhbHD/46j221jTxw4ThSEyN/OomOqBCISMgdMiibv113OL85ZzSrK6o5/Y8f8qNp8ymtDK/7B098tJL3vyrj9lNHcGDvTL/jeEaFQER8ERdnTD64P+/8zzFcdeRgps8p4djfvscf311OXaP/9w8Wrd/Gb95Yygkj8rlo4n5+x/GUCoGI+CozOYEfnzycN79/NEcMy+G3/1jKN373Hi/PKaHZp9HJtQ3N3Pj8XHqkJnDXOaMjehnKzlAhEJGwMCgnjUcuKeS5qw6lZ1oi339hHifc+y/+NnddyAvCnTMXs7y0irsnj4m6rqIdUSEQkbBy2JBe/P36I3jgwnEE4owbn5/L8ff+i78WrQ3JJaO3F2/iyU9Wc+URgzhyWK7nxwsHFu5dt3ZWWFjoioqK/I4hIiHQ0uJ4Y9FG7nt7GUs2VpKVksDZ4/tx0cT9PJnrZ+nGSr792KfkZSTxt+sPJyk+0O3H8IuZzXLOFXa4TYVARMKdc45Piit49rM1/GPRRhqbHQcP7Mnkwv6cPKoPad2wStjCddu4+PHPSIqP47mrDmVwbno3JA8fKgQiEjXKq+qZNquEF79YS3F5NWmJAU4d3ZfJBxcwfr+eXbqxO2v1Fi7/0+dkJifw7FUTGdArzYPk/lIhEJGo45xj1uotvFi0llfnb6CmoZkhuWlMLuzPWeP7kZeR3Knvef+rMq55ehZ5GUk8c9Wh9OuR4nFyf6gQiEhUq65v4rUFG3jxi7UUrd6CGfTvmcqwvHSG5qezf14GI/tlMSQ3jfhAHM0tjn9+uZGpH6ykaPUWhual8+yVE8nL7FzxiEQqBCISM1aUVfH6/A0s3VTJ8tIqisuqaQiuh5CcEMeIPpmUVzWwZnMNBT1T+O/DB3H+wf275T5DONtdIYjun1xEYs6Q3HS+e9ywHa+bmltYWV7NwvXbWFCynYXrttGvRwq3nnQgJxzUO2rWHd4XnhcCMwsARcA659ypO21LAp4EJgAVwPnOuVVeZxKR2BEfiGNYfgbD8jM4a5zfacJTKAaU3Qgs3sW2K4AtzrmhwL3Ar0OQR0RE2vG0EJhZAXAKMHUXu5wB/CX4fBpwnEX7pB4iImHG6xbB74EfArtaubofsBbAOdcEbAN67byTmU0xsyIzKyorK/MoqohIbPKsEJjZqUCpc27Wvn6Xc+5R51yhc64wNzc25v4QEQkVL1sEhwOnm9kq4HngG2b29E77rAP6A5hZPJBF601jEREJEc8KgXPuVudcgXNuIHAB8I5z7uKddpsBXBZ8fm5wn8ga2CAiEuFCPo7AzP4PKHLOzQAeB54ys+XAZloLhoiIhFBICoFz7j3gveDz29u9XwecF4oMIiLSsYibYsLMyoCttPYwapPV7nVHz9v+zAHKu3jo9t+7N9s7en9Xedu/7mifrubfU/bd7bOrfB293t25B+/yd/Xc7/xa577z2fa0vavnvv3zcPp3u7fZ2z8Pl3PfwznXcW8b51zEPYBHd/W6o+ft/izqrmN2dntH7+8qb0eZuyP/nrLvTf6unnsv83f13Hcys859CM99R/nD4d/t3mYPxd+dfTn3Oz8idanKv+/mdUfPd96/O47Z2e0dvb+rvO1f726fvdWZz3c2fzSd+51f69zvOUNnt3f13Ld/Hk759zZ7Z469J16e+6+JuEtD+8LMitwuZt+LBMrvn0jODpGdP5KzQ2Tkj9QWQVc96neAfaT8/onk7BDZ+SM5O0RA/phqEYiIyH+KtRaBiIjsRIVARCTGqRCIiMQ4FYIgMzvSzB42s6lm9rHfefaWmcWZ2S/N7H4zu2zPnwgfZnaMmX0QPP/H+J2nK8wsLThV+ql73jt8mNnw4HmfZmbf8TvP3jKzM83sMTN7wcxO8DvP3jKzwWb2uJlN8zNHVBQCM3vCzErNbOFO759oZkvNbLmZ3bK773DOfeCcuwZ4lX8vlhMS3ZGf1kV+CoBGoMSrrDvrpuwOqAKSCWF26Lb8AD8CXvQmZce66e/94uDf+8m0zhgcMt2U/xXn3FXANcD5XubdWTflL3bOXeFt0k7o6oi9cHoARwHjgYXt3gsAK4DBQCIwDxgBjKL1f/btH3ntPvcikBFp+YFbgKuDn50WYdnjgp/LB56JwHN/PK0TJl4OnBpJ2YOfOR2YCVwYaee+3efuBsZHcP6Q/Zvt6BHy2Ue94Jx738wG7vT2IcBy51wxgJk9D5zhnLsT6LD5bmb7Aducc5Ve5t1Zd+Q3sxKgIfiy2cO4X9Nd5z5oC5DkSdBd6KZzfwyQRus/+Foze905t6tV+bpNd5171zoT8Awzew141sPIOx+3O869AXcBM51zsz2O/DXd/HffV1FRCHZhxzKYQSXAxD185grgT54l2jt7m386cL+ZHQm872WwTtir7GZ2NvAtoAfwgKfJOmev8jvnbgMws8uB8lAUgd3Y23N/DHA2rQX4dS+DddLe/r3/LvBNIMvMhjrnHvYyXCfs7fnvBfwSGGdmtwYLRshFcyHYa865O/zO0FXOuRpaC1nEcc5Np7WQRTTn3J/9zrC3XLsp4iORc+4+4D6/c3SVc66C1vsbvoqKm8W7sGMZzKCC4HuRIpLzR3J2iOz8kZwdlN8X0VwIvgCGmdkgM0uk9WbeDJ8z7Y1Izh/J2SGy80dydlB+f/h5p7ob794/B2zg310nrwi+fzLwFa138W/zO2c05o/k7JGeP5KzK394PTTpnIhIjIvmS0MiItIJKgQiIjFOhUBEJMapEIiIxDgVAhGRGKdCICIS41QIJCqYWVWIj9cta1YE12LYZmZzzWyJmf2uE58508xGdMfxRUCFQKRDZrbbebicc5O68XAfOOfGAuOAU81sT+sCnEnrTKci3UKFQKKWmQ0xszfMbJa1roB2YPD908zsMzObY2ZvmVl+8P2fmtlTZvYR8FTw9RNm9p6ZFZvZDe2+uyr45zHB7dOCv9E/E5waGTM7OfjeLDO7z8xe3V1e51wtMJfWGSwxs6vM7Aszm2dmL5lZqplNonX9gN8GWxFDdvVzinSWCoFEs0eB7zrnJgD/AzwYfP9D4FDn3DjgeeCH7T4zAvimc+7bwdcH0jpF9iHAHWaW0MFxxgHfC352MHC4mSUDjwAnBY+fu6ewZtYTGMa/pxGf7pw72Dk3BlhM6xQGH9M6d83NzrmxzrkVu/k5RTpF01BLVDKzdGAS8NfgL+jw70VvCoAXzKwPratIrWz30RnB38zbvOacqwfqzayU1lXUdl5O83PnXEnwuHOBgbQuvVnsnGv77ueAKbuIe6SZzaO1CPzeObcx+P5IM/sFres0pAP/2MufU6RTVAgkWsUBW4PX3nd2P3CPc25GcGGWn7bbVr3TvvXtnjfT8b+ZzuyzOx845041s0HAp2b2onNuLvBn4Ezn3LzgojfHdPDZ3f2cIp2iS0MSlZxz24GVZnYetC5paGZjgpuz+Pcc8Zd5FGEpMLjdUoZ7XFg92Hq4C/hR8K0MYEPwctRF7XatDG7b088p0ikqBBItUs2spN3jJlr/53lF8LLLIuCM4L4/pfVSyiyg3IswwctL1wJvBI9TCWzrxEcfBo4KFpD/BT4DPgKWtNvneeDm4M3uIez65xTpFE1DLeIRM0t3zlUFexH9EVjmnLvX71wiO1OLQMQ7VwVvHi+i9XLUI/7GEemYWgQiIjFOLQIRkRinQiAiEuNUCEREYpwKgYhIjFMhEBGJcSoEIiIx7v8BFPZ8YGPPSlYAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"learn.remove_cb(ActivationStats)\n",
"learn.lr_find()"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"SuggestedLRs(lr_min=0.04365158379077912, lr_steep=0.3630780577659607)"
]
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEOCAYAAACEiBAqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAr0UlEQVR4nO3deXxV9Z3/8dcn+0YSSMIW9k1kXyK7iFXbqghW3Nu6VEXQsWKnndrp/FqnnY4dx1ZFiqi1brVqpeqggltd2NwIguz7vmaBhCSQkOT7+yM3NqYhBMm5596b9/PxuI/knnPuue/cB+Sds32POecQEZGWK8rvACIi4i8VgYhIC6ciEBFp4VQEIiItnIpARKSFUxGIiLRwnhWBmZ1hZivqPIrNbEa9ZSaYWVGdZX7hVR4REWlYjFcrds5tAIYAmFk0sAd4pYFFFznnJnqVQ0REGhesXUPnAVucczuC9H4iItJEwSqCq4HnTzBvtJmtNLMFZtY/SHlERCTAvB5iwszigL1Af+fcgXrzUoFq51yJmV0EPOSc693AOqYCUwGSk5OH9+3b19PMIiKRJjc3N985l9XQvGAUwWTgdufcN5uw7HYgxzmXf6JlcnJy3LJly5oxoYhI5DOzXOdcTkPzgrFr6BpOsFvIzNqbmQW+HxHIUxCETCIiEuDZWUMAZpYMXADcWmfaNADn3BzgcmC6mVUCR4GrnYZDFREJKk+LwDlXCmTUmzanzvezgFleZhARkcbpymIRkRZORSAi0sKpCEREWjgVgYhIGHhn7QE2HyzxZN0qAhGREOec47bncpmbu9uT9asIRERCXPGxSo5XOTJT4jxZv4pARCTEFZZWANAmWUUgItIiFZSUA5CREu/J+lUEIiIhriCwRZChLQIRkZapoCRQBDpGICLSMhWW1uwa0jECEZEWKr+kglbxMcTHRHuyfhWBiEiIKyytoI1Hu4VARSAiEvIKSss9O1AMKgIRkZBXUFJBm2RvTh0FFYGISMgrKK3w7KpiUBGIiIS06mrHodIKz84YAhWBiEhIKz52nMpq59lVxaAiEBEJaV5fVQwqAhGRkOb1VcXgYRGY2RlmtqLOo9jMZtRbxsxsppltNrMvzGyYV3lERMKR11cVA8R4tWLn3AZgCICZRQN7gFfqLXYh0DvwGAk8EvgqIiLUXFUMkBkBxwjOA7Y453bUmz4ZeMbV+BhIN7MOQcokIhLyau9F0DopDHcN1XM18HwD07OBXXWe7w5MExERau5FkJoQQ1yMd7+uPS8CM4sDJgEvncY6pprZMjNblpeX13zhRERCXEFphaenjkJwtgguBJY75w40MG8P0LnO806BaV/hnHvMOZfjnMvJysryKKaISOgpLK3w9NRRCE4RXEPDu4UA5gHXBc4eGgUUOef2BSGTiEhYqBlnKIyLwMySgQuAl+tMm2Zm0wJP5wNbgc3A48BtXuYREQk3wdg15NnpowDOuVIgo960OXW+d8DtXmYQEQlX1dWOQ2WRsWtIRES+hqKjx6mqdp5eVQwqAhGRkFUQhKuKQUUgIhKyCoJwVTGoCEREQlbtyKPaIhARaaG+HIJaxwhERFqmgpKaYwRejjMEKgIRkZBVWFpBelIssdHe/qpWEYiIhKhgXFUMKgIRkZBVUFpOZrK3ZwyBikBEJGRpi0BEpIUrLK3w/IwhUBGIiISkqmpHYRDGGQIVgYhISDpcVoFzeD7yKKgIRERCUrCuKgYVgYhISKodZ0jHCEREWqjakUczdPqoiEjLVBikcYZARSAiEpLySyow836cIVARiIiEpMLSclonxREdZZ6/l4pARCQEFZYG56pi8LgIzCzdzOaa2XozW2dmo+vNn2BmRWa2IvD4hZd5RETCRX6QhpcAiPF4/Q8BbzrnLjezOCCpgWUWOecmepxDRCSsFJZW0LttSlDey7MiMLM0YDxwA4BzrgKo8Or9REQiSbDGGQJvdw11B/KAJ83sczP7o5klN7DcaDNbaWYLzKy/h3lERMJCVbXjUFkFbYJwDQF4WwQxwDDgEefcUKAUuLveMsuBrs65wcDDwKsNrcjMpprZMjNblpeX52FkERH/fTnOUAQcLN4N7HbOfRJ4PpeaYviSc67YOVcS+H4+EGtmmfVX5Jx7zDmX45zLycrK8jCyiIj/gjnOEHhYBM65/cAuMzsjMOk8YG3dZcysvZlZ4PsRgTwFXmUSEQkHX44zFCFnDd0BPBc4Y2grcKOZTQNwzs0BLgemm1klcBS42jnnPM4kIhLSaoeXaBOkg8WeFoFzbgWQU2/ynDrzZwGzvMwgIhJuCoM44BzoymIRkZBTe4ygdVJsUN5PRSAiEmIKSytIT4olJjo4v6JVBCIiIaYgiMNLgIpARCTkFJSWB+2MIVARiIiEnGCOPAoqAhGRkFNTBME5YwhUBCIiIaW62nGo7DiZQbqGAFQEIiIhpejocaqqnXYNiYi0VMEeZwhUBCIiIaWgJLhXFYOKQEQkpBRqi0BEpGWr3TUUrLuTgYpARCSkFH45zpCKQESkRSosrSA1IYa4mOD9elYRiIiEkILSCjJSgnegGFQEIiIhpaCkPKgHikFFICISUoI9zhCoCEREQkpBaUVQRx4FFYGISMhwznFIWwQiIi1X8dFKKoM8zhB4XARmlm5mc81svZmtM7PR9eabmc00s81m9oWZDfMyj4hIKCsI3LQ+M8hnDcV4vP6HgDedc5ebWRyQVG/+hUDvwGMk8Ejga7Pbll/K22v20zUjic5tkujSJolWCcG5MbSISFP4MbwEeFgEZpYGjAduAHDOVQAV9RabDDzjnHPAx4EtiA7OuX3NneeL3Ye5d8H6r0zr3CaRSwZ15NKh2fRp14pATvYVHWN7QSmZKfF0ap1IUpzXfSkiAvklEVYEQHcgD3jSzAYDucCdzrnSOstkA7vqPN8dmNbsRTB5SDbn9m3LzoIydhaWsaOgjI+2FjDnwy3M/mALZ3ZIJSkumo37j3CkvPIrr81Miadzm0Sy0xPp1DqJ7NaJdEpPpEN6Ah3TE0nVloWININCH8YZAm+LIAYYBtzhnPvEzB4C7gb+36muyMymAlMBunTp8rUDpSbEMiA7jQHZaQBMn9CTvCPlvP7FXuav2oeZ1WwdtG9Ft4wkCksr2H3oKLsKa8pj1Z4i3lqzn+NV7ivrTYmPoW1qPFkp8WS2qvnaJjmO1slxZAQeHdISaZcWT3xM9NfOLyKRrTBwjCCStgh2A7udc58Ens+lpgjq2gN0rvO8U2DaVzjnHgMeA8jJyXH155+OrFbx3Di2OzeO7d6k5aurHQePlLPn8FH2FR1l7+Gj7D18jLwj5eQdKWft3mLyj5T/01ZFrYzkODq1TqRLRjJd2yTRNSOJHlnJ9MxKIT2Ig0yJSOgpKK0gJT4m6H8welYEzrn9ZrbLzM5wzm0AzgPW1ltsHvAvZvYCNQeJi7w4PtCcoqKM9mkJtE9LAFqfcLmKymoOl1VQWFZB3pFy9hcdY3/RMfYWHWP3oTJW7jrM/FX7qKr+R69lJMfRLTOZjumJdExPIDs9kc5tkuiVlUJ2eiJRURaEn1BE/FJYWhH03ULg/VlDdwDPBc4Y2grcaGbTAJxzc4D5wEXAZqAMuNHjPEETFxNF29QE2qYm0Ld9w8scr6pm96GjbMsvYcvBUrbml7Atv5Qvdh/mrdXHqKiq/nLZhNgoemSmcGaHVPp3TGVAdhr9OqaSEq8D2SKRwo/hJcDjInDOrQBy6k2eU2e+A273MkMoi42OontmMt0zk/lG36/Oq6525JeUs72gjM0HS9iSV8KmgyV8uDGPvy3fDUCUwfCurTm3b1u+0bctZ7RrhZm2GkTCVX5JBdnpCUF/X/05GaKiouzLLYoR3dt8Zd7B4mOs2VtM7o5DvL/hIPe9uYH73txA+9QExvXO5OzemYztlRn0i1JE5PQUlpYzMDs16O+rIghDtQVxbt+2/PhbZ3Cg+Bjvrz/Iwk15vLP2AHNza7YYhnRO5+KBHbhwYHs6ta5/LZ+IhBLnXGDXUPD/gFMRRIB2qQlcPaILV4/oQlW1Y/WeIhZuzOPNNfv5zfx1/Gb+OoZ0TueyYdlMGtxRZyeJhKAj5ZUcr3JBH3kUVAQRJzrKGNw5ncGd07njvN5szy9l/up9zFuxl1/83xr+6/V1XNC/HVfldGZcr0ydiSQSIgp9uqoYVAQRr1tmMrdN6MVtE3qxek8Rc3N38+qKPbzxxT56ZiVz49juXDYsW8NoiPiswKerikHDULcoA7LTuGdSfz759/N44KrBJMXF8B+vrmb0ve9x/1sbOFRafygoEQmWL4eX0DECCYb4mGi+M7QTlw7JJnfHIZ5YvI0/fLCZJ5ds4/uju3HL2d2DfvNskZZuf/ExwJ8tgiYVgZklA0edc9Vm1gfoCyxwzh33NJ14yszI6daGnG5t2HjgCLPe28yjC7fw9NLt3Hx2d249p6cuWBMJks93HCIzJY4OacG/jqCpu4YWAglmlg28DXwfeMqrUBJ8fdq1YuY1Q3nnrnM478y2PPzeZib87wc8/+lOKutc4Swi3vhkWyEjurfx5aLQphaBOefKgMuA2c65K4D+3sUSv/Rqm8Ksa4fxym1j6JaRxM9eXsXFMxezfOchv6OJRKw9h4+y5/BRRnRrc/KFPdDkIgjcZvK7wBuBaRpPOYIN7dKal6aN5pHvDuPIseNMeWQpv3ptLWUVDY+qKiJf32fbCgE4q3toF8EM4GfAK865NWbWA3jfs1QSEsyMCwd24K27xvPdkV3405JtfPvBRSzdku93NJGI8sm2QlolxNC3ffCHl4AmFoFz7kPn3CTn3P+YWRSQ75z7ocfZJES0Sojlvy4dyAtTRxFlcO3jn3DPvDUcrajyO5pIRPhseyE5XVsT7dMFnk0qAjP7i5mlBs4eWg2sNbOfeBtNQs2oHhksuHM8N4zpxlNLt3PxzEU6diBymgpKytl8sIQR3TN8y9DUXUP9nHPFwKXAAmruR/x9r0JJ6EqMi+aeSf35y80jKa+s5vJHlvL7tzfozCKRr+mz7TV/TI3ofuIbXXmtqUUQa2ax1BTBvMD1A816y0gJL2N6ZfLmjLP5ztBOzHxvM9c+/gn7io76HUsk7Hy6rZD4mCgGZqf7lqGpRfAosB1IBhaaWVeg2KtQEh5aJcTyuysH88BVg1m9t4iLHlrE39cd8DuWSFj5dHsBQ7ukExfj34g/TT1YPNM5l+2cu8jV2AGc63E2CRPfGdqJ1+8YR4e0RG56ehn3vbn+K/diFpGGHTl2nLV7i309PgBNP1icZma/N7NlgcfvqNk6EAGgR1YKL982hqvP6szsD7Zw41OfcbhMg9iJNCZ3xyGqHYz06fqBWk3dFvkTcAS4MvAoBp70KpSEp4TYaH47ZRD//Z2BfLQln0mzlrBun/YgipzIZ9sLiYkyhnZJ9zVHU4ugp3Pul865rYHHfwI9vAwm4evakV14YepoyiuruGz2Uhas2ud3JJGQ9Om2QgZkp/l+P5CmFsFRMxtX+8TMxgInPUXEzLab2SozW2FmyxqYP8HMigLzV5jZL5oeXULZ8K6tee2OcfTt0Irpzy3noXc34ZyOG4jUOna8ipW7ihjh824haPr9CKYBz5hZWuD5IeD6Jr72XOdcY2MSLHLOTWziuiSMtG2VwPO3jOLfX1nFA+9uZOOBI9x/xWAS4zRMlci6fcVUVFUzrIt/1w/UalIROOdWAoPNLDXwvNjMZgBfeJhNIkBCbDS/u2Iwfdu34t4F69lRWMrj1+XQIS3R72givlq9pwiAgZ3STrKk907pxFXnXHHgCmOAHzXlJcDbZpZrZlNPsMxoM1tpZgvMrMGhrc1sau0ZS3l5eacSWUKAmTF1fE+euD6H7fllTJ61hBW7DvsdS8RXq/cU0zoplo4+3IimvtO5gqEpoyONc84NAy4Ebjez8fXmLwe6OucGAw8Drza0EufcY865HOdcTlZW1mlEFj99o287Xr5tDPGxUVz16Ee8tnKv35FEfLNqTxEDstN8uRFNfadTBCc98uec2xP4ehB4BRhRb36xc64k8P18aoayyDyNTBLi+rRrxau3jWVQpzTueP5zHnx3ow4iS4tz7HgVGw8cYWC2/7uF4CRFYGZHzKy4gccRoONJXptsZq1qvwe+Sc3IpXWXaW+BOjSzEYE8Bafx80gYyEiJ5883j2TKsE48+O4mZry4gmPHNaS1tBwbDxyhstoxIESKoNGDxc65Vqex7nbAK4Hf8zHAX5xzb5rZtMC65wCXA9PNrJKa01GvdvrzsEWIj4nm/isG0SMrmf99awO7Dx3lse8PJyMl3u9oIp5bVXugOByK4HQ457YCgxuYPqfO97OAWV5lkNBmZtx+bi+6ZSTzo7+u4NLZS3j6xhH0yErxO5qIp1bvKSItMZZOrUPj7Dn/hrsTCbh4UAdemDqKsvIqpjyyVDe7kYi3ek8xA7JTQ+JAMagIJEQM7dKav00fQ2piLNc+/jHvrtVw1hKZKiqr2bD/SMgcHwAVgYSQbpnJ/G36GPq0a8XUZ5fxwqc7/Y4k0uw2HjhCRVU1AzqqCEQalJkSz/O3jOLs3lnc/fIqHv1wi9+RRJrV6hA7UAwqAglByfExPH5dDhcP6sC9C9Zz/1sbdK2BRIxVe4polRBD14wkv6N8yd+xT0VOIC4miplXD6VVfAyz3t9MSXklv5jYj6io0Di4JvJ1rd5bTP+OoXOgGFQEEsKio4x7LxtISnwMf1y8jSPHKvmfKQOJidaGrISn41XVrNtXzPWju/od5StUBBLSzIyfX3wmrRJieeDdjZRVVPLg1UOIj9FQ1hJ+Nh8soaKyOqTOGAIdI5AwYGbceX5vfjGxHwtW7+eWZ3I5WqEhKST81F5RrCIQ+Zp+MK47900ZxOJNeVz3p08oOnrc70gip2T1niKS46LpnpHsd5SvUBFIWLnyrM48fM0wVuw6zFWPfsSB4mN+RxJpstwdh+jfMS3kTnpQEUjYuXhQB568YQS7CsuY8shStuaV+B1J5KQ+2VrAmr3FXDyog99R/omKQMLSuN6ZPD91FEcrqrh8zkes1B3PJMTNen8zmSnxXHVWZ7+j/BMVgYStQZ3SmTt9DElx0Vzz+Mf8fZ3GJ5LQtHLXYRZtyufms7uTEBt6Z7ypCCSsdc9M5uXpY+iZlcItzyzjqSXb/I4k8k/+8P5mUhNi+N6o0Lp+oJaKQMJe29QEXrx1FOed2Y57XlvLf762hqpqDUkhoWHD/iO8vfYAN4ztTkp8aF66pSKQiJAUF8Oc7w3nB2O78+SS7dz6bC6l5ZV+xxJh9gebSYqL5sYx3fyOckIqAokY0VHGLy7px68m9+e99Qe4UqeXis+255fy2sq9fG9UV1onx/kd54RUBBJxrhvdjSeuP4vt+aVMnrWENXuL/I4kLdSfP95BTFQUN4/r7neURqkIJCKd27ctL00bgxlcMecj3t9w0O9I0gK9t+Ego3pm0DY1we8ojfK0CMxsu5mtMrMVZrasgflmZjPNbLOZfWFmw7zMIy1Lv46pvHr7WLpnJnPz07rjmQTXzoIytuaVcu4ZWX5HOalgbBGc65wb4pzLaWDehUDvwGMq8EgQ8kgL0i41gRdvHc3YXpnc/fIqfv/ORt3kRoLig401W6ETzmjrc5KT83vX0GTgGVfjYyDdzELv+msJaynxMTxxfQ5XDO/EzL9v4idzv+B4VbXfsSTCvb/+IN0ykuieGVoDzDXE6yJwwNtmlmtmUxuYnw3sqvN8d2DaV5jZVDNbZmbL8vLyPIoqkSw2Oor7Lh/EjPN7Mzd3Nz946jOOHNPopeKNY8er+GhrQVhsDYD3RTDOOTeMml1At5vZ+K+zEufcY865HOdcTlZW6O9vk9BkZsw4vw/3TRnE0i0FXPXoxzq9VDzx8dYCjh2vZkIYHB8Aj4vAObcn8PUg8Aowot4ie4C6IzB1CkwT8cyVZ3Xmietz2FFQymWzl7LxwBG/I0mE+WBDHgmxUYzqkeF3lCbxrAjMLNnMWtV+D3wTWF1vsXnAdYGzh0YBRc65fV5lEqk14Yy2vHjraCqqqpkyeykfbtQuR2k+7284yOgeGSE5wFxDvNwiaAcsNrOVwKfAG865N81smplNCywzH9gKbAYeB27zMI/IVwzITuPV28eS3TqRG5/8lKeWbNMZRXLatuWXsqOgjHP7hsfxAfDw5vXOua3A4Aamz6nzvQNu9yqDyMlkpyfyt+ljuPOFFdzz2lo255Xwy0v6Exvt9wl1Eq7eXx84bbRP+BSB/rVLi5ccH8Nj3x/OtHN68uePd3LdE59SWFrhdywJU+9vOEiPrGS6ZCT5HaXJVAQiQFSUcfeFffn9lYPJ3XmISbMWs25fsd+xJMyUVVTyybZCzg2T00ZrqQhE6rhsWCf+eutoKiqruWz2Uhas0rkL0nQfby2gojJ8ThutpSIQqWdI53Reu2McZ7RvxfTnlvO7tzdQrRvdSBN8GDht9KxubfyOckpUBCINaJeawAtTR3HF8E48/N5mbnlmGcW6EllOYuGmfEaF0WmjtVQEIieQEBvNfZcP4leT+/Phxjwu/cMStuSV+B1LQtTOgjK25Zcyvnd47RYCFYFIo8yM60Z34883j6So7DiTZy3RcQNp0Iebai5KPCfMjg+AikCkSUb1yOC1O8bRq20K059bzq9fX6sRTOUrFm7MIzs9kR5hMNpofSoCkSbqmJ7IX28dzQ1juvHE4m1c/djH7C/SoHUCFZXVfLSlgPF9sjAzv+OcMhWByCmIi4ninkn9mXnNUNbtK+bChxby3voDfscSny3feYiS8krO6RN+u4VARSDytUwa3JHX7hhH+7REfvDUMn79+loqKrWrqKVauDGP6ChjTK/wGG20PhWByNfUMyuFV24bw3Wju/LE4m1MeWQp2/NL/Y4lPli4KY9hXdJJTYj1O8rXoiIQOQ0JsdH8avIA5nxvODsLy7h45iJe/Vy31GhJ8kvKWb2nOGx3C4GKQKRZfHtAe+bfeTb9OqYy48UV/OtfV1JaXul3LAmCRYHTRserCEQkOz2R528ZxZ3n9eaVz3cz8eHFrNh12O9Y4rGFG/NpkxzHgI5pfkf52lQEIs0oJjqKuy7ow19uGUX58SqmPLKUh97dRKWuOYhI1dWORZvyGNcrk6io8DtttJaKQMQDo3pksGDGeCYN7sgD725kypyP2KrhKSLO57sOk19SwXlnhtew0/WpCEQ8kpYYywNXDWHWtUPZnl/KhQ8tYs6HW7R1EEHeWXuAmChjQpjdf6A+FYGIxyYO6sjbd43nnD5Z/HbBei6dvYQ1e4v8jiXN4J21+xnZow1pieF52mgtFYFIELRLTeDR7w9n9neHsb+onEmzlnDfm+spr6zyO5p8TVvzStiSV8oFZ7bzO8pp87wIzCzazD43s9cbmHeDmeWZ2YrA42av84j4xcy4aGAH3v3ReL4zNJvZH2xh4kydWRSu3llbM7TI+f1UBE1xJ7CukfkvOueGBB5/DEIeEV+lJ8Vx/xWDefLGsygpr+Sy2Uu4d/46jlZo6yCcvLP2AP06pNKpdfjcpP5EPC0CM+sEXAzoF7xIPeee0Za37hrPVWd14dGFW/nmgx/ywYaDfseSJsgvKSd35yEuiICtAfB+i+BB4N+Axk6TmGJmX5jZXDPr7HEekZCSmhDLvZcN5IWpo4iLjuKGJz/j9r8s52CxhrcOZe+tO4hzqAhOxswmAgedc7mNLPYa0M05Nwh4B3j6BOuaambLzGxZXl6eB2lF/DWqRwbz7zybf72gD++sPcB5v/uQJ5ds06mmIerttQfITk+kf8dUv6M0Cy+3CMYCk8xsO/AC8A0z+3PdBZxzBc658sDTPwLDG1qRc+4x51yOcy4nKyt8x/MQaUx8TDR3nNebt2aMZ2jX1vzna2uZNGsJuTsO+R1N6jhaUcXizXmcf2bbsLwJTUM8KwLn3M+cc52cc92Aq4H3nHPfq7uMmXWo83QSjR9UFmkRumcm8/SNZzH7u8MoLK1gyiNL+fFLK7W7KEQs2pTHsePVXNCvvd9Rmk1MsN/QzH4FLHPOzQN+aGaTgEqgELgh2HlEQlHtqabj+2Tx8N838acl21iwah//8o3e/GBcN+Jjov2O2CI553gpdzetEmIY2aON33GajTnn/M5wSnJyctyyZcv8jiESVNvyS/nNG+t4d90BurRJ4mcX9uXbA9pHzK6JcPH8pzv52cur+Mm3zuD2c3v5HeeUmFmucy6noXm6slgkDHTPTOaP1+fw7E0jSIiNYvpzy7ny0Y9YqYvRgmbdvmLumbeGs3tnMv2cnn7HaVYqApEwcnbvLOb/8Gz++zsD2ZZfyuQ/LOGuF1ew5/BRv6NFtJLySm5/bvmXAwmG85DTDVERiISZmOgorh3Zhfd/PIHpE3oyf9U+zr3/A+5dsI6io8f9jhdxnHP8/JVVbC8oZeY1Q8lMifc7UrNTEYiEqVYJsfz0231578cTmDioA48t3MqE/32fJxZv02B2zWjeyr3834q93HV+H0b1yPA7jidUBCJhLjs9kd9fOYTX7xjHgOw0fv36Wr5x/4f8LXc3VdXhdTJIqCktr+S/569jUKc0bguzg8OnQkUgEiH6d0zj2ZtG8uebRtImOY5/fWklF89cxFtr9hNuZweGijkfbuFAcTm/vKQf0RF2XKAuFYFIhBnXO5P/u30ss64dSnllNbc+m8tFMxfz5up9VGsLocl2FZbx2MKtTB7SkeFdI+eagYaoCEQiUFSUMXFQR965azy/v3Iwx45XMe3Py7lo5iLe+EKF0BS/XbAeM/jpt/v6HcVzKgKRCBYTHcVlwzrxzl3jeeCqwVRUVXP7X5bzzQcX8n8r9ugYwgl8srWAN1btY/o5veiYnuh3HM/pymKRFqSq2jF/1T5mvbeZDQeO0C0jiVvG92DKsE4kxGrYCqj5jCbNWszhsuO8+6NzSIyLjM9FVxaLCADRUcYlgzuy4M6zmfO94aQlxvLzV1Yz7n/eY9Z7mzhcVuF3RN/95dOdrNlbzN0X9o2YEjgZbRGItGDOOT7eWsijC7fwwYY8EmOjuWxYNjeO7Uavtq38jhd0BSXlnHv/BwzITuO5m0dG1FhOjW0RBH30UREJHWbG6J4ZjO6Zwfr9xTy5eDsv5e7muU92Mr5PFj8Y243xvbMibkiFE/ntgvUcPV7Fryb3j6gSOBltEYjIVxSUlPP8pzt55qMdHDxSTs+sZG4Y250pw7JJiovcvx1zdxQy5ZGPmHZOT+6+MPLOFGpsi0BFICINqqisZv6qfTyxeBur9hSRmhDDd4Zmc83ILvRtHxm3aKxVWVXNJbOWcLisgnd/dA7J8ZFXeNo1JCKnLC4mikuHZjN5SEeW7TjEsx/t4PlPd/H0RzsY0jmda0Z05uJBHUmJgF+af/54B+v2FTP7u8MisgRORlsEItJkhaUVvLx8Ny98tovNB0tIjI3mooEduDKnEyO6twnL/eq7Csv49oMLGda1Nc/8YERY/gxNoV1DItKsnHMs33mYubm7eG3lPkrKK+makcTlwzoxZXinsLkIq7racc3jH7NmbzFvzjibTq2T/I7kGRWBiHimrKKSBav281LuLj7eWogZjO2ZyUUDO3BBv3ZktQrd8fufWLyNX7++lvumDOLKszr7HcdTKgIRCYqdBWXMXb6bVz/fw87CMszgrK5t+NaA9nyrf7uQ+ot788ESLp65iHG9Mvnj9TkRu0uolq9FYGbRwDJgj3NuYr158cAzwHCgALjKObe9sfWpCERCn3OO9fuP8Obq/by1Zj/r9x8BYEB2Kt/q156LB3WgR1aKb/kqq6qZMucjdhSU8vaM8bRNTfAtS7D4fdbQncA6oKHzzW4CDjnnepnZ1cD/AFcFIZOIeMjMOLNDKmd2SOWuC/qwPb+Ut9bUlMLv3tnI797ZyMDsNCYN7sglgzvSPi14v4hLyiv5j1dWsXLXYWZdO7RFlMDJeLpFYGadgKeB3wA/amCL4C3gHufcR2YWA+wHslwjobRFIBLe9hcd4/Uv9jJv5V6+2F2EGYzo1oZLBnfkwgHtyfDwnsCf7zzEjBdXsKuwjB+e15sZ5/fx7L1CjW+7hsxsLnAv0Ar4cQNFsBr4tnNud+D5FmCkcy7/ROtUEYhEjq15JcxbuZfXVu5lS14p0VHGmJ4ZfKt/e77Zr12z/bVeVe2Y/f5mHvz7JtqnJvDAVUMY0T2ybzZTny9FYGYTgYucc7eZ2QROowjMbCowFaBLly7Dd+zY4UlmEfFH7TGF11bu5Y1V+9hRUAbA0C7pnNe3LWf3zmJAdtrXul3k3sNHmfHiCj7dVsikwR359aUDSEuMbe4fIeT5VQT3At8HKoEEao4RvOyc+16dZbRrSES+wjnHpoMlvL1mP2+tOcCqPUUApCXGMrZXBiO6tWF41zac2aEVMdGNj6T/5ur9/PRvX3C8qppfTR7A5cM7BeNHCEm+nz7ayBbB7cBA59y0wMHiy5xzVza2LhWBSMuSX1LOks35LNqUz5LN+ewrOgZAYmw0Z3ZoRXJ8DLHRUcRGGzFRUZhBlBlHjh3n/Q15DMxOY+Y1Q+memezzT+Ivv88aqh/mV8Ay59w84AngWTPbDBQCVwc7j4iEtsyUeCYPyWbykGycc+wtOkbujkMs33GI9fuLKSmv5HhVNZVVjuNV1TgH1c7hgFvP6cG/XnAGcTG6B1djdEGZiEgLoFtViojICakIRERaOBWBiEgLpyIQEWnhVAQiIi2cikBEpIVTEYiItHAqAhGRFi7sLigzszzgMFBUZ3JanecNfV/7NRM44cimjai7zlNZpinTGstbd5pX2U80v7Gc9Z/78Zk3NXdTstb9vu40v7OH62cearlPtExL+//Z1TmX1eC7OOfC7gE8dqLnDX1f5+uy5ni/pi7TlGmN5Q1G9hPNbyxnKHzmTc3dlKyNfPb6zCMgd1P+XZxK9nD+/3miR7juGnqtkecNfV9/+dN9v6Yu05RpJ8vrdfYTzW8sZ/3nfnzmTc1df5rXuZuyjkj/zEMt94mWaYn/PxsUdruGToeZLXMnGGsj1IVr9nDNDeGbXbmDL5yzQ8s7WPyY3wFOQ7hmD9fcEL7ZlTv4wjl7y9oiEBGRf9bStghERKQeFYGISAunIhARaeFUBAFmdraZzTGzP5rZUr/zNJWZRZnZb8zsYTO73u88p8LMJpjZosDnPsHvPKfCzJLNbJmZTTz50qHDzM4MfN5zzWy633mayswuNbPHzexFM/um33lOhZn1MLMnzGyu31lOJCKKwMz+ZGYHzWx1venfNrMNZrbZzO5ubB3OuUXOuWnA68DTXuatk++0cwOTgU7AcWC3V1nra6bsDigBEghS9mbKDfBT4K/epGxYM/07Xxf4d34lMNbLvHXyNUfuV51ztwDTgKu8zFtXM2Xf6py7ydukp+nrXA0Xag9gPDAMWF1nWjSwBegBxAErgX7AQGp+2dd9tK3zur8CrcIlN3A3cGvgtXPD6TMHogKvawc8F0a5LwCuBm4AJobTZx54zSRgAXBtOOUOvO53wLBw+8wDrwva/89TfcQQAZxzC82sW73JI4DNzrmtAGb2AjDZOXcv0ODmvJl1AYqcc0e8zFurOXKb2W6gIvC0ysO4X9Fcn3nAISDek6D1NNNnPgFIpuY//1Ezm++cq/YyNzTfZ+6cmwfMM7M3gL94GLn2/ZrjMzfgt8AC59xyjyN/qZn/nYesiCiCE8gGdtV5vhsYeZLX3AQ86VmipjnV3C8DD5vZ2cBCL4M1wSllN7PLgG8B6cAsT5M17pRyO+d+DmBmNwD5wSiBRpzqZz4BuIya4p3vZbCTONV/53cA5wNpZtbLOTfHy3AncaqfeQbwG2Comf0sUBghJZKL4JQ5537pd4ZT5Zwro6bAwo5z7mVqiiwsOeee8jvDqXLOfQB84HOMU+acmwnM9DvH1+GcK6Dm2EbIioiDxSewB+hc53mnwLRQF665IXyzh2tuCN/s4Zobwjt7gyK5CD4DeptZdzOLo+bg3jyfMzVFuOaG8M0errkhfLOHa24I7+wN8/todXM8gOeBffzjFMqbAtMvAjZSc4T/537njJTc4Zw9XHOHc/ZwzR3u2U/loUHnRERauEjeNSQiIk2gIhARaeFUBCIiLZyKQESkhVMRiIi0cCoCEZEWTkUgEcHMSoL8fs1yz4rAPRmKzGyFma03s/ub8JpLzaxfc7y/CKgIRBpkZo2Ow+WcG9OMb7fIOTcEGApMNLOT3SfgUmpGPhVpFioCiVhm1tPM3jSzXKu5E1rfwPRLzOwTM/vczN41s3aB6feY2bNmtgR4NvD8T2b2gZltNbMf1ll3SeDrhMD8uYG/6J8LDJmMmV0UmJZrZjPN7PXG8jrnjgIrqBndEjO7xcw+M7OVZvY3M0syszHU3E/gfwNbET1P9HOKNJWKQCLZY8AdzrnhwI+B2YHpi4FRzrmhwAvAv9V5TT/gfOfcNYHnfakZKnsE8Eszi23gfYYCMwKv7QGMNbME4FHgwsD7Z50srJm1Bnrzj+HEX3bOneWcGwyso2Z4g6XUjGvzE+fcEOfclkZ+TpEm0TDUEpHMLAUYA7wU+AMd/nHzm07Ai2bWgZo7TG2r89J5gb/Ma73hnCsHys3sIDV3U6t/W81PnXO7A++7AuhGzS04tzrnatf9PDD1BHHPNrOV1JTAg865/YHpA8zsv6i5X0MK8NYp/pwiTaIikEgVBRwO7Huv72Hg9865eYEbtdxTZ15pvWXL63xfRcP/Z5qyTGMWOecmmll34GMz+6tzbgXwFHCpc25l4CY4Exp4bWM/p0iTaNeQRCTnXDGwzcyugJpbHZrZ4MDsNP4xfvz1HkXYAPSoc5vDk95wPbD18Fvgp4FJrYB9gd1R362z6JHAvJP9nCJNoiKQSJFkZrvrPH5EzS/PmwK7XdYAkwPL3kPNrpRcIN+LMIHdS7cBbwbe5whQ1ISXzgHGBwrk/wGfAEuA9XWWeQH4SeBgd09O/HOKNImGoRbxiJmlOOdKAmcR/QHY5Jx7wO9cIvVpi0DEO7cEDh6voWZ31KP+xhFpmLYIRERaOG0RiIi0cCoCEZEWTkUgItLCqQhERFo4FYGISAunIhARaeH+P1rQvbvrb8JaAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"admin_init(learn.model, scales)\n",
"learn.lr_find()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Tracking"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmorgan\u001b[0m (use `wandb login --relogin` to force relogin)\n"
]
}
],
"source": [
"# hide\n",
"!wandb login"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Load Experiment Tracking with Weights & Biases:"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"import wandb\n",
"\n",
"WANDB_NAME = f'admin_init_bte2'\n",
"GROUP = 'admin'\n",
"NOTES = 'Early ADMIN testing'\n",
"CONFIG = {}\n",
"TAGS = ['admin']"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmorgan\u001b[0m (use `wandb login --relogin` to force relogin)\n"
]
},
{
"data": {
"text/html": [
"\n",
" Tracking run with wandb version 0.10.19<br/>\n",
" Syncing run <strong style=\"color:#cdcd00\">admin_init_bte2</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
" Project page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox</a><br/>\n",
" Run page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/2o27v7d4\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox/runs/2o27v7d4</a><br/>\n",
" Run data is saved locally in <code>/home/morgan/ml/projects/transformers_sandbox/nbs/exploration/wandb/run-20210220_234952-2o27v7d4</code><br/><br/>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<h1>Run(2o27v7d4)</h1><iframe src=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/2o27v7d4\" style=\"border:none;width:100%;height:400px\"></iframe>"
],
"text/plain": [
"<wandb.sdk.wandb_run.Run at 0x7fb00ef616a0>"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#hide_output\n",
"wandb.init(reinit=True, project=\"transformers_sandbox\", entity=\"fastai_community\", \n",
" name=WANDB_NAME, group=GROUP, notes=NOTES, tags=TAGS, config=CONFIG)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>perplexity</th>\n",
" <th>bpc</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>2.117492</td>\n",
" <td>2.035515</td>\n",
" <td>0.404338</td>\n",
" <td>7.656197</td>\n",
" <td>2.936628</td>\n",
" <td>07:32</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>1.601835</td>\n",
" <td>1.523893</td>\n",
" <td>0.554345</td>\n",
" <td>4.590059</td>\n",
" <td>2.198513</td>\n",
" <td>07:29</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>1.452392</td>\n",
" <td>1.390810</td>\n",
" <td>0.589804</td>\n",
" <td>4.018106</td>\n",
" <td>2.006516</td>\n",
" <td>07:28</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>1.347416</td>\n",
" <td>1.315846</td>\n",
" <td>0.613917</td>\n",
" <td>3.727902</td>\n",
" <td>1.898364</td>\n",
" <td>07:29</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>1.275978</td>\n",
" <td>1.247816</td>\n",
" <td>0.633069</td>\n",
" <td>3.482728</td>\n",
" <td>1.800218</td>\n",
" <td>07:28</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>1.227183</td>\n",
" <td>1.200533</td>\n",
" <td>0.647163</td>\n",
" <td>3.321887</td>\n",
" <td>1.732003</td>\n",
" <td>07:27</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>1.188224</td>\n",
" <td>1.171363</td>\n",
" <td>0.655419</td>\n",
" <td>3.226386</td>\n",
" <td>1.689919</td>\n",
" <td>07:28</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>1.147837</td>\n",
" <td>1.153088</td>\n",
" <td>0.661562</td>\n",
" <td>3.167962</td>\n",
" <td>1.663555</td>\n",
" <td>07:28</td>\n",
" </tr>\n",
" <tr>\n",
" <td>8</td>\n",
" <td>1.123723</td>\n",
" <td>1.142603</td>\n",
" <td>0.665061</td>\n",
" <td>3.134917</td>\n",
" <td>1.648427</td>\n",
" <td>07:29</td>\n",
" </tr>\n",
" <tr>\n",
" <td>9</td>\n",
" <td>1.113917</td>\n",
" <td>1.141676</td>\n",
" <td>0.665857</td>\n",
" <td>3.132012</td>\n",
" <td>1.647090</td>\n",
" <td>07:28</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with valid_loss value: 2.035515308380127.\n",
"Better model found at epoch 1 with valid_loss value: 1.523892879486084.\n",
"Better model found at epoch 2 with valid_loss value: 1.390810489654541.\n",
"Better model found at epoch 3 with valid_loss value: 1.3158457279205322.\n",
"Better model found at epoch 4 with valid_loss value: 1.2478158473968506.\n",
"Better model found at epoch 5 with valid_loss value: 1.2005329132080078.\n",
"Better model found at epoch 6 with valid_loss value: 1.1713625192642212.\n",
"Better model found at epoch 7 with valid_loss value: 1.1530883312225342.\n",
"Better model found at epoch 8 with valid_loss value: 1.142602562904358.\n",
"Better model found at epoch 9 with valid_loss value: 1.1416757106781006.\n"
]
}
],
"source": [
"admin_init(learn.model, scales)\n",
"learn.fit_one_cycle(n_epochs, 1e-3, cbs=[WandbCallback(log_model=False, log_preds=False)])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train - SPUniTokenizer\n",
"\n",
"Train a tokenizer with the same dictionary length as the T5 tokenizer"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"SentencePiece Unigram Tokenizer"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Reusing dataset wikitext (/home/morgan/.cache/huggingface/datasets/wikitext/wikitext-103-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91)\n",
"Loading cached processed dataset at /home/morgan/.cache/huggingface/datasets/wikitext/wikitext-103-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91/cache-e9eccb98202e10a0.arrow\n"
]
}
],
"source": [
"big_train_ds = load_dataset('wikitext', name='wikitext-103-raw-v1', split='train')\n",
"big_train_ds = big_train_ds.filter(lambda x: x['text'] != '')\n",
"\n",
"spu_tok = SPUniTokenizer(vcb_sz=32093, add_eos=True, add_bos=True)\n",
"spu_tok.train(big_train_ds['text'])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Save Trained Tokenizer"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"def save_obj(obj, name ):\n",
" with open(name + '.pkl', 'wb') as f:\n",
" pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)\n",
"\n",
"def load_obj(name ):\n",
" with open(name + '.pkl', 'rb') as f:\n",
" return pickle.load(f)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"# save_obj(spu_tok)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"spu_tok = load_obj('sentencepieceunigram_wiki103_32093')"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"tok = spu_tok"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"#Run setup\n",
"n_epochs = 10\n",
"bs = 12\n",
"sl = 512\n",
"n_layers = 6\n",
"pad_id = tok.pad_token_id\n",
"vocab_size = tok.vocab_size"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Tokenize Data"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Loading cached processed dataset at /home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91/cache-4267024924ef223b.arrow\n",
"Loading cached processed dataset at /home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91/cache-916f06ba6ec602c9.arrow\n"
]
}
],
"source": [
"def encode(examples): return {'token_ids' : spu_tok(examples['text'])}\n",
"\n",
"tok_ds = train_ds.map(encode, batched=False)\n",
"val_tok_ds = valid_ds.map(encode, batched=False)\n",
"all_toks = tok_ds['token_ids'] + val_tok_ds['token_ids']\n",
"df['toks'] = all_toks "
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>text</th>\n",
" <th>text_</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Coulthard and Button collided on lap 18 when Button attempted to pass the Red Bull on the inside at turn eight ; the Honda lost its front wing and retired a lap later after two pit stops . Hamilton continued his climb back through the field ; he moved from 18th , passing Piquet , Davidson , Sutil and Bourdais in separate manoeuvres , to sit in 14th by the time he pitted on lap 31 . Piquet retired on lap 42 with transmission failure , requiring a gearbox change before the next race . Anderson was involved in filmmaking at a young age and never really had an alternative plan to directing films . He made his first movie when he was eight years old and started making movies on a Betamax video camera which his dad bought in 1982 when he was twelve years old . He</td>\n",
" <td>Coulthard and Button collided on lap 18 when Button attempted to pass the Red Bull on the inside at turn eight ; the Honda lost its front wing and retired a lap later after two pit stops . Hamilton continued his climb back through the field ; he moved from 18th , passing Piquet , Davidson , Sutil and Bourdais in separate manoeuvres , to sit in 14th by the time he pitted on lap 31 . Piquet retired on lap 42 with transmission failure , requiring a gearbox change before the next race . Anderson was involved in filmmaking at a young age and never really had an alternative plan to directing films . He made his first movie when he was eight years old and started making movies on a Betamax video camera which his dad bought in 1982 when he was twelve years old . He</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>subdue and molest his victims . The song ends with the narrator turning inward with the lyrics : \" And in my best behavior , I am really just like him / Look beneath the floorboards for the secrets I have hid . \" Stevens stated in a 2009 interview with Paste that \" we 're all capable of what [ Gacy ] did . \" Technically , it is known as a speaker that easily reveals poor quality in recordings . Recording engineers sought to dull its treble response by hanging tissue paper in front of it , resulting in what became known as the \" tissue paper effect \" , a type of comb filtering . The NS @-@ 10 has been used to monitor a large number of successful recordings by numerous artists , leading Gizmodo to refer to it as \" the most important loudspeaker</td>\n",
" <td>and molest his victims . The song ends with the narrator turning inward with the lyrics : \" And in my best behavior , I am really just like him / Look beneath the floorboards for the secrets I have hid . \" Stevens stated in a 2009 interview with Paste that \" we 're all capable of what [ Gacy ] did . \" Technically , it is known as a speaker that easily reveals poor quality in recordings . Recording engineers sought to dull its treble response by hanging tissue paper in front of it , resulting in what became known as the \" tissue paper effect \" , a type of comb filtering . The NS @-@ 10 has been used to monitor a large number of successful recordings by numerous artists , leading Gizmodo to refer to it as \" the most important loudspeaker you</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"df['lens'] = df['toks'].apply(len)\n",
"splits = ColSplitter()(df)\n",
"tfms = [attrgetter(\"text\"), spu_tok]\n",
"# tfms = [attrgetter(\"toks\")]\n",
"dsets = Datasets(df, [tfms], splits=splits, dl_type=LMDataLoader)\n",
"dl_kwargs = [{'lens':df['lens'].values[splits[0]]},\n",
" {'val_lens':df['lens'].values[splits[1]]}]\n",
"dls = dsets.dataloaders(bs=bs, seq_len=sl, dl_kwargs=dl_kwargs, shuffle_train=True)\n",
"dls.show_batch(max_n=2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Tracking"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"WANDB_NAME = f'admin_init_spu'\n",
"GROUP = 'admin'\n",
"NOTES = 'SentencePieceUnigram tokenizer trained on wiki 103, 32100 vcb_sz, Early ADMIN testing'\n",
"CONFIG = {}\n",
"TAGS = ['admin','spu']"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmorgan\u001b[0m (use `wandb login --relogin` to force relogin)\n"
]
},
{
"data": {
"text/html": [
"\n",
" Tracking run with wandb version 0.10.19<br/>\n",
" Syncing run <strong style=\"color:#cdcd00\">admin_init_spu</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
" Project page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox</a><br/>\n",
" Run page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/896aogrk\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox/runs/896aogrk</a><br/>\n",
" Run data is saved locally in <code>/home/morgan/ml/projects/transformers_sandbox/nbs/exploration/wandb/run-20210221_183500-896aogrk</code><br/><br/>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<h1>Run(896aogrk)</h1><iframe src=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/896aogrk\" style=\"border:none;width:100%;height:400px\"></iframe>"
],
"text/plain": [
"<wandb.sdk.wandb_run.Run at 0x7f5b92f19e50>"
]
},
"execution_count": 32,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#hide_output\n",
"wandb.init(reinit=True, project=\"transformers_sandbox\", entity=\"fastai_community\", \n",
" name=WANDB_NAME, group=GROUP, notes=NOTES, tags=TAGS, config=CONFIG)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Train SPU"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>perplexity</th>\n",
" <th>bpc</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>5.331179</td>\n",
" <td>5.275365</td>\n",
" <td>0.253242</td>\n",
" <td>195.461777</td>\n",
" <td>7.610743</td>\n",
" <td>02:44</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>4.771585</td>\n",
" <td>4.862295</td>\n",
" <td>0.290732</td>\n",
" <td>129.320679</td>\n",
" <td>7.014809</td>\n",
" <td>02:43</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>4.419650</td>\n",
" <td>4.540506</td>\n",
" <td>0.322093</td>\n",
" <td>93.738213</td>\n",
" <td>6.550565</td>\n",
" <td>02:46</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>4.108190</td>\n",
" <td>4.304454</td>\n",
" <td>0.344261</td>\n",
" <td>74.028770</td>\n",
" <td>6.210014</td>\n",
" <td>02:44</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>3.865303</td>\n",
" <td>4.159061</td>\n",
" <td>0.357382</td>\n",
" <td>64.011414</td>\n",
" <td>6.000257</td>\n",
" <td>02:45</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>3.655769</td>\n",
" <td>4.056620</td>\n",
" <td>0.368161</td>\n",
" <td>57.778694</td>\n",
" <td>5.852466</td>\n",
" <td>02:44</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>3.459788</td>\n",
" <td>3.989344</td>\n",
" <td>0.376501</td>\n",
" <td>54.019436</td>\n",
" <td>5.755407</td>\n",
" <td>02:45</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>3.292417</td>\n",
" <td>3.953279</td>\n",
" <td>0.382756</td>\n",
" <td>52.105968</td>\n",
" <td>5.703377</td>\n",
" <td>02:46</td>\n",
" </tr>\n",
" <tr>\n",
" <td>8</td>\n",
" <td>3.169972</td>\n",
" <td>3.935873</td>\n",
" <td>0.385500</td>\n",
" <td>51.206848</td>\n",
" <td>5.678265</td>\n",
" <td>02:47</td>\n",
" </tr>\n",
" <tr>\n",
" <td>9</td>\n",
" <td>3.099260</td>\n",
" <td>3.935462</td>\n",
" <td>0.385990</td>\n",
" <td>51.185768</td>\n",
" <td>5.677671</td>\n",
" <td>02:44</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with valid_loss value: 5.275364875793457.\n",
"Better model found at epoch 1 with valid_loss value: 4.862295150756836.\n",
"Better model found at epoch 2 with valid_loss value: 4.540505886077881.\n",
"Better model found at epoch 3 with valid_loss value: 4.3044538497924805.\n",
"Better model found at epoch 4 with valid_loss value: 4.159061431884766.\n",
"Better model found at epoch 5 with valid_loss value: 4.056620121002197.\n",
"Better model found at epoch 6 with valid_loss value: 3.9893438816070557.\n",
"Better model found at epoch 7 with valid_loss value: 3.953279495239258.\n",
"Better model found at epoch 8 with valid_loss value: 3.93587327003479.\n",
"Better model found at epoch 9 with valid_loss value: 3.9354615211486816.\n"
]
}
],
"source": [
"admin_init(learn.model, scales)\n",
"learn.fit_one_cycle(n_epochs, 1e-3, cbs=[WandbCallback(log_model=False, log_preds=False)])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train SPU - 20e"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"SuggestedLRs(lr_min=0.04365158379077912, lr_steep=0.3630780577659607)"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAApoElEQVR4nO3deXhU9d3+8fcnO1kIkIV9B5F9C2jBBetaq4LWKioKiKLSuvWnrT5t1dr26WYf910RBdeqtWqta90VISAgoMi+h4QlG9mT7++PTDTGCSQhM2dmcr+ua67MnHNmzp0A+XDOdzPnHCIiIg1FeR1ARERCkwqEiIj4pQIhIiJ+qUCIiIhfKhAiIuKXCoSIiPgV43WA1pKenu769OnjdQwRkbCyZMmS3c65DH/7IqZA9OnTh+zsbK9jiIiEFTPb3Ng+3WISERG/VCBERMQvFQgREfFLBUJERPxSgRAREb9UIERExK+I6ebamiqra4iNblrtdM5RVF5FXlE5eUXlFJVVfWd/SkIMPTsl0qV9AtFRFoi4IiIBoQLhU1Vdw9tf5vLEp5v4ZP0ekuKiSU+JJyM5noTYaEoqqiipqKakopryqmoqqx2VVTWUV9VQUV1z0M+PjTa6prYjKT6G2GgjJsqIi4kiOT6m9pEQQ8fEOLqkJtAttR1dOyTQNbUd7RNiMFNhEZHga/MForCskgULN/Pkwi1szy+lW2oClx3bj8oqR15xObuLytlfUUVSXAzpyfEkxkUTHxNNbIwRGx1FXEwU6UnxZKTEk54cT2q7WOp+nzsH+0oq2LqvhG37Stm+r5Syymoqq2uoqnGUV9awI7+M/RVVFJdVkV9aSXXNdxdwSoyLpktqAl1TE+jVKYneaYn07pRI77Ta50nxbf6PUEQCpM3/dqmoquGOt9eS1bsjvz1tCCcMziSmibeXWltVdQ15xeXsyC9lR34ZOQVl7CwoY1dhGdvzS3ljVQ5791d85z3pyfH0SastGH3TE+mTnkSftCT6ZSSRGNfm/3hF5BBYpCw5mpWV5Vo61UZuYRmZ7RNaOVFgFJVVsnlPSe1j73427y5h0579bN5TQk5h2XeO7d6hHQMykxnUJYWs3h0Z37cTHRLjPEouIqHIzJY457L87lOBiBwlFVVs3lPCxt37WZ9bzNrcYtb5HnXtJId3SeHIfmkcNSCdI/unkaxbVCJt2oEKhH47RJDEuBgGd23P4K7tv7O9rLKaFdsKWLRxD59t3Mszi7cw75NNxEQZY3p15LjDMzl5aGf6ZSR7lFxEQpGuINqgsspqlm7ex4frdvPh2jxWbi8E4LDOyZwytAtnj+1Jr7REj1OKSDDoFpMc0Pb8Ut5clcMbq3JYtHEvDjj2sAymHdGb4w7P1PgNkQimAiFNtrOglGcWbeWZxVvYVVhO9w7tOP+IXpw7rifpyfFexxORVqYCIc1WWV3DO1/uYv7CzXy8bg9x0VGcOrwLMyb2ZVTPDl7HE5FWokZqabbY6ChOGdaVU4Z1ZV1uMQsWbuaFJdt4adkOxvftxOXH9mPSYZlE6faTSMTSFYQ0WXF5Fc8u3sqjH25gR0EZh3VO5rqTBnHikM6aDkQkTB3oCkKzuUqTJcfHMOuovrz/y+O4/dyRVNc4Zs9fwrRHP2NNTpHX8USklalASLPFRkdx5ugevH7NMdxy+hBWbi/kR3d+wE3/WklhWaXX8USklahASIvFRkcxY2Jf3rtuEtOO7M2ChZs54e/v8/rKHK+jiUgrUIGQQ9YxKY5bJw/jpZ9NJD05nssXLOGy+dnkFJQd/M0iErJUIKTVjOjRgX/9fCI3/Ohw3luTx8l3fMAbq3Q1IRKuVCCkVcVGR3H5sf15/Zpj6NUpkcvmL+G3L62krLLa62gi0kwqEBIQfdOTeOGKCVx6dF/mL9zMlHs/ZkNesdexRKQZAlYgzGyumeWa2cp62zqZ2Vtmttb3tWMj7602s2W+x8uByiiBFRcTxa9/PITHZo4jt6ics+7/hOxNe72OJSJNFMgriHnAKQ223QC845wbCLzje+1PqXNulO9xRgAzShAcNyiTl+ZMpGNiHOc/8hn/+WKn15FEpAkCViCccx8ADf+7OBl43Pf8cWBKoM4voaVXWiIvXDGBYd3aM+eppTz60UavI4nIQQS7DaKzc67uv485QOdGjksws2wzW2hmUxr7MDOb7TsuOy8vr7WzSivrlBTHU5ceyUlDOvP7V1dz6yurqa6JjKleRCKRZ43UrnYSqMZ+O/T2zQ1yPnCHmfVv5DMecs5lOeeyMjIyAhVVWlFCbDT3XTCWmRP7MPfjjfzsyaXq4SQSooJdIHaZWVcA39dcfwc557b7vm4A3gNGByugBF50lHHz6UP57WlDeGN1Duc9vJA9xeVexxKRBoJdIF4GpvueTwf+1fAAM+toZvG+5+nARGB10BJK0Mw6qi/3XzCG1TsKOfuBTzXyWiTEBLKb69PAp8AgM9tmZrOAPwMnmtla4ATfa8wsy8we8b11MJBtZsuBd4E/O+dUICLUKcO68tSlR5BXVM75Dy8kt1BFQiRUaD0ICQnZm/Zy0dxFdElN4JnZR5KZkuB1JJE2QetBSMjL6tOJeTPHk1NQxvkPf0ZekdokRLymAiEhY3zfTsydMY7t+0qZ9shn7N1f4XUkkTZNBUJCypH90nh0ehab9uxn2iOfkV+iIiHiFRUICTkTBqTz8EVZrMst5sJHF1FQqlXqRLygAiEh6ZjDMnjgwjF8lVPIjMcWUVxe5XUkkTZHBUJC1g8P78y954/hi20FXDxvMaUVGnEtEkwqEBLSThrahdvPHcXiTXu5bMESyqtUJESCRQVCQt7pI7vxl7NG8MHXeVz51OdUVtd4HUmkTVCBkLBwzrie3HL6EN5cvYvr/rGcGs0CKxJwMV4HEGmqGRP7UlJZzV9fX0PHxDhuPn0IZuZ1LJGIpQIhYeWKY/uzp7iCRz/aSEZKPD87boDXkUQilgqEhBUz49enDmZ3cTl/e2MNGSnxnJPV0+tYIhFJBULCTlSU8bezR7J3fwU3vvgFaUlxHD+4scUJRaSl1EgtYSkuJor7p41lSNf2zHlyKZ9t2ON1JJGIowIhYSs5PoZ5M8fRo2M7Zj2ezRfbCryOJBJRVCAkrKUlx7PgkiNIbRfL9McWsS63yOtIIhFDBULCXtfUdjx5yRFERxnTHlnE1r0lXkcSiQgqEBIR+qQnMX/WeEorq5k5bzEFJZoBVuRQqUBIxDi8S3sevHAsm/fs54onl1BRpSk5RA6FCoRElCP7pfHXs0fwyfo9/M8/vyBS1lwX8YLGQUjEOXN0DzbvKeGOt9fSu1MiVx4/0OtIImFJBUIi0tXHD2TLnhL+/tbX9MtI5scjunodSSTs6BaTRCQz408/Gc7Y3h257h/LWbVDYyREmksFQiJWfEw0D0wbS4fEWGY/sYTdxeVeRxIJKyoQEtEyUuJ56MIsdheXc8UC9WwSaQ4VCIl4w3uk8tezR7B40z5ufnmlejaJNJEaqaVNmDyqO2tyirjvvfUMyExh1lF9vY4kEvICdgVhZnPNLNfMVtbb1snM3jKztb6vHRt573TfMWvNbHqgMkrbct1JgzhlaBf+8O/VvPPlLq/jiIS8QN5imgec0mDbDcA7zrmBwDu+199hZp2Am4EjgPHAzY0VEpHmiIoybj93FMO6pXLl05+zekeh15FEQlrACoRz7gNgb4PNk4HHfc8fB6b4eevJwFvOub3OuX3AW3y/0Ii0SLu4aB6ZnkVqu1gueXwxuYVlXkcSCVnBbqTu7Jzb6XueA/hbBqw7sLXe622+bSKtonP7BB6ZnkV+aSUz5y2mqEwT+4n441kvJlfbleSQupOY2Wwzyzaz7Ly8vFZKJm3B0G6p3HvBGNbkFDH7iSWUVVZ7HUkk5AS7QOwys64Avq+5fo7ZDtRfhb6Hb9v3OOcecs5lOeeyMjIyWj2sRLbjBmVy209H8umGPVzzzDKqa9T9VaS+YBeIl4G6XknTgX/5OeYN4CQz6+hrnD7Jt02k1U0Z3Z2bThvC66ty+M1Lmv1VpL6AjYMws6eBSUC6mW2jtmfSn4HnzGwWsBk4x3dsFnC5c+4S59xeM/s9sNj3Ubc65xo2dou0mouP6sue/eXc++56MlMSuPbEw7yOJBISAlYgnHPnNbLreD/HZgOX1Hs9F5gboGgi33PdSYPIKSjnznfW0rNTImeP7eF1JBHPaSS1CL7ZX88azs6CUm54YQXdUhOYMCDd61gintJcTCI+cTFR3D9tLH3Tk7hswRLW7iryOpKIp1QgROpJbRfLYzPHkRAbzYzHNJBO2jYVCJEGenRMZO70cewrqWDGYxpIJ22XCoSIH8N7+AbS7SpizpNLtY6EtEkqECKNOG5QJn86azgfrt3NDS+s0BgJaXPUi0nkAM7J6smugjL+/tbXdO2QwPUnH+51JJGgUYEQOYif/3AAOwpKuffd9fRNT9YYCWkzdItJ5CDMjFsnD2NC/zRufHEFizdpYL+0DSoQIk0QGx3FfReMoUfHRC6bv4Qte0q8jiQScCoQIk3UITGOR6dnUV3jmPX4YgrV/VUinAqESDP0y0jm/mlj2Lh7P3MWqPurRDYVCJFmmtA/nT//ZAQfrdvNL59fTo3WkZAIpV5MIi1w9tge7Cos429vrKFzagI3/miw15FEWp0KhEgLzZnUn5yCMh58fwNd2icwc2JfryOJtCoVCJEWMjNuOWMouwrLuPXV1XRpn8CPhnf1OpZIq1EbhMghiI4y7jpvNKN7duDqZ5eRrTESEkFUIEQOUUJsNI9MH0f3Du245Ils1ucVex1JpFWoQIi0gk5JccybOY5oM2Y8toi8onKvI4kcMhUIkVbSOy2JR2eMI6+onEseX0xpRbXXkUQOiQqESCsa1bMDd583hhXbC/jFc8s0RkLCmgqESCs7cUhnfvPjIfxnZQ5/eeMrr+OItJi6uYoEwMUT+7BxdzEPvr+BPmlJnDe+l9eRRJpNBUIkAMyMW04fyta9pfzmpZX07JjIUQPTvY4l0iy6xSQSIDHRUdxz/mgGZiZz+YIlrN5R6HUkkWZRgRAJoJSEWB6bOY6UhBhmzlvE9vxSryOJNJkKhEiAdU1tx7yZ4ympqGbG3EUUlGgdCQkPTSoQZpZkZlG+54eZ2RlmFhvYaCKRY1CXFB66MIvNe0q4dH42ZZUaIyGhr6lXEB8ACWbWHXgTuBCY19KTmtnVZrbSzFaZ2TV+9k8yswIzW+Z73NTSc4mEih/0T+O2c0ayaONefvn8CpzTGAkJbU3txWTOuRIzmwXc55z7q5kta8kJzWwYcCkwHqgAXjezV51z6xoc+qFz7rSWnEMkVJ0xshtb95bwtzfW0CctkV+cNMjrSCKNauoVhJnZD4ALgH/7tkW38JyDgc+ccyXOuSrgfeCsFn6WSNiZM6k/52T14K7/ruOFJdu8jiPSqKYWiGuAG4F/OudWmVk/4N0WnnMlcLSZpZlZInAq0NPPcT8ws+Vm9h8zG+rvg8xstpllm1l2Xl5eC+OIBJeZ8Ycpw/lBvzRueHEFCzfs8TqSiF/W3PugvsbqZOdcizt1+25VzQH2A6uAcufcNfX2twdqnHPFZnYqcKdzbuCBPjMrK8tlZ2e3NJJI0BWUVHLW/R+zu7iCf1z+Aw7rnOJ1JGmDzGyJcy7L376m9mJ6yszam1kStVcAq83s+pYGcs496pwb65w7BtgHfN1gf6Fzrtj3/DUg1sw0DFUiSmpiLPNmjicuJooLH/2MrXtLvI4k8h1NvcU0xHfFMAX4D9CX2p5MLWJmmb6vvahtf3iqwf4uZma+5+N9OXUdLhGnZ6dEnrh4PKUV1Vw0dxG7i7WOhISOphaIWN+4hynAy865SuBQ+ui9YGargVeAnznn8s3scjO73Lf/bGClmS0H7gKmOvUJlAg1uGt75s4Yx86CUmY8toiiMg2kk9DQpDYIM7sK+BWwHPgx0AtY4Jw7OrDxmk5tEBLu3v0ql0ufyGZ8307f3HoSCbRDboNwzt3lnOvunDvV1doMHNeqKUXauOMOz+QvPxnBJ+v38KsXNJBOvNekgXJmlgrcDBzj2/Q+cCtQEKBcIm3ST8b2YEd+KX9/62t6dGzH/9NAOvFQU69h5wJFwDm+RyHwWKBCibRlP//hAKaO68nd/13HM4u2eB1H2rCmTrXR3zn3k3qvf9fSqTZE5MDMjN9PGcbOgjJ+/dJKOqcmcNygTK9jSRvU1CuIUjM7qu6FmU0ENLG9SIDERkdx7wVjOLxLCnMWLOXzLfu8jiRtUFMLxOXAvWa2ycw2AfcAlwUslYiQHB/DvJnjyWwfz8XzFrM+r9jrSNLGNLUX03Ln3EhgBDDCOTca+GFAk4kIGSnxPHHxeKKjjIseXUROQZnXkaQNaVZHa98UGHVzMP0iAHlEpIHeaUnMmzmegtJKps9dRKEG0kmQHMpIHGu1FCJyQMO6p/LghWNZn1fMz5/6nKrqGq8jSRtwKAVCo3hEgmjigHT+MGUYH3ydxx/+/aXXcaQNOGA3VzMrwn8hMKBdQBKJSKOmju/FutxiHvloI/0zk7nwyN5eR5IIdsAC4ZzTBPUiIebGUwezYfd+bnl5FX3TkjhqoGbCl8DQbGAiYSY6yrhz6igGZCQz58klbN6z3+tIEqFUIETCUEpCLI9MzyIqypj9xBL2l1d5HUk8Mu/jjTz4/vqAfLYKhEiY6tkpkbvPG83a3CKuf365Zn9to15dsZN31+QG5LNVIETC2NEDM/jVKYfz2hc5PPD+Bq/jiAd2FpTRrUNg+gypQIiEudnH9OO0EV356xtf8f7XeV7HkSCqqq4hp7CM7ioQIuKPmfHXs0cwqHMK1z67TNNxtCG5ReVU1zhdQYhI4xLjYrj3gjGUVVZz1dMaad1W7MivnVRbBUJEDqh/RjJ/PHMYizbt5Y6313odR4Jgu69AdO+QEJDPV4EQiSBnju7BuVk9ufe9dXyg9oiItyO/9nZi11RdQYhIE9xyxlAOy6xtj9hVqPaISLYjv5QOibEkxTd1cdDmUYEQiTDt4qK594LRlFSoPSLS7cgvpVuArh5ABUIkIg3ITOGPZw7js41qj4hk2/NLA9ZADSoQIhHrrDHftkdofERk2pFfGrAGalCBEIlov5s89JvxETsLSr2OI62oqKySwrIquuoKQkRaIiE2mnsvGEO5xkdEnJ2+AZG6xSQiLdY/I5n/PWs4izft4//e+trrONJKAj0GAjwqEGZ2tZmtNLNVZnaNn/1mZneZ2TozW2FmYzyIKRIxJo/qznnje3Lfe+vVHhEhAj2KGjwoEGY2DLgUGA+MBE4zswENDvsRMND3mA3cH9SQIhHoptNq2yN+ofEREWFHfinRUUZmSmRdQQwGPnPOlTjnqoD3gbMaHDMZeMLVWgh0MLOuwQ4qEknqj4+4+pnPqa7R+hHhbEd+GV3aJxAdZQE7hxcFYiVwtJmlmVkicCrQs8Ex3YGt9V5v8237DjObbWbZZpadl6fLZpGDGZCZwu+nDGPhhr3c9Y7GR4Sz7fmlAZvmu07QC4Rz7kvgL8CbwOvAMqC6hZ/1kHMuyzmXlZGR0XohRSLY2WN7cNaY7tz937Us3rTX6zjSQjvyS+kWwAZq8KiR2jn3qHNurHPuGGAf0LBrxXa+e1XRw7dNRFrBrZOH0bNTItc8s4yC0kqv40gzVdc4cgK4klwdr3oxZfq+9qK2/eGpBoe8DFzk6810JFDgnNsZ5JgiESs5PoY7zh1FTmEZv3lppdazDjN5ReVUBXChoDpejYN4wcxWA68AP3PO5ZvZ5WZ2uW//a8AGYB3wMDDHo5wiEWt0r45ce8JAXlm+gxeX6gI9nHw7BiKwBSIwc8QehHPuaD/bHqj33AE/C2ookTboikkD+GDtbm7610rG9u5In/QkryNJEwRjDARoJLVImxYdZdx+7iiio4xrn1umqTjCxLcFIgIbqUUkdHTv0I4/njmcz7fkc8+767yOI02wI7+UlIQYUhJiA3oeFQgR4fSR3ThzdHfu/u86lm7Z53UcOYjt+WUBb38AFQgR8fnd5KF0aZ/Atc8uo7i8yus4cgA7ArxQUB0VCBEBoH1CLHdMHcXWvSXc+soqr+PIAewoKKVramDbH0AFQkTqGdenE3MmDeC57G289oWGHoWi/eVV5JdU6gpCRILv6hMGMrJnB254YcU3/e0ldNStDKg2CBEJutjoKO6aOorqGse1zyzTrK8hZnt+4FeSq6MCISLf0zstid9PGcaiTXu5V11fQ8r2fbVXEGqDEBHPnDWmB1NGdePOd9ayZLNmfQ0Vq3YUkBwfoysIEfHW76cMo1uHBK56ehmFZZr1NRR8viWfUT07BHShoDoqECLSqJSEWO6cOrp21td/atZXr+0vr+KrnELG9OoQlPOpQIjIAY3p1ZFrjh/Iy8t38M/PNeurl5Zvy6fGwejeHYNyPhUIETmoOccNYHyfTvz2pZVs3rPf6zht1udb8gEY01MFQkRCRHSUcfvUUURFGVc/s4xKzfrqiaWb99E/I4nUxMBO0ldHBUJEmqR7h3b86azhLNuaz21vrvE6TpvjnOPzrfmM7hWcqwdQgRCRZjhtRDfOP6IXD76/gVdX7PA6TpuyeU8Je/dXMEYFQkRC1c2nD2FMrw5c/48VfLmz0Os4bUbdNOxjencI2jlVIESkWeJjonlg2lhSEmKYPT+b/JIKryO1CUu37CM5PoaBmSlBO6cKhIg0W2b7BB64cCy7Csq58unPtVRpECzdnM/InqlBGSBXRwVCRFpkTK+O3Dp5KB+u3c3/vvaV13EiWklF3QC54LU/AMQE9WwiElGmju/Fml1FzP14I/0zk7jgiN5eR4pIy7cWUOMIeoHQFYSIHJLf/HgIkwZlcNO/VvHxut1ex4lIdQ3Uo4M0xUYdFQgROSTRUcbd542mf0YSVyxYwvq8Yq8jRZzPt+yjX0YSHRLjgnpeFQgROWQpCbE8On0csdFRXDxvMbuLy72OFDGccyzdks/oIE2vUZ8KhIi0ip6dEnnooix2FZYxfe4iijQ9eKtYn7e/doBcEMc/1FGBEJFWM7Z3R+6fNpY1OUVc+kQ2ZZXVXkcKe2+t3gXApEGZQT+3JwXCzK41s1VmttLMnjazhAb7Z5hZnpkt8z0u8SKniDTfcYMyue2nI1m4YS9XP6MxEofqjVU5DO+eSvcgrCDXUNALhJl1B64Cspxzw4BoYKqfQ591zo3yPR4JakgROSRTRnfn5tOH8MaqXfxaCw212M6CUpZtzeeUYV08Ob9X4yBigHZmVgkkApr1SyTCzJzYl737K7j7v+vISInnupMHeR0p7Ly5qvb20slDvSkQQb+CcM5tB24DtgA7gQLn3Jt+Dv2Jma0ws+fNrGdQQ4pIq/jFiYdx3vie3PPuOuZ9vNHrOGHn9ZU5DMhMZkBmsifn9+IWU0dgMtAX6AYkmdm0Boe9AvRxzo0A3gIeb+SzZptZtpll5+XlBTK2iLSAmfH7ycM4aUhnfvfqal5erpsFTbV3fwWLNu3l5KGdPcvgRSP1CcBG51yec64SeBGYUP8A59we51xdR+pHgLH+Psg595BzLss5l5WRkRHQ0CLSMjHRUdx13mjG9enE/3tuGe+uyfU6Ulh4+8tdVNc4Thna1bMMXhSILcCRZpZoZgYcD3xZ/wAzq/8TOaPhfhEJLwmx0Tx8URaHdU7hsieW8OaqHK8jhbw3VubQvUM7hnVv71kGL9ogPgOeB5YCX/gyPGRmt5rZGb7DrvJ1g11ObY+nGcHOKSKtK7VdLE9dciSDu7VnzpNL+feKnV5HClnF5VV8uG43Jw3tTO3/o73hSS8m59zNwM0NNt9Ub/+NwI1BDSUiAZeaGMuCWeOZ+dhirnx6KZXVo5gyurvXsULOe2tyqaiq4RSPei/V0UhqEQmqlIRYHr94PEf0TePa55bx/JJtXkcKOa+vzCEtKY6sPp08zaECISJBlxQfw9wZ4zhqQDrXP7+c5xZv9TpSyCivqua9NXmcOKRzUFeP80cFQkQ80S6utuH6qAHp/PKFFTyzaIvXkULCwg17KS6v4sQh3nVvraMCISKeqevddOxhGdzw4hcsWLjZ60iee2t1Du1io5k4IN3rKCoQIuKthNhoHrxwLD88PJPfvLSSO99e22bnbnLO8fbqXI45LJ2E2Giv46hAiIj36orET8b04Pa3v+Z//vlFm5wF9ovtBeQUlnHiEG97L9XxarI+EZHviI2O4rafjqBragL3vLuO3MJy7j5/NIlxbefX1FurdxFl8MPDg7/2gz+6ghCRkGFmXHfyIP4wZRjvrsnl3AcXklNQ5nWsoHlr9S6y+nSiU1Jw155ujAqEiIScaUf25uGLstiQV8wZ93zE8q35XkcKuK17S/gqp4gTB3vfe6mOCoSIhKTjB3fmxTkTiYuJ4pwHP434mWDrlhYNhe6tdVQgRCRkDeqSwr9+NpGRPTpw1dOf86f/fBmxjddvrd7FwMxk+qQneR3lGyoQIhLS0pLjWXDJEVxwRC8efH8DF81dxJ7i8oO/MYzkl9Su/RBKVw+gAiEiYSAuJoo/njmcv549guzN+zj97shql3h3TS7VNU4FQkSkpc7J6skLl0/AzDjr/k/4n39+QW5R+PdyennZDjq3j2dkjw5eR/kOFQgRCSvDe6Ty6pVHMe2IXjy3eCuT/vYed7z9NfvLq7yO1iLb80t57+s8zsnqSZTHk/M1pAIhImGnY1Icv5s8jLd+cSyTBmVwx9trOe6293gueys1NeE1Tcezvplsz8nq6XGS71OBEJGw1Tc9ifsuGMsLV0ygW4d2/PL5FZx+z0cs3LDH62hNUlVdwz+yt3L0wAx6dkr0Os73qECISNgb27sj/5wzgTunjmLf/gqmPrSQqQ99ypurcqgO4SuK97/OY2dBGeePD72rB9BcTCISIcyMyaO6c/LQLsz/dDPzPtnE7PlL6NUpkZkT+3D+Eb2Ij/F+htT6nl60hfTkeI4PodHT9ekKQkQiSkJsNJce04/3r5/EfReMITMlnt+9spqTb/+At1fvCpmpxHMKyvjvV7n8NKsHsdGh+as4NFOJiByimOgoTh3eleevmMDjF48nJjqKS57I5qK5i1i1o8DzQvFc9lZqHEwdF5q3l0C3mESkDTj2sAwmXH008z/dzO1vf82P7/qI/hlJnDq8Kz8a1pXBXVMwa34X002797NtXyl5xWXkFZVTWlFDn/REBmam0C8j6TuL/tTUOCqqa6iorqG8soZnF2/lqAHp9E4Lnak1GjKvq2hrycrKctnZ2V7HEJEQt29/Bf/+YievfbGThRv2UOOgW2oCRw/M4KiB6Uzon0Zacnyj7y+rrOY/K3fyxKeb+XxLfqPHmUFKfAyV1Y7K6hqq/DSW33P+aE4b0a01vq0WM7Mlzrksv/tUIESkrdpTXM6bq3fxwdd5fLxuN4VltYPtkuKiyUiJJyMlntR2ccREGdHRhgGfrN/D3v0V9E1P4oIjejG8e+o3x8bFRLFpdwlrc4tYu6uYgtJK4mKiiI02YqKiiIuJIt73SE2M47ThXT0fHKcCISJyEFXVNXyxvYDFm/ayq7Cc3KJy8orKKCitorqm9gqgusYxqHMKF/6gNxP7p3v+y701HKhAqA1CRITaRu3RvToyuldHr6OEDPViEhERv1QgRETEL08KhJlda2arzGylmT1tZgkN9seb2bNmts7MPjOzPl7kFBFpy4JeIMysO3AVkOWcGwZEA1MbHDYL2OecGwDcDvwluClFRMSrW0wxQDsziwESgYarkU8GHvc9fx443loyikVERFos6AXCObcduA3YAuwECpxzbzY4rDuw1Xd8FVAApDX8LDObbWbZZpadl5cX2OAiIm2MF7eYOlJ7hdAX6AYkmdm0lnyWc+4h51yWcy4rIyOjNWOKiLR5XtxiOgHY6JzLc85VAi8CExocsx3oCeC7DZUKhMcKICIiEcKLgXJbgCPNLBEoBY4HGg6BfhmYDnwKnA381x1kyPeSJUt2m9lmaotJgW9zc56nA7tb8P3U/6zm7G+4/UCv/T1X7uYd05RtB8pbf5t+5k3bf7Cfuf59Nj/Xwfa3JHfvRs/inAv6A/gd8BWwEpgPxAO3Amf49icA/wDWAYuAfs347Ida8hzIbuH38lBL9jfcfqDXjeRV7mYc05RtB8qrn/mh525KVv3Mvcnd2MOTqTacczcDNzfYfFO9/WXAT1v48a8cwvNDPV9z9jfcfqDX/p4rd/OOacq2g+XVz7x5+w/2M9e/z8YFO7dfETNZ36Eys2zXyIRVoUy5gy9cs4drbgjf7OGau46m2vjWQ14HaCHlDr5wzR6uuSF8s4drbkBXECIi0ghdQYiIiF8qECIi4pcKhIiI+KUCcRBmdrSZPWBmj5jZJ17naQ4zizKzP5rZ3WY23es8TWVmk8zsQ9/PfZLXeZrLzJJ8c4Sd5nWWpjKzwb6f9/NmdoXXeZrKzKaY2cO+5QFO8jpPc5hZPzN71Mye9zpLYyK6QJjZXDPLNbOVDbafYmZrfOtN3HCgz3DOfeicuxx4lW9nmA241shO7ZxXPYBKYFugstbXSrkdUEztgMmg5IZWyw7wK+C5wKT8vlb6e/6l7+/5OcDEQOatl681cr/knLsUuBw4N5B562ul7Bucc7MCm/TQRHQvJjM7htpfNE+42rUnMLNo4GvgRGp/+SwGzqN2XYo/NfiIi51zub73PQfMcs4VhUt232Ofc+5BM3veOXd2mOTe7ZyrMbPOwP855y4IdO5WzD6S2pmHE3zfx6vhkNs5l2tmZwBXAPOdc0+FS27f+/4OPOmcWxro3AHIHpR/my3hyUjqYHHOfeBnNbrxwDrn3AYAM3sGmOyc+xPg95aAmfWidlryoBQHaJ3sZrYNqPC9rA5g3G+01s/cZx+107AERSv9zCcBScAQoNTMXnPO1YR6bt/nvAy8bGb/BgJeIFrp523An4H/BKs4QKv/PQ9ZEV0gGvHNWhM+24AjDvKeWcBjAUvUdM3N/iJwt5kdDXwQyGAH0azcZnYWcDLQAbgnoMkOrlnZnXO/BjCzGfiuhAKarnHN/ZlPAs6itiC/FshgB9Hcv+NXUjtDdKqZDXDOPRDIcAfR3J95GvBHYLSZ3egrJCGlLRaIZvPNHRV2nHMl1Ba3sOKce5Ha4ha2nHPzvM7QHM6594D3PI7RbM65u4C7vM7REs65PdS2nYSsiG6kbsQ3a0349PBtCwfhmj1cc0P4Zlfu4Avn7H61xQKxGBhoZn3NLA6YSu36E+EgXLOHa24I3+zKHXzhnN2/lsxVHi4P4Glq172u6+Y5y7f9VGp7G6wHfu11zkjKHq65wzm7cit7oB4R3c1VRERari3eYhIRkSZQgRAREb9UIERExC8VCBER8UsFQkRE/FKBEBERv1QgJKKZWXGQz9cqa4ZY7ZoYBWa2zMy+MrPbmvCeKWY2pDXOLwIqECLNYmYHnL/MOTehFU/3oXNuFDAaOM3MDrZOwxRqZ5EVaRUqENLmmFl/M3vdzJZY7cp1h/u2n25mn5nZ52b2tm89CszsFjObb2YfA/N9r+ea2XtmtsHMrqr32cW+r5N8+5/3XQE86ZuaGjM71bdtiZndZWYHXDPCOVcKLKN2tlDM7FIzW2xmy83sBTNLNLMJwBnA33xXHf0b+z5FmkoFQtqih4ArnXNjgeuA+3zbPwKOdM6NBp4BflnvPUOAE5xz5/leH07tlOTjgZvNLNbPeUYD1/je2w+YaGYJwIPAj3znzzhYWDPrCAzk2ynbX3TOjXPOjQS+pHaah0+onffneufcKOfc+gN8nyJNoum+pU0xs2RgAvAP33/o4dtFiXoAz5pZVyAO2FjvrS/7/idf59/OuXKg3Mxygc58f3nURc65bb7zLgP6ULsK2QbnXN1nPw3MbiTu0Wa2nNricIdzLse3fZiZ/YHa9TKSgTea+X2KNIkKhLQ1UUC+795+Q3dTu8Tpy74FdG6pt29/g2PL6z2vxv+/paYccyAfOudOM7O+wEIze845twyYB0xxzi33LUw0yc97D/R9ijSJbjFJm+KcKwQ2mtlPoXbJSjMb6dudyrfz908PUIQ1QL96y1Wee7A3+K42/gz8yrcpBdjpu61Vf73uIt++g32fIk2iAiGRLtHMttV7/ILaX6qzfLdvVgGTfcfeQu0tmSXA7kCE8d2mmgO87jtPEVDQhLc+ABzjKyy/BT4DPga+qnfMM8D1vkb2/jT+fYo0iab7FgkyM0t2zhX7ejXdC6x1zt3udS6RhnQFIRJ8l/oarVdRe1vrQW/jiPinKwgREfFLVxAiIuKXCoSIiPilAiEiIn6pQIiIiF8qECIi4pcKhIiI+PX/AWtLmDCh60YdAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"learn.remove_cb(ActivationStats)\n",
"admin_init(learn.model, scales)\n",
"learn.lr_find()"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"WANDB_NAME = f'admin_init_spu_20e'\n",
"GROUP = 'admin'\n",
"NOTES = 'SentencePieceUnigram tokenizer trained on wiki 103, 20e, 32100 vcb_sz, Early ADMIN testing'\n",
"CONFIG = {}\n",
"TAGS = ['admin','spu']"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmorgan\u001b[0m (use `wandb login --relogin` to force relogin)\n"
]
},
{
"data": {
"text/html": [
"\n",
" Tracking run with wandb version 0.10.19<br/>\n",
" Syncing run <strong style=\"color:#cdcd00\">admin_init_spu_20e</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
" Project page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox</a><br/>\n",
" Run page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/3ks1ewq1\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox/runs/3ks1ewq1</a><br/>\n",
" Run data is saved locally in <code>/home/morgan/ml/projects/transformers_sandbox/nbs/exploration/wandb/run-20210221_193133-3ks1ewq1</code><br/><br/>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<h1>Run(3ks1ewq1)</h1><iframe src=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/3ks1ewq1\" style=\"border:none;width:100%;height:400px\"></iframe>"
],
"text/plain": [
"<wandb.sdk.wandb_run.Run at 0x7fc180055d30>"
]
},
"execution_count": 33,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#hide_output\n",
"wandb.init(reinit=True, project=\"transformers_sandbox\", entity=\"fastai_community\", \n",
" name=WANDB_NAME, group=GROUP, notes=NOTES, tags=TAGS, config=CONFIG)"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>perplexity</th>\n",
" <th>bpc</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>5.516002</td>\n",
" <td>5.488830</td>\n",
" <td>0.231003</td>\n",
" <td>241.973953</td>\n",
" <td>7.918708</td>\n",
" <td>01:41</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>4.964672</td>\n",
" <td>4.980829</td>\n",
" <td>0.285725</td>\n",
" <td>145.595062</td>\n",
" <td>7.185818</td>\n",
" <td>01:43</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>4.570927</td>\n",
" <td>4.636693</td>\n",
" <td>0.315706</td>\n",
" <td>103.202446</td>\n",
" <td>6.689333</td>\n",
" <td>01:45</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>4.294974</td>\n",
" <td>4.476145</td>\n",
" <td>0.328804</td>\n",
" <td>87.895203</td>\n",
" <td>6.457713</td>\n",
" <td>01:44</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>4.072177</td>\n",
" <td>4.286242</td>\n",
" <td>0.347858</td>\n",
" <td>72.692741</td>\n",
" <td>6.183739</td>\n",
" <td>01:46</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>3.882438</td>\n",
" <td>4.174093</td>\n",
" <td>0.354857</td>\n",
" <td>64.980888</td>\n",
" <td>6.021944</td>\n",
" <td>01:43</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>3.688582</td>\n",
" <td>4.089933</td>\n",
" <td>0.362383</td>\n",
" <td>59.735886</td>\n",
" <td>5.900526</td>\n",
" <td>01:42</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>3.555619</td>\n",
" <td>4.045366</td>\n",
" <td>0.370683</td>\n",
" <td>57.132111</td>\n",
" <td>5.836230</td>\n",
" <td>01:44</td>\n",
" </tr>\n",
" <tr>\n",
" <td>8</td>\n",
" <td>3.410418</td>\n",
" <td>4.018274</td>\n",
" <td>0.375051</td>\n",
" <td>55.605038</td>\n",
" <td>5.797144</td>\n",
" <td>01:42</td>\n",
" </tr>\n",
" <tr>\n",
" <td>9</td>\n",
" <td>3.273482</td>\n",
" <td>3.989733</td>\n",
" <td>0.379958</td>\n",
" <td>54.040470</td>\n",
" <td>5.755968</td>\n",
" <td>01:46</td>\n",
" </tr>\n",
" <tr>\n",
" <td>10</td>\n",
" <td>3.123741</td>\n",
" <td>3.992993</td>\n",
" <td>0.383907</td>\n",
" <td>54.216911</td>\n",
" <td>5.760671</td>\n",
" <td>01:41</td>\n",
" </tr>\n",
" <tr>\n",
" <td>11</td>\n",
" <td>2.994096</td>\n",
" <td>3.990759</td>\n",
" <td>0.385993</td>\n",
" <td>54.095951</td>\n",
" <td>5.757449</td>\n",
" <td>01:42</td>\n",
" </tr>\n",
" <tr>\n",
" <td>12</td>\n",
" <td>2.877162</td>\n",
" <td>4.005277</td>\n",
" <td>0.387474</td>\n",
" <td>54.887009</td>\n",
" <td>5.778393</td>\n",
" <td>01:42</td>\n",
" </tr>\n",
" <tr>\n",
" <td>13</td>\n",
" <td>2.746851</td>\n",
" <td>4.023293</td>\n",
" <td>0.390281</td>\n",
" <td>55.884834</td>\n",
" <td>5.804385</td>\n",
" <td>01:45</td>\n",
" </tr>\n",
" <tr>\n",
" <td>14</td>\n",
" <td>2.615041</td>\n",
" <td>4.046219</td>\n",
" <td>0.392509</td>\n",
" <td>57.180866</td>\n",
" <td>5.837461</td>\n",
" <td>01:44</td>\n",
" </tr>\n",
" <tr>\n",
" <td>15</td>\n",
" <td>2.524932</td>\n",
" <td>4.072429</td>\n",
" <td>0.393079</td>\n",
" <td>58.699383</td>\n",
" <td>5.875273</td>\n",
" <td>01:45</td>\n",
" </tr>\n",
" <tr>\n",
" <td>16</td>\n",
" <td>2.434323</td>\n",
" <td>4.087971</td>\n",
" <td>0.392887</td>\n",
" <td>59.618816</td>\n",
" <td>5.897696</td>\n",
" <td>01:42</td>\n",
" </tr>\n",
" <tr>\n",
" <td>17</td>\n",
" <td>2.361165</td>\n",
" <td>4.097982</td>\n",
" <td>0.393126</td>\n",
" <td>60.218670</td>\n",
" <td>5.912139</td>\n",
" <td>01:46</td>\n",
" </tr>\n",
" <tr>\n",
" <td>18</td>\n",
" <td>2.310880</td>\n",
" <td>4.103701</td>\n",
" <td>0.393512</td>\n",
" <td>60.564026</td>\n",
" <td>5.920389</td>\n",
" <td>01:43</td>\n",
" </tr>\n",
" <tr>\n",
" <td>19</td>\n",
" <td>2.289599</td>\n",
" <td>4.104567</td>\n",
" <td>0.393593</td>\n",
" <td>60.616467</td>\n",
" <td>5.921638</td>\n",
" <td>01:44</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with valid_loss value: 5.488830089569092.\n",
"Better model found at epoch 1 with valid_loss value: 4.980829238891602.\n",
"Better model found at epoch 2 with valid_loss value: 4.636692523956299.\n",
"Better model found at epoch 3 with valid_loss value: 4.476145267486572.\n",
"Better model found at epoch 4 with valid_loss value: 4.28624153137207.\n",
"Better model found at epoch 5 with valid_loss value: 4.174093246459961.\n",
"Better model found at epoch 6 with valid_loss value: 4.089932918548584.\n",
"Better model found at epoch 7 with valid_loss value: 4.045366287231445.\n",
"Better model found at epoch 8 with valid_loss value: 4.018273830413818.\n",
"Better model found at epoch 9 with valid_loss value: 3.9897332191467285.\n"
]
}
],
"source": [
"learn.remove_cb(ActivationStats)\n",
"admin_init(learn.model, scales)\n",
"learn.fit_one_cycle(20, 1e-3, cbs=[WandbCallback(log_model=False, log_preds=False)])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Train - T5 Tokenizer"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"T5 SentencePiece BPE Pretrained Tokenizer"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"from transformers import T5TokenizerFast\n",
"\n",
"class T5TokenizerTransform(Transform):\n",
" def __init__(self):\n",
" self.tok = T5TokenizerFast.from_pretrained('t5-small')\n",
" self.pad_token_id = self.tok.pad_token_id\n",
" self.eos_token_id = self.tok.eos_token_id\n",
" self.bos_token_id = self.tok.bos_token_id\n",
" self.unk_token_id = self.tok.unk_token_id\n",
" self.vcb_size = self.tok.vocab_size\n",
" \n",
" def __call__(self, o, **kwargs):\n",
" return LMTensorText(self.tok(o)['input_ids'])\n",
" \n",
" def encodes(self, o):\n",
" return LMTensorText(self.tok(o)['input_ids'])\n",
" \n",
" def decodes(self, o):\n",
" return TitledStr(self.tok.decode(o.numpy()))\n",
" \n",
" @property\n",
" def vocab_size(self): return self.vcb_size"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"t5_tok = T5TokenizerTransform()\n",
"tok = t5_tok"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Loading cached processed dataset at /home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91/cache-bd8ef24658a2bf53.arrow\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4aba2cec8c844241ae3221804c011c4a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=2461.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Token indices sequence length is longer than the specified maximum sequence length for this model (585 > 512). Running this sequence through the model will result in indexing errors\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"def encode(examples): return {'token_ids' : t5_tok(examples['text'])}\n",
"\n",
"tok_ds = train_ds.map(encode, batched=False)\n",
"val_tok_ds = valid_ds.map(encode, batched=False)\n",
"all_toks = tok_ds['token_ids'] + val_tok_ds['token_ids']\n",
"df['toks'] = all_toks "
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"#Run setup\n",
"n_epochs = 10\n",
"bs = 12\n",
"sl = 512\n",
"n_layers = 6\n",
"pad_id = tok.pad_token_id\n",
"vocab_size = tok.vocab_size"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>text</th>\n",
" <th>text_</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>It has been estimated that average @-@ sized bacteria contain about 2 million proteins per cell ( e.g. E. coli and Staphylococcus aureus ). Smaller bacteria, such as Mycoplasma or spirochetes contain fewer molecules, namely on the order of 50 @,@ 000 to 1 million. By contrast, eukaryotic cells are larger and thus contain much more protein. For instance, yeast cells were estimated to contain about 50 million proteins and human cells on the order of 1 to 3 billion. Note that bacterial genomes encode about 10 times fewer proteins than humans ( e.g. small bacteria &lt;unk&gt; 1 @,@ 000, E. coli : &lt;unk&gt; 4 @,@ 000, yeast : &lt;unk&gt; 6 @,@ 000, human : &lt;unk&gt; 20 @,@ 000 ).&lt;/s&gt; Even though cadmium and its compounds are toxic in certain forms and concentrations, the British Pharmaceutical Codex from 1907 states that cadmium iodide was used as a medication to treat</td>\n",
" <td>has been estimated that average @-@ sized bacteria contain about 2 million proteins per cell ( e.g. E. coli and Staphylococcus aureus ). Smaller bacteria, such as Mycoplasma or spirochetes contain fewer molecules, namely on the order of 50 @,@ 000 to 1 million. By contrast, eukaryotic cells are larger and thus contain much more protein. For instance, yeast cells were estimated to contain about 50 million proteins and human cells on the order of 1 to 3 billion. Note that bacterial genomes encode about 10 times fewer proteins than humans ( e.g. small bacteria &lt;unk&gt; 1 @,@ 000, E. coli : &lt;unk&gt; 4 @,@ 000, yeast : &lt;unk&gt; 6 @,@ 000, human : &lt;unk&gt; 20 @,@ 000 ).&lt;/s&gt; Even though cadmium and its compounds are toxic in certain forms and concentrations, the British Pharmaceutical Codex from 1907 states that cadmium iodide was used as a medication to treat \"</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>, collided with the back of the Renault, knocking the McLaren's front wing off the car. Suffering handling difficulties, Hamilton returned to the pit @-@ lane for a new nose section, and rejoined in 18th place. Räikkönen took second place when he passed Kubica on lap three ; Heidfeld took fourth when he passed Trulli and Kovalainen in separate manoeuvres. Further down the field, Vettel retired from the race on the first lap after twice colliding with other cars ; Button, Sutil and Coulthard pitted to repair early damage.&lt;/s&gt; = = Wheelchair basketball = =&lt;/s&gt; Jardine then ordered his team to move to bodyline positions immediately after Woodfull's injury. Jardine wrote that Larwood had asked for the field, while Larwood said that it was Jardine's decision. The capacity Saturday afternoon crowd viewed this as hitting a man when he was down. Journalist – cricketer Dick Whitington wrote that Jardine's actions</td>\n",
" <td>collided with the back of the Renault, knocking the McLaren's front wing off the car. Suffering handling difficulties, Hamilton returned to the pit @-@ lane for a new nose section, and rejoined in 18th place. Räikkönen took second place when he passed Kubica on lap three ; Heidfeld took fourth when he passed Trulli and Kovalainen in separate manoeuvres. Further down the field, Vettel retired from the race on the first lap after twice colliding with other cars ; Button, Sutil and Coulthard pitted to repair early damage.&lt;/s&gt; = = Wheelchair basketball = =&lt;/s&gt; Jardine then ordered his team to move to bodyline positions immediately after Woodfull's injury. Jardine wrote that Larwood had asked for the field, while Larwood said that it was Jardine's decision. The capacity Saturday afternoon crowd viewed this as hitting a man when he was down. Journalist – cricketer Dick Whitington wrote that Jardine's actions were</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"df['lens'] = df['toks'].apply(len)\n",
"splits = ColSplitter()(df)\n",
"tfms = [attrgetter(\"text\"), t5_tok]\n",
"dsets = Datasets(df, [tfms], splits=splits, dl_type=LMDataLoader)\n",
"dl_kwargs = [{'lens':df['lens'].values[splits[0]]},\n",
" {'val_lens':df['lens'].values[splits[1]]}]\n",
"dls = dsets.dataloaders(bs=bs, seq_len=sl, dl_kwargs=dl_kwargs, shuffle_train=True)\n",
"dls.show_batch(max_n=2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### LR Find"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"SuggestedLRs(lr_min=0.05248074531555176, lr_steep=0.43651583790779114)"
]
},
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAneElEQVR4nO3deXhU9d3+8fdnspKNkIU97CA7QSLuS9XWHdy3qrhUHn2sWmtttfapS3+2atUqVusuaqt1qbZorXWr4i5hR0TCTlizEUISsn5/f2S0ESeQhJk5mZn7dV1zZeacM3Pu5CK5Odv3mHMOERGRXfm8DiAiIl2TCkJERAJSQYiISEAqCBERCUgFISIiAakgREQkoHivAwRLTk6OGzRokNcxREQiyty5c0udc7mB5kVNQQwaNIjCwkKvY4iIRBQzW9vWPO1iEhGRgFQQIiISkApCREQCUkGIiEhAKggREQlIBSEiIgGpIHZRWdPA3LUVNDVrGHQRiW1Rcx3E3nDOMW/dNp79bB2vLdpIXWMzfbsnc9Z+Azhrvzx6d0/2OqKISNjFfEGsL6/h0qcLWba5itTEOE6f1J99B/Tg7ws28Ie3lzPj3SLy8zIZkpPK4NxUBmal0tDUTHl1PRU19WyvbaDZgcPhHNQ3NlNT30R1fSO19U30zEhmcHYKA7NTGZCdQlZqIlkpiWR0SyDOZ15/+yIibYr5gujdPZne3ZO54MBBTMnvS1pSy4/ktEn9WVtWzV/nrGfu2greW17Ci3OLv/Ven0F6cgI+AzPDgIQ4H6lJcaQmxZMU72PB+gr+uWgju+6xMoPMbglkpyWRlZpIdmoi2WmJ5KQlkZ2WRI+UBBLjfCTE+0jwtXxmVmoiWamJpCXFY6ZyEZHQsmi55WhBQYEL9VAbVTsbWF9eS3KCj6zURDKSE/C1YyugvrGZ9RU1FFfUUuHf8qiorqe8pp7y6nrKdtRTVl1P2Y46Kmoa9vh5ifE+clITyU5LIictkZ7pyQzITmFAVgqD/Fsq3bslBONbFpEoZ2ZznXMFgebF/BZER6QnJzC6b8f/8CbG+xiam8bQ3LQ9LtvQ1OwvkQYampppaGqmvrGZ6vpGyqsbKK+uo6y6ntKqekp31LG1qo7FG7ZTuqPuW5+TlZrIwOyWwhjWM43hPdPYp3c6eT1S2lVqIiIqiC4mIc5Hz4xkemZ07MB4dV0j68prWFtWw7ryalaX1rC2rJpPV5XxyvwN3yyXkhjH2H7dmZiXSX5eJpMG9aBnug7Ci8h3qSCiRGpSPKP6ZDCqT8Z35lXtbKBo6w6KtlSxdON2FhRX8uRHa6hvagZgn17pHDo8h0OG53DAkGySE+LCHV9EuiAdg4hRdY1NfLmpik9XlfFBUQlz1lRQ39hMcoKPQ4blcOTIXhw5sqdO8RWJcrs7BqGCEABq65v4fE05/1m2lbe/3EJxRS0A+XmZHDu2N8eM6c3gnFSPU4pIsKkgpEOcc6zYuoM3l27h319sZlFxJQAje6czNb8fU/L70i+zm8cpRSQYVBCyVzZsq+WNJZt5bdFG5q/bBsDkQVmcu/8AThjfh4Q4jdgiEqlUEBI068pqmLVwAy/P28Cq0mp6ZyQz7aBBnDt5AN1TdO2FSKRRQUjQNTc73i8q4fEPVvPhilJSEuO4+ODBXHroEBWFSARRQUhILd24nQfeW8E/F20iIzme6YcN4cKDB38zbImIdF27KwjtPJa9NrpvBg+cuy+vX3Uokwdncdebyznq7vd4Y8kmouU/ICKxSAUhQTO6bwaPTduPv11+EFmpSVz253lMf2YumyprvY4mIp2ggpCgmzSwB7N+fDA3HDeSD4pKOPru93nq4zU06yZMIhFFBSEhkRDn438OH8pb1xzOvgN7cNOsLzj9oY8p2lLldTQRaScVhIRUXlYKT188mXvOnMCq0mqOn/EB9769nPrGZq+jicgeqCAk5MyMU/ftz9s/PZxjx/bh3reLOOXBj/hqs7YmRLoyFYSETU5aEvefM5GHzpvE5sqdnHT/h/zpvZU06diESJekgpCwO3Zsb9685jCOHNmTO95YxjmPfkp5db3XsURkFyErCDN7wsy2mtmSVtOyzOwtMyvyf+3RxnubzGyB/zErVBnFO9lpSfzpvH25+4wJLFi/jakPfKgD2CJdTCi3IGYCx+4y7XrgHefccOAd/+tAap1z+f7HlBBmFA+ZGadN6s/z0w+gtr6ZUx/8mP98tdXrWCLiF7KCcM7NBsp3mTwVeMr//Cng5FCtXyLHxAEt103kZaVwycw5PPvZOq8jiQjhPwbRyzm3yf98M9CrjeWSzazQzD41s5Pb+jAzm+5frrCkpCTYWSWM+mZ246XLD+TwEbn88pXFPDp7ldeRRGKeZwepXcsgPW2dvjLQP3jUucC9Zja0jc94xDlX4JwryM3NDVVUCZOUxHgePr+AE8b14bbXv+QPby3XWE4iHgr3cJtbzKyPc26TmfUBAu5wds5t8H9dZWbvAROBleGLKV5JjPcx45yJdEuM4753iqiua+TGE0ZhZl5HE4k54d6CmAVM8z+fBvxj1wXMrIeZJfmf5wAHA0vDllA8F+cz7jxtPNMOHMhjH67mlleXaktCxAMh24Iws+eAI4AcMysGbgJuB14ws0uAtcCZ/mULgMuccz8CRgEPm1kzLQV2u3NOBRFjfD7j5iljiI/z8fiHqwG46aTR2pIQCaOQFYRz7pw2Zh0VYNlC4Ef+5x8D40KVSyKHmfGrE0ZhwGMfrsY5x81TxqgkRMJEt/ySLs3MuPGEUfh8xiOzV9Hs4NapKgmRcFBBSJdnZtxw3EjM4OH3V9HsHL+ZOhafTyUhEkoqCIkIZsb1x44kzowH/QP8/faUcSoJkRBSQUjEMDOuO2Yf4nzG/e+uoNk5bj91vEpCJERUEBJRzIyffn8EPjPue6cI5+CO01QSIqGggpCIY2Zc8/0ROGDGO0XEx/m47WQdkxAJNhWERKxrjh5OY1MzD763knif6ewmkSBTQUjE+vqYRGOz45HZq4iPM359oi6mEwkWFYREtK9PgW1scjzxUcsV1yoJkeBQQUjEMzP+78RRADzx0Wqc07AcIsGggpCo8HVJ+KxlWI5m57hFw3KI7BUVhESNr4fliPMZD89ehdOwHCJ7RQUhUcXMuP64kQA8PHsVPVIS+OkP9vE4lUhkUkFI1Pm6JLbVNDDj3RX0SE3kooMHex1LJOKoICQqmRm3nTKWipp6bnl1KVmpiUzN7+d1LJGI4tk9qUVCLT6u5falBwzJ4toXFvLeVwHvcCsibVBBSFRLTojj0QsKGNErnSv+Mo8lGyq9jiQSMVQQEvXSkxN48qL9yExJ5KKZc1hfXuN1JJGIoIKQmNArI5mZF+1HXUMTFz75Odtq6r2OJNLlqSAkZgzvlc4jFxSwvryW6U/Ppa6xyetIIl2aCkJiygFDsrnrzAl8vqacG/62GOec15FEuiyd5ioxZ8qEvqwuqeYPby9nWK80/veIYV5HEumSVBASk646ahgrS3Zw5xtfMSQnjWPH9vY6kkiXo11MEpPMjDtPH09+XibXPL9Ap7+KBKCCkJiVnBDHIxdMokdKAj96qpDNlTu9jiTSpaggJKb1TE/m8Qv3o2pnAxfPnEN1XaPXkUS6DBWExLxRfTJ44If78tWWKq56bj5NzTqzSQRUECIAHLFPT26eMoZ3lm3lN68t9TqOSJegs5hE/M4/YCBrS6t57MPVDMxO0RDhEvNUECKt3HD8KNaV1/Cb15aS1yOFo0f38jqSiGe0i0mklTifce/Z+Yzt150rn5uv018lpoWsIMzsCTPbamZLWk3LMrO3zKzI/7VHG++d5l+myMymhSqjSCApifE8Nq2ArNRELp45h43bar2OJOKJUG5BzASO3WXa9cA7zrnhwDv+199iZlnATcD+wGTgpraKRCRUeqYn88SF+1Fb38TFM+ewQ6e/SgwKWUE452YD5btMngo85X/+FHBygLceA7zlnCt3zlUAb/HdohEJuX16p/PAD/elaOsOrtbprxKDwn0MopdzbpP/+WYg0BHAfsD6Vq+L/dNEwu6wEbncdNJo3lm2lTvfWOZ1HJGw8uwsJuecM7O9+i+ZmU0HpgMMGDAgKLlEdnXBgYMo2rKDh2evYmjPNM4syPM6kkhYhHsLYouZ9QHwfw10F/kNQOvfwP7+ad/hnHvEOVfgnCvIzc0NeliRr/36pNEcMiyHG19ZzGeryryOIxIW4S6IWcDXZyVNA/4RYJl/Az8wsx7+g9M/8E8T8UxCnI8Hzt2XvKwULv/LPN3XWmJCKE9zfQ74BNjHzIrN7BLgduD7ZlYEHO1/jZkVmNljAM65cuA3wBz/41b/NBFPdU9J4LELCmhsaubSpwt1ZpNEPYuWWy4WFBS4wsJCr2NIDJi9vIQLn/yco0f14qHzJuHzmdeRRDrNzOY65woCzdOV1CIddNiIXH51wmjeXLqFP7y93Os4IiGjsZhEOuGigwfx1eYq7n93BcN6pjE1X2diS/TRFoRIJ5gZt548hsmDs7juxUU6s0mikgpCpJOS4uN45PxJ9M/qxvRn5rJi6w6vI4kElQpCZC9kpiQy88LJxPuMi2Z+TumOOq8jiQSNCkJkLw3ITuHxC/ejpKqOS54qpKZep79KdFBBiARBfl4m9509kcXF27j8z/Oob2z2OpLIXlNBiATJMWN6c9sp43h/eQk/e3EhzRr9VSKcTnMVCaJzJg+goqaeO9/4iqzURG46aTRmupBOIpMKQiTILj98KOU76nnsw9VkpyZy5VHDvY4k0ikqCJEgMzN+efwoyqvrufut5QzITtGFdBKRVBAiIeDzGbefNp4N22q57sVF9MvsRsGgLK9jiXSIDlKLhEhivI+HzptEvx4tF9KtK9MQ4RJZVBAiIdQjNZEnLtyPpmbHRTM/p7KmwetIIu2mghAJscE5qTx8/iTWlddwxbPzaGjSNRISGVQQImFwwJBsbjtlHB+uKOXWV5d6HUekXXSQWiRMzizIY+XWHTw8exXDeqYx7aBBXkcS2S0VhEgY/fzYkawsqeaWV79gcE4qh43I9TqSSJu0i0kkjOJ8xn1n57NP7wyu+Ms8irZUeR1JpE0qCJEwS02K5/FpBSQnxnHxU3M0RLh0We0qCDNLNTOf//kIM5tiZgmhjSYSvfpmduOxCwooqapj+tOF7Gxo8jqSyHe0dwtiNpBsZv2AN4HzgZmhCiUSCybkZXLPmfnMW7eN615ahHMa/VW6lvYWhDnnaoBTgQedc2cAY0IXSyQ2HD+uDz8/dh9eXbiRP7xd5HUckW9p71lMZmYHAj8ELvFPiwtNJJHYcvnhQ1lVUs2Md4oY3jONkyb09TqSCND+LYifADcArzjnvjCzIcB/QpZKJIaYGbedMpaCgT342YsLWVS8zetIIkA7C8I5975zbopz7g7/wepS59xVIc4mEjOS4uN46PxJ5KQlcenThWzZvtPrSCLtPovpWTPLMLNUYAmw1MyuC200kdiSk5bEoxcUULWzUWc2SZfQ3l1Mo51z24GTgX8Bg2k5k0lEgmh03wz+cFY+C4sruVb3tRaPtbcgEvzXPZwMzHLONQD6lysSAseM6c0Nx43kn4s2cce/l3kdR2JYe89iehhYAywEZpvZQGB7qEKJxLrphw1hfUUND7+/irweKZx3wECvI0kMaldBOOdmADNaTVprZt8LTSQRMTNuPmkMG7ft5Nf/WELfzGSOHNnL61gSY9p7kLq7md1jZoX+x91AaoizicS0+Dgf958zkdF9M/jxs/NZulEb7RJe7T0G8QRQBZzpf2wHnuzsSs3sajNbYmZfmNlPAsw/wswqzWyB//Hrzq5LJJKlJsXzxLT9yEhO4NKnCzWwn4RVewtiqHPuJufcKv/jFmBIZ1ZoZmOBS4HJwATgRDMbFmDRD5xz+f7HrZ1Zl0g06JmRzKMXFFBWXcdlz8ylrlGnv0p4tLcgas3skK9fmNnBQG0n1zkK+Mw5V+OcawTep2WMJxFpw7j+3bn7jHwK11Zw4ytLNLCfhEV7C+Iy4AEzW2Nma4A/Av/TyXUuAQ41s2wzSwGOB/ICLHegmS00s3+ZWcCBAc1s+tfHRUpKSjoZRyQynDC+D1cfNZyX5hbz+IervY4jMaC9Q20sdM5NAMYD451zE4EjO7NC59yXwB20DBv+BrAA2HWbeR4w0L/O+4G/t/FZjzjnCpxzBbm5unWjRL+rjxrOsWN687t/LWPeugqv40iU69Ad5Zxz2/1XVAP8tLMrdc497pyb5Jw7DKgAlgdYzw7/89dpuVAvp7PrE4kWPp9x5xnj6dM9mauem09lbYPXkSSK7c0tR63TbzTr6f86gJbjD8/uMr+3mZn/+WR/zrLORxWJHhnJCcw4ZyKbKnfyy5cX63iEhMzeFMTe/Kv8m5ktBV4FrnDObTOzy8zsMv/804ElZraQlgv0znb6LRD5xr4DenDtD0bwz8Wb+Ouc9V7HkShlu/u7a2ZVBC4CA7o559o7VEfIFRQUuMLCQq9jiIRNc7Pjgic+p3BtOa/++BCG90r3OpJEIDOb65wrCDRvt1sQzrl051xGgEd6VyoHkVjk8xn3nDWB1MR4rnxuvoYHl6Dbm11MIuKxnunJ3HXGBJZtruL2f2nkVwkuFYRIhPveyJ5cdPAgZn68hneXbfE6jkQRFYRIFLj+uJGM6pPBz15cxFbdrlSCRAUhEgWS4uO4/5x8auobdSc6CRoVhEiUGNYznV+fOIYPikr582drvY4jUUAFIRJFzpmcx+Ejcvnd68tYW1btdRyJcCoIkShiZtx+2jji44zrXlykXU2yV1QQIlGmT/du3HTSGD5fU87Mj9d4HUcimApCJAqdtm8/jhzZkzv/vYzVpdrVJJ2jghCJQmbG704dR2Kcj+t0VpN0kgpCJEr1ykjmppPGULi2Qmc1SaeoIESi2Kn79uOwEbnc8a9lFFfUeB1HIowKQiSKmRm/PWUsDnQva+kwFYRIlOvfI4WfH7MP7y8v4ZX5G7yOIxFEBSESA84/cBCTBvbg1teWUlJV53UciRAqCJEYEOcz7jhtHDV1Tdw0a4nXcSRCqCBEYsSwnulcffRwXl+8mdcXb/I6jkQAFYRIDJl+2BDG9svg1/9YQnl1vddxpItTQYjEkIQ4H78/fQLbahq45dUvvI4jXZwKQiTGjOqTwRXfG8Y/FmzkraW6A520TQUhEoOu+N4wRvZO58ZXFrOtRruaJDAVhEgMSoz3cdcZEyivruf//qFdTRKYCkIkRo3t152rjxrOqws3MmvhRq/jSBekghCJYZcfMZSJAzL51SuL2Vy50+s40sWoIERiWHycj3vOzKehyXHdSws1VpN8iwpCJMYNzknllyeM4oOiUp75VMOCy3+pIESE8/YfwOEjcvnt61/qDnTyDRWEiGBm3HHaeBLjfPzsxYU06Q50ggpCRPx6d0/mlqljmLu2gsc+WOV1HOkCVBAi8o2T8/vxg9G9uPvN5SzfUuV1HPGYCkJEvmFm3HbKOFKT4rj2hYU0NDV7HUk85ElBmNnVZrbEzL4ws58EmG9mNsPMVpjZIjPb14OYIjEpNz2J204Zx+INlfzx3RVexxEPhb0gzGwscCkwGZgAnGhmw3ZZ7DhguP8xHfhTWEOKxLjjx/XhlIn9uP/dIuauLfc6jnjEiy2IUcBnzrka51wj8D5w6i7LTAWedi0+BTLNrE+4g4rEslumjqFvZjeu/usCqnY2eB1HPOBFQSwBDjWzbDNLAY4H8nZZph+wvtXrYv+0bzGz6WZWaGaFJSUlIQssEosykhO47+x8Nm6r5SYN6BeTwl4QzrkvgTuAN4E3gAVAUyc/6xHnXIFzriA3Nzd4IUUEgEkDs7jyyOG8PH8D/1iwwes4EmaeHKR2zj3unJvknDsMqACW77LIBr69VdHfP01EwuzKI4ex74BMfvXKEtaX13gdR8LIq7OYevq/DqDl+MOzuywyC7jAfzbTAUClc053WRfxQHycj/vOngjANc8voFGnvsYMr66D+JuZLQVeBa5wzm0zs8vM7DL//NeBVcAK4FHgfz3KKSJAXlYK/++UsRSureCP/9Gpr7Ei3ouVOucODTDtoVbPHXBFWEOJyG5Nze/H+1+VMOOdIg4elsN+g7K8jiQhpiupRaTdbpk6hv49UvjJXxdQWatTX6OdCkJE2i09OYEZ50xky/ad3PDyIt1gKMqpIESkQ/LzMrnumH14ffFmHtWor1FNBSEiHTb9sCEcP643t/9rGR+tKPU6joSICkJEOszMuPP0CQzNTePHz86juELXR0QjFYSIdEpaUjyPXFBAY7Pjsj/PZWdDpwZEkC5MBSEinTY4J5V7z8pnyYbt/OzFhTTrVqVRRQUhInvlqFG9uP64kby2aBN3vfmV13EkiDy5UE5Eosv/HDaEdeU1PPjeSvKyUjhn8gCvI0kQqCBEZK+ZGbdOGcOGilp+9fcl9M3sxuEjNMJyOLxYuJ7GZheSUtYuJhEJivg4Hw/8cF9G9Ernir/M44uNlV5HiglPfLSGVxduDMlnqyBEJGjSkuJ58sL9yEiO58In52h48BArr67ny03bOWhodkg+XwUhIkHVu3syT108mbqGJqY9+TkV1fVeR4pan60qA+BAFYSIRIrhvdJ5bNp+FFfUcslTc6it1zUSofDxyjJSEuMY3z8zJJ+vghCRkJg8OIsZZ+czf/02rnxuHg260VDQfbyylMmDs0iIC82fchWEiITMsWP7cOuUMbz95VZ+8dIiXUgXRFu372RlSTUHDgnN7iXQaa4iEmLnHziIbTUN3P3WcjK6JXDTSaMxM69jRbxP/McfDhqaE7J1qCBEJOR+fOQwttU28PiHq8lMSeAnR4/wOlLE+2RlGRnJ8YzumxGydaggRCTkzIwbjx9FZW0D975dREpiHNMPG+p1rIj28coy9h+STZwvdFtjKggRCQufz7j91HHUNjTx29eXUd/YzI+PHO51rIhUXFHDuvIaLjp4UEjXo4IQkbCJj/Nx31n5JMb5uOvN5dQ3Oa45eriOSXTQJytDe/3D11QQIhJW8XE+7jpjAglxxox3iqhvbOYXx+6jkuiAT1aWkZ2ayIie6SFdjwpCRMIuzmfcfup4EuN9PPT+SmrqG7n5pDH4Qrg/PVo45/hkVRkHDMkO+c9LBSEinvD5jN9MHUtqYjwPz15F1c5Gfn/6eOJDdNFXtFhTVsOmyp0h370EKggR8ZCZcf1xI8nolsDv//0VO+oauf+ciSQnxHkdrcv6NMTjL7WmqhYRT5kZV3xvGLdOHcNbS7dw0ZNzqKxt8DpWlzV/XQU9UhIYkpMa8nWpIESkS7jgwEHce1Y+hWvLOfOhT9i4rdbrSF3SgvXbmJCXGZaD+ioIEekyTp7Yj5kXTWbjtlpOffBjlm3e7nWkLqVqZwNFW3cwMa9HWNanghCRLuXgYTm8cNmBAJzxp09476utHifqOhYVV+Ic5A/IDMv6VBAi0uWM6pPBK1ccRP+sFC6aOYd7316ukWBp2b0EkB+i+z/sSgUhIl1Sn+7dePnygzhlYj/ufbuIi2bOifm7081ft40hual0T0kIy/o8KQgzu8bMvjCzJWb2nJkl7zL/QjMrMbMF/sePvMgpIt7qlhjH3WdM4LenjOOTlWWceP+HzF5e4nUsTzjnWLC+gvy8zLCtM+wFYWb9gKuAAufcWCAOODvAos875/L9j8fCGlJEugwz49z9B/DiZQeSFO/jgic+56cvLIi5rYniilpKd9QzMZoLwi8e6GZm8UAKsNGjHCISISbkZfL61Ydy5ZHDmLVgI0fd8z5/m1scM8cmvj7+MHFAeM5gAg8Kwjm3AbgLWAdsAiqdc28GWPQ0M1tkZi+ZWV5YQ4pIl5ScEMe1P9iH1646hAFZKVz74kJOvP9DPiwq9TpayM1ft42keB/79A7tAH2tebGLqQcwFRgM9AVSzey8XRZ7FRjknBsPvAU81cZnTTezQjMrLCmJzf2SIrFoZO8MXr78IO47O5/tOxs47/HPOP/xz1i6MXqvm1iwvoJx/bqTEMaxqrzYxXQ0sNo5V+KcawBeBg5qvYBzrsw5V+d/+RgwKdAHOececc4VOOcKcnNzQxpaRLoWn8+Ymt+Pd649nF+dMIpFxZWccP8H/PT5BRRX1HgdL6jqG5tZsnE7E8N0/cPXvBisbx1wgJmlALXAUUBh6wXMrI9zbpP/5RTgy/BGFJFIkRQfx48OHcIZk/J48P0VPPnRGl5btIlz9x/ACeP7MDEvM+JHiP1y03bqG5vJD9MV1F8Le0E45z4zs5eAeUAjMB94xMxuBQqdc7OAq8xsin9+OXBhuHOKSGTpnpLADceNYtqBg7jnreU88+laZn68hvSkeA4als1Ro3px7NjeZCSH5xqCYPrmArkwb0GYc9FxBkBBQYErLCzc84IiEhMqaxv4eEUps4tKmL28lA3bakmM93H0qJ5Mze/HAUOy6d4tMsrimucX8NGKUj775VFBH6TPzOY65woCzdP9IEQkKnXvlsBx4/pw3Lg+OOdYWFzJ3+dv4NWFG3l98WYA+mV2Y1SfDEb1SWdIbiqDslMZkpMWtiuV22v+upYL5MJ9W1YVhIhEPTMjPy+T/LxMbjxhFJ+tKmfRhm18uamKLzdt591lW2h9OUVivI+keB9J8XEkxfvITU+iX49u9M/sRr8e3ejX6mt6CHdZNTc7/vifFawpq+GH+w8M2XraooIQkZiSEOfjkOE5HDI855tpdY1NrC+vZXVpNatLd1C2o566xmbqGpvZ2dDElu07+WJDJW99sYX6puZvfd7A7BROGt+XKfl9GdGr5RqFytoGlm7czqbKWgZmpzK8V1qHj31U7Wzg2hcW8ubSLZyc35fzDwx/QegYhIhIOzU3O0p31FG8rZYNFbUUV9Ty8cpSPlpRSrODYT3T2NnQRHHFd2921DsjmSG5qQzISmFAdgoDslIY0SudITmp3zrLqrqukcUbKrnxlcWsKavhxuNHcdHBg0K2e2l3xyBUECIie2lr1U5eX7SJt7/cSveUBMb0zWB0nwz6ZXZjbVkNRVt3ULSlitVl1awvr6F0x3/HkUqM9zG8Zxp9uiezYusO1pbX4BxkpSbywLn7hvze0yoIEZEuZEddI2vLqlm+pYplm6pYumk7W7bvZGhumv+geQb7DepBZkpiyLPoLCYRkS4kLSmeMX27M6Zvd5jodZq2RfblhSIiEjIqCBERCUgFISIiAakgREQkIBWEiIgEpIIQEZGAVBAiIhKQCkJERAKKmiupzawEWAt0Byr9k1s/b/060DI5QEfvfL7r57d3mT1N29Pz1tNCkbut+cHK3ZnMu8u1p/lt/TsI9Fq595xrT/M7kztYv5O7y7W7+e2ZFsrcXv4tyXTOBb5ns3Muqh7AI4Get34daBla7mbX6XV1ZJk9TdvT812mBT13W/ODlbszmYOZe3evldub3MH6nexs7vZMC2XurvC3JNAjGncxvdrG89avd7dMZ9fVkWX2NG1Pz/cmc3ve39b8aMm9u9fK3fb62ju/M7mD9TvZnvd35ndy19cx8bckanYx7S0zK3RtDFjVlUVi7kjMDModbsrtvWjcguisR7wO0EmRmDsSM4Nyh5tye0xbECIiEpC2IEREJCAVhIiIBKSCEBGRgFQQe2Bmh5rZQ2b2mJl97HWe9jIzn5ndZmb3m9k0r/O0l5kdYWYf+H/mR3idpyPMLNXMCs3sRK+ztJeZjfL/rF8ys8u9ztNeZnaymT1qZs+b2Q+8ztMeZjbEzB43s5e8ztJeUV0QZvaEmW01syW7TD/WzL4ysxVmdv3uPsM594Fz7jLgNeCpUOZtlW+vcwNTgf5AA1AcqqytBSm3A3YAyURWboBfAC+EJuV3Benf95f+f99nAgeHMm+rfMHI/Xfn3KXAZcBZoczrzxaMzKucc5eENmlwRfVZTGZ2GC1/bJ52zo31T4sDlgPfp+UP0BzgHCAO+N0uH3Gxc26r/30vAJc456oiIbf/UeGce9jMXnLOnR4huUudc81m1gu4xzn3wwjJPQHIpqXYSp1zr0VCbufcVjObAlwOPOOcezZScvvfdzfwF+fcvAjKHJbfx2CI9zpAKDnnZpvZoF0mTwZWOOdWAZjZX4GpzrnfAQF3DZjZAKAyHOUAwcltZsVAvf9lUwjjfiNYP2+/CiApJEF3EaSf9xFAKjAaqDWz151zzV09t/9zZgGzzOyfQMgLIkg/bwNuB/4V6nKAoP/bjhhRXRBt6Aesb/W6GNh/D++5BHgyZInap6O5XwbuN7NDgdmhDLYHHcptZqcCxwCZwB9Dmmz3OpTbOXcjgJldiH8rKKTp2tbRn/cRwKm0lPHroQy2Bx39930lcDTQ3cyGOeceCmW4NnT0Z50N3AZMNLMb/EXSpcViQXSYc+4mrzN0lHOuhpZiiyjOuZdpKbeI5Jyb6XWGjnDOvQe853GMDnPOzQBmeJ2jI5xzZbQcM4kYUX2Qug0bgLxWr/v7p3V1yh1eyh1ekZg7EjN3SCwWxBxguJkNNrNE4GxglseZ2kO5w0u5wysSc0di5o7p6LjlkfQAngM28d9TPS/xTz+elrMPVgI3ep1TuZVbubt27kjMHIxHVJ/mKiIinReLu5hERKQdVBAiIhKQCkJERAJSQYiISEAqCBERCUgFISIiAakgJKqZ2Y4wry8o9wyxlvtiVJrZAjNbZmZ3teM9J5vZ6GCsXwRUECIdYma7Hb/MOXdQEFf3gXMuH5gInGhme7pfw8m0jCYrEhQqCIk5ZjbUzN4ws7nWcve6kf7pJ5nZZ2Y238ze9t+TAjO72cyeMbOPgGf8r58ws/fMbJWZXdXqs3f4vx7hn/+SfwvgL/4hqjGz4/3T5prZDDPb7b0jnHO1wAJaRg/FzC41szlmttDM/mZmKWZ2EDAF+L1/q2NoW9+nSHupICQWPQJc6ZybBPwMeNA//UPgAOfcROCvwM9bvWc0cLRz7hz/65G0DEs+GbjJzBICrGci8BP/e4cAB5tZMvAwcJx//bl7CmtmPYDh/HfY9pedc/s55yYAX9Iy7MPHtIwDdJ1zLt85t3I336dIu2i4b4kpZpYGHAS86P8PPfz3xkT9gefNrA+QCKxu9dZZ/v/Jf+2fzrk6oM7MtgK9+O4tUj93zhX717sAGETLXclWOee+/uzngOltxD3UzBbSUg73Ouc2+6ePNbP/R8s9M9KAf3fw+xRpFxWExBofsM2/b39X99Nym9NZ/hvp3NxqXvUuy9a1et5E4N+l9iyzOx845040s8HAp2b2gnNuATATONk5t9B/g6IjArx3d9+nSLtoF5PEFOfcdmC1mZ0BLbeuNLMJ/tnd+e94/tNCFOErYEir21eetac3+Lc2bgd+4Z+UDmzy79Zqfc/uKv+8PX2fIu2igpBol2Jmxa0eP6Xlj+ol/t03XwBT/cveTMsumblAaSjC+HdT/S/whn89VUBlO976EHCYv1j+D/gM+AhY1mqZvwLX+Q+yD6Xt71OkXTTct0iYmVmac26H/6ymB4Ai59wfvM4lsittQYiE36X+g9Zf0LJb62Fv44gEpi0IEREJSFsQIiISkApCREQCUkGIiEhAKggREQlIBSEiIgGpIEREJKD/D1E91U6LYjx2AAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"learn.remove_cb(ActivationStats)\n",
"admin_init(learn.model, scales)\n",
"learn.lr_find()"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"WANDB_NAME = f'admin_init_t5_20e'\n",
"GROUP = 'admin'\n",
"NOTES = 'T5Tokenizer tokenizer, 20e, 32100 vcb_sz, Early ADMIN testing'\n",
"CONFIG = {}\n",
"TAGS = ['admin','t5']"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmorgan\u001b[0m (use `wandb login --relogin` to force relogin)\n"
]
},
{
"data": {
"text/html": [
"\n",
" Tracking run with wandb version 0.10.19<br/>\n",
" Syncing run <strong style=\"color:#cdcd00\">admin_init_t5_20e</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
" Project page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox</a><br/>\n",
" Run page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/2k8d2xge\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox/runs/2k8d2xge</a><br/>\n",
" Run data is saved locally in <code>/home/morgan/ml/projects/transformers_sandbox/nbs/exploration/wandb/run-20210221_220517-2k8d2xge</code><br/><br/>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<h1>Run(2k8d2xge)</h1><iframe src=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/2k8d2xge\" style=\"border:none;width:100%;height:400px\"></iframe>"
],
"text/plain": [
"<wandb.sdk.wandb_run.Run at 0x7f58280433a0>"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#hide_output\n",
"wandb.init(reinit=True, project=\"transformers_sandbox\", entity=\"fastai_community\", \n",
" name=WANDB_NAME, group=GROUP, notes=NOTES, tags=TAGS, config=CONFIG)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" <div>\n",
" <style>\n",
" /* Turns off some styling */\n",
" progress {\n",
" /* gets rid of default border in Firefox and Opera. */\n",
" border: none;\n",
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
" background-size: auto;\n",
" }\n",
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
" background: #F44336;\n",
" }\n",
" </style>\n",
" <progress value='0' class='' max='20' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
" 0.00% [0/20 00:00<00:00]\n",
" </div>\n",
" \n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>perplexity</th>\n",
" <th>bpc</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" </tbody>\n",
"</table><p>\n",
"\n",
" <div>\n",
" <style>\n",
" /* Turns off some styling */\n",
" progress {\n",
" /* gets rid of default border in Firefox and Opera. */\n",
" border: none;\n",
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
" background-size: auto;\n",
" }\n",
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
" background: #F44336;\n",
" }\n",
" </style>\n",
" <progress value='8' class='' max='466' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
" 1.72% [8/466 00:01<01:39 9.5600]\n",
" </div>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"learn.remove_cb(ActivationStats)\n",
"admin_init(learn.model, scales)\n",
"learn.fit_one_cycle(20, 1e-3, cbs=[WandbCallback(log_model=False, log_preds=False)])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Converted 00_core.ipynb.\n",
"Converted 01_layers.ipynb.\n",
"Converted 02_attention.ipynb.\n",
"Converted 03_models.transformer.ipynb.\n",
"Converted 04a_models.reformer.ipynb.\n",
"Converted 04x_models.xtransformer.ipynb.\n",
"Converted 05_tokenizers.ipynb.\n",
"Converted 06_data.ipynb.\n",
"Converted 07_metrics.ipynb.\n",
"Converted 08_optimizers.ipynb.\n",
"Converted 09_tracking.ipynb.\n",
"Converted 10_config.ipynb.\n",
"Converted index.ipynb.\n"
]
}
],
"source": [
"#hide\n",
"from nbdev.export import notebook2script; notebook2script()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.1"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"#default_exp xtransformer"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"#hide\n",
"from nbdev.showdoc import *"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"#hide\n",
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"#!pip install datasets"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"#export\n",
"from fastai.basics import *\n",
"from fastai.text.all import *\n",
"from fastai.callback import *\n",
"from fastai.callback.wandb import *\n",
"\n",
"from transformers_sandbox.core import *\n",
"from transformers_sandbox.layers import *\n",
"from transformers_sandbox.attention import *\n",
"from transformers_sandbox.tokenizers import ByteTextTokenizer, SPUniTokenizer\n",
"from transformers_sandbox.metrics import bpc\n",
"from transformers_sandbox.transformer import LMMixin, EncDecMixin, TransformerLM\n",
"from transformers_sandbox.config import ConfigBase, update_sig\n",
"from transformers_sandbox.xtransformer import *\n",
"\n",
"import wandb\n",
"from datasets import load_dataset"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# ADMIN Init\n",
"\n",
"> ADMIN Initialisation"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"📕 **Paper**: https://arxiv.org/abs/2004.08249\n",
"\n",
"Transformers have proved effective in many NLP tasks. However, their training requires non-trivial efforts regarding designing cutting-edge optimizers and learning rate schedulers carefully (e.g., conventional SGD fails to train Transformers effectively). Our objective here is to understand what complicates Transformer training from both empirical and theoretical perspectives. \n",
"\n",
"Our analysis reveals that unbalanced gradients are not the root cause of the instability of training. Instead, we identify an amplification effect that influences training substantially -- for each layer in a multi-layer Transformer model, heavy dependency on its residual branch makes training unstable, since it amplifies small parameter perturbations (e.g., parameter updates) and results in significant disturbances in the model output. \n",
"\n",
"Yet we observe that a light dependency limits the model potential and leads to inferior trained models. Inspired by our analysis, we propose Admin (**Ad**aptive **m**odel **in**itialization) to stabilize stabilize the early stage's training and unleash its full potential in the late stage. Extensive experiments show that Admin is more stable, converges faster, and leads to better performance. Implementations are released at: [this https URL](https://github.com/LiyuanLucasLiu/Transforemr-Clinic)."
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"#export\n",
"class AdminResidual(Module):\n",
" def __init__(self, sublayer, d_model):\n",
" self.sublayer = sublayer\n",
" self.w = torch.nn.Parameter(torch.ones(d_model))\n",
" def forward(self, x, *args, **kwargs):\n",
" return x*self.w + self.sublayer(x, *args, **kwargs)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"#export\n",
"class TransformerEncoderBlockAdmin(Module):\n",
" \"\"\"\n",
" Bacis transformer encoder block. Consists of multi-head attention and positional \n",
" feedforward layers\n",
" \"\"\"\n",
" def __init__(self,\n",
" d_model:int, \n",
" n_heads:int = 8, \n",
" d_ff:int = None, \n",
" attn_dropout:float = 0.1,\n",
" ff_dropout:float = 0.1,\n",
" causal:bool = False, \n",
" attn_bias:bool = False, \n",
" prenorm:bool=False,\n",
" shared_qk:bool=False):\n",
" store_attr('attn_dropout') # mb separate argument attn_post_dropout\n",
" \n",
" self.attn = PostNorm(d_model, AdminResidual(Attention(d_model, n_heads=n_heads, causal=causal, dropout=attn_dropout, bias=attn_bias, shared_qk=shared_qk), d_model))\n",
" self.ff = PostNorm(d_model, AdminResidual(FeedForward(d_model, d_ff=d_ff, dropout=ff_dropout), d_model))\n",
" \n",
" def forward(self, x, mask=None):\n",
" out = self.attn(x, mask=mask)\n",
" return self.ff(out)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"# bs = 4\n",
"# sl = 128\n",
"# d = 64\n",
"# x = torch.randn(bs, sl, d)\n",
"# m = TransformerEncoderBlockAdmin(d)\n",
"# out = m(x)\n",
"# assert (out.size() == (bs, sl, d))\n",
"# out.shape"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"#export\n",
"class TransformerEncoderAdmin(Module):\n",
" \"\"\"Stack of TransformerEncoderBlocks\"\"\"\n",
" def __init__(self, \n",
" d_model, \n",
" n_layers=6, \n",
" n_heads=8, \n",
" d_ff=None,\n",
" ff_dropout=0.1, \n",
" attn_dropout=0.1,\n",
" attn_bias=False,\n",
" causal=False, \n",
" prenorm=False,\n",
" shared_qk:bool=False,\n",
" final_norm=None):\n",
" store_attr('d_model')\n",
" self.layers = nn.ModuleList([]) \n",
" for _ in range(n_layers):\n",
" self.layers.append(TransformerEncoderBlockAdmin(d_model, n_heads, causal=causal, \n",
" d_ff=d_ff, attn_dropout=attn_dropout, ff_dropout=ff_dropout, \n",
" prenorm=prenorm, attn_bias=attn_bias, shared_qk=shared_qk))\n",
" self.norm = None if final_norm is None else final_norm(d_model)\n",
" \n",
" def forward(self, x, mask=None):\n",
" for layer in self.layers: x = layer(x, mask=mask)\n",
" if self.norm is not None: x = self.norm(x)\n",
" return x"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"#export\n",
"class TransformerLMAdmin(Module, LMMixin):\n",
" \"\"\"\n",
" tmp\n",
" Basic Transformer for language modelling\n",
" \n",
" Parameters:\n",
" * vocab_sz: int\n",
" * d_model: int - inner dimension of the model\n",
" * n_layers: int (default: 6) \n",
" * n_heads: int (default: 8)\n",
" * d_ff: int - inner dimension of the pointwise FeedForward net, if None defaults to 4*d_model\n",
" * attn_dropout: float - attention dropout\n",
" * ff_dropout: float - feed-forward dropout\n",
" * emb_dropout: float - embedding dropout\n",
" * causal: bool (default: True) - if True does causal masking automatically\n",
" * max_seq_len: int (default: 512)\n",
" * tie_weights: bool - if True target embedding weights are used for computation output projection\n",
" * prenorm: bool - wether to use PreNorm or PostNorm\n",
" * attn_bias: bool - wether to allow biases in attention projection layers\n",
" * pad_idx: int - padding token id, required for autogeneration of padding mask\n",
" * pos_enc: str from {'absolute', 'fixed', 'axial'} - type of positional encoding to use\n",
" * axial_shape: tuple - [optional] should be factors of max_seq_len\n",
" * axial_emb_dims: tuple - [optional] axial embedding components, should sum to d_model\n",
" Inputs:\n",
" * x - input ids, shape [bs, sl]\n",
" * mask - optional boolean mask, shape [bs, sl]\n",
" Returns:\n",
" * logits - target token logits, shape [bs, sl, vocab_sz]\n",
" \"\"\"\n",
" def __init__(self, \n",
" vocab_sz:int, \n",
" d_model:int, \n",
" n_layers:int=6,\n",
" n_heads:int=8,\n",
" d_ff:int=None,\n",
" attn_dropout:float=0.1,\n",
" ff_dropout:float=0.1,\n",
" emb_dropout:float=0.1,\n",
" tie_weights:bool=True,\n",
" causal:bool=True,\n",
" pos_enc:str='absolute',\n",
" max_seq_len:int=512,\n",
" axial_shape:tuple=None,\n",
" axial_emb_dims:tuple=None,\n",
" pad_idx:int=None,\n",
" prenorm:bool=False,\n",
" attn_bias:bool=False,\n",
" shared_qk:bool=False):\n",
" store_attr()\n",
" self.emb = TransformerEmbedding(vocab_sz, d_model, max_seq_len, dropout=emb_dropout, \n",
" pos_enc=pos_enc, axial_shape=axial_shape, \n",
" axial_emb_dims=axial_emb_dims)\n",
" final_norm = None\n",
" self.encoder = TransformerEncoderAdmin(d_model, n_layers, n_heads, causal=causal, d_ff=d_ff,\n",
" attn_dropout=attn_dropout, ff_dropout=ff_dropout,\n",
" prenorm=prenorm, attn_bias=attn_bias,\n",
" shared_qk=shared_qk, final_norm=final_norm)\n",
" self.proj = nn.Linear(d_model, vocab_sz)\n",
" if tie_weights: self.proj.weight = self.emb.emb.weight\n",
" \n",
" def forward(self, x, mask=None):\n",
" x = self.emb(x)\n",
" x = self.encoder(x, mask=mask)\n",
" return self.proj(x)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"@patch(cls_method=True)\n",
"def from_config(cls:TransformerLMAdmin, config):\n",
" return cls(**config.dict())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Data"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Reusing dataset wikitext (/home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91)\n",
"Loading cached processed dataset at /home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91/cache-0b8034a134b9567b.arrow\n",
"Reusing dataset wikitext (/home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91)\n",
"Loading cached processed dataset at /home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91/cache-d23112255b6f8a24.arrow\n"
]
}
],
"source": [
"train_ds = load_dataset('wikitext', name='wikitext-2-raw-v1', split='train')\n",
"\n",
"train_ds = train_ds.filter(lambda x: x['text'] != '')\n",
"train_df = train_ds.data.to_pandas()\n",
"\n",
"valid_ds = load_dataset('wikitext', name='wikitext-2-raw-v1', split='validation')\n",
"\n",
"valid_ds = valid_ds.filter(lambda x: x['text'] != '')\n",
"valid_df = valid_ds.data.to_pandas()\n",
"\n",
"df = pd.concat([train_df, valid_df])\n",
"df['is_valid'] = [False]*len(train_df) + [True]*len(valid_df)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Tokenizer + DataLoaders"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"ByteTextTokenizer"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"tok = ByteTextTokenizer(is_lm=True, add_bos=False, add_eos=False)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"#Run setup\n",
"n_epochs = 10\n",
"bs = 16\n",
"sl = 512\n",
"n_layers = 6\n",
"pad_id = tok.pad_token_id\n",
"vocab_size = tok.vocab_size"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"df['toks'] = df['text'].apply(tok)\n",
"df['lens'] = df['toks'].apply(len)\n",
"splits = ColSplitter()(df)\n",
"tfms = [attrgetter(\"text\"), tok]\n",
"dsets = Datasets(df, [tfms], splits=splits, dl_type=LMDataLoader)\n",
"dl_kwargs = [{'lens':df['lens'].values[splits[0]]},\n",
" {'val_lens':df['lens'].values[splits[1]]}]\n",
"dls = dsets.dataloaders(bs=bs, seq_len=sl, dl_kwargs=dl_kwargs, shuffle_train=True, n_workers=2)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>text</th>\n",
" <th>text_</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>The film is set in 2034 , two years after the events of 2nd GIG . Togusa is now the team leader for Public Security Section 9 , which has increased considerably in size . Section 9 deals with a series of complicated incidents , including the assassination of Ka Rum , a former dictator of the Siak Republic , which leads to a terrorist plot using children as vectors for a cybernetic virus . Investigations reveal that a hacker nicknamed the \" The Puppeteer \" is behind the entire series of events . \\n Mozambiqu</td>\n",
" <td>The film is set in 2034 , two years after the events of 2nd GIG . Togusa is now the team leader for Public Security Section 9 , which has increased considerably in size . Section 9 deals with a series of complicated incidents , including the assassination of Ka Rum , a former dictator of the Siak Republic , which leads to a terrorist plot using children as vectors for a cybernetic virus . Investigations reveal that a hacker nicknamed the \" The Puppeteer \" is behind the entire series of events . \\n Mozambique</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>združenje ) its association with Yugoslavia on 25 June 1991 . The European Economic Community and the Conference on Security and Cooperation in Europe urged Croatian authorities to place a three @-@ month moratorium on the decision . Croatia agreed to freeze its independence declaration for three months , initially easing tensions . Nonetheless , the Croatian War of Independence escalated further . On 7 October , the eve of expiration of the moratorium , the Yugoslav Air Force attacked Banski dvori , the m</td>\n",
" <td>druženje ) its association with Yugoslavia on 25 June 1991 . The European Economic Community and the Conference on Security and Cooperation in Europe urged Croatian authorities to place a three @-@ month moratorium on the decision . Croatia agreed to freeze its independence declaration for three months , initially easing tensions . Nonetheless , the Croatian War of Independence escalated further . On 7 October , the eve of expiration of the moratorium , the Yugoslav Air Force attacked Banski dvori , the ma</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"(16, None)"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#collapse_output\n",
"dls.bs, dls.show_batch(max_n=2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Setup"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"#config = XConfig(vocab_sz=vocab_size, d_model=512, n_layers=n_layers, max_seq_len=512, pad_idx=pad_id)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"class CharLMConfig(ConfigBase):\n",
" _model = TransformerLM\n",
" _d = {\n",
" 'vocab_sz':256,\n",
" 'd_model':512,\n",
" 'n_layers':6,\n",
" 'n_heads':8,\n",
" 'd_ff':4096,\n",
" 'attn_dropout':0.1,\n",
" 'ff_dropout':0.1,\n",
" 'emb_dropout':0.1,\n",
" 'tie_weights':True,\n",
" 'causal':True,\n",
" 'pos_enc':'absolute',\n",
" 'max_seq_len':512,\n",
" 'axial_shape':None,\n",
" 'axial_emb_dims':None,\n",
" 'pad_idx':None,\n",
" 'prenorm':False,\n",
" 'attn_bias':False,\n",
" 'shared_qk':False,\n",
" }\n",
" @update_sig(_d)\n",
" def __init__(self, **kwargs):\n",
" super().__init__(**kwargs)\n",
"\n",
"config = CharLMConfig(vocab_sz=vocab_size, d_model=512, n_layers=n_layers, \n",
" max_seq_len=512, pad_idx=pad_id)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Learner"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"learn = Learner(dls, TransformerLMAdmin.from_config(config),\n",
" loss_func=CrossEntropyLossFlat(ignore_index=pad_id), #opt_func=Adam,\n",
" cbs = [\n",
" #GradientClip(1.0),\n",
" SaveModelCallback(with_opt=True)],\n",
" metrics=[accuracy, perplexity, bpc]).to_fp16()"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"class BreakFitCallback(Callback):\n",
" order=-1\n",
" \"Cancels fit after one batch before weight update\"\n",
" def before_step(self):\n",
" self.model.zero_grad(set_to_none=True)\n",
" raise CancelStepException\n",
" def after_step(self):\n",
" raise CancelBatchException\n",
" def after_batch(self):\n",
" print('Fit canceled')\n",
" raise CancelFitException"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"def res_submodules(model):\n",
" return [m.sublayer for m in learn.model.modules() if isinstance(m, AdminResidual)]\n",
"\n",
"def res_modules(model):\n",
" return [m for m in learn.model.modules() if isinstance(m, AdminResidual)]"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"learn.add_cb(ActivationStats(modules=res_submodules(learn.model)));"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"12"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(learn.activation_stats.modules)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"train_eval -10\n",
"recorder 50\n",
"progress 60\n",
"save_model 60\n",
"mixed_precision 10\n",
"activation_stats -20\n"
]
}
],
"source": [
"for cb in learn.cbs:\n",
" print(cb.name, cb.order)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>perplexity</th>\n",
" <th>bpc</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Fit canceled\n"
]
}
],
"source": [
"with learn.added_cbs(BreakFitCallback()), learn.removed_cbs(SaveModelCallback):\n",
" learn.fit(1, 1e-3)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(#12) [{'mean': 0.00481663690879941, 'std': 0.3442692756652832, 'near_zero': 0.6015446980794271},{'mean': -0.0025428251828998327, 'std': 0.3987179100513458, 'near_zero': 0.5950142542521158},{'mean': 0.004401894751936197, 'std': 0.19933182001113892, 'near_zero': 0.6423584620157877},{'mean': 0.003396979533135891, 'std': 0.4013838768005371, 'near_zero': 0.5894660949707031},{'mean': 0.009360421448946, 'std': 0.26095426082611084, 'near_zero': 0.6073331832885742},{'mean': -0.0018820768455043435, 'std': 0.3994253873825073, 'near_zero': 0.5938955942789713},{'mean': -0.02019776962697506, 'std': 0.3356610834598541, 'near_zero': 0.6348047256469727},{'mean': 0.013347254134714603, 'std': 0.40092015266418457, 'near_zero': 0.5801426569620768},{'mean': -0.024423303082585335, 'std': 0.3910963237285614, 'near_zero': 0.621729850769043},{'mean': 0.005235261749476194, 'std': 0.3965536057949066, 'near_zero': 0.587321917215983}...]"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"learn.activation_stats.stats[0]"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([0.11852133, 0.15897597, 0.03973317, 0.16110902, 0.06809713,\n",
" 0.15954064, 0.11266836, 0.16073697, 0.15295633, 0.15725476,\n",
" 0.24887388, 0.15945325])"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def variances(learn):\n",
" return np.array([stat['std']**2 for stat in learn.activation_stats.stats[0]])\n",
"\n",
"variances(learn)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([0.34426928, 0.52678013, 0.56323217, 0.69162092, 0.73921352,\n",
" 0.84022453, 0.90479038, 0.98963761, 1.06411415, 1.13560279,\n",
" 1.24034978, 1.30304291])"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def _init_scales(vars):\n",
" return np.sqrt(np.cumsum(vars))\n",
"scales = _init_scales(variances(learn))\n",
"scales"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Define ADMIN Initialisation"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"def admin_init(model, scales):\n",
" ms = res_modules(model)\n",
" for m, s in zip(ms, scales):\n",
" m.w.data *= s"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## LR Find"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"SuggestedLRs(lr_min=0.03019951581954956, lr_steep=0.25118863582611084)"
]
},
"execution_count": 40,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAr2UlEQVR4nO3dd5xU5b3H8c9vZ3uFZQtlka6CdFZR7DEaexeN/V4VjRpNzDXRmKvJTdEUNVFjRZPYNYiGqBhji70sHQQElra0LbTt9bl/7CxZyQLLsmfOlO/79ZoXM3POzPnuEfztc85TzDmHiIjErji/A4iIiL9UCEREYpwKgYhIjFMhEBGJcSoEIiIxToVARCTGxfsdYG/l5OS4gQMH+h1DRCSizJo1q9w5l9vRtogrBAMHDqSoqMjvGCIiEcXMVu9qmy4NiYjEOBUCEZEYp0IgIhLjVAhERGKcCoGISIxTIRARiXExUwi21zXy6vz1NDS1+B1FRCSsRNw4gq6auWADP3ppATnpSZx/cAEXHLwf/bNTAXDOUVnfRF1D8479HeAcOBzOgRn0zkzGzHz6CUREvBEzheDcCf3Jy0zmmU/X8NB7K3jwvRUckJ9BZV0TZVX1nWop9M5M5rjheXxzRD6HDe5FckIgBMlFRLxlkbZCWWFhodvXkcXrt9by/BdrWVCyley0JHLSE8lJTyIlMUD7X/gNwwziDOqbWvh4eQXvLyujpqGZxEAc+VlJ5KYnkZuRRM/URAJxRnycERdnpCYG6JmaSK/0RLLTkijomcKA7FTiAzFzNU5EwoiZzXLOFXa4LRYLwb6oa2zmk+IKPl1RwabtdZRV1VO6vZ6ttY20tDiaWhwtLY6axmaaW75+bhMCxsBeaQzJTSczJZ7E+DiS4gOkJcUzpiCLgwdlk5mc4NNPJiLRbHeFIGYuDXWX5IQAxx6Qx7EH5O12v5YWx/a6RiqqG9hc3cDqihqWl1axvLSKZaWVVNc309DcQn1jM7WNzbS41pbHiL6ZFA7IZlh+OsPyMhial052WmKIfjoRiUUqBB6JizN6pCbSIzWRIblw8MDsXe5b19jM7DVb+Kx4M58WV/DCF2upbfz3jeteaYkMy09n//wM9s/PYHifTIb3ySA1Uf/5RGTf6f8kYSA5IcCkITlMGpIDtLYm1m+r3dGCWF5axVebKpk+ex1V9U1Aa+thcG46I/tmMnFwL44YmrOjF5SIyN5QIQhDcXFGQc9UCnqmcky7S1DOOdZvq2Px+u0sXL+Nheu28/GKCl6Zux6AQTlpHDUsh28d1JtDBmXrxrSIdIpuFkc45xwryqr4YFk5Hywr55MVFdQ2NpOTnsi3DurNqaP7MnFQNnFxGv8gEsvUayiG1DY0897SUl5dsIF3FpdS29hM/+wUJk/oz7mFBfTJSvE7ooj4QIUgRtU0NPHmok288MVaPimuIM7gmAPyuOSwARw9LFetBJEYokIgrK6o5q9FJbxQtJayynoG9ErlkkMHcF5hf7JSNHZBJNqpEMgODU0tvLFoI09+vIqi1VvITI7n6qOHcPmkgaQlqe+ASLRSIZAOLVy3jd+/9RVvLS6lV1oi3zlmCBcfOkBzKIlEIRUC2a3Za7Zwz5tf8eHycvpkJXPjccM4d0KBup+KRJHdFQL9SxfG79eTp6+cyLNXTSQ/M5lbpi/g+Hvf5+/z1hNpvyiIyN5TIZAdJg3J4eVrJ/HYpYUkBuL47nNzuGjqZ6wqr/Y7moh4SIVAvsbMOH5EPq/feCS/OmsUC9Zt41u/f58H31tOY7NWdxOJRioE0qFAnHHhxP1466ajOfaAPH7zxlLOeOAjisuq/I4mIt1MhUB2Kz8zmYcvmcDDF09gw7ZaTn/gI15fsMHvWCLSjVQIpFNOHNmbV284kqF56Vz7zGx+/uqXulQkEiVUCKTT+vVI4cWrD+Oywwbw+IcrufCxT6moqvc7lojsIxUC2SuJ8XH87IyR/OGCscwr2cZZD37M8lLdNxDx2herNlO6vc6T71YhkC45Y2w/np9yKDUNTZz94Ed8tLzc70giUau5xXHR1M+Y+uFKT75fhUC6bPx+PXn52sPpnZXMZU98zl+L1vodSSQqrd9aS0NTC4Ny0jz5fhUC2Sf9s1OZ9p1JHDq4FzdPm88zn632O5JI1CkODuocHImFwMx6mNk0M1tiZovN7LCdtpuZ3Wdmy81svpmN9zKPeCMzOYGplxXyjQPzuO3lhfzl41V+RxKJKiuD43cG5UZgIQD+ALzhnDsQGAMs3mn7ScCw4GMK8JDHecQjyQkBHr54AieMyOeOGYuY+kGx35FEokZxeTUZSfHkpid58v2eFQIzywKOAh4HcM41OOe27rTbGcCTrtWnQA8z6+NVJvFWYnwcf7xoPCeP6s0vXlusYiDSTYrLqhmUm4aZN6sKetkiGASUAX8yszlmNtXMdm7X9APa32EsCb4nESohEMd9F4zbUQxe/EI3kEX21cryas/uD4C3hSAeGA885JwbB1QDt3Tli8xsipkVmVlRWVlZd2YUD8QH4rj3/LEcOSyHW6bPZ6ampBDpstqGZtZtrWVwbrpnx/CyEJQAJc65z4Kvp9FaGNpbB/Rv97og+N7XOOcedc4VOucKc3NzPQkr3SspPsAjl0xg3H49ufH5uXywTAVcpCtWVbT2GPKq6yh4WAiccxuBtWZ2QPCt44Avd9ptBnBpsPfQocA255x+fYwSqYnxPHHZwQzOTWPKk7OYX7LV70giEae4LNh11KMeQ+B9r6HvAs+Y2XxgLPArM7vGzK4Jbn8dKAaWA48B13qcR0IsKzWBJ684hOy0RKY8OcuzIfIi0WplebDraCS2CACcc3ODl3RGO+fOdM5tcc497Jx7OLjdOeeuc84Ncc6Ncs5pMeIolJeRzGOXFrKttpGrn55FXWOz35FEIkZxWTV9spJJTYz37BgaWSwhMaJvJvdMHsOcNVv5ySsLtRaySCcVl1d72hoAFQIJoZNG9eF73xzGtFklPO7R5Fki0cQ5R3FZlaf3B0CFQELshm8M46SRvfnV64v5WDOWiuxWRXUD2+uaGJzjXddRUCGQEIuLM3533hgG56Zzw/Nz2LhNN49FdmVlcLI5r+YYaqNCICGXlhTPwxePp6ahmeufna0lL0V2oTg42dwQtQgkGg3Ny+Cuc0ZTtHoLv565xO84ImGpuLyaxEAc/XqmeHocFQLxzelj+nLZYQOY+uFKTUMh0oHismoG9EolEOfNZHNtVAjEVz8+ZThj+/fg5mnzWVNR43cckbCyMgRdR0GFQHyWFB/g/m+PwwxueH6O7heIBDU1t7C6otrTyebaqBCI7/pnp3LX2aOZu3Yrd7/5ld9xRMLCuq21NDY7z8cQgAqBhIlTRvfh24f05+F/rdBMpSK0m2xOl4Ykltx+6kEMy0vn+y/Mo7yq3u84Ir5aEew6qktDElNSEgPcf+E4Kusa+Z+/ztN8RBLTVpZXk5WSQM/UBM+PpUIgYeXA3pn8+OThvLe0jKc/Xe13HBHfFJe19hjyap3i9lQIJOxcetgAjto/l1++vnhH81gk1qyu8Had4vZUCCTsmBm/PXc0yQkBvv/CXHUplZhT19jMhu117NcrNSTHUyGQsJSfmcydZ41ifsk27nt7md9xREKqZEsNzsHAXmoRSIw7aVQfzhlfwB/fXc6s1Vv8jiMSMqvKW0fZD1CLQAR+evoI+mSlcPO0eVriUmLG6s1thUAtAhEykhO465xRFJdV8wddIpIYsbqimozk+JB0HQUVAokARw7LZXJhAY++X8yCkm1+xxHx3OqKGgb2Ck3XUVAhkAhx2ykj6JWWyM3T5tHQpF5EEt1WV1SHrMcQqBBIhMhKSeCXZ41iycZKHnpvhd9xRDzT1NxCyZZaBqoQiPyn40fkc/qYvjzw7jKWbNzudxwRT6zfWkdTi2NAdmhuFIMKgUSYO04bQUZyAj+aNp8mDTSTKLSqonXW0VB1HQUVAokwvdKT+OnpBzGvZBtPfLTS7zgi3a6t6+jAEE0vASoEEoFOG92Hbw7P5+43v2JlebXfcUS61eryapIT4sjLSArZMVUIJOKYGb88aySJ8XHc8tJ8Wlo0XbVEj1UVNQzIDl3XUVAhkAiVn5nMT04ZzmcrN/Ps52v8jiPSbdZsrg7p/QFQIZAINrmwP0cMzeGumUtYv7XW7zgi+6ylxbG6okaFQKSzzIw7zx5Fc4vjf19ZqBXNJOJtqqyjvqklZHMMtVEhkIjWPzuVH5ywP28vKeW1BRv8jiOyT1ZXhHbW0TYqBBLxLp80kNEFWfx0xiK21jT4HUeky1YHxxCEah2CNioEEvHiA3HcdfZottQ08qvXF/sdR6TLVlfUkBAw+mQlh/S4nhYCM1tlZgvMbK6ZFXWw/Rgz2xbcPtfMbvcyj0SvEX0zmXLUYF4sKuHj5eV+xxHpktUVNRT0TCU+ENrf0UNxtGOdc2Odc4W72P5BcPtY59z/hSCPRKkbjxvGwF6p3PryAi1iIxFpVUXou46CLg1JFElOCPCrs0axuqKGB95Z7ncckb3inGNNcB2CUPO6EDjgTTObZWZTdrHPYWY2z8xmmtlBHueRKDdpaA5nj+/HI++v4KtNlX7HEem0zdUNVNY3sV929LUIjnDOjQdOAq4zs6N22j4bGOCcGwPcD7zS0ZeY2RQzKzKzorKyMk8DS+S77eThpCfFc9vLCzT9hESMf082F2WFwDm3LvhnKfAycMhO27c756qCz18HEswsp4PvedQ5V+icK8zNzfUyskSBXulJ/Pjk4XyxagsvFq31O45Ip6zeMf10FF0aMrM0M8toew6cACzcaZ/eFpxZycwOCeap8CqTxI5zJxQwcVA2v3p9MWWV9X7HEdmjVeU1mEFBz5SQH9vLFkE+8KGZzQM+B15zzr1hZteY2TXBfc4FFgb3uQ+4wGmeAOkGrTOUjqKusYWfv/ql33FE9mjt5hr6ZqWQFB8I+bHjvfpi51wxMKaD9x9u9/wB4AGvMkhsG5qXznXHDuXet77ijLF9OW54vt+RRHZpzeYa+meHvjUA6j4qUe47xwzhgPwMbnt5IZV1jX7HEdmlNZtrfOkxBCoEEuUS4+P49bmjKa2s466ZS/yOI9KhusZmSivr6d9ThUDEE2P79+CKIwbxzGdr+LRYfREk/JRsae06up8Po4pBhUBixE3HH8B+2anc8tJ8TT8hYWdNcAxBf10aEvFOSmKAu84ZxaqKGu596yu/44h8zdrNrSvs6dKQiMcmDcnh/ML+TP1gJYvWb/M7jsgOazbXkJIQICc90ZfjqxBITLn15APpmZrArdMX0KzpJyRMtPUYCo6vDTkVAokpPVITueO0g5hfso0/f7zK7zgiQOtgMr/GEIAKgcSgU0f34dgDcrn7zaU7emuI+MU5FywE/twfABUCiUFmxs/PHAnA/76yEM1qIn7aXN1AdUOzb4PJQIVAYlRBz1R+cMIBvLu0jBnz1vsdR2LYjq6jPvUYAhUCiWGXTxrI2P49+OmMRZqhVHyzdktr11G/BpOBCoHEsECc8bvzRlPd0KxLROKbtZHSIgiuLRAXfL6/mZ1uZgneRhPx3tC8DL73zWG8sWgjry3Y4HcciUFrKmrISU8iJTH000+36WyL4H0g2cz6AW8ClwB/9iqUSChNOXIwowuyuP1vi6io0iUiCa21W2rYz8euo9D5QmDOuRrgbOBB59x5gBaal6gQH4jjt+eOobKukdtnLPI7jsQYP6efbtPpQmBmhwEXAa8F3/OvHSPSzQ7oncGNxw3jtfkbeHW+ehFJaDQ2t7B+a62vYwig84Xge8CtwMvOuUVmNhh417NUIj645ughjCnI4ievLKR0e53fcSQGbNhaR4vzb9bRNp0qBM65fznnTnfO/Tp407jcOXeDx9lEQio+EMfdk8dS29DMLdMXqBeReK5tDEFEXBoys2fNLNPM0oCFwJdmdrO30URCb2heOj888UDeWVLKi0Vr/Y4jUc7vdQjadPbS0Ajn3HbgTGAmMIjWnkMiUee/Jg3k0MHZ/N/fv9zRx1vEC2u31JAQMHpnJvuao7OFICE4buBMYIZzrhFQu1miUlyc8dtzx2Bm/M9f59Gi6arFI2s211DQM5VAnD/TT7fpbCF4BFgFpAHvm9kAYLtXoUT81j87ldtPHcFnKzfzxEcr/Y4jUWrt5hoKevo7hgA6f7P4PudcP+fcya7VauBYj7OJ+Oq8wgK+OTyP3/xjKcs2VfodR6LQ2jAYQwCdv1mcZWb3mFlR8HE3ra0DkahlZtx59mjSk+L5/otzaWxu8TuSRJHtdY1sqWmMnEIAPAFUApODj+3An7wKJRIucjOS+NVZI1m4bjv3v7Pc7zgSRdaGSY8h6HwhGOKcu8M5Vxx8/AwY7GUwkXBx4sg+nD2+H398dzlz1271O45EibVhMoYAOl8Ias3siLYXZnY4UOtNJJHwc8dpB5GfkcRNL8yltqHZ7zgSBVaWBwuBj+sQtOlsIbgG+KOZrTKzVcADwNWepRIJM1kpCfz2vDEUl1fz6zeW+B1HosCKsiryMpLITPZ/Rv/O9hqa55wbA4wGRjvnxgHf8DSZSJg5fGgOl08ayJ8/XsWHy8r9jiMRbkVZFUNy0/2OAezlCmXOue3BEcYAN3mQRySs/ejEAxmcm8bN0+axrbbR7zgSoZxzrCitYkheeHS+3JelKv0dCifig5TEAPdMHktpZT0/09oF0kXlVQ1sr2uKzBbBTjTuXmLS2P49uO7YoUyfs46ZWt5SumBFWRVAZBQCM6s0s+0dPCqBviHKKBJ2vvuNoYzql8Ut0xewYZs60Mne2VEI8iKgEDjnMpxzmR08Mpxz8aEKKRJuEgJx/OGCsTQ0tXDTC/No1sR0shdWlFaTkhCgj8+zjrbZl0tDexTsbrrAzOaaWVEH283M7jOz5WY238zGe5lHpDsNzk3nZ6cfxCfFFTz6frHfcSSCrCirYnBuGnE+zzraxtNCEHSsc26sc66wg20nAcOCjynAQyHII9Jtziss4ORRvbn7zaXML9nqdxyJEOHUdRRCUwh25wzgyeCMpp8CPcysj8+ZRDrNzLjzrNHkZSRxw3NzqK5v8juShLnahmbWba2NqULggDfNbJaZTelgez+g/XqAJcH3vsbMprTNfFpWVuZRVJGuyUpN4N7zx7Jmcw23aq1j2YOV5dU4R9iMIQDvC8ERzrnxtF4Cus7MjurKlzjnHnXOFTrnCnNzc7s3oUg3mDi4Fz844QBmzFvPU5+u9juOhLFw6zoKHhcC59y64J+lwMvAITvtsg7o3+51QfA9kYjznaOHcNyBefz81S+ZvWaL33EkTK0oq8IMBuXEQIvAzNLMLKPtOXACsHCn3WYAlwZ7Dx0KbHPOaYSORKS4OOOeyWPpnZXMdc/MpqKq3u9IEoZWlFVT0DOF5ISA31F28LJFkA98aGbzgM+B15xzb5jZNWZ2TXCf14FiYDnwGHCth3lEPJeVmsBDF02gorqBG5+fq/EF8h9WlIZXjyEAzwaFOeeKgTEdvP9wu+cOuM6rDCJ+GNkvi1+cMZIfvjSfX7+xhB+fPNzvSBImWlocxeVVHDakl99Rvkajg0U8MPng/ixYt41H3y/mwN4ZnD2+wO9IEgbWb6ulrrEl7FoEfo8jEIlat582gkMHZ3PL9AVa4lKA1vsDAEPDZI6hNioEIh5JCMTx4EUTyMtI4uqniijdXud3JPHZitK2rqPh02MIVAhEPJWdlshjlxZSWdfElKdmUdeo9Y5j2YqyKnqkJpCdluh3lK9RIRDx2PA+mdwzeSzzSrby/Rfm0qKeRDGrbY4hs/CYbK6NCoFICJw4sje3nTycmQs3ctcbS/yOIz5ZUVYddpeFQL2GRELmiiMGsXZzDY++X0z/nilccthAvyNJCG2rbaSssj7segyBCoFIyJgZt592EOu21nLHjEX07ZHCccPz/Y4lIbK8NPzmGGqjS0MiIRSIM+779jgO6pvF9c/O0RoGMWTpxkoADuid4XOS/6RCIBJiqYnxPH55Ib3SE/nvP3/BmooavyNJCCzduJ20xAD9eqT4HeU/qBCI+CAvI5m//PchNLU4LvvT52yubvA7knhsycZKDuidETbLU7anQiDikyG56Uy9tJD1W2u58i9fUNugMQbRyjkXLASZfkfpkAqBiI8KB2bzhwvGMmftVq5/djaNzS1+RxIPbNpez7baRg4Mw/sDoEIg4rsTR/bh52eM5O0lpdz04jxNXR2FlmzcDoTnjWJQ91GRsHDxoQOoqm/irplLSEsMcOfZo8Ju9Kl0XVuPoXBtEagQiISJa44eQlVdEw+8u5y0pHh+cspwFYMosXRjJfmZSfRIDa85htqoEIiEkR+csD9V9U08/uFK0pLiuen4/f2OJN0gnG8UgwqBSFgxM24/dQTV9U3c9/YyUhMDXHP0EL9jyT5oam5heVkVRwzL8TvKLqkQiISZuDjjrnNGU9fUwl0zl5CSEOCySQP9jiVdtKqimoamFg7ID8/7A6BCIBKWAnHGPZPHUNfYzB0zFpGSEGDywf39jiVdsKTtRnGf8C0E6j4qEqYSAnE8cOE4jhyWw4+mz+dvc9f5HUm6YOnGSgJxFnbLU7anQiASxpLiAzx6SSETB2Vz04vzeH3BBr8jyV5asrGSQTlpJMUH/I6ySyoEImEuJTHA45cdzNj+PbjhuTm89eUmvyPJXlganGMonKkQiESAtKR4/vRfB3NQ30yufWY27y0t9TuSdEJVfRNrNtdwYBjfKAYVApGIkZmcwJP/PZGheelMeWoW//qqzO9IsgdfbQrfNQjaUyEQiSBZqQk8c+VEhuSmc9WTRWoZhLl/Ty0RvoPJQIVAJOL0TEvk2SsnMiwvnSlPzuLdJSoG4WrpxkpSEwMU9Ay/xWjaUyEQiUA90xJ55sqJ7N87naufmsU7S3QDORwt2bid/fPDczGa9lQIRCJUj9REnrniUA7oncHVT81iprqWhhXnHEs3VjI8jAeStVEhEIlgWakJPH3lREb1y+L65+bwyhwNOgsXs9dsYUtNI4UDsv2OskcqBCIRLislgaeumMghA7P5/otzee7zNX5HEuCVOetJTojjhIPy/Y6yRyoEIlGgbZzB0fvncuv0Bfzpo5V+R4ppjc0tvLZgA98cnk9GcoLfcfZIhUAkSiQnBHjkkgl866B8fvb3L3nwveV+R4pZHywrY3N1A2eO7ed3lE5RIRCJIknxAf544XjOGNuX37yxlHveXIpzWgM51F6es54eqQkctX+u31E6xfNCYGYBM5tjZq92sO1yMyszs7nBx5Ve5xGJdvGBOO6ZPJbzC/tz3zvL+eVri1UMQqiqvol/frmRU0f3ITE+Mn7XDsV6BDcCi4FdDa17wTl3fQhyiMSMQJxx59mjSEkMMPXDlVQ3NPGLM0cRCPP+7NHgzUUbqWtsiZjLQuBxi8DMCoBTgKleHkdE/lNcnHHHaSO47tghPPf5Wq5/djb1Tc1+x4p6r8xdT0HPFCYM6Ol3lE7zut3ye+CHQMtu9jnHzOab2TQz63AJJjObYmZFZlZUVqaJtkQ6y8y4+VsH8pNThjNz4Ub+609fUFXf5HesqFVaWceHy8o4Y2xfzCKn9eVZITCzU4FS59ys3ez2d2Cgc2408E/gLx3t5Jx71DlX6JwrzM2NjJsvIuHkyiMHc/d5Y/hs5WYufOxTyqvq/Y4UlV6dt4EWR0RdFgJvWwSHA6eb2SrgeeAbZvZ0+x2ccxXOuba/kVOBCR7mEYlp50wo4NFLJvDVpkrOfvBjVpRV+R0p6rwydx0j+mQyLMzXH9iZZ4XAOXerc67AOTcQuAB4xzl3cft9zKxPu5en03pTWUQ8ctzwfJ6fchjV9U2c89DHfLFqs9+RosayTZXML9nG2eMjqzUAPowjMLP/M7PTgy9vMLNFZjYPuAG4PNR5RGLN2P49ePnaw8lOTeSiqZ/x6vz1fkeKCi/NXkcgzjgjwi4LAVik9S8uLCx0RUVFfscQiXhbqhu46skiilZv4QfH78/13xgaUTc4w0lzi2PSXW8zsm8Wj19+sN9xOmRms5xzhR1ti4zRDiLS7XqmJfL0lRM5a1w/7v7nV9zw/FzqGtW9tCs+Wl7Opu31nD2+wO8oXRKKAWUiEqaSEwLcM3kM++dn8Jt/LGFNRTWPXlpIfmay39EiykuzS8hMjue44Xl+R+kStQhEYpyZ8Z1jhvDoJYUsK63itPs/ZPaaLX7HihiVdY38Y9FGThvTl+SEgN9xukSFQEQAOH5EPtOvnURyQoALHvmU57WuQafMXNA6pcQ5EyLzshCoEIhIOwf2zmTG9YczcXA2t0xfwG0vL6ChaXcTA8i02SUMykljXP8efkfpMhUCEfmaHqmJ/Pm/DuGao4fwzGdrmPzIJ6zbWut3rLC0dnMNn6/czDnj+0V0jysVAhH5D4E445aTDuTBi8azvLSKk//wAW8v3uR3rLDz0uwSzOCsCO0t1EaFQER26eRRfXj1u0fQr0cKV/yliDtfX0xjsy4VATjnmD57HYcO6kW/Hil+x9knKgQislsDc9KYfu0kLpy4H4+8X8z5j3xCyZYav2P5btbqLazZXBPRN4nbqBCIyB4lJwT41VmjuO/b4/hqU+ulopkLNvgdy1cvzS4hJSHAiSN7+x1ln6kQiEinnT6mL6/fcCSDctL4zjOz+ckrC6htiL3RyHWNzbw6fwMnjexNelLkj8tVIRCRvbJfr1T+es0krj5qME9/uoZT7v+AuWu3+h0rpN5avInKuqaInVJiZyoEIrLXEuPjuPXk4Txz5UTqGpo556GPuefNpTFzI/mlWSX0yUrmsCG9/I7SLVQIRKTLDh+awxvfP4ozx/bjvneWc9aDH7FsU6XfsTxVWlnH+8vKOXNcPwJxkTt2oD0VAhHZJ5nJCdw9eQwPXzyB9VvrOOX+D5n6QTEtLZE1xX1nzZi7nuYWxzkRuADNrqgQiEi3OHFkb/7xvaM4algOv3htMRdN/Swqu5m+NHsdYwqyGJoXWctR7o4KgYh0m9yMJB67tJBfnzOK+SVb+da97zP1g2KaouTewZfrt7N4w/aouUncRoVARLqVmXH+wfvxxveO4pBB2fzitcWc9sBHET+1dVNzC3fMWEhaYoDTxvT1O063UiEQEU/0z07licsP5uGLx7O1poFzHvqYH06bR+n2Or+jdckf3l7GF6u28MuzRpGdluh3nG6lQiAinjEzThzZh3/edDRXHTmYl+es45jfvcd9by+LqIFoHy0v54F3l3PehALOHBc9N4nbqBCIiOfSk+L58cnDeeumozl6/1zu+edXHPu793hlzjqcC+/eRWWV9XzvhbkMyU3nZ2cc5HccT6gQiEjIDOiVxkMXT+DFqw8jNyOJ770wl3Mf/oQFJdv8jtahlhbHD/46j221jTxw4ThSEyN/OomOqBCISMgdMiibv113OL85ZzSrK6o5/Y8f8qNp8ymtDK/7B098tJL3vyrj9lNHcGDvTL/jeEaFQER8ERdnTD64P+/8zzFcdeRgps8p4djfvscf311OXaP/9w8Wrd/Gb95Yygkj8rlo4n5+x/GUCoGI+CozOYEfnzycN79/NEcMy+G3/1jKN373Hi/PKaHZp9HJtQ3N3Pj8XHqkJnDXOaMjehnKzlAhEJGwMCgnjUcuKeS5qw6lZ1oi339hHifc+y/+NnddyAvCnTMXs7y0irsnj4m6rqIdUSEQkbBy2JBe/P36I3jgwnEE4owbn5/L8ff+i78WrQ3JJaO3F2/iyU9Wc+URgzhyWK7nxwsHFu5dt3ZWWFjoioqK/I4hIiHQ0uJ4Y9FG7nt7GUs2VpKVksDZ4/tx0cT9PJnrZ+nGSr792KfkZSTxt+sPJyk+0O3H8IuZzXLOFXa4TYVARMKdc45Piit49rM1/GPRRhqbHQcP7Mnkwv6cPKoPad2wStjCddu4+PHPSIqP47mrDmVwbno3JA8fKgQiEjXKq+qZNquEF79YS3F5NWmJAU4d3ZfJBxcwfr+eXbqxO2v1Fi7/0+dkJifw7FUTGdArzYPk/lIhEJGo45xj1uotvFi0llfnb6CmoZkhuWlMLuzPWeP7kZeR3Knvef+rMq55ehZ5GUk8c9Wh9OuR4nFyf6gQiEhUq65v4rUFG3jxi7UUrd6CGfTvmcqwvHSG5qezf14GI/tlMSQ3jfhAHM0tjn9+uZGpH6ykaPUWhual8+yVE8nL7FzxiEQqBCISM1aUVfH6/A0s3VTJ8tIqisuqaQiuh5CcEMeIPpmUVzWwZnMNBT1T+O/DB3H+wf275T5DONtdIYjun1xEYs6Q3HS+e9ywHa+bmltYWV7NwvXbWFCynYXrttGvRwq3nnQgJxzUO2rWHd4XnhcCMwsARcA659ypO21LAp4EJgAVwPnOuVVeZxKR2BEfiGNYfgbD8jM4a5zfacJTKAaU3Qgs3sW2K4AtzrmhwL3Ar0OQR0RE2vG0EJhZAXAKMHUXu5wB/CX4fBpwnEX7pB4iImHG6xbB74EfArtaubofsBbAOdcEbAN67byTmU0xsyIzKyorK/MoqohIbPKsEJjZqUCpc27Wvn6Xc+5R51yhc64wNzc25v4QEQkVL1sEhwOnm9kq4HngG2b29E77rAP6A5hZPJBF601jEREJEc8KgXPuVudcgXNuIHAB8I5z7uKddpsBXBZ8fm5wn8ga2CAiEuFCPo7AzP4PKHLOzQAeB54ys+XAZloLhoiIhFBICoFz7j3gveDz29u9XwecF4oMIiLSsYibYsLMyoCttPYwapPV7nVHz9v+zAHKu3jo9t+7N9s7en9Xedu/7mifrubfU/bd7bOrfB293t25B+/yd/Xc7/xa577z2fa0vavnvv3zcPp3u7fZ2z8Pl3PfwznXcW8b51zEPYBHd/W6o+ft/izqrmN2dntH7+8qb0eZuyP/nrLvTf6unnsv83f13Hcys859CM99R/nD4d/t3mYPxd+dfTn3Oz8idanKv+/mdUfPd96/O47Z2e0dvb+rvO1f726fvdWZz3c2fzSd+51f69zvOUNnt3f13Ld/Hk759zZ7Z469J16e+6+JuEtD+8LMitwuZt+LBMrvn0jODpGdP5KzQ2Tkj9QWQVc96neAfaT8/onk7BDZ+SM5O0RA/phqEYiIyH+KtRaBiIjsRIVARCTGqRCIiMQ4FYIgMzvSzB42s6lm9rHfefaWmcWZ2S/N7H4zu2zPnwgfZnaMmX0QPP/H+J2nK8wsLThV+ql73jt8mNnw4HmfZmbf8TvP3jKzM83sMTN7wcxO8DvP3jKzwWb2uJlN8zNHVBQCM3vCzErNbOFO759oZkvNbLmZ3bK773DOfeCcuwZ4lX8vlhMS3ZGf1kV+CoBGoMSrrDvrpuwOqAKSCWF26Lb8AD8CXvQmZce66e/94uDf+8m0zhgcMt2U/xXn3FXANcD5XubdWTflL3bOXeFt0k7o6oi9cHoARwHjgYXt3gsAK4DBQCIwDxgBjKL1f/btH3ntPvcikBFp+YFbgKuDn50WYdnjgp/LB56JwHN/PK0TJl4OnBpJ2YOfOR2YCVwYaee+3efuBsZHcP6Q/Zvt6BHy2Ue94Jx738wG7vT2IcBy51wxgJk9D5zhnLsT6LD5bmb7Aducc5Ve5t1Zd+Q3sxKgIfiy2cO4X9Nd5z5oC5DkSdBd6KZzfwyQRus/+Foze905t6tV+bpNd5171zoT8Awzew141sPIOx+3O869AXcBM51zsz2O/DXd/HffV1FRCHZhxzKYQSXAxD185grgT54l2jt7m386cL+ZHQm872WwTtir7GZ2NvAtoAfwgKfJOmev8jvnbgMws8uB8lAUgd3Y23N/DHA2rQX4dS+DddLe/r3/LvBNIMvMhjrnHvYyXCfs7fnvBfwSGGdmtwYLRshFcyHYa865O/zO0FXOuRpaC1nEcc5Np7WQRTTn3J/9zrC3XLsp4iORc+4+4D6/c3SVc66C1vsbvoqKm8W7sGMZzKCC4HuRIpLzR3J2iOz8kZwdlN8X0VwIvgCGmdkgM0uk9WbeDJ8z7Y1Izh/J2SGy80dydlB+f/h5p7ob794/B2zg310nrwi+fzLwFa138W/zO2c05o/k7JGeP5KzK394PTTpnIhIjIvmS0MiItIJKgQiIjFOhUBEJMapEIiIxDgVAhGRGKdCICIS41QIJCqYWVWIj9cta1YE12LYZmZzzWyJmf2uE58508xGdMfxRUCFQKRDZrbbebicc5O68XAfOOfGAuOAU81sT+sCnEnrTKci3UKFQKKWmQ0xszfMbJa1roB2YPD908zsMzObY2ZvmVl+8P2fmtlTZvYR8FTw9RNm9p6ZFZvZDe2+uyr45zHB7dOCv9E/E5waGTM7OfjeLDO7z8xe3V1e51wtMJfWGSwxs6vM7Aszm2dmL5lZqplNonX9gN8GWxFDdvVzinSWCoFEs0eB7zrnJgD/AzwYfP9D4FDn3DjgeeCH7T4zAvimc+7bwdcH0jpF9iHAHWaW0MFxxgHfC352MHC4mSUDjwAnBY+fu6ewZtYTGMa/pxGf7pw72Dk3BlhM6xQGH9M6d83NzrmxzrkVu/k5RTpF01BLVDKzdGAS8NfgL+jw70VvCoAXzKwPratIrWz30RnB38zbvOacqwfqzayU1lXUdl5O83PnXEnwuHOBgbQuvVnsnGv77ueAKbuIe6SZzaO1CPzeObcx+P5IM/sFres0pAP/2MufU6RTVAgkWsUBW4PX3nd2P3CPc25GcGGWn7bbVr3TvvXtnjfT8b+ZzuyzOx845041s0HAp2b2onNuLvBn4Ezn3LzgojfHdPDZ3f2cIp2iS0MSlZxz24GVZnYetC5paGZjgpuz+Pcc8Zd5FGEpMLjdUoZ7XFg92Hq4C/hR8K0MYEPwctRF7XatDG7b088p0ikqBBItUs2spN3jJlr/53lF8LLLIuCM4L4/pfVSyiyg3IswwctL1wJvBI9TCWzrxEcfBo4KFpD/BT4DPgKWtNvneeDm4M3uIez65xTpFE1DLeIRM0t3zlUFexH9EVjmnLvX71wiO1OLQMQ7VwVvHi+i9XLUI/7GEemYWgQiIjFOLQIRkRinQiAiEuNUCEREYpwKgYhIjFMhEBGJcSoEIiIx7v8BFPZ8YGPPSlYAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"learn.remove_cb(ActivationStats)\n",
"learn.lr_find()"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"SuggestedLRs(lr_min=0.04365158379077912, lr_steep=0.3630780577659607)"
]
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEOCAYAAACEiBAqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAr0UlEQVR4nO3deXxV9Z3/8dcn+0YSSMIW9k1kXyK7iFXbqghW3Nu6VEXQsWKnndrp/FqnnY4dx1ZFiqi1brVqpeqggltd2NwIguz7vmaBhCSQkOT7+yM3NqYhBMm5596b9/PxuI/knnPuue/cB+Sds32POecQEZGWK8rvACIi4i8VgYhIC6ciEBFp4VQEIiItnIpARKSFUxGIiLRwnhWBmZ1hZivqPIrNbEa9ZSaYWVGdZX7hVR4REWlYjFcrds5tAIYAmFk0sAd4pYFFFznnJnqVQ0REGhesXUPnAVucczuC9H4iItJEwSqCq4HnTzBvtJmtNLMFZtY/SHlERCTAvB5iwszigL1Af+fcgXrzUoFq51yJmV0EPOSc693AOqYCUwGSk5OH9+3b19PMIiKRJjc3N985l9XQvGAUwWTgdufcN5uw7HYgxzmXf6JlcnJy3LJly5oxoYhI5DOzXOdcTkPzgrFr6BpOsFvIzNqbmQW+HxHIUxCETCIiEuDZWUMAZpYMXADcWmfaNADn3BzgcmC6mVUCR4GrnYZDFREJKk+LwDlXCmTUmzanzvezgFleZhARkcbpymIRkRZORSAi0sKpCEREWjgVgYhIGHhn7QE2HyzxZN0qAhGREOec47bncpmbu9uT9asIRERCXPGxSo5XOTJT4jxZv4pARCTEFZZWANAmWUUgItIiFZSUA5CREu/J+lUEIiIhriCwRZChLQIRkZapoCRQBDpGICLSMhWW1uwa0jECEZEWKr+kglbxMcTHRHuyfhWBiEiIKyytoI1Hu4VARSAiEvIKSss9O1AMKgIRkZBXUFJBm2RvTh0FFYGISMgrKK3w7KpiUBGIiIS06mrHodIKz84YAhWBiEhIKz52nMpq59lVxaAiEBEJaV5fVQwqAhGRkOb1VcXgYRGY2RlmtqLOo9jMZtRbxsxsppltNrMvzGyYV3lERMKR11cVA8R4tWLn3AZgCICZRQN7gFfqLXYh0DvwGAk8EvgqIiLUXFUMkBkBxwjOA7Y453bUmz4ZeMbV+BhIN7MOQcokIhLyau9F0DopDHcN1XM18HwD07OBXXWe7w5MExERau5FkJoQQ1yMd7+uPS8CM4sDJgEvncY6pprZMjNblpeX13zhRERCXEFphaenjkJwtgguBJY75w40MG8P0LnO806BaV/hnHvMOZfjnMvJysryKKaISOgpLK3w9NRRCE4RXEPDu4UA5gHXBc4eGgUUOef2BSGTiEhYqBlnKIyLwMySgQuAl+tMm2Zm0wJP5wNbgc3A48BtXuYREQk3wdg15NnpowDOuVIgo960OXW+d8DtXmYQEQlX1dWOQ2WRsWtIRES+hqKjx6mqdp5eVQwqAhGRkFUQhKuKQUUgIhKyCoJwVTGoCEREQlbtyKPaIhARaaG+HIJaxwhERFqmgpKaYwRejjMEKgIRkZBVWFpBelIssdHe/qpWEYiIhKhgXFUMKgIRkZBVUFpOZrK3ZwyBikBEJGRpi0BEpIUrLK3w/IwhUBGIiISkqmpHYRDGGQIVgYhISDpcVoFzeD7yKKgIRERCUrCuKgYVgYhISKodZ0jHCEREWqjakUczdPqoiEjLVBikcYZARSAiEpLySyow836cIVARiIiEpMLSclonxREdZZ6/l4pARCQEFZYG56pi8LgIzCzdzOaa2XozW2dmo+vNn2BmRWa2IvD4hZd5RETCRX6QhpcAiPF4/Q8BbzrnLjezOCCpgWUWOecmepxDRCSsFJZW0LttSlDey7MiMLM0YDxwA4BzrgKo8Or9REQiSbDGGQJvdw11B/KAJ83sczP7o5klN7DcaDNbaWYLzKy/h3lERMJCVbXjUFkFbYJwDQF4WwQxwDDgEefcUKAUuLveMsuBrs65wcDDwKsNrcjMpprZMjNblpeX52FkERH/fTnOUAQcLN4N7HbOfRJ4PpeaYviSc67YOVcS+H4+EGtmmfVX5Jx7zDmX45zLycrK8jCyiIj/gjnOEHhYBM65/cAuMzsjMOk8YG3dZcysvZlZ4PsRgTwFXmUSEQkHX44zFCFnDd0BPBc4Y2grcKOZTQNwzs0BLgemm1klcBS42jnnPM4kIhLSaoeXaBOkg8WeFoFzbgWQU2/ynDrzZwGzvMwgIhJuCoM44BzoymIRkZBTe4ygdVJsUN5PRSAiEmIKSytIT4olJjo4v6JVBCIiIaYgiMNLgIpARCTkFJSWB+2MIVARiIiEnGCOPAoqAhGRkFNTBME5YwhUBCIiIaW62nGo7DiZQbqGAFQEIiIhpejocaqqnXYNiYi0VMEeZwhUBCIiIaWgJLhXFYOKQEQkpBRqi0BEpGWr3TUUrLuTgYpARCSkFH45zpCKQESkRSosrSA1IYa4mOD9elYRiIiEkILSCjJSgnegGFQEIiIhpaCkPKgHikFFICISUoI9zhCoCEREQkpBaUVQRx4FFYGISMhwznFIWwQiIi1X8dFKKoM8zhB4XARmlm5mc81svZmtM7PR9eabmc00s81m9oWZDfMyj4hIKCsI3LQ+M8hnDcV4vP6HgDedc5ebWRyQVG/+hUDvwGMk8Ejga7Pbll/K22v20zUjic5tkujSJolWCcG5MbSISFP4MbwEeFgEZpYGjAduAHDOVQAV9RabDDzjnHPAx4EtiA7OuX3NneeL3Ye5d8H6r0zr3CaRSwZ15NKh2fRp14pATvYVHWN7QSmZKfF0ap1IUpzXfSkiAvklEVYEQHcgD3jSzAYDucCdzrnSOstkA7vqPN8dmNbsRTB5SDbn9m3LzoIydhaWsaOgjI+2FjDnwy3M/mALZ3ZIJSkumo37j3CkvPIrr81Miadzm0Sy0xPp1DqJ7NaJdEpPpEN6Ah3TE0nVloWININCH8YZAm+LIAYYBtzhnPvEzB4C7gb+36muyMymAlMBunTp8rUDpSbEMiA7jQHZaQBMn9CTvCPlvP7FXuav2oeZ1WwdtG9Ft4wkCksr2H3oKLsKa8pj1Z4i3lqzn+NV7ivrTYmPoW1qPFkp8WS2qvnaJjmO1slxZAQeHdISaZcWT3xM9NfOLyKRrTBwjCCStgh2A7udc58Ens+lpgjq2gN0rvO8U2DaVzjnHgMeA8jJyXH155+OrFbx3Di2OzeO7d6k5aurHQePlLPn8FH2FR1l7+Gj7D18jLwj5eQdKWft3mLyj5T/01ZFrYzkODq1TqRLRjJd2yTRNSOJHlnJ9MxKIT2Ig0yJSOgpKK0gJT4m6H8welYEzrn9ZrbLzM5wzm0AzgPW1ltsHvAvZvYCNQeJi7w4PtCcoqKM9mkJtE9LAFqfcLmKymoOl1VQWFZB3pFy9hcdY3/RMfYWHWP3oTJW7jrM/FX7qKr+R69lJMfRLTOZjumJdExPIDs9kc5tkuiVlUJ2eiJRURaEn1BE/FJYWhH03ULg/VlDdwDPBc4Y2grcaGbTAJxzc4D5wEXAZqAMuNHjPEETFxNF29QE2qYm0Ld9w8scr6pm96GjbMsvYcvBUrbml7Atv5Qvdh/mrdXHqKiq/nLZhNgoemSmcGaHVPp3TGVAdhr9OqaSEq8D2SKRwo/hJcDjInDOrQBy6k2eU2e+A273MkMoi42OontmMt0zk/lG36/Oq6525JeUs72gjM0HS9iSV8KmgyV8uDGPvy3fDUCUwfCurTm3b1u+0bctZ7RrhZm2GkTCVX5JBdnpCUF/X/05GaKiouzLLYoR3dt8Zd7B4mOs2VtM7o5DvL/hIPe9uYH73txA+9QExvXO5OzemYztlRn0i1JE5PQUlpYzMDs16O+rIghDtQVxbt+2/PhbZ3Cg+Bjvrz/Iwk15vLP2AHNza7YYhnRO5+KBHbhwYHs6ta5/LZ+IhBLnXGDXUPD/gFMRRIB2qQlcPaILV4/oQlW1Y/WeIhZuzOPNNfv5zfx1/Gb+OoZ0TueyYdlMGtxRZyeJhKAj5ZUcr3JBH3kUVAQRJzrKGNw5ncGd07njvN5szy9l/up9zFuxl1/83xr+6/V1XNC/HVfldGZcr0ydiSQSIgp9uqoYVAQRr1tmMrdN6MVtE3qxek8Rc3N38+qKPbzxxT56ZiVz49juXDYsW8NoiPiswKerikHDULcoA7LTuGdSfz759/N44KrBJMXF8B+vrmb0ve9x/1sbOFRafygoEQmWL4eX0DECCYb4mGi+M7QTlw7JJnfHIZ5YvI0/fLCZJ5ds4/uju3HL2d2DfvNskZZuf/ExwJ8tgiYVgZklA0edc9Vm1gfoCyxwzh33NJ14yszI6daGnG5t2HjgCLPe28yjC7fw9NLt3Hx2d249p6cuWBMJks93HCIzJY4OacG/jqCpu4YWAglmlg28DXwfeMqrUBJ8fdq1YuY1Q3nnrnM478y2PPzeZib87wc8/+lOKutc4Swi3vhkWyEjurfx5aLQphaBOefKgMuA2c65K4D+3sUSv/Rqm8Ksa4fxym1j6JaRxM9eXsXFMxezfOchv6OJRKw9h4+y5/BRRnRrc/KFPdDkIgjcZvK7wBuBaRpPOYIN7dKal6aN5pHvDuPIseNMeWQpv3ptLWUVDY+qKiJf32fbCgE4q3toF8EM4GfAK865NWbWA3jfs1QSEsyMCwd24K27xvPdkV3405JtfPvBRSzdku93NJGI8sm2QlolxNC3ffCHl4AmFoFz7kPn3CTn3P+YWRSQ75z7ocfZJES0Sojlvy4dyAtTRxFlcO3jn3DPvDUcrajyO5pIRPhseyE5XVsT7dMFnk0qAjP7i5mlBs4eWg2sNbOfeBtNQs2oHhksuHM8N4zpxlNLt3PxzEU6diBymgpKytl8sIQR3TN8y9DUXUP9nHPFwKXAAmruR/x9r0JJ6EqMi+aeSf35y80jKa+s5vJHlvL7tzfozCKRr+mz7TV/TI3ofuIbXXmtqUUQa2ax1BTBvMD1A816y0gJL2N6ZfLmjLP5ztBOzHxvM9c+/gn7io76HUsk7Hy6rZD4mCgGZqf7lqGpRfAosB1IBhaaWVeg2KtQEh5aJcTyuysH88BVg1m9t4iLHlrE39cd8DuWSFj5dHsBQ7ukExfj34g/TT1YPNM5l+2cu8jV2AGc63E2CRPfGdqJ1+8YR4e0RG56ehn3vbn+K/diFpGGHTl2nLV7i309PgBNP1icZma/N7NlgcfvqNk6EAGgR1YKL982hqvP6szsD7Zw41OfcbhMg9iJNCZ3xyGqHYz06fqBWk3dFvkTcAS4MvAoBp70KpSEp4TYaH47ZRD//Z2BfLQln0mzlrBun/YgipzIZ9sLiYkyhnZJ9zVHU4ugp3Pul865rYHHfwI9vAwm4evakV14YepoyiuruGz2Uhas2ud3JJGQ9Om2QgZkp/l+P5CmFsFRMxtX+8TMxgInPUXEzLab2SozW2FmyxqYP8HMigLzV5jZL5oeXULZ8K6tee2OcfTt0Irpzy3noXc34ZyOG4jUOna8ipW7ihjh824haPr9CKYBz5hZWuD5IeD6Jr72XOdcY2MSLHLOTWziuiSMtG2VwPO3jOLfX1nFA+9uZOOBI9x/xWAS4zRMlci6fcVUVFUzrIt/1w/UalIROOdWAoPNLDXwvNjMZgBfeJhNIkBCbDS/u2Iwfdu34t4F69lRWMrj1+XQIS3R72givlq9pwiAgZ3STrKk907pxFXnXHHgCmOAHzXlJcDbZpZrZlNPsMxoM1tpZgvMrMGhrc1sau0ZS3l5eacSWUKAmTF1fE+euD6H7fllTJ61hBW7DvsdS8RXq/cU0zoplo4+3IimvtO5gqEpoyONc84NAy4Ebjez8fXmLwe6OucGAw8Drza0EufcY865HOdcTlZW1mlEFj99o287Xr5tDPGxUVz16Ee8tnKv35FEfLNqTxEDstN8uRFNfadTBCc98uec2xP4ehB4BRhRb36xc64k8P18aoayyDyNTBLi+rRrxau3jWVQpzTueP5zHnx3ow4iS4tz7HgVGw8cYWC2/7uF4CRFYGZHzKy4gccRoONJXptsZq1qvwe+Sc3IpXWXaW+BOjSzEYE8Bafx80gYyEiJ5883j2TKsE48+O4mZry4gmPHNaS1tBwbDxyhstoxIESKoNGDxc65Vqex7nbAK4Hf8zHAX5xzb5rZtMC65wCXA9PNrJKa01GvdvrzsEWIj4nm/isG0SMrmf99awO7Dx3lse8PJyMl3u9oIp5bVXugOByK4HQ457YCgxuYPqfO97OAWV5lkNBmZtx+bi+6ZSTzo7+u4NLZS3j6xhH0yErxO5qIp1bvKSItMZZOrUPj7Dn/hrsTCbh4UAdemDqKsvIqpjyyVDe7kYi3ek8xA7JTQ+JAMagIJEQM7dKav00fQ2piLNc+/jHvrtVw1hKZKiqr2bD/SMgcHwAVgYSQbpnJ/G36GPq0a8XUZ5fxwqc7/Y4k0uw2HjhCRVU1AzqqCEQalJkSz/O3jOLs3lnc/fIqHv1wi9+RRJrV6hA7UAwqAglByfExPH5dDhcP6sC9C9Zz/1sbdK2BRIxVe4polRBD14wkv6N8yd+xT0VOIC4miplXD6VVfAyz3t9MSXklv5jYj6io0Di4JvJ1rd5bTP+OoXOgGFQEEsKio4x7LxtISnwMf1y8jSPHKvmfKQOJidaGrISn41XVrNtXzPWju/od5StUBBLSzIyfX3wmrRJieeDdjZRVVPLg1UOIj9FQ1hJ+Nh8soaKyOqTOGAIdI5AwYGbceX5vfjGxHwtW7+eWZ3I5WqEhKST81F5RrCIQ+Zp+MK47900ZxOJNeVz3p08oOnrc70gip2T1niKS46LpnpHsd5SvUBFIWLnyrM48fM0wVuw6zFWPfsSB4mN+RxJpstwdh+jfMS3kTnpQEUjYuXhQB568YQS7CsuY8shStuaV+B1J5KQ+2VrAmr3FXDyog99R/omKQMLSuN6ZPD91FEcrqrh8zkes1B3PJMTNen8zmSnxXHVWZ7+j/BMVgYStQZ3SmTt9DElx0Vzz+Mf8fZ3GJ5LQtHLXYRZtyufms7uTEBt6Z7ypCCSsdc9M5uXpY+iZlcItzyzjqSXb/I4k8k/+8P5mUhNi+N6o0Lp+oJaKQMJe29QEXrx1FOed2Y57XlvLf762hqpqDUkhoWHD/iO8vfYAN4ztTkp8aF66pSKQiJAUF8Oc7w3nB2O78+SS7dz6bC6l5ZV+xxJh9gebSYqL5sYx3fyOckIqAokY0VHGLy7px68m9+e99Qe4UqeXis+255fy2sq9fG9UV1onx/kd54RUBBJxrhvdjSeuP4vt+aVMnrWENXuL/I4kLdSfP95BTFQUN4/r7neURqkIJCKd27ctL00bgxlcMecj3t9w0O9I0gK9t+Ego3pm0DY1we8ojfK0CMxsu5mtMrMVZrasgflmZjPNbLOZfWFmw7zMIy1Lv46pvHr7WLpnJnPz07rjmQTXzoIytuaVcu4ZWX5HOalgbBGc65wb4pzLaWDehUDvwGMq8EgQ8kgL0i41gRdvHc3YXpnc/fIqfv/ORt3kRoLig401W6ETzmjrc5KT83vX0GTgGVfjYyDdzELv+msJaynxMTxxfQ5XDO/EzL9v4idzv+B4VbXfsSTCvb/+IN0ykuieGVoDzDXE6yJwwNtmlmtmUxuYnw3sqvN8d2DaV5jZVDNbZmbL8vLyPIoqkSw2Oor7Lh/EjPN7Mzd3Nz946jOOHNPopeKNY8er+GhrQVhsDYD3RTDOOTeMml1At5vZ+K+zEufcY865HOdcTlZW6O9vk9BkZsw4vw/3TRnE0i0FXPXoxzq9VDzx8dYCjh2vZkIYHB8Aj4vAObcn8PUg8Aowot4ie4C6IzB1CkwT8cyVZ3Xmietz2FFQymWzl7LxwBG/I0mE+WBDHgmxUYzqkeF3lCbxrAjMLNnMWtV+D3wTWF1vsXnAdYGzh0YBRc65fV5lEqk14Yy2vHjraCqqqpkyeykfbtQuR2k+7284yOgeGSE5wFxDvNwiaAcsNrOVwKfAG865N81smplNCywzH9gKbAYeB27zMI/IVwzITuPV28eS3TqRG5/8lKeWbNMZRXLatuWXsqOgjHP7hsfxAfDw5vXOua3A4Aamz6nzvQNu9yqDyMlkpyfyt+ljuPOFFdzz2lo255Xwy0v6Exvt9wl1Eq7eXx84bbRP+BSB/rVLi5ccH8Nj3x/OtHN68uePd3LdE59SWFrhdywJU+9vOEiPrGS6ZCT5HaXJVAQiQFSUcfeFffn9lYPJ3XmISbMWs25fsd+xJMyUVVTyybZCzg2T00ZrqQhE6rhsWCf+eutoKiqruWz2Uhas0rkL0nQfby2gojJ8ThutpSIQqWdI53Reu2McZ7RvxfTnlvO7tzdQrRvdSBN8GDht9KxubfyOckpUBCINaJeawAtTR3HF8E48/N5mbnlmGcW6EllOYuGmfEaF0WmjtVQEIieQEBvNfZcP4leT+/Phxjwu/cMStuSV+B1LQtTOgjK25Zcyvnd47RYCFYFIo8yM60Z34883j6So7DiTZy3RcQNp0Iebai5KPCfMjg+AikCkSUb1yOC1O8bRq20K059bzq9fX6sRTOUrFm7MIzs9kR5hMNpofSoCkSbqmJ7IX28dzQ1juvHE4m1c/djH7C/SoHUCFZXVfLSlgPF9sjAzv+OcMhWByCmIi4ninkn9mXnNUNbtK+bChxby3voDfscSny3feYiS8krO6RN+u4VARSDytUwa3JHX7hhH+7REfvDUMn79+loqKrWrqKVauDGP6ChjTK/wGG20PhWByNfUMyuFV24bw3Wju/LE4m1MeWQp2/NL/Y4lPli4KY9hXdJJTYj1O8rXoiIQOQ0JsdH8avIA5nxvODsLy7h45iJe/Vy31GhJ8kvKWb2nOGx3C4GKQKRZfHtAe+bfeTb9OqYy48UV/OtfV1JaXul3LAmCRYHTRserCEQkOz2R528ZxZ3n9eaVz3cz8eHFrNh12O9Y4rGFG/NpkxzHgI5pfkf52lQEIs0oJjqKuy7ow19uGUX58SqmPLKUh97dRKWuOYhI1dWORZvyGNcrk6io8DtttJaKQMQDo3pksGDGeCYN7sgD725kypyP2KrhKSLO57sOk19SwXlnhtew0/WpCEQ8kpYYywNXDWHWtUPZnl/KhQ8tYs6HW7R1EEHeWXuAmChjQpjdf6A+FYGIxyYO6sjbd43nnD5Z/HbBei6dvYQ1e4v8jiXN4J21+xnZow1pieF52mgtFYFIELRLTeDR7w9n9neHsb+onEmzlnDfm+spr6zyO5p8TVvzStiSV8oFZ7bzO8pp87wIzCzazD43s9cbmHeDmeWZ2YrA42av84j4xcy4aGAH3v3ReL4zNJvZH2xh4kydWRSu3llbM7TI+f1UBE1xJ7CukfkvOueGBB5/DEIeEV+lJ8Vx/xWDefLGsygpr+Sy2Uu4d/46jlZo6yCcvLP2AP06pNKpdfjcpP5EPC0CM+sEXAzoF7xIPeee0Za37hrPVWd14dGFW/nmgx/ywYaDfseSJsgvKSd35yEuiICtAfB+i+BB4N+Axk6TmGJmX5jZXDPr7HEekZCSmhDLvZcN5IWpo4iLjuKGJz/j9r8s52CxhrcOZe+tO4hzqAhOxswmAgedc7mNLPYa0M05Nwh4B3j6BOuaambLzGxZXl6eB2lF/DWqRwbz7zybf72gD++sPcB5v/uQJ5ds06mmIerttQfITk+kf8dUv6M0Cy+3CMYCk8xsO/AC8A0z+3PdBZxzBc658sDTPwLDG1qRc+4x51yOcy4nKyt8x/MQaUx8TDR3nNebt2aMZ2jX1vzna2uZNGsJuTsO+R1N6jhaUcXizXmcf2bbsLwJTUM8KwLn3M+cc52cc92Aq4H3nHPfq7uMmXWo83QSjR9UFmkRumcm8/SNZzH7u8MoLK1gyiNL+fFLK7W7KEQs2pTHsePVXNCvvd9Rmk1MsN/QzH4FLHPOzQN+aGaTgEqgELgh2HlEQlHtqabj+2Tx8N838acl21iwah//8o3e/GBcN+Jjov2O2CI553gpdzetEmIY2aON33GajTnn/M5wSnJyctyyZcv8jiESVNvyS/nNG+t4d90BurRJ4mcX9uXbA9pHzK6JcPH8pzv52cur+Mm3zuD2c3v5HeeUmFmucy6noXm6slgkDHTPTOaP1+fw7E0jSIiNYvpzy7ny0Y9YqYvRgmbdvmLumbeGs3tnMv2cnn7HaVYqApEwcnbvLOb/8Gz++zsD2ZZfyuQ/LOGuF1ew5/BRv6NFtJLySm5/bvmXAwmG85DTDVERiISZmOgorh3Zhfd/PIHpE3oyf9U+zr3/A+5dsI6io8f9jhdxnHP8/JVVbC8oZeY1Q8lMifc7UrNTEYiEqVYJsfz0231578cTmDioA48t3MqE/32fJxZv02B2zWjeyr3834q93HV+H0b1yPA7jidUBCJhLjs9kd9fOYTX7xjHgOw0fv36Wr5x/4f8LXc3VdXhdTJIqCktr+S/569jUKc0bguzg8OnQkUgEiH6d0zj2ZtG8uebRtImOY5/fWklF89cxFtr9hNuZweGijkfbuFAcTm/vKQf0RF2XKAuFYFIhBnXO5P/u30ss64dSnllNbc+m8tFMxfz5up9VGsLocl2FZbx2MKtTB7SkeFdI+eagYaoCEQiUFSUMXFQR965azy/v3Iwx45XMe3Py7lo5iLe+EKF0BS/XbAeM/jpt/v6HcVzKgKRCBYTHcVlwzrxzl3jeeCqwVRUVXP7X5bzzQcX8n8r9ugYwgl8srWAN1btY/o5veiYnuh3HM/pymKRFqSq2jF/1T5mvbeZDQeO0C0jiVvG92DKsE4kxGrYCqj5jCbNWszhsuO8+6NzSIyLjM9FVxaLCADRUcYlgzuy4M6zmfO94aQlxvLzV1Yz7n/eY9Z7mzhcVuF3RN/95dOdrNlbzN0X9o2YEjgZbRGItGDOOT7eWsijC7fwwYY8EmOjuWxYNjeO7Uavtq38jhd0BSXlnHv/BwzITuO5m0dG1FhOjW0RBH30UREJHWbG6J4ZjO6Zwfr9xTy5eDsv5e7muU92Mr5PFj8Y243xvbMibkiFE/ntgvUcPV7Fryb3j6gSOBltEYjIVxSUlPP8pzt55qMdHDxSTs+sZG4Y250pw7JJiovcvx1zdxQy5ZGPmHZOT+6+MPLOFGpsi0BFICINqqisZv6qfTyxeBur9hSRmhDDd4Zmc83ILvRtHxm3aKxVWVXNJbOWcLisgnd/dA7J8ZFXeNo1JCKnLC4mikuHZjN5SEeW7TjEsx/t4PlPd/H0RzsY0jmda0Z05uJBHUmJgF+af/54B+v2FTP7u8MisgRORlsEItJkhaUVvLx8Ny98tovNB0tIjI3mooEduDKnEyO6twnL/eq7Csv49oMLGda1Nc/8YERY/gxNoV1DItKsnHMs33mYubm7eG3lPkrKK+makcTlwzoxZXinsLkIq7racc3jH7NmbzFvzjibTq2T/I7kGRWBiHimrKKSBav281LuLj7eWogZjO2ZyUUDO3BBv3ZktQrd8fufWLyNX7++lvumDOLKszr7HcdTKgIRCYqdBWXMXb6bVz/fw87CMszgrK5t+NaA9nyrf7uQ+ot788ESLp65iHG9Mvnj9TkRu0uolq9FYGbRwDJgj3NuYr158cAzwHCgALjKObe9sfWpCERCn3OO9fuP8Obq/by1Zj/r9x8BYEB2Kt/q156LB3WgR1aKb/kqq6qZMucjdhSU8vaM8bRNTfAtS7D4fdbQncA6oKHzzW4CDjnnepnZ1cD/AFcFIZOIeMjMOLNDKmd2SOWuC/qwPb+Ut9bUlMLv3tnI797ZyMDsNCYN7sglgzvSPi14v4hLyiv5j1dWsXLXYWZdO7RFlMDJeLpFYGadgKeB3wA/amCL4C3gHufcR2YWA+wHslwjobRFIBLe9hcd4/Uv9jJv5V6+2F2EGYzo1oZLBnfkwgHtyfDwnsCf7zzEjBdXsKuwjB+e15sZ5/fx7L1CjW+7hsxsLnAv0Ar4cQNFsBr4tnNud+D5FmCkcy7/ROtUEYhEjq15JcxbuZfXVu5lS14p0VHGmJ4ZfKt/e77Zr12z/bVeVe2Y/f5mHvz7JtqnJvDAVUMY0T2ybzZTny9FYGYTgYucc7eZ2QROowjMbCowFaBLly7Dd+zY4UlmEfFH7TGF11bu5Y1V+9hRUAbA0C7pnNe3LWf3zmJAdtrXul3k3sNHmfHiCj7dVsikwR359aUDSEuMbe4fIeT5VQT3At8HKoEEao4RvOyc+16dZbRrSES+wjnHpoMlvL1mP2+tOcCqPUUApCXGMrZXBiO6tWF41zac2aEVMdGNj6T/5ur9/PRvX3C8qppfTR7A5cM7BeNHCEm+nz7ayBbB7cBA59y0wMHiy5xzVza2LhWBSMuSX1LOks35LNqUz5LN+ewrOgZAYmw0Z3ZoRXJ8DLHRUcRGGzFRUZhBlBlHjh3n/Q15DMxOY+Y1Q+memezzT+Ivv88aqh/mV8Ay59w84AngWTPbDBQCVwc7j4iEtsyUeCYPyWbykGycc+wtOkbujkMs33GI9fuLKSmv5HhVNZVVjuNV1TgH1c7hgFvP6cG/XnAGcTG6B1djdEGZiEgLoFtViojICakIRERaOBWBiEgLpyIQEWnhVAQiIi2cikBEpIVTEYiItHAqAhGRFi7sLigzszzgMFBUZ3JanecNfV/7NRM44cimjai7zlNZpinTGstbd5pX2U80v7Gc9Z/78Zk3NXdTstb9vu40v7OH62cearlPtExL+//Z1TmX1eC7OOfC7gE8dqLnDX1f5+uy5ni/pi7TlGmN5Q1G9hPNbyxnKHzmTc3dlKyNfPb6zCMgd1P+XZxK9nD+/3miR7juGnqtkecNfV9/+dN9v6Yu05RpJ8vrdfYTzW8sZ/3nfnzmTc1df5rXuZuyjkj/zEMt94mWaYn/PxsUdruGToeZLXMnGGsj1IVr9nDNDeGbXbmDL5yzQ8s7WPyY3wFOQ7hmD9fcEL7ZlTv4wjl7y9oiEBGRf9bStghERKQeFYGISAunIhARaeFUBAFmdraZzTGzP5rZUr/zNJWZRZnZb8zsYTO73u88p8LMJpjZosDnPsHvPKfCzJLNbJmZTTz50qHDzM4MfN5zzWy633mayswuNbPHzexFM/um33lOhZn1MLMnzGyu31lOJCKKwMz+ZGYHzWx1venfNrMNZrbZzO5ubB3OuUXOuWnA68DTXuatk++0cwOTgU7AcWC3V1nra6bsDigBEghS9mbKDfBT4K/epGxYM/07Xxf4d34lMNbLvHXyNUfuV51ztwDTgKu8zFtXM2Xf6py7ydukp+nrXA0Xag9gPDAMWF1nWjSwBegBxAErgX7AQGp+2dd9tK3zur8CrcIlN3A3cGvgtXPD6TMHogKvawc8F0a5LwCuBm4AJobTZx54zSRgAXBtOOUOvO53wLBw+8wDrwva/89TfcQQAZxzC82sW73JI4DNzrmtAGb2AjDZOXcv0ODmvJl1AYqcc0e8zFurOXKb2W6gIvC0ysO4X9Fcn3nAISDek6D1NNNnPgFIpuY//1Ezm++cq/YyNzTfZ+6cmwfMM7M3gL94GLn2/ZrjMzfgt8AC59xyjyN/qZn/nYesiCiCE8gGdtV5vhsYeZLX3AQ86VmipjnV3C8DD5vZ2cBCL4M1wSllN7PLgG8B6cAsT5M17pRyO+d+DmBmNwD5wSiBRpzqZz4BuIya4p3vZbCTONV/53cA5wNpZtbLOTfHy3AncaqfeQbwG2Comf0sUBghJZKL4JQ5537pd4ZT5Zwro6bAwo5z7mVqiiwsOeee8jvDqXLOfQB84HOMU+acmwnM9DvH1+GcK6Dm2EbIioiDxSewB+hc53mnwLRQF665IXyzh2tuCN/s4Zobwjt7gyK5CD4DeptZdzOLo+bg3jyfMzVFuOaG8M0errkhfLOHa24I7+wN8/todXM8gOeBffzjFMqbAtMvAjZSc4T/537njJTc4Zw9XHOHc/ZwzR3u2U/loUHnRERauEjeNSQiIk2gIhARaeFUBCIiLZyKQESkhVMRiIi0cCoCEZEWTkUgEcHMSoL8fs1yz4rAPRmKzGyFma03s/ub8JpLzaxfc7y/CKgIRBpkZo2Ow+WcG9OMb7fIOTcEGApMNLOT3SfgUmpGPhVpFioCiVhm1tPM3jSzXKu5E1rfwPRLzOwTM/vczN41s3aB6feY2bNmtgR4NvD8T2b2gZltNbMf1ll3SeDrhMD8uYG/6J8LDJmMmV0UmJZrZjPN7PXG8jrnjgIrqBndEjO7xcw+M7OVZvY3M0syszHU3E/gfwNbET1P9HOKNJWKQCLZY8AdzrnhwI+B2YHpi4FRzrmhwAvAv9V5TT/gfOfcNYHnfakZKnsE8Eszi23gfYYCMwKv7QGMNbME4FHgwsD7Z50srJm1Bnrzj+HEX3bOneWcGwyso2Z4g6XUjGvzE+fcEOfclkZ+TpEm0TDUEpHMLAUYA7wU+AMd/nHzm07Ai2bWgZo7TG2r89J5gb/Ma73hnCsHys3sIDV3U6t/W81PnXO7A++7AuhGzS04tzrnatf9PDD1BHHPNrOV1JTAg865/YHpA8zsv6i5X0MK8NYp/pwiTaIikEgVBRwO7Huv72Hg9865eYEbtdxTZ15pvWXL63xfRcP/Z5qyTGMWOecmmll34GMz+6tzbgXwFHCpc25l4CY4Exp4bWM/p0iTaNeQRCTnXDGwzcyugJpbHZrZ4MDsNP4xfvz1HkXYAPSoc5vDk95wPbD18Fvgp4FJrYB9gd1R362z6JHAvJP9nCJNoiKQSJFkZrvrPH5EzS/PmwK7XdYAkwPL3kPNrpRcIN+LMIHdS7cBbwbe5whQ1ISXzgHGBwrk/wGfAEuA9XWWeQH4SeBgd09O/HOKNImGoRbxiJmlOOdKAmcR/QHY5Jx7wO9cIvVpi0DEO7cEDh6voWZ31KP+xhFpmLYIRERaOG0RiIi0cCoCEZEWTkUgItLCqQhERFo4FYGISAunIhARaeH+P1rQvbvrb8JaAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"admin_init(learn.model, scales)\n",
"learn.lr_find()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Tracking"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmorgan\u001b[0m (use `wandb login --relogin` to force relogin)\n"
]
}
],
"source": [
"# hide\n",
"!wandb login"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Load Experiment Tracking with Weights & Biases:"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"import wandb\n",
"\n",
"WANDB_NAME = f'admin_init_bte2'\n",
"GROUP = 'admin'\n",
"NOTES = 'Early ADMIN testing'\n",
"CONFIG = {}\n",
"TAGS = ['admin']"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmorgan\u001b[0m (use `wandb login --relogin` to force relogin)\n"
]
},
{
"data": {
"text/html": [
"\n",
" Tracking run with wandb version 0.10.19<br/>\n",
" Syncing run <strong style=\"color:#cdcd00\">admin_init_bte2</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
" Project page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox</a><br/>\n",
" Run page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/2o27v7d4\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox/runs/2o27v7d4</a><br/>\n",
" Run data is saved locally in <code>/home/morgan/ml/projects/transformers_sandbox/nbs/exploration/wandb/run-20210220_234952-2o27v7d4</code><br/><br/>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<h1>Run(2o27v7d4)</h1><iframe src=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/2o27v7d4\" style=\"border:none;width:100%;height:400px\"></iframe>"
],
"text/plain": [
"<wandb.sdk.wandb_run.Run at 0x7fb00ef616a0>"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#hide_output\n",
"wandb.init(reinit=True, project=\"transformers_sandbox\", entity=\"fastai_community\", \n",
" name=WANDB_NAME, group=GROUP, notes=NOTES, tags=TAGS, config=CONFIG)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>perplexity</th>\n",
" <th>bpc</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>2.117492</td>\n",
" <td>2.035515</td>\n",
" <td>0.404338</td>\n",
" <td>7.656197</td>\n",
" <td>2.936628</td>\n",
" <td>07:32</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>1.601835</td>\n",
" <td>1.523893</td>\n",
" <td>0.554345</td>\n",
" <td>4.590059</td>\n",
" <td>2.198513</td>\n",
" <td>07:29</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>1.452392</td>\n",
" <td>1.390810</td>\n",
" <td>0.589804</td>\n",
" <td>4.018106</td>\n",
" <td>2.006516</td>\n",
" <td>07:28</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>1.347416</td>\n",
" <td>1.315846</td>\n",
" <td>0.613917</td>\n",
" <td>3.727902</td>\n",
" <td>1.898364</td>\n",
" <td>07:29</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>1.275978</td>\n",
" <td>1.247816</td>\n",
" <td>0.633069</td>\n",
" <td>3.482728</td>\n",
" <td>1.800218</td>\n",
" <td>07:28</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>1.227183</td>\n",
" <td>1.200533</td>\n",
" <td>0.647163</td>\n",
" <td>3.321887</td>\n",
" <td>1.732003</td>\n",
" <td>07:27</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>1.188224</td>\n",
" <td>1.171363</td>\n",
" <td>0.655419</td>\n",
" <td>3.226386</td>\n",
" <td>1.689919</td>\n",
" <td>07:28</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>1.147837</td>\n",
" <td>1.153088</td>\n",
" <td>0.661562</td>\n",
" <td>3.167962</td>\n",
" <td>1.663555</td>\n",
" <td>07:28</td>\n",
" </tr>\n",
" <tr>\n",
" <td>8</td>\n",
" <td>1.123723</td>\n",
" <td>1.142603</td>\n",
" <td>0.665061</td>\n",
" <td>3.134917</td>\n",
" <td>1.648427</td>\n",
" <td>07:29</td>\n",
" </tr>\n",
" <tr>\n",
" <td>9</td>\n",
" <td>1.113917</td>\n",
" <td>1.141676</td>\n",
" <td>0.665857</td>\n",
" <td>3.132012</td>\n",
" <td>1.647090</td>\n",
" <td>07:28</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with valid_loss value: 2.035515308380127.\n",
"Better model found at epoch 1 with valid_loss value: 1.523892879486084.\n",
"Better model found at epoch 2 with valid_loss value: 1.390810489654541.\n",
"Better model found at epoch 3 with valid_loss value: 1.3158457279205322.\n",
"Better model found at epoch 4 with valid_loss value: 1.2478158473968506.\n",
"Better model found at epoch 5 with valid_loss value: 1.2005329132080078.\n",
"Better model found at epoch 6 with valid_loss value: 1.1713625192642212.\n",
"Better model found at epoch 7 with valid_loss value: 1.1530883312225342.\n",
"Better model found at epoch 8 with valid_loss value: 1.142602562904358.\n",
"Better model found at epoch 9 with valid_loss value: 1.1416757106781006.\n"
]
}
],
"source": [
"admin_init(learn.model, scales)\n",
"learn.fit_one_cycle(n_epochs, 1e-3, cbs=[WandbCallback(log_model=False, log_preds=False)])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train - SPUniTokenizer\n",
"\n",
"Train a tokenizer with the same dictionary length as the T5 tokenizer"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"SentencePiece Unigram Tokenizer"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Reusing dataset wikitext (/home/morgan/.cache/huggingface/datasets/wikitext/wikitext-103-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91)\n",
"Loading cached processed dataset at /home/morgan/.cache/huggingface/datasets/wikitext/wikitext-103-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91/cache-e9eccb98202e10a0.arrow\n"
]
}
],
"source": [
"big_train_ds = load_dataset('wikitext', name='wikitext-103-raw-v1', split='train')\n",
"big_train_ds = big_train_ds.filter(lambda x: x['text'] != '')\n",
"\n",
"spu_tok = SPUniTokenizer(vcb_sz=32093, add_eos=True, add_bos=True)\n",
"spu_tok.train(big_train_ds['text'])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Save Trained Tokenizer"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"def save_obj(obj, name ):\n",
" with open(name + '.pkl', 'wb') as f:\n",
" pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)\n",
"\n",
"def load_obj(name ):\n",
" with open(name + '.pkl', 'rb') as f:\n",
" return pickle.load(f)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"# save_obj(spu_tok)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"spu_tok = load_obj('sentencepieceunigram_wiki103_32093')"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"tok = spu_tok"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"#Run setup\n",
"n_epochs = 10\n",
"bs = 12\n",
"sl = 512\n",
"n_layers = 6\n",
"pad_id = tok.pad_token_id\n",
"vocab_size = tok.vocab_size"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Tokenize Data"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Loading cached processed dataset at /home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91/cache-4267024924ef223b.arrow\n",
"Loading cached processed dataset at /home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91/cache-916f06ba6ec602c9.arrow\n"
]
}
],
"source": [
"def encode(examples): return {'token_ids' : spu_tok(examples['text'])}\n",
"\n",
"tok_ds = train_ds.map(encode, batched=False)\n",
"val_tok_ds = valid_ds.map(encode, batched=False)\n",
"all_toks = tok_ds['token_ids'] + val_tok_ds['token_ids']\n",
"df['toks'] = all_toks "
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>text</th>\n",
" <th>text_</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Coulthard and Button collided on lap 18 when Button attempted to pass the Red Bull on the inside at turn eight ; the Honda lost its front wing and retired a lap later after two pit stops . Hamilton continued his climb back through the field ; he moved from 18th , passing Piquet , Davidson , Sutil and Bourdais in separate manoeuvres , to sit in 14th by the time he pitted on lap 31 . Piquet retired on lap 42 with transmission failure , requiring a gearbox change before the next race . Anderson was involved in filmmaking at a young age and never really had an alternative plan to directing films . He made his first movie when he was eight years old and started making movies on a Betamax video camera which his dad bought in 1982 when he was twelve years old . He</td>\n",
" <td>Coulthard and Button collided on lap 18 when Button attempted to pass the Red Bull on the inside at turn eight ; the Honda lost its front wing and retired a lap later after two pit stops . Hamilton continued his climb back through the field ; he moved from 18th , passing Piquet , Davidson , Sutil and Bourdais in separate manoeuvres , to sit in 14th by the time he pitted on lap 31 . Piquet retired on lap 42 with transmission failure , requiring a gearbox change before the next race . Anderson was involved in filmmaking at a young age and never really had an alternative plan to directing films . He made his first movie when he was eight years old and started making movies on a Betamax video camera which his dad bought in 1982 when he was twelve years old . He</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>subdue and molest his victims . The song ends with the narrator turning inward with the lyrics : \" And in my best behavior , I am really just like him / Look beneath the floorboards for the secrets I have hid . \" Stevens stated in a 2009 interview with Paste that \" we 're all capable of what [ Gacy ] did . \" Technically , it is known as a speaker that easily reveals poor quality in recordings . Recording engineers sought to dull its treble response by hanging tissue paper in front of it , resulting in what became known as the \" tissue paper effect \" , a type of comb filtering . The NS @-@ 10 has been used to monitor a large number of successful recordings by numerous artists , leading Gizmodo to refer to it as \" the most important loudspeaker</td>\n",
" <td>and molest his victims . The song ends with the narrator turning inward with the lyrics : \" And in my best behavior , I am really just like him / Look beneath the floorboards for the secrets I have hid . \" Stevens stated in a 2009 interview with Paste that \" we 're all capable of what [ Gacy ] did . \" Technically , it is known as a speaker that easily reveals poor quality in recordings . Recording engineers sought to dull its treble response by hanging tissue paper in front of it , resulting in what became known as the \" tissue paper effect \" , a type of comb filtering . The NS @-@ 10 has been used to monitor a large number of successful recordings by numerous artists , leading Gizmodo to refer to it as \" the most important loudspeaker you</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"df['lens'] = df['toks'].apply(len)\n",
"splits = ColSplitter()(df)\n",
"tfms = [attrgetter(\"text\"), spu_tok]\n",
"# tfms = [attrgetter(\"toks\")]\n",
"dsets = Datasets(df, [tfms], splits=splits, dl_type=LMDataLoader)\n",
"dl_kwargs = [{'lens':df['lens'].values[splits[0]]},\n",
" {'val_lens':df['lens'].values[splits[1]]}]\n",
"dls = dsets.dataloaders(bs=bs, seq_len=sl, dl_kwargs=dl_kwargs, shuffle_train=True)\n",
"dls.show_batch(max_n=2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Tracking"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"WANDB_NAME = f'admin_init_spu'\n",
"GROUP = 'admin'\n",
"NOTES = 'SentencePieceUnigram tokenizer trained on wiki 103, 32100 vcb_sz, Early ADMIN testing'\n",
"CONFIG = {}\n",
"TAGS = ['admin','spu']"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmorgan\u001b[0m (use `wandb login --relogin` to force relogin)\n"
]
},
{
"data": {
"text/html": [
"\n",
" Tracking run with wandb version 0.10.19<br/>\n",
" Syncing run <strong style=\"color:#cdcd00\">admin_init_spu</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
" Project page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox</a><br/>\n",
" Run page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/896aogrk\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox/runs/896aogrk</a><br/>\n",
" Run data is saved locally in <code>/home/morgan/ml/projects/transformers_sandbox/nbs/exploration/wandb/run-20210221_183500-896aogrk</code><br/><br/>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<h1>Run(896aogrk)</h1><iframe src=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/896aogrk\" style=\"border:none;width:100%;height:400px\"></iframe>"
],
"text/plain": [
"<wandb.sdk.wandb_run.Run at 0x7f5b92f19e50>"
]
},
"execution_count": 32,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#hide_output\n",
"wandb.init(reinit=True, project=\"transformers_sandbox\", entity=\"fastai_community\", \n",
" name=WANDB_NAME, group=GROUP, notes=NOTES, tags=TAGS, config=CONFIG)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Train SPU"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>perplexity</th>\n",
" <th>bpc</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>5.331179</td>\n",
" <td>5.275365</td>\n",
" <td>0.253242</td>\n",
" <td>195.461777</td>\n",
" <td>7.610743</td>\n",
" <td>02:44</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>4.771585</td>\n",
" <td>4.862295</td>\n",
" <td>0.290732</td>\n",
" <td>129.320679</td>\n",
" <td>7.014809</td>\n",
" <td>02:43</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>4.419650</td>\n",
" <td>4.540506</td>\n",
" <td>0.322093</td>\n",
" <td>93.738213</td>\n",
" <td>6.550565</td>\n",
" <td>02:46</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>4.108190</td>\n",
" <td>4.304454</td>\n",
" <td>0.344261</td>\n",
" <td>74.028770</td>\n",
" <td>6.210014</td>\n",
" <td>02:44</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>3.865303</td>\n",
" <td>4.159061</td>\n",
" <td>0.357382</td>\n",
" <td>64.011414</td>\n",
" <td>6.000257</td>\n",
" <td>02:45</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>3.655769</td>\n",
" <td>4.056620</td>\n",
" <td>0.368161</td>\n",
" <td>57.778694</td>\n",
" <td>5.852466</td>\n",
" <td>02:44</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>3.459788</td>\n",
" <td>3.989344</td>\n",
" <td>0.376501</td>\n",
" <td>54.019436</td>\n",
" <td>5.755407</td>\n",
" <td>02:45</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>3.292417</td>\n",
" <td>3.953279</td>\n",
" <td>0.382756</td>\n",
" <td>52.105968</td>\n",
" <td>5.703377</td>\n",
" <td>02:46</td>\n",
" </tr>\n",
" <tr>\n",
" <td>8</td>\n",
" <td>3.169972</td>\n",
" <td>3.935873</td>\n",
" <td>0.385500</td>\n",
" <td>51.206848</td>\n",
" <td>5.678265</td>\n",
" <td>02:47</td>\n",
" </tr>\n",
" <tr>\n",
" <td>9</td>\n",
" <td>3.099260</td>\n",
" <td>3.935462</td>\n",
" <td>0.385990</td>\n",
" <td>51.185768</td>\n",
" <td>5.677671</td>\n",
" <td>02:44</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with valid_loss value: 5.275364875793457.\n",
"Better model found at epoch 1 with valid_loss value: 4.862295150756836.\n",
"Better model found at epoch 2 with valid_loss value: 4.540505886077881.\n",
"Better model found at epoch 3 with valid_loss value: 4.3044538497924805.\n",
"Better model found at epoch 4 with valid_loss value: 4.159061431884766.\n",
"Better model found at epoch 5 with valid_loss value: 4.056620121002197.\n",
"Better model found at epoch 6 with valid_loss value: 3.9893438816070557.\n",
"Better model found at epoch 7 with valid_loss value: 3.953279495239258.\n",
"Better model found at epoch 8 with valid_loss value: 3.93587327003479.\n",
"Better model found at epoch 9 with valid_loss value: 3.9354615211486816.\n"
]
}
],
"source": [
"admin_init(learn.model, scales)\n",
"learn.fit_one_cycle(n_epochs, 1e-3, cbs=[WandbCallback(log_model=False, log_preds=False)])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train SPU - 20e"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"SuggestedLRs(lr_min=0.04365158379077912, lr_steep=0.3630780577659607)"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAApoElEQVR4nO3deXhU9d3+8fcnO1kIkIV9B5F9C2jBBetaq4LWKioKiKLSuvWnrT5t1dr26WYf910RBdeqtWqta90VISAgoMi+h4QlG9mT7++PTDTGCSQhM2dmcr+ua67MnHNmzp0A+XDOdzPnHCIiIg1FeR1ARERCkwqEiIj4pQIhIiJ+qUCIiIhfKhAiIuKXCoSIiPgV43WA1pKenu769OnjdQwRkbCyZMmS3c65DH/7IqZA9OnTh+zsbK9jiIiEFTPb3Ng+3WISERG/VCBERMQvFQgREfFLBUJERPxSgRAREb9UIERExK+I6ebamiqra4iNblrtdM5RVF5FXlE5eUXlFJVVfWd/SkIMPTsl0qV9AtFRFoi4IiIBoQLhU1Vdw9tf5vLEp5v4ZP0ekuKiSU+JJyM5noTYaEoqqiipqKakopryqmoqqx2VVTWUV9VQUV1z0M+PjTa6prYjKT6G2GgjJsqIi4kiOT6m9pEQQ8fEOLqkJtAttR1dOyTQNbUd7RNiMFNhEZHga/MForCskgULN/Pkwi1szy+lW2oClx3bj8oqR15xObuLytlfUUVSXAzpyfEkxkUTHxNNbIwRGx1FXEwU6UnxZKTEk54cT2q7WOp+nzsH+0oq2LqvhG37Stm+r5Syymoqq2uoqnGUV9awI7+M/RVVFJdVkV9aSXXNdxdwSoyLpktqAl1TE+jVKYneaYn07pRI77Ta50nxbf6PUEQCpM3/dqmoquGOt9eS1bsjvz1tCCcMziSmibeXWltVdQ15xeXsyC9lR34ZOQVl7CwoY1dhGdvzS3ljVQ5791d85z3pyfH0SastGH3TE+mTnkSftCT6ZSSRGNfm/3hF5BBYpCw5mpWV5Vo61UZuYRmZ7RNaOVFgFJVVsnlPSe1j73427y5h0579bN5TQk5h2XeO7d6hHQMykxnUJYWs3h0Z37cTHRLjPEouIqHIzJY457L87lOBiBwlFVVs3lPCxt37WZ9bzNrcYtb5HnXtJId3SeHIfmkcNSCdI/unkaxbVCJt2oEKhH47RJDEuBgGd23P4K7tv7O9rLKaFdsKWLRxD59t3Mszi7cw75NNxEQZY3p15LjDMzl5aGf6ZSR7lFxEQpGuINqgsspqlm7ex4frdvPh2jxWbi8E4LDOyZwytAtnj+1Jr7REj1OKSDDoFpMc0Pb8Ut5clcMbq3JYtHEvDjj2sAymHdGb4w7P1PgNkQimAiFNtrOglGcWbeWZxVvYVVhO9w7tOP+IXpw7rifpyfFexxORVqYCIc1WWV3DO1/uYv7CzXy8bg9x0VGcOrwLMyb2ZVTPDl7HE5FWokZqabbY6ChOGdaVU4Z1ZV1uMQsWbuaFJdt4adkOxvftxOXH9mPSYZlE6faTSMTSFYQ0WXF5Fc8u3sqjH25gR0EZh3VO5rqTBnHikM6aDkQkTB3oCkKzuUqTJcfHMOuovrz/y+O4/dyRVNc4Zs9fwrRHP2NNTpHX8USklalASLPFRkdx5ugevH7NMdxy+hBWbi/kR3d+wE3/WklhWaXX8USklahASIvFRkcxY2Jf3rtuEtOO7M2ChZs54e/v8/rKHK+jiUgrUIGQQ9YxKY5bJw/jpZ9NJD05nssXLOGy+dnkFJQd/M0iErJUIKTVjOjRgX/9fCI3/Ohw3luTx8l3fMAbq3Q1IRKuVCCkVcVGR3H5sf15/Zpj6NUpkcvmL+G3L62krLLa62gi0kwqEBIQfdOTeOGKCVx6dF/mL9zMlHs/ZkNesdexRKQZAlYgzGyumeWa2cp62zqZ2Vtmttb3tWMj7602s2W+x8uByiiBFRcTxa9/PITHZo4jt6ics+7/hOxNe72OJSJNFMgriHnAKQ223QC845wbCLzje+1PqXNulO9xRgAzShAcNyiTl+ZMpGNiHOc/8hn/+WKn15FEpAkCViCccx8ADf+7OBl43Pf8cWBKoM4voaVXWiIvXDGBYd3aM+eppTz60UavI4nIQQS7DaKzc67uv485QOdGjksws2wzW2hmUxr7MDOb7TsuOy8vr7WzSivrlBTHU5ceyUlDOvP7V1dz6yurqa6JjKleRCKRZ43UrnYSqMZ+O/T2zQ1yPnCHmfVv5DMecs5lOeeyMjIyAhVVWlFCbDT3XTCWmRP7MPfjjfzsyaXq4SQSooJdIHaZWVcA39dcfwc557b7vm4A3gNGByugBF50lHHz6UP57WlDeGN1Duc9vJA9xeVexxKRBoJdIF4GpvueTwf+1fAAM+toZvG+5+nARGB10BJK0Mw6qi/3XzCG1TsKOfuBTzXyWiTEBLKb69PAp8AgM9tmZrOAPwMnmtla4ATfa8wsy8we8b11MJBtZsuBd4E/O+dUICLUKcO68tSlR5BXVM75Dy8kt1BFQiRUaD0ICQnZm/Zy0dxFdElN4JnZR5KZkuB1JJE2QetBSMjL6tOJeTPHk1NQxvkPf0ZekdokRLymAiEhY3zfTsydMY7t+0qZ9shn7N1f4XUkkTZNBUJCypH90nh0ehab9uxn2iOfkV+iIiHiFRUICTkTBqTz8EVZrMst5sJHF1FQqlXqRLygAiEh6ZjDMnjgwjF8lVPIjMcWUVxe5XUkkTZHBUJC1g8P78y954/hi20FXDxvMaUVGnEtEkwqEBLSThrahdvPHcXiTXu5bMESyqtUJESCRQVCQt7pI7vxl7NG8MHXeVz51OdUVtd4HUmkTVCBkLBwzrie3HL6EN5cvYvr/rGcGs0CKxJwMV4HEGmqGRP7UlJZzV9fX0PHxDhuPn0IZuZ1LJGIpQIhYeWKY/uzp7iCRz/aSEZKPD87boDXkUQilgqEhBUz49enDmZ3cTl/e2MNGSnxnJPV0+tYIhFJBULCTlSU8bezR7J3fwU3vvgFaUlxHD+4scUJRaSl1EgtYSkuJor7p41lSNf2zHlyKZ9t2ON1JJGIowIhYSs5PoZ5M8fRo2M7Zj2ezRfbCryOJBJRVCAkrKUlx7PgkiNIbRfL9McWsS63yOtIIhFDBULCXtfUdjx5yRFERxnTHlnE1r0lXkcSiQgqEBIR+qQnMX/WeEorq5k5bzEFJZoBVuRQqUBIxDi8S3sevHAsm/fs54onl1BRpSk5RA6FCoRElCP7pfHXs0fwyfo9/M8/vyBS1lwX8YLGQUjEOXN0DzbvKeGOt9fSu1MiVx4/0OtIImFJBUIi0tXHD2TLnhL+/tbX9MtI5scjunodSSTs6BaTRCQz408/Gc7Y3h257h/LWbVDYyREmksFQiJWfEw0D0wbS4fEWGY/sYTdxeVeRxIJKyoQEtEyUuJ56MIsdheXc8UC9WwSaQ4VCIl4w3uk8tezR7B40z5ufnmlejaJNJEaqaVNmDyqO2tyirjvvfUMyExh1lF9vY4kEvICdgVhZnPNLNfMVtbb1snM3jKztb6vHRt573TfMWvNbHqgMkrbct1JgzhlaBf+8O/VvPPlLq/jiIS8QN5imgec0mDbDcA7zrmBwDu+199hZp2Am4EjgPHAzY0VEpHmiIoybj93FMO6pXLl05+zekeh15FEQlrACoRz7gNgb4PNk4HHfc8fB6b4eevJwFvOub3OuX3AW3y/0Ii0SLu4aB6ZnkVqu1gueXwxuYVlXkcSCVnBbqTu7Jzb6XueA/hbBqw7sLXe622+bSKtonP7BB6ZnkV+aSUz5y2mqEwT+4n441kvJlfbleSQupOY2Wwzyzaz7Ly8vFZKJm3B0G6p3HvBGNbkFDH7iSWUVVZ7HUkk5AS7QOwys64Avq+5fo7ZDtRfhb6Hb9v3OOcecs5lOeeyMjIyWj2sRLbjBmVy209H8umGPVzzzDKqa9T9VaS+YBeIl4G6XknTgX/5OeYN4CQz6+hrnD7Jt02k1U0Z3Z2bThvC66ty+M1Lmv1VpL6AjYMws6eBSUC6mW2jtmfSn4HnzGwWsBk4x3dsFnC5c+4S59xeM/s9sNj3Ubc65xo2dou0mouP6sue/eXc++56MlMSuPbEw7yOJBISAlYgnHPnNbLreD/HZgOX1Hs9F5gboGgi33PdSYPIKSjnznfW0rNTImeP7eF1JBHPaSS1CL7ZX88azs6CUm54YQXdUhOYMCDd61gintJcTCI+cTFR3D9tLH3Tk7hswRLW7iryOpKIp1QgROpJbRfLYzPHkRAbzYzHNJBO2jYVCJEGenRMZO70cewrqWDGYxpIJ22XCoSIH8N7+AbS7SpizpNLtY6EtEkqECKNOG5QJn86azgfrt3NDS+s0BgJaXPUi0nkAM7J6smugjL+/tbXdO2QwPUnH+51JJGgUYEQOYif/3AAOwpKuffd9fRNT9YYCWkzdItJ5CDMjFsnD2NC/zRufHEFizdpYL+0DSoQIk0QGx3FfReMoUfHRC6bv4Qte0q8jiQScCoQIk3UITGOR6dnUV3jmPX4YgrV/VUinAqESDP0y0jm/mlj2Lh7P3MWqPurRDYVCJFmmtA/nT//ZAQfrdvNL59fTo3WkZAIpV5MIi1w9tge7Cos429vrKFzagI3/miw15FEWp0KhEgLzZnUn5yCMh58fwNd2icwc2JfryOJtCoVCJEWMjNuOWMouwrLuPXV1XRpn8CPhnf1OpZIq1EbhMghiI4y7jpvNKN7duDqZ5eRrTESEkFUIEQOUUJsNI9MH0f3Du245Ils1ucVex1JpFWoQIi0gk5JccybOY5oM2Y8toi8onKvI4kcMhUIkVbSOy2JR2eMI6+onEseX0xpRbXXkUQOiQqESCsa1bMDd583hhXbC/jFc8s0RkLCmgqESCs7cUhnfvPjIfxnZQ5/eeMrr+OItJi6uYoEwMUT+7BxdzEPvr+BPmlJnDe+l9eRRJpNBUIkAMyMW04fyta9pfzmpZX07JjIUQPTvY4l0iy6xSQSIDHRUdxz/mgGZiZz+YIlrN5R6HUkkWZRgRAJoJSEWB6bOY6UhBhmzlvE9vxSryOJNJkKhEiAdU1tx7yZ4ympqGbG3EUUlGgdCQkPTSoQZpZkZlG+54eZ2RlmFhvYaCKRY1CXFB66MIvNe0q4dH42ZZUaIyGhr6lXEB8ACWbWHXgTuBCY19KTmtnVZrbSzFaZ2TV+9k8yswIzW+Z73NTSc4mEih/0T+O2c0ayaONefvn8CpzTGAkJbU3txWTOuRIzmwXc55z7q5kta8kJzWwYcCkwHqgAXjezV51z6xoc+qFz7rSWnEMkVJ0xshtb95bwtzfW0CctkV+cNMjrSCKNauoVhJnZD4ALgH/7tkW38JyDgc+ccyXOuSrgfeCsFn6WSNiZM6k/52T14K7/ruOFJdu8jiPSqKYWiGuAG4F/OudWmVk/4N0WnnMlcLSZpZlZInAq0NPPcT8ws+Vm9h8zG+rvg8xstpllm1l2Xl5eC+OIBJeZ8Ycpw/lBvzRueHEFCzfs8TqSiF/W3PugvsbqZOdcizt1+25VzQH2A6uAcufcNfX2twdqnHPFZnYqcKdzbuCBPjMrK8tlZ2e3NJJI0BWUVHLW/R+zu7iCf1z+Aw7rnOJ1JGmDzGyJcy7L376m9mJ6yszam1kStVcAq83s+pYGcs496pwb65w7BtgHfN1gf6Fzrtj3/DUg1sw0DFUiSmpiLPNmjicuJooLH/2MrXtLvI4k8h1NvcU0xHfFMAX4D9CX2p5MLWJmmb6vvahtf3iqwf4uZma+5+N9OXUdLhGnZ6dEnrh4PKUV1Vw0dxG7i7WOhISOphaIWN+4hynAy865SuBQ+ui9YGargVeAnznn8s3scjO73Lf/bGClmS0H7gKmOvUJlAg1uGt75s4Yx86CUmY8toiiMg2kk9DQpDYIM7sK+BWwHPgx0AtY4Jw7OrDxmk5tEBLu3v0ql0ufyGZ8307f3HoSCbRDboNwzt3lnOvunDvV1doMHNeqKUXauOMOz+QvPxnBJ+v38KsXNJBOvNekgXJmlgrcDBzj2/Q+cCtQEKBcIm3ST8b2YEd+KX9/62t6dGzH/9NAOvFQU69h5wJFwDm+RyHwWKBCibRlP//hAKaO68nd/13HM4u2eB1H2rCmTrXR3zn3k3qvf9fSqTZE5MDMjN9PGcbOgjJ+/dJKOqcmcNygTK9jSRvU1CuIUjM7qu6FmU0ENLG9SIDERkdx7wVjOLxLCnMWLOXzLfu8jiRtUFMLxOXAvWa2ycw2AfcAlwUslYiQHB/DvJnjyWwfz8XzFrM+r9jrSNLGNLUX03Ln3EhgBDDCOTca+GFAk4kIGSnxPHHxeKKjjIseXUROQZnXkaQNaVZHa98UGHVzMP0iAHlEpIHeaUnMmzmegtJKps9dRKEG0kmQHMpIHGu1FCJyQMO6p/LghWNZn1fMz5/6nKrqGq8jSRtwKAVCo3hEgmjigHT+MGUYH3ydxx/+/aXXcaQNOGA3VzMrwn8hMKBdQBKJSKOmju/FutxiHvloI/0zk7nwyN5eR5IIdsAC4ZzTBPUiIebGUwezYfd+bnl5FX3TkjhqoGbCl8DQbGAiYSY6yrhz6igGZCQz58klbN6z3+tIEqFUIETCUEpCLI9MzyIqypj9xBL2l1d5HUk8Mu/jjTz4/vqAfLYKhEiY6tkpkbvPG83a3CKuf365Zn9to15dsZN31+QG5LNVIETC2NEDM/jVKYfz2hc5PPD+Bq/jiAd2FpTRrUNg+gypQIiEudnH9OO0EV356xtf8f7XeV7HkSCqqq4hp7CM7ioQIuKPmfHXs0cwqHMK1z67TNNxtCG5ReVU1zhdQYhI4xLjYrj3gjGUVVZz1dMaad1W7MivnVRbBUJEDqh/RjJ/PHMYizbt5Y6313odR4Jgu69AdO+QEJDPV4EQiSBnju7BuVk9ufe9dXyg9oiItyO/9nZi11RdQYhIE9xyxlAOy6xtj9hVqPaISLYjv5QOibEkxTd1cdDmUYEQiTDt4qK594LRlFSoPSLS7cgvpVuArh5ABUIkIg3ITOGPZw7js41qj4hk2/NLA9ZADSoQIhHrrDHftkdofERk2pFfGrAGalCBEIlov5s89JvxETsLSr2OI62oqKySwrIquuoKQkRaIiE2mnsvGEO5xkdEnJ2+AZG6xSQiLdY/I5n/PWs4izft4//e+trrONJKAj0GAjwqEGZ2tZmtNLNVZnaNn/1mZneZ2TozW2FmYzyIKRIxJo/qznnje3Lfe+vVHhEhAj2KGjwoEGY2DLgUGA+MBE4zswENDvsRMND3mA3cH9SQIhHoptNq2yN+ofEREWFHfinRUUZmSmRdQQwGPnPOlTjnqoD3gbMaHDMZeMLVWgh0MLOuwQ4qEknqj4+4+pnPqa7R+hHhbEd+GV3aJxAdZQE7hxcFYiVwtJmlmVkicCrQs8Ex3YGt9V5v8237DjObbWbZZpadl6fLZpGDGZCZwu+nDGPhhr3c9Y7GR4Sz7fmlAZvmu07QC4Rz7kvgL8CbwOvAMqC6hZ/1kHMuyzmXlZGR0XohRSLY2WN7cNaY7tz937Us3rTX6zjSQjvyS+kWwAZq8KiR2jn3qHNurHPuGGAf0LBrxXa+e1XRw7dNRFrBrZOH0bNTItc8s4yC0kqv40gzVdc4cgK4klwdr3oxZfq+9qK2/eGpBoe8DFzk6810JFDgnNsZ5JgiESs5PoY7zh1FTmEZv3lppdazDjN5ReVUBXChoDpejYN4wcxWA68AP3PO5ZvZ5WZ2uW//a8AGYB3wMDDHo5wiEWt0r45ce8JAXlm+gxeX6gI9nHw7BiKwBSIwc8QehHPuaD/bHqj33AE/C2ookTboikkD+GDtbm7610rG9u5In/QkryNJEwRjDARoJLVImxYdZdx+7iiio4xrn1umqTjCxLcFIgIbqUUkdHTv0I4/njmcz7fkc8+767yOI02wI7+UlIQYUhJiA3oeFQgR4fSR3ThzdHfu/u86lm7Z53UcOYjt+WUBb38AFQgR8fnd5KF0aZ/Atc8uo7i8yus4cgA7ArxQUB0VCBEBoH1CLHdMHcXWvSXc+soqr+PIAewoKKVramDbH0AFQkTqGdenE3MmDeC57G289oWGHoWi/eVV5JdU6gpCRILv6hMGMrJnB254YcU3/e0ldNStDKg2CBEJutjoKO6aOorqGse1zyzTrK8hZnt+4FeSq6MCISLf0zstid9PGcaiTXu5V11fQ8r2fbVXEGqDEBHPnDWmB1NGdePOd9ayZLNmfQ0Vq3YUkBwfoysIEfHW76cMo1uHBK56ehmFZZr1NRR8viWfUT07BHShoDoqECLSqJSEWO6cOrp21td/atZXr+0vr+KrnELG9OoQlPOpQIjIAY3p1ZFrjh/Iy8t38M/PNeurl5Zvy6fGwejeHYNyPhUIETmoOccNYHyfTvz2pZVs3rPf6zht1udb8gEY01MFQkRCRHSUcfvUUURFGVc/s4xKzfrqiaWb99E/I4nUxMBO0ldHBUJEmqR7h3b86azhLNuaz21vrvE6TpvjnOPzrfmM7hWcqwdQgRCRZjhtRDfOP6IXD76/gVdX7PA6TpuyeU8Je/dXMEYFQkRC1c2nD2FMrw5c/48VfLmz0Os4bUbdNOxjencI2jlVIESkWeJjonlg2lhSEmKYPT+b/JIKryO1CUu37CM5PoaBmSlBO6cKhIg0W2b7BB64cCy7Csq58unPtVRpECzdnM/InqlBGSBXRwVCRFpkTK+O3Dp5KB+u3c3/vvaV13EiWklF3QC54LU/AMQE9WwiElGmju/Fml1FzP14I/0zk7jgiN5eR4pIy7cWUOMIeoHQFYSIHJLf/HgIkwZlcNO/VvHxut1ex4lIdQ3Uo4M0xUYdFQgROSTRUcbd542mf0YSVyxYwvq8Yq8jRZzPt+yjX0YSHRLjgnpeFQgROWQpCbE8On0csdFRXDxvMbuLy72OFDGccyzdks/oIE2vUZ8KhIi0ip6dEnnooix2FZYxfe4iijQ9eKtYn7e/doBcEMc/1FGBEJFWM7Z3R+6fNpY1OUVc+kQ2ZZXVXkcKe2+t3gXApEGZQT+3JwXCzK41s1VmttLMnjazhAb7Z5hZnpkt8z0u8SKniDTfcYMyue2nI1m4YS9XP6MxEofqjVU5DO+eSvcgrCDXUNALhJl1B64Cspxzw4BoYKqfQ591zo3yPR4JakgROSRTRnfn5tOH8MaqXfxaCw212M6CUpZtzeeUYV08Ob9X4yBigHZmVgkkApr1SyTCzJzYl737K7j7v+vISInnupMHeR0p7Ly5qvb20slDvSkQQb+CcM5tB24DtgA7gQLn3Jt+Dv2Jma0ws+fNrGdQQ4pIq/jFiYdx3vie3PPuOuZ9vNHrOGHn9ZU5DMhMZkBmsifn9+IWU0dgMtAX6AYkmdm0Boe9AvRxzo0A3gIeb+SzZptZtpll5+XlBTK2iLSAmfH7ycM4aUhnfvfqal5erpsFTbV3fwWLNu3l5KGdPcvgRSP1CcBG51yec64SeBGYUP8A59we51xdR+pHgLH+Psg595BzLss5l5WRkRHQ0CLSMjHRUdx13mjG9enE/3tuGe+uyfU6Ulh4+8tdVNc4Thna1bMMXhSILcCRZpZoZgYcD3xZ/wAzq/8TOaPhfhEJLwmx0Tx8URaHdU7hsieW8OaqHK8jhbw3VubQvUM7hnVv71kGL9ogPgOeB5YCX/gyPGRmt5rZGb7DrvJ1g11ObY+nGcHOKSKtK7VdLE9dciSDu7VnzpNL+feKnV5HClnF5VV8uG43Jw3tTO3/o73hSS8m59zNwM0NNt9Ub/+NwI1BDSUiAZeaGMuCWeOZ+dhirnx6KZXVo5gyurvXsULOe2tyqaiq4RSPei/V0UhqEQmqlIRYHr94PEf0TePa55bx/JJtXkcKOa+vzCEtKY6sPp08zaECISJBlxQfw9wZ4zhqQDrXP7+c5xZv9TpSyCivqua9NXmcOKRzUFeP80cFQkQ80S6utuH6qAHp/PKFFTyzaIvXkULCwg17KS6v4sQh3nVvraMCISKeqevddOxhGdzw4hcsWLjZ60iee2t1Du1io5k4IN3rKCoQIuKthNhoHrxwLD88PJPfvLSSO99e22bnbnLO8fbqXI45LJ2E2Giv46hAiIj36orET8b04Pa3v+Z//vlFm5wF9ovtBeQUlnHiEG97L9XxarI+EZHviI2O4rafjqBragL3vLuO3MJy7j5/NIlxbefX1FurdxFl8MPDg7/2gz+6ghCRkGFmXHfyIP4wZRjvrsnl3AcXklNQ5nWsoHlr9S6y+nSiU1Jw155ujAqEiIScaUf25uGLstiQV8wZ93zE8q35XkcKuK17S/gqp4gTB3vfe6mOCoSIhKTjB3fmxTkTiYuJ4pwHP434mWDrlhYNhe6tdVQgRCRkDeqSwr9+NpGRPTpw1dOf86f/fBmxjddvrd7FwMxk+qQneR3lGyoQIhLS0pLjWXDJEVxwRC8efH8DF81dxJ7i8oO/MYzkl9Su/RBKVw+gAiEiYSAuJoo/njmcv549guzN+zj97shql3h3TS7VNU4FQkSkpc7J6skLl0/AzDjr/k/4n39+QW5R+PdyennZDjq3j2dkjw5eR/kOFQgRCSvDe6Ty6pVHMe2IXjy3eCuT/vYed7z9NfvLq7yO1iLb80t57+s8zsnqSZTHk/M1pAIhImGnY1Icv5s8jLd+cSyTBmVwx9trOe6293gueys1NeE1Tcezvplsz8nq6XGS71OBEJGw1Tc9ifsuGMsLV0ygW4d2/PL5FZx+z0cs3LDH62hNUlVdwz+yt3L0wAx6dkr0Os73qECISNgb27sj/5wzgTunjmLf/gqmPrSQqQ99ypurcqgO4SuK97/OY2dBGeePD72rB9BcTCISIcyMyaO6c/LQLsz/dDPzPtnE7PlL6NUpkZkT+3D+Eb2Ij/F+htT6nl60hfTkeI4PodHT9ekKQkQiSkJsNJce04/3r5/EfReMITMlnt+9spqTb/+At1fvCpmpxHMKyvjvV7n8NKsHsdGh+as4NFOJiByimOgoTh3eleevmMDjF48nJjqKS57I5qK5i1i1o8DzQvFc9lZqHEwdF5q3l0C3mESkDTj2sAwmXH008z/dzO1vf82P7/qI/hlJnDq8Kz8a1pXBXVMwa34X002797NtXyl5xWXkFZVTWlFDn/REBmam0C8j6TuL/tTUOCqqa6iorqG8soZnF2/lqAHp9E4Lnak1GjKvq2hrycrKctnZ2V7HEJEQt29/Bf/+YievfbGThRv2UOOgW2oCRw/M4KiB6Uzon0Zacnyj7y+rrOY/K3fyxKeb+XxLfqPHmUFKfAyV1Y7K6hqq/DSW33P+aE4b0a01vq0WM7Mlzrksv/tUIESkrdpTXM6bq3fxwdd5fLxuN4VltYPtkuKiyUiJJyMlntR2ccREGdHRhgGfrN/D3v0V9E1P4oIjejG8e+o3x8bFRLFpdwlrc4tYu6uYgtJK4mKiiI02YqKiiIuJIt73SE2M47ThXT0fHKcCISJyEFXVNXyxvYDFm/ayq7Cc3KJy8orKKCitorqm9gqgusYxqHMKF/6gNxP7p3v+y701HKhAqA1CRITaRu3RvToyuldHr6OEDPViEhERv1QgRETEL08KhJlda2arzGylmT1tZgkN9seb2bNmts7MPjOzPl7kFBFpy4JeIMysO3AVkOWcGwZEA1MbHDYL2OecGwDcDvwluClFRMSrW0wxQDsziwESgYarkU8GHvc9fx443loyikVERFos6AXCObcduA3YAuwECpxzbzY4rDuw1Xd8FVAApDX8LDObbWbZZpadl5cX2OAiIm2MF7eYOlJ7hdAX6AYkmdm0lnyWc+4h51yWcy4rIyOjNWOKiLR5XtxiOgHY6JzLc85VAi8CExocsx3oCeC7DZUKhMcKICIiEcKLgXJbgCPNLBEoBY4HGg6BfhmYDnwKnA381x1kyPeSJUt2m9lmaotJgW9zc56nA7tb8P3U/6zm7G+4/UCv/T1X7uYd05RtB8pbf5t+5k3bf7Cfuf59Nj/Xwfa3JHfvRs/inAv6A/gd8BWwEpgPxAO3Amf49icA/wDWAYuAfs347Ida8hzIbuH38lBL9jfcfqDXjeRV7mYc05RtB8qrn/mh525KVv3Mvcnd2MOTqTacczcDNzfYfFO9/WXAT1v48a8cwvNDPV9z9jfcfqDX/p4rd/OOacq2g+XVz7x5+w/2M9e/z8YFO7dfETNZ36Eys2zXyIRVoUy5gy9cs4drbgjf7OGau46m2vjWQ14HaCHlDr5wzR6uuSF8s4drbkBXECIi0ghdQYiIiF8qECIi4pcKhIiI+KUCcRBmdrSZPWBmj5jZJ17naQ4zizKzP5rZ3WY23es8TWVmk8zsQ9/PfZLXeZrLzJJ8c4Sd5nWWpjKzwb6f9/NmdoXXeZrKzKaY2cO+5QFO8jpPc5hZPzN71Mye9zpLYyK6QJjZXDPLNbOVDbafYmZrfOtN3HCgz3DOfeicuxx4lW9nmA241shO7ZxXPYBKYFugstbXSrkdUEztgMmg5IZWyw7wK+C5wKT8vlb6e/6l7+/5OcDEQOatl681cr/knLsUuBw4N5B562ul7Bucc7MCm/TQRHQvJjM7htpfNE+42rUnMLNo4GvgRGp/+SwGzqN2XYo/NfiIi51zub73PQfMcs4VhUt232Ofc+5BM3veOXd2mOTe7ZyrMbPOwP855y4IdO5WzD6S2pmHE3zfx6vhkNs5l2tmZwBXAPOdc0+FS27f+/4OPOmcWxro3AHIHpR/my3hyUjqYHHOfeBnNbrxwDrn3AYAM3sGmOyc+xPg95aAmfWidlryoBQHaJ3sZrYNqPC9rA5g3G+01s/cZx+107AERSv9zCcBScAQoNTMXnPO1YR6bt/nvAy8bGb/BgJeIFrp523An4H/BKs4QKv/PQ9ZEV0gGvHNWhM+24AjDvKeWcBjAUvUdM3N/iJwt5kdDXwQyGAH0azcZnYWcDLQAbgnoMkOrlnZnXO/BjCzGfiuhAKarnHN/ZlPAs6itiC/FshgB9Hcv+NXUjtDdKqZDXDOPRDIcAfR3J95GvBHYLSZ3egrJCGlLRaIZvPNHRV2nHMl1Ba3sOKce5Ha4ha2nHPzvM7QHM6594D3PI7RbM65u4C7vM7REs65PdS2nYSsiG6kbsQ3a0349PBtCwfhmj1cc0P4Zlfu4Avn7H61xQKxGBhoZn3NLA6YSu36E+EgXLOHa24I3+zKHXzhnN2/lsxVHi4P4Glq172u6+Y5y7f9VGp7G6wHfu11zkjKHq65wzm7cit7oB4R3c1VRERari3eYhIRkSZQgRAREb9UIERExC8VCBER8UsFQkRE/FKBEBERv1QgJKKZWXGQz9cqa4ZY7ZoYBWa2zMy+MrPbmvCeKWY2pDXOLwIqECLNYmYHnL/MOTehFU/3oXNuFDAaOM3MDrZOwxRqZ5EVaRUqENLmmFl/M3vdzJZY7cp1h/u2n25mn5nZ52b2tm89CszsFjObb2YfA/N9r+ea2XtmtsHMrqr32cW+r5N8+5/3XQE86ZuaGjM71bdtiZndZWYHXDPCOVcKLKN2tlDM7FIzW2xmy83sBTNLNLMJwBnA33xXHf0b+z5FmkoFQtqih4ArnXNjgeuA+3zbPwKOdM6NBp4BflnvPUOAE5xz5/leH07tlOTjgZvNLNbPeUYD1/je2w+YaGYJwIPAj3znzzhYWDPrCAzk2ynbX3TOjXPOjQS+pHaah0+onffneufcKOfc+gN8nyJNoum+pU0xs2RgAvAP33/o4dtFiXoAz5pZVyAO2FjvrS/7/idf59/OuXKg3Mxygc58f3nURc65bb7zLgP6ULsK2QbnXN1nPw3MbiTu0Wa2nNricIdzLse3fZiZ/YHa9TKSgTea+X2KNIkKhLQ1UUC+795+Q3dTu8Tpy74FdG6pt29/g2PL6z2vxv+/paYccyAfOudOM7O+wEIze845twyYB0xxzi33LUw0yc97D/R9ijSJbjFJm+KcKwQ2mtlPoXbJSjMb6dudyrfz908PUIQ1QL96y1Wee7A3+K42/gz8yrcpBdjpu61Vf73uIt++g32fIk2iAiGRLtHMttV7/ILaX6qzfLdvVgGTfcfeQu0tmSXA7kCE8d2mmgO87jtPEVDQhLc+ABzjKyy/BT4DPga+qnfMM8D1vkb2/jT+fYo0iab7FgkyM0t2zhX7ejXdC6x1zt3udS6RhnQFIRJ8l/oarVdRe1vrQW/jiPinKwgREfFLVxAiIuKXCoSIiPilAiEiIn6pQIiIiF8qECIi4pcKhIiI+PX/AWtLmDCh60YdAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"learn.remove_cb(ActivationStats)\n",
"admin_init(learn.model, scales)\n",
"learn.lr_find()"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"WANDB_NAME = f'admin_init_spu_20e'\n",
"GROUP = 'admin'\n",
"NOTES = 'SentencePieceUnigram tokenizer trained on wiki 103, 20e, 32100 vcb_sz, Early ADMIN testing'\n",
"CONFIG = {}\n",
"TAGS = ['admin','spu']"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmorgan\u001b[0m (use `wandb login --relogin` to force relogin)\n"
]
},
{
"data": {
"text/html": [
"\n",
" Tracking run with wandb version 0.10.19<br/>\n",
" Syncing run <strong style=\"color:#cdcd00\">admin_init_spu_20e</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
" Project page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox</a><br/>\n",
" Run page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/3ks1ewq1\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox/runs/3ks1ewq1</a><br/>\n",
" Run data is saved locally in <code>/home/morgan/ml/projects/transformers_sandbox/nbs/exploration/wandb/run-20210221_193133-3ks1ewq1</code><br/><br/>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<h1>Run(3ks1ewq1)</h1><iframe src=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/3ks1ewq1\" style=\"border:none;width:100%;height:400px\"></iframe>"
],
"text/plain": [
"<wandb.sdk.wandb_run.Run at 0x7fc180055d30>"
]
},
"execution_count": 33,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#hide_output\n",
"wandb.init(reinit=True, project=\"transformers_sandbox\", entity=\"fastai_community\", \n",
" name=WANDB_NAME, group=GROUP, notes=NOTES, tags=TAGS, config=CONFIG)"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>perplexity</th>\n",
" <th>bpc</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>5.516002</td>\n",
" <td>5.488830</td>\n",
" <td>0.231003</td>\n",
" <td>241.973953</td>\n",
" <td>7.918708</td>\n",
" <td>01:41</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>4.964672</td>\n",
" <td>4.980829</td>\n",
" <td>0.285725</td>\n",
" <td>145.595062</td>\n",
" <td>7.185818</td>\n",
" <td>01:43</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>4.570927</td>\n",
" <td>4.636693</td>\n",
" <td>0.315706</td>\n",
" <td>103.202446</td>\n",
" <td>6.689333</td>\n",
" <td>01:45</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>4.294974</td>\n",
" <td>4.476145</td>\n",
" <td>0.328804</td>\n",
" <td>87.895203</td>\n",
" <td>6.457713</td>\n",
" <td>01:44</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>4.072177</td>\n",
" <td>4.286242</td>\n",
" <td>0.347858</td>\n",
" <td>72.692741</td>\n",
" <td>6.183739</td>\n",
" <td>01:46</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>3.882438</td>\n",
" <td>4.174093</td>\n",
" <td>0.354857</td>\n",
" <td>64.980888</td>\n",
" <td>6.021944</td>\n",
" <td>01:43</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>3.688582</td>\n",
" <td>4.089933</td>\n",
" <td>0.362383</td>\n",
" <td>59.735886</td>\n",
" <td>5.900526</td>\n",
" <td>01:42</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>3.555619</td>\n",
" <td>4.045366</td>\n",
" <td>0.370683</td>\n",
" <td>57.132111</td>\n",
" <td>5.836230</td>\n",
" <td>01:44</td>\n",
" </tr>\n",
" <tr>\n",
" <td>8</td>\n",
" <td>3.410418</td>\n",
" <td>4.018274</td>\n",
" <td>0.375051</td>\n",
" <td>55.605038</td>\n",
" <td>5.797144</td>\n",
" <td>01:42</td>\n",
" </tr>\n",
" <tr>\n",
" <td>9</td>\n",
" <td>3.273482</td>\n",
" <td>3.989733</td>\n",
" <td>0.379958</td>\n",
" <td>54.040470</td>\n",
" <td>5.755968</td>\n",
" <td>01:46</td>\n",
" </tr>\n",
" <tr>\n",
" <td>10</td>\n",
" <td>3.123741</td>\n",
" <td>3.992993</td>\n",
" <td>0.383907</td>\n",
" <td>54.216911</td>\n",
" <td>5.760671</td>\n",
" <td>01:41</td>\n",
" </tr>\n",
" <tr>\n",
" <td>11</td>\n",
" <td>2.994096</td>\n",
" <td>3.990759</td>\n",
" <td>0.385993</td>\n",
" <td>54.095951</td>\n",
" <td>5.757449</td>\n",
" <td>01:42</td>\n",
" </tr>\n",
" <tr>\n",
" <td>12</td>\n",
" <td>2.877162</td>\n",
" <td>4.005277</td>\n",
" <td>0.387474</td>\n",
" <td>54.887009</td>\n",
" <td>5.778393</td>\n",
" <td>01:42</td>\n",
" </tr>\n",
" <tr>\n",
" <td>13</td>\n",
" <td>2.746851</td>\n",
" <td>4.023293</td>\n",
" <td>0.390281</td>\n",
" <td>55.884834</td>\n",
" <td>5.804385</td>\n",
" <td>01:45</td>\n",
" </tr>\n",
" <tr>\n",
" <td>14</td>\n",
" <td>2.615041</td>\n",
" <td>4.046219</td>\n",
" <td>0.392509</td>\n",
" <td>57.180866</td>\n",
" <td>5.837461</td>\n",
" <td>01:44</td>\n",
" </tr>\n",
" <tr>\n",
" <td>15</td>\n",
" <td>2.524932</td>\n",
" <td>4.072429</td>\n",
" <td>0.393079</td>\n",
" <td>58.699383</td>\n",
" <td>5.875273</td>\n",
" <td>01:45</td>\n",
" </tr>\n",
" <tr>\n",
" <td>16</td>\n",
" <td>2.434323</td>\n",
" <td>4.087971</td>\n",
" <td>0.392887</td>\n",
" <td>59.618816</td>\n",
" <td>5.897696</td>\n",
" <td>01:42</td>\n",
" </tr>\n",
" <tr>\n",
" <td>17</td>\n",
" <td>2.361165</td>\n",
" <td>4.097982</td>\n",
" <td>0.393126</td>\n",
" <td>60.218670</td>\n",
" <td>5.912139</td>\n",
" <td>01:46</td>\n",
" </tr>\n",
" <tr>\n",
" <td>18</td>\n",
" <td>2.310880</td>\n",
" <td>4.103701</td>\n",
" <td>0.393512</td>\n",
" <td>60.564026</td>\n",
" <td>5.920389</td>\n",
" <td>01:43</td>\n",
" </tr>\n",
" <tr>\n",
" <td>19</td>\n",
" <td>2.289599</td>\n",
" <td>4.104567</td>\n",
" <td>0.393593</td>\n",
" <td>60.616467</td>\n",
" <td>5.921638</td>\n",
" <td>01:44</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with valid_loss value: 5.488830089569092.\n",
"Better model found at epoch 1 with valid_loss value: 4.980829238891602.\n",
"Better model found at epoch 2 with valid_loss value: 4.636692523956299.\n",
"Better model found at epoch 3 with valid_loss value: 4.476145267486572.\n",
"Better model found at epoch 4 with valid_loss value: 4.28624153137207.\n",
"Better model found at epoch 5 with valid_loss value: 4.174093246459961.\n",
"Better model found at epoch 6 with valid_loss value: 4.089932918548584.\n",
"Better model found at epoch 7 with valid_loss value: 4.045366287231445.\n",
"Better model found at epoch 8 with valid_loss value: 4.018273830413818.\n",
"Better model found at epoch 9 with valid_loss value: 3.9897332191467285.\n"
]
}
],
"source": [
"learn.remove_cb(ActivationStats)\n",
"admin_init(learn.model, scales)\n",
"learn.fit_one_cycle(20, 1e-3, cbs=[WandbCallback(log_model=False, log_preds=False)])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Train - T5 Tokenizer"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"T5 SentencePiece BPE Pretrained Tokenizer"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"from transformers import T5TokenizerFast\n",
"\n",
"class T5TokenizerTransform(Transform):\n",
" def __init__(self):\n",
" self.tok = T5TokenizerFast.from_pretrained('t5-small')\n",
" self.pad_token_id = self.tok.pad_token_id\n",
" self.eos_token_id = self.tok.eos_token_id\n",
" self.bos_token_id = self.tok.bos_token_id\n",
" self.unk_token_id = self.tok.unk_token_id\n",
" self.vcb_size = self.tok.vocab_size\n",
" \n",
" def __call__(self, o, **kwargs):\n",
" return LMTensorText(self.tok(o)['input_ids'])\n",
" \n",
" def encodes(self, o):\n",
" return LMTensorText(self.tok(o)['input_ids'])\n",
" \n",
" def decodes(self, o):\n",
" return TitledStr(self.tok.decode(o.numpy()))\n",
" \n",
" @property\n",
" def vocab_size(self): return self.vcb_size"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"t5_tok = T5TokenizerTransform()\n",
"tok = t5_tok"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Loading cached processed dataset at /home/morgan/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/47c57a6745aa5ce8e16a5355aaa4039e3aa90d1adad87cef1ad4e0f29e74ac91/cache-bd8ef24658a2bf53.arrow\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4aba2cec8c844241ae3221804c011c4a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=2461.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Token indices sequence length is longer than the specified maximum sequence length for this model (585 > 512). Running this sequence through the model will result in indexing errors\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"def encode(examples): return {'token_ids' : t5_tok(examples['text'])}\n",
"\n",
"tok_ds = train_ds.map(encode, batched=False)\n",
"val_tok_ds = valid_ds.map(encode, batched=False)\n",
"all_toks = tok_ds['token_ids'] + val_tok_ds['token_ids']\n",
"df['toks'] = all_toks "
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"#Run setup\n",
"n_epochs = 10\n",
"bs = 12\n",
"sl = 512\n",
"n_layers = 6\n",
"pad_id = tok.pad_token_id\n",
"vocab_size = tok.vocab_size"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>text</th>\n",
" <th>text_</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>It has been estimated that average @-@ sized bacteria contain about 2 million proteins per cell ( e.g. E. coli and Staphylococcus aureus ). Smaller bacteria, such as Mycoplasma or spirochetes contain fewer molecules, namely on the order of 50 @,@ 000 to 1 million. By contrast, eukaryotic cells are larger and thus contain much more protein. For instance, yeast cells were estimated to contain about 50 million proteins and human cells on the order of 1 to 3 billion. Note that bacterial genomes encode about 10 times fewer proteins than humans ( e.g. small bacteria &lt;unk&gt; 1 @,@ 000, E. coli : &lt;unk&gt; 4 @,@ 000, yeast : &lt;unk&gt; 6 @,@ 000, human : &lt;unk&gt; 20 @,@ 000 ).&lt;/s&gt; Even though cadmium and its compounds are toxic in certain forms and concentrations, the British Pharmaceutical Codex from 1907 states that cadmium iodide was used as a medication to treat</td>\n",
" <td>has been estimated that average @-@ sized bacteria contain about 2 million proteins per cell ( e.g. E. coli and Staphylococcus aureus ). Smaller bacteria, such as Mycoplasma or spirochetes contain fewer molecules, namely on the order of 50 @,@ 000 to 1 million. By contrast, eukaryotic cells are larger and thus contain much more protein. For instance, yeast cells were estimated to contain about 50 million proteins and human cells on the order of 1 to 3 billion. Note that bacterial genomes encode about 10 times fewer proteins than humans ( e.g. small bacteria &lt;unk&gt; 1 @,@ 000, E. coli : &lt;unk&gt; 4 @,@ 000, yeast : &lt;unk&gt; 6 @,@ 000, human : &lt;unk&gt; 20 @,@ 000 ).&lt;/s&gt; Even though cadmium and its compounds are toxic in certain forms and concentrations, the British Pharmaceutical Codex from 1907 states that cadmium iodide was used as a medication to treat \"</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>, collided with the back of the Renault, knocking the McLaren's front wing off the car. Suffering handling difficulties, Hamilton returned to the pit @-@ lane for a new nose section, and rejoined in 18th place. Räikkönen took second place when he passed Kubica on lap three ; Heidfeld took fourth when he passed Trulli and Kovalainen in separate manoeuvres. Further down the field, Vettel retired from the race on the first lap after twice colliding with other cars ; Button, Sutil and Coulthard pitted to repair early damage.&lt;/s&gt; = = Wheelchair basketball = =&lt;/s&gt; Jardine then ordered his team to move to bodyline positions immediately after Woodfull's injury. Jardine wrote that Larwood had asked for the field, while Larwood said that it was Jardine's decision. The capacity Saturday afternoon crowd viewed this as hitting a man when he was down. Journalist – cricketer Dick Whitington wrote that Jardine's actions</td>\n",
" <td>collided with the back of the Renault, knocking the McLaren's front wing off the car. Suffering handling difficulties, Hamilton returned to the pit @-@ lane for a new nose section, and rejoined in 18th place. Räikkönen took second place when he passed Kubica on lap three ; Heidfeld took fourth when he passed Trulli and Kovalainen in separate manoeuvres. Further down the field, Vettel retired from the race on the first lap after twice colliding with other cars ; Button, Sutil and Coulthard pitted to repair early damage.&lt;/s&gt; = = Wheelchair basketball = =&lt;/s&gt; Jardine then ordered his team to move to bodyline positions immediately after Woodfull's injury. Jardine wrote that Larwood had asked for the field, while Larwood said that it was Jardine's decision. The capacity Saturday afternoon crowd viewed this as hitting a man when he was down. Journalist – cricketer Dick Whitington wrote that Jardine's actions were</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"df['lens'] = df['toks'].apply(len)\n",
"splits = ColSplitter()(df)\n",
"tfms = [attrgetter(\"text\"), t5_tok]\n",
"dsets = Datasets(df, [tfms], splits=splits, dl_type=LMDataLoader)\n",
"dl_kwargs = [{'lens':df['lens'].values[splits[0]]},\n",
" {'val_lens':df['lens'].values[splits[1]]}]\n",
"dls = dsets.dataloaders(bs=bs, seq_len=sl, dl_kwargs=dl_kwargs, shuffle_train=True)\n",
"dls.show_batch(max_n=2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### LR Find"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"SuggestedLRs(lr_min=0.05248074531555176, lr_steep=0.43651583790779114)"
]
},
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAneElEQVR4nO3deXhU9d3+8fdnspKNkIU97CA7QSLuS9XWHdy3qrhUHn2sWmtttfapS3+2atUqVusuaqt1qbZorXWr4i5hR0TCTlizEUISsn5/f2S0ESeQhJk5mZn7dV1zZeacM3Pu5CK5Odv3mHMOERGRXfm8DiAiIl2TCkJERAJSQYiISEAqCBERCUgFISIiAakgREQkoHivAwRLTk6OGzRokNcxREQiyty5c0udc7mB5kVNQQwaNIjCwkKvY4iIRBQzW9vWPO1iEhGRgFQQIiISkApCREQCUkGIiEhAKggREQlIBSEiIgGpIHZRWdPA3LUVNDVrGHQRiW1Rcx3E3nDOMW/dNp79bB2vLdpIXWMzfbsnc9Z+Azhrvzx6d0/2OqKISNjFfEGsL6/h0qcLWba5itTEOE6f1J99B/Tg7ws28Ie3lzPj3SLy8zIZkpPK4NxUBmal0tDUTHl1PRU19WyvbaDZgcPhHNQ3NlNT30R1fSO19U30zEhmcHYKA7NTGZCdQlZqIlkpiWR0SyDOZ15/+yIibYr5gujdPZne3ZO54MBBTMnvS1pSy4/ktEn9WVtWzV/nrGfu2greW17Ci3OLv/Ven0F6cgI+AzPDgIQ4H6lJcaQmxZMU72PB+gr+uWgju+6xMoPMbglkpyWRlZpIdmoi2WmJ5KQlkZ2WRI+UBBLjfCTE+0jwtXxmVmoiWamJpCXFY6ZyEZHQsmi55WhBQYEL9VAbVTsbWF9eS3KCj6zURDKSE/C1YyugvrGZ9RU1FFfUUuHf8qiorqe8pp7y6nrKdtRTVl1P2Y46Kmoa9vh5ifE+clITyU5LIictkZ7pyQzITmFAVgqD/Fsq3bslBONbFpEoZ2ZznXMFgebF/BZER6QnJzC6b8f/8CbG+xiam8bQ3LQ9LtvQ1OwvkQYampppaGqmvrGZ6vpGyqsbKK+uo6y6ntKqekp31LG1qo7FG7ZTuqPuW5+TlZrIwOyWwhjWM43hPdPYp3c6eT1S2lVqIiIqiC4mIc5Hz4xkemZ07MB4dV0j68prWFtWw7ryalaX1rC2rJpPV5XxyvwN3yyXkhjH2H7dmZiXSX5eJpMG9aBnug7Ci8h3qSCiRGpSPKP6ZDCqT8Z35lXtbKBo6w6KtlSxdON2FhRX8uRHa6hvagZgn17pHDo8h0OG53DAkGySE+LCHV9EuiAdg4hRdY1NfLmpik9XlfFBUQlz1lRQ39hMcoKPQ4blcOTIXhw5sqdO8RWJcrs7BqGCEABq65v4fE05/1m2lbe/3EJxRS0A+XmZHDu2N8eM6c3gnFSPU4pIsKkgpEOcc6zYuoM3l27h319sZlFxJQAje6czNb8fU/L70i+zm8cpRSQYVBCyVzZsq+WNJZt5bdFG5q/bBsDkQVmcu/8AThjfh4Q4jdgiEqlUEBI068pqmLVwAy/P28Cq0mp6ZyQz7aBBnDt5AN1TdO2FSKRRQUjQNTc73i8q4fEPVvPhilJSEuO4+ODBXHroEBWFSARRQUhILd24nQfeW8E/F20iIzme6YcN4cKDB38zbImIdF27KwjtPJa9NrpvBg+cuy+vX3Uokwdncdebyznq7vd4Y8kmouU/ICKxSAUhQTO6bwaPTduPv11+EFmpSVz253lMf2YumyprvY4mIp2ggpCgmzSwB7N+fDA3HDeSD4pKOPru93nq4zU06yZMIhFFBSEhkRDn438OH8pb1xzOvgN7cNOsLzj9oY8p2lLldTQRaScVhIRUXlYKT188mXvOnMCq0mqOn/EB9769nPrGZq+jicgeqCAk5MyMU/ftz9s/PZxjx/bh3reLOOXBj/hqs7YmRLoyFYSETU5aEvefM5GHzpvE5sqdnHT/h/zpvZU06diESJekgpCwO3Zsb9685jCOHNmTO95YxjmPfkp5db3XsURkFyErCDN7wsy2mtmSVtOyzOwtMyvyf+3RxnubzGyB/zErVBnFO9lpSfzpvH25+4wJLFi/jakPfKgD2CJdTCi3IGYCx+4y7XrgHefccOAd/+tAap1z+f7HlBBmFA+ZGadN6s/z0w+gtr6ZUx/8mP98tdXrWCLiF7KCcM7NBsp3mTwVeMr//Cng5FCtXyLHxAEt103kZaVwycw5PPvZOq8jiQjhPwbRyzm3yf98M9CrjeWSzazQzD41s5Pb+jAzm+5frrCkpCTYWSWM+mZ246XLD+TwEbn88pXFPDp7ldeRRGKeZwepXcsgPW2dvjLQP3jUucC9Zja0jc94xDlX4JwryM3NDVVUCZOUxHgePr+AE8b14bbXv+QPby3XWE4iHgr3cJtbzKyPc26TmfUBAu5wds5t8H9dZWbvAROBleGLKV5JjPcx45yJdEuM4753iqiua+TGE0ZhZl5HE4k54d6CmAVM8z+fBvxj1wXMrIeZJfmf5wAHA0vDllA8F+cz7jxtPNMOHMhjH67mlleXaktCxAMh24Iws+eAI4AcMysGbgJuB14ws0uAtcCZ/mULgMuccz8CRgEPm1kzLQV2u3NOBRFjfD7j5iljiI/z8fiHqwG46aTR2pIQCaOQFYRz7pw2Zh0VYNlC4Ef+5x8D40KVSyKHmfGrE0ZhwGMfrsY5x81TxqgkRMJEt/ySLs3MuPGEUfh8xiOzV9Hs4NapKgmRcFBBSJdnZtxw3EjM4OH3V9HsHL+ZOhafTyUhEkoqCIkIZsb1x44kzowH/QP8/faUcSoJkRBSQUjEMDOuO2Yf4nzG/e+uoNk5bj91vEpCJERUEBJRzIyffn8EPjPue6cI5+CO01QSIqGggpCIY2Zc8/0ROGDGO0XEx/m47WQdkxAJNhWERKxrjh5OY1MzD763knif6ewmkSBTQUjE+vqYRGOz45HZq4iPM359oi6mEwkWFYREtK9PgW1scjzxUcsV1yoJkeBQQUjEMzP+78RRADzx0Wqc07AcIsGggpCo8HVJ+KxlWI5m57hFw3KI7BUVhESNr4fliPMZD89ehdOwHCJ7RQUhUcXMuP64kQA8PHsVPVIS+OkP9vE4lUhkUkFI1Pm6JLbVNDDj3RX0SE3kooMHex1LJOKoICQqmRm3nTKWipp6bnl1KVmpiUzN7+d1LJGI4tk9qUVCLT6u5falBwzJ4toXFvLeVwHvcCsibVBBSFRLTojj0QsKGNErnSv+Mo8lGyq9jiQSMVQQEvXSkxN48qL9yExJ5KKZc1hfXuN1JJGIoIKQmNArI5mZF+1HXUMTFz75Odtq6r2OJNLlqSAkZgzvlc4jFxSwvryW6U/Ppa6xyetIIl2aCkJiygFDsrnrzAl8vqacG/62GOec15FEuiyd5ioxZ8qEvqwuqeYPby9nWK80/veIYV5HEumSVBASk646ahgrS3Zw5xtfMSQnjWPH9vY6kkiXo11MEpPMjDtPH09+XibXPL9Ap7+KBKCCkJiVnBDHIxdMokdKAj96qpDNlTu9jiTSpaggJKb1TE/m8Qv3o2pnAxfPnEN1XaPXkUS6DBWExLxRfTJ44If78tWWKq56bj5NzTqzSQRUECIAHLFPT26eMoZ3lm3lN68t9TqOSJegs5hE/M4/YCBrS6t57MPVDMxO0RDhEvNUECKt3HD8KNaV1/Cb15aS1yOFo0f38jqSiGe0i0mklTifce/Z+Yzt150rn5uv018lpoWsIMzsCTPbamZLWk3LMrO3zKzI/7VHG++d5l+myMymhSqjSCApifE8Nq2ArNRELp45h43bar2OJOKJUG5BzASO3WXa9cA7zrnhwDv+199iZlnATcD+wGTgpraKRCRUeqYn88SF+1Fb38TFM+ewQ6e/SgwKWUE452YD5btMngo85X/+FHBygLceA7zlnCt3zlUAb/HdohEJuX16p/PAD/elaOsOrtbprxKDwn0MopdzbpP/+WYg0BHAfsD6Vq+L/dNEwu6wEbncdNJo3lm2lTvfWOZ1HJGw8uwsJuecM7O9+i+ZmU0HpgMMGDAgKLlEdnXBgYMo2rKDh2evYmjPNM4syPM6kkhYhHsLYouZ9QHwfw10F/kNQOvfwP7+ad/hnHvEOVfgnCvIzc0NeliRr/36pNEcMiyHG19ZzGeryryOIxIW4S6IWcDXZyVNA/4RYJl/Az8wsx7+g9M/8E8T8UxCnI8Hzt2XvKwULv/LPN3XWmJCKE9zfQ74BNjHzIrN7BLgduD7ZlYEHO1/jZkVmNljAM65cuA3wBz/41b/NBFPdU9J4LELCmhsaubSpwt1ZpNEPYuWWy4WFBS4wsJCr2NIDJi9vIQLn/yco0f14qHzJuHzmdeRRDrNzOY65woCzdOV1CIddNiIXH51wmjeXLqFP7y93Os4IiGjsZhEOuGigwfx1eYq7n93BcN6pjE1X2diS/TRFoRIJ5gZt548hsmDs7juxUU6s0mikgpCpJOS4uN45PxJ9M/qxvRn5rJi6w6vI4kElQpCZC9kpiQy88LJxPuMi2Z+TumOOq8jiQSNCkJkLw3ITuHxC/ejpKqOS54qpKZep79KdFBBiARBfl4m9509kcXF27j8z/Oob2z2OpLIXlNBiATJMWN6c9sp43h/eQk/e3EhzRr9VSKcTnMVCaJzJg+goqaeO9/4iqzURG46aTRmupBOIpMKQiTILj98KOU76nnsw9VkpyZy5VHDvY4k0ikqCJEgMzN+efwoyqvrufut5QzITtGFdBKRVBAiIeDzGbefNp4N22q57sVF9MvsRsGgLK9jiXSIDlKLhEhivI+HzptEvx4tF9KtK9MQ4RJZVBAiIdQjNZEnLtyPpmbHRTM/p7KmwetIIu2mghAJscE5qTx8/iTWlddwxbPzaGjSNRISGVQQImFwwJBsbjtlHB+uKOXWV5d6HUekXXSQWiRMzizIY+XWHTw8exXDeqYx7aBBXkcS2S0VhEgY/fzYkawsqeaWV79gcE4qh43I9TqSSJu0i0kkjOJ8xn1n57NP7wyu+Ms8irZUeR1JpE0qCJEwS02K5/FpBSQnxnHxU3M0RLh0We0qCDNLNTOf//kIM5tiZgmhjSYSvfpmduOxCwooqapj+tOF7Gxo8jqSyHe0dwtiNpBsZv2AN4HzgZmhCiUSCybkZXLPmfnMW7eN615ahHMa/VW6lvYWhDnnaoBTgQedc2cAY0IXSyQ2HD+uDz8/dh9eXbiRP7xd5HUckW9p71lMZmYHAj8ELvFPiwtNJJHYcvnhQ1lVUs2Md4oY3jONkyb09TqSCND+LYifADcArzjnvjCzIcB/QpZKJIaYGbedMpaCgT342YsLWVS8zetIIkA7C8I5975zbopz7g7/wepS59xVIc4mEjOS4uN46PxJ5KQlcenThWzZvtPrSCLtPovpWTPLMLNUYAmw1MyuC200kdiSk5bEoxcUULWzUWc2SZfQ3l1Mo51z24GTgX8Bg2k5k0lEgmh03wz+cFY+C4sruVb3tRaPtbcgEvzXPZwMzHLONQD6lysSAseM6c0Nx43kn4s2cce/l3kdR2JYe89iehhYAywEZpvZQGB7qEKJxLrphw1hfUUND7+/irweKZx3wECvI0kMaldBOOdmADNaTVprZt8LTSQRMTNuPmkMG7ft5Nf/WELfzGSOHNnL61gSY9p7kLq7md1jZoX+x91AaoizicS0+Dgf958zkdF9M/jxs/NZulEb7RJe7T0G8QRQBZzpf2wHnuzsSs3sajNbYmZfmNlPAsw/wswqzWyB//Hrzq5LJJKlJsXzxLT9yEhO4NKnCzWwn4RVewtiqHPuJufcKv/jFmBIZ1ZoZmOBS4HJwATgRDMbFmDRD5xz+f7HrZ1Zl0g06JmRzKMXFFBWXcdlz8ylrlGnv0p4tLcgas3skK9fmNnBQG0n1zkK+Mw5V+OcawTep2WMJxFpw7j+3bn7jHwK11Zw4ytLNLCfhEV7C+Iy4AEzW2Nma4A/Av/TyXUuAQ41s2wzSwGOB/ICLHegmS00s3+ZWcCBAc1s+tfHRUpKSjoZRyQynDC+D1cfNZyX5hbz+IervY4jMaC9Q20sdM5NAMYD451zE4EjO7NC59yXwB20DBv+BrAA2HWbeR4w0L/O+4G/t/FZjzjnCpxzBbm5unWjRL+rjxrOsWN687t/LWPeugqv40iU69Ad5Zxz2/1XVAP8tLMrdc497pyb5Jw7DKgAlgdYzw7/89dpuVAvp7PrE4kWPp9x5xnj6dM9mauem09lbYPXkSSK7c0tR63TbzTr6f86gJbjD8/uMr+3mZn/+WR/zrLORxWJHhnJCcw4ZyKbKnfyy5cX63iEhMzeFMTe/Kv8m5ktBV4FrnDObTOzy8zsMv/804ElZraQlgv0znb6LRD5xr4DenDtD0bwz8Wb+Ouc9V7HkShlu/u7a2ZVBC4CA7o559o7VEfIFRQUuMLCQq9jiIRNc7Pjgic+p3BtOa/++BCG90r3OpJEIDOb65wrCDRvt1sQzrl051xGgEd6VyoHkVjk8xn3nDWB1MR4rnxuvoYHl6Dbm11MIuKxnunJ3HXGBJZtruL2f2nkVwkuFYRIhPveyJ5cdPAgZn68hneXbfE6jkQRFYRIFLj+uJGM6pPBz15cxFbdrlSCRAUhEgWS4uO4/5x8auobdSc6CRoVhEiUGNYznV+fOIYPikr582drvY4jUUAFIRJFzpmcx+Ejcvnd68tYW1btdRyJcCoIkShiZtx+2jji44zrXlykXU2yV1QQIlGmT/du3HTSGD5fU87Mj9d4HUcimApCJAqdtm8/jhzZkzv/vYzVpdrVJJ2jghCJQmbG704dR2Kcj+t0VpN0kgpCJEr1ykjmppPGULi2Qmc1SaeoIESi2Kn79uOwEbnc8a9lFFfUeB1HIowKQiSKmRm/PWUsDnQva+kwFYRIlOvfI4WfH7MP7y8v4ZX5G7yOIxFEBSESA84/cBCTBvbg1teWUlJV53UciRAqCJEYEOcz7jhtHDV1Tdw0a4nXcSRCqCBEYsSwnulcffRwXl+8mdcXb/I6jkQAFYRIDJl+2BDG9svg1/9YQnl1vddxpItTQYjEkIQ4H78/fQLbahq45dUvvI4jXZwKQiTGjOqTwRXfG8Y/FmzkraW6A520TQUhEoOu+N4wRvZO58ZXFrOtRruaJDAVhEgMSoz3cdcZEyivruf//qFdTRKYCkIkRo3t152rjxrOqws3MmvhRq/jSBekghCJYZcfMZSJAzL51SuL2Vy50+s40sWoIERiWHycj3vOzKehyXHdSws1VpN8iwpCJMYNzknllyeM4oOiUp75VMOCy3+pIESE8/YfwOEjcvnt61/qDnTyDRWEiGBm3HHaeBLjfPzsxYU06Q50ggpCRPx6d0/mlqljmLu2gsc+WOV1HOkCVBAi8o2T8/vxg9G9uPvN5SzfUuV1HPGYCkJEvmFm3HbKOFKT4rj2hYU0NDV7HUk85ElBmNnVZrbEzL4ws58EmG9mNsPMVpjZIjPb14OYIjEpNz2J204Zx+INlfzx3RVexxEPhb0gzGwscCkwGZgAnGhmw3ZZ7DhguP8xHfhTWEOKxLjjx/XhlIn9uP/dIuauLfc6jnjEiy2IUcBnzrka51wj8D5w6i7LTAWedi0+BTLNrE+4g4rEslumjqFvZjeu/usCqnY2eB1HPOBFQSwBDjWzbDNLAY4H8nZZph+wvtXrYv+0bzGz6WZWaGaFJSUlIQssEosykhO47+x8Nm6r5SYN6BeTwl4QzrkvgTuAN4E3gAVAUyc/6xHnXIFzriA3Nzd4IUUEgEkDs7jyyOG8PH8D/1iwwes4EmaeHKR2zj3unJvknDsMqACW77LIBr69VdHfP01EwuzKI4ex74BMfvXKEtaX13gdR8LIq7OYevq/DqDl+MOzuywyC7jAfzbTAUClc053WRfxQHycj/vOngjANc8voFGnvsYMr66D+JuZLQVeBa5wzm0zs8vM7DL//NeBVcAK4FHgfz3KKSJAXlYK/++UsRSureCP/9Gpr7Ei3ouVOucODTDtoVbPHXBFWEOJyG5Nze/H+1+VMOOdIg4elsN+g7K8jiQhpiupRaTdbpk6hv49UvjJXxdQWatTX6OdCkJE2i09OYEZ50xky/ad3PDyIt1gKMqpIESkQ/LzMrnumH14ffFmHtWor1FNBSEiHTb9sCEcP643t/9rGR+tKPU6joSICkJEOszMuPP0CQzNTePHz86juELXR0QjFYSIdEpaUjyPXFBAY7Pjsj/PZWdDpwZEkC5MBSEinTY4J5V7z8pnyYbt/OzFhTTrVqVRRQUhInvlqFG9uP64kby2aBN3vfmV13EkiDy5UE5Eosv/HDaEdeU1PPjeSvKyUjhn8gCvI0kQqCBEZK+ZGbdOGcOGilp+9fcl9M3sxuEjNMJyOLxYuJ7GZheSUtYuJhEJivg4Hw/8cF9G9Ernir/M44uNlV5HiglPfLSGVxduDMlnqyBEJGjSkuJ58sL9yEiO58In52h48BArr67ny03bOWhodkg+XwUhIkHVu3syT108mbqGJqY9+TkV1fVeR4pan60qA+BAFYSIRIrhvdJ5bNp+FFfUcslTc6it1zUSofDxyjJSEuMY3z8zJJ+vghCRkJg8OIsZZ+czf/02rnxuHg260VDQfbyylMmDs0iIC82fchWEiITMsWP7cOuUMbz95VZ+8dIiXUgXRFu372RlSTUHDgnN7iXQaa4iEmLnHziIbTUN3P3WcjK6JXDTSaMxM69jRbxP/McfDhqaE7J1qCBEJOR+fOQwttU28PiHq8lMSeAnR4/wOlLE+2RlGRnJ8YzumxGydaggRCTkzIwbjx9FZW0D975dREpiHNMPG+p1rIj28coy9h+STZwvdFtjKggRCQufz7j91HHUNjTx29eXUd/YzI+PHO51rIhUXFHDuvIaLjp4UEjXo4IQkbCJj/Nx31n5JMb5uOvN5dQ3Oa45eriOSXTQJytDe/3D11QQIhJW8XE+7jpjAglxxox3iqhvbOYXx+6jkuiAT1aWkZ2ayIie6SFdjwpCRMIuzmfcfup4EuN9PPT+SmrqG7n5pDH4Qrg/PVo45/hkVRkHDMkO+c9LBSEinvD5jN9MHUtqYjwPz15F1c5Gfn/6eOJDdNFXtFhTVsOmyp0h370EKggR8ZCZcf1xI8nolsDv//0VO+oauf+ciSQnxHkdrcv6NMTjL7WmqhYRT5kZV3xvGLdOHcNbS7dw0ZNzqKxt8DpWlzV/XQU9UhIYkpMa8nWpIESkS7jgwEHce1Y+hWvLOfOhT9i4rdbrSF3SgvXbmJCXGZaD+ioIEekyTp7Yj5kXTWbjtlpOffBjlm3e7nWkLqVqZwNFW3cwMa9HWNanghCRLuXgYTm8cNmBAJzxp09476utHifqOhYVV+Ic5A/IDMv6VBAi0uWM6pPBK1ccRP+sFC6aOYd7316ukWBp2b0EkB+i+z/sSgUhIl1Sn+7dePnygzhlYj/ufbuIi2bOifm7081ft40hual0T0kIy/o8KQgzu8bMvjCzJWb2nJkl7zL/QjMrMbMF/sePvMgpIt7qlhjH3WdM4LenjOOTlWWceP+HzF5e4nUsTzjnWLC+gvy8zLCtM+wFYWb9gKuAAufcWCAOODvAos875/L9j8fCGlJEugwz49z9B/DiZQeSFO/jgic+56cvLIi5rYniilpKd9QzMZoLwi8e6GZm8UAKsNGjHCISISbkZfL61Ydy5ZHDmLVgI0fd8z5/m1scM8cmvj7+MHFAeM5gAg8Kwjm3AbgLWAdsAiqdc28GWPQ0M1tkZi+ZWV5YQ4pIl5ScEMe1P9iH1646hAFZKVz74kJOvP9DPiwq9TpayM1ft42keB/79A7tAH2tebGLqQcwFRgM9AVSzey8XRZ7FRjknBsPvAU81cZnTTezQjMrLCmJzf2SIrFoZO8MXr78IO47O5/tOxs47/HPOP/xz1i6MXqvm1iwvoJx/bqTEMaxqrzYxXQ0sNo5V+KcawBeBg5qvYBzrsw5V+d/+RgwKdAHOececc4VOOcKcnNzQxpaRLoWn8+Ymt+Pd649nF+dMIpFxZWccP8H/PT5BRRX1HgdL6jqG5tZsnE7E8N0/cPXvBisbx1wgJmlALXAUUBh6wXMrI9zbpP/5RTgy/BGFJFIkRQfx48OHcIZk/J48P0VPPnRGl5btIlz9x/ACeP7MDEvM+JHiP1y03bqG5vJD9MV1F8Le0E45z4zs5eAeUAjMB94xMxuBQqdc7OAq8xsin9+OXBhuHOKSGTpnpLADceNYtqBg7jnreU88+laZn68hvSkeA4als1Ro3px7NjeZCSH5xqCYPrmArkwb0GYc9FxBkBBQYErLCzc84IiEhMqaxv4eEUps4tKmL28lA3bakmM93H0qJ5Mze/HAUOy6d4tMsrimucX8NGKUj775VFBH6TPzOY65woCzdP9IEQkKnXvlsBx4/pw3Lg+OOdYWFzJ3+dv4NWFG3l98WYA+mV2Y1SfDEb1SWdIbiqDslMZkpMWtiuV22v+upYL5MJ9W1YVhIhEPTMjPy+T/LxMbjxhFJ+tKmfRhm18uamKLzdt591lW2h9OUVivI+keB9J8XEkxfvITU+iX49u9M/sRr8e3ejX6mt6CHdZNTc7/vifFawpq+GH+w8M2XraooIQkZiSEOfjkOE5HDI855tpdY1NrC+vZXVpNatLd1C2o566xmbqGpvZ2dDElu07+WJDJW99sYX6puZvfd7A7BROGt+XKfl9GdGr5RqFytoGlm7czqbKWgZmpzK8V1qHj31U7Wzg2hcW8ubSLZyc35fzDwx/QegYhIhIOzU3O0p31FG8rZYNFbUUV9Ty8cpSPlpRSrODYT3T2NnQRHHFd2921DsjmSG5qQzISmFAdgoDslIY0SudITmp3zrLqrqukcUbKrnxlcWsKavhxuNHcdHBg0K2e2l3xyBUECIie2lr1U5eX7SJt7/cSveUBMb0zWB0nwz6ZXZjbVkNRVt3ULSlitVl1awvr6F0x3/HkUqM9zG8Zxp9uiezYusO1pbX4BxkpSbywLn7hvze0yoIEZEuZEddI2vLqlm+pYplm6pYumk7W7bvZGhumv+geQb7DepBZkpiyLPoLCYRkS4kLSmeMX27M6Zvd5jodZq2RfblhSIiEjIqCBERCUgFISIiAakgREQkIBWEiIgEpIIQEZGAVBAiIhKQCkJERAKKmiupzawEWAt0Byr9k1s/b/060DI5QEfvfL7r57d3mT1N29Pz1tNCkbut+cHK3ZnMu8u1p/lt/TsI9Fq595xrT/M7kztYv5O7y7W7+e2ZFsrcXv4tyXTOBb5ns3Muqh7AI4Get34daBla7mbX6XV1ZJk9TdvT812mBT13W/ODlbszmYOZe3evldub3MH6nexs7vZMC2XurvC3JNAjGncxvdrG89avd7dMZ9fVkWX2NG1Pz/cmc3ve39b8aMm9u9fK3fb62ju/M7mD9TvZnvd35ndy19cx8bckanYx7S0zK3RtDFjVlUVi7kjMDModbsrtvWjcguisR7wO0EmRmDsSM4Nyh5tye0xbECIiEpC2IEREJCAVhIiIBKSCEBGRgFQQe2Bmh5rZQ2b2mJl97HWe9jIzn5ndZmb3m9k0r/O0l5kdYWYf+H/mR3idpyPMLNXMCs3sRK+ztJeZjfL/rF8ys8u9ztNeZnaymT1qZs+b2Q+8ztMeZjbEzB43s5e8ztJeUV0QZvaEmW01syW7TD/WzL4ysxVmdv3uPsM594Fz7jLgNeCpUOZtlW+vcwNTgf5AA1AcqqytBSm3A3YAyURWboBfAC+EJuV3Benf95f+f99nAgeHMm+rfMHI/Xfn3KXAZcBZoczrzxaMzKucc5eENmlwRfVZTGZ2GC1/bJ52zo31T4sDlgPfp+UP0BzgHCAO+N0uH3Gxc26r/30vAJc456oiIbf/UeGce9jMXnLOnR4huUudc81m1gu4xzn3wwjJPQHIpqXYSp1zr0VCbufcVjObAlwOPOOcezZScvvfdzfwF+fcvAjKHJbfx2CI9zpAKDnnZpvZoF0mTwZWOOdWAZjZX4GpzrnfAQF3DZjZAKAyHOUAwcltZsVAvf9lUwjjfiNYP2+/CiApJEF3EaSf9xFAKjAaqDWz151zzV09t/9zZgGzzOyfQMgLIkg/bwNuB/4V6nKAoP/bjhhRXRBt6Aesb/W6GNh/D++5BHgyZInap6O5XwbuN7NDgdmhDLYHHcptZqcCxwCZwB9Dmmz3OpTbOXcjgJldiH8rKKTp2tbRn/cRwKm0lPHroQy2Bx39930lcDTQ3cyGOeceCmW4NnT0Z50N3AZMNLMb/EXSpcViQXSYc+4mrzN0lHOuhpZiiyjOuZdpKbeI5Jyb6XWGjnDOvQe853GMDnPOzQBmeJ2jI5xzZbQcM4kYUX2Qug0bgLxWr/v7p3V1yh1eyh1ekZg7EjN3SCwWxBxguJkNNrNE4GxglseZ2kO5w0u5wysSc0di5o7p6LjlkfQAngM28d9TPS/xTz+elrMPVgI3ep1TuZVbubt27kjMHIxHVJ/mKiIinReLu5hERKQdVBAiIhKQCkJERAJSQYiISEAqCBERCUgFISIiAakgJKqZ2Y4wry8o9wyxlvtiVJrZAjNbZmZ3teM9J5vZ6GCsXwRUECIdYma7Hb/MOXdQEFf3gXMuH5gInGhme7pfw8m0jCYrEhQqCIk5ZjbUzN4ws7nWcve6kf7pJ5nZZ2Y238ze9t+TAjO72cyeMbOPgGf8r58ws/fMbJWZXdXqs3f4vx7hn/+SfwvgL/4hqjGz4/3T5prZDDPb7b0jnHO1wAJaRg/FzC41szlmttDM/mZmKWZ2EDAF+L1/q2NoW9+nSHupICQWPQJc6ZybBPwMeNA//UPgAOfcROCvwM9bvWc0cLRz7hz/65G0DEs+GbjJzBICrGci8BP/e4cAB5tZMvAwcJx//bl7CmtmPYDh/HfY9pedc/s55yYAX9Iy7MPHtIwDdJ1zLt85t3I336dIu2i4b4kpZpYGHAS86P8PPfz3xkT9gefNrA+QCKxu9dZZ/v/Jf+2fzrk6oM7MtgK9+O4tUj93zhX717sAGETLXclWOee+/uzngOltxD3UzBbSUg73Ouc2+6ePNbP/R8s9M9KAf3fw+xRpFxWExBofsM2/b39X99Nym9NZ/hvp3NxqXvUuy9a1et5E4N+l9iyzOx845040s8HAp2b2gnNuATATONk5t9B/g6IjArx3d9+nSLtoF5PEFOfcdmC1mZ0BLbeuNLMJ/tnd+e94/tNCFOErYEir21eetac3+Lc2bgd+4Z+UDmzy79Zqfc/uKv+8PX2fIu2igpBol2Jmxa0eP6Xlj+ol/t03XwBT/cveTMsumblAaSjC+HdT/S/whn89VUBlO976EHCYv1j+D/gM+AhY1mqZvwLX+Q+yD6Xt71OkXTTct0iYmVmac26H/6ymB4Ai59wfvM4lsittQYiE36X+g9Zf0LJb62Fv44gEpi0IEREJSFsQIiISkApCREQCUkGIiEhAKggREQlIBSEiIgGpIEREJKD/D1E91U6LYjx2AAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"learn.remove_cb(ActivationStats)\n",
"admin_init(learn.model, scales)\n",
"learn.lr_find()"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"WANDB_NAME = f'admin_init_t5_20e'\n",
"GROUP = 'admin'\n",
"NOTES = 'T5Tokenizer tokenizer, 20e, 32100 vcb_sz, Early ADMIN testing'\n",
"CONFIG = {}\n",
"TAGS = ['admin','t5']"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmorgan\u001b[0m (use `wandb login --relogin` to force relogin)\n"
]
},
{
"data": {
"text/html": [
"\n",
" Tracking run with wandb version 0.10.19<br/>\n",
" Syncing run <strong style=\"color:#cdcd00\">admin_init_t5_20e</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
" Project page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox</a><br/>\n",
" Run page: <a href=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/2k8d2xge\" target=\"_blank\">https://wandb.ai/fastai_community/transformers_sandbox/runs/2k8d2xge</a><br/>\n",
" Run data is saved locally in <code>/home/morgan/ml/projects/transformers_sandbox/nbs/exploration/wandb/run-20210221_220517-2k8d2xge</code><br/><br/>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<h1>Run(2k8d2xge)</h1><iframe src=\"https://wandb.ai/fastai_community/transformers_sandbox/runs/2k8d2xge\" style=\"border:none;width:100%;height:400px\"></iframe>"
],
"text/plain": [
"<wandb.sdk.wandb_run.Run at 0x7f58280433a0>"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#hide_output\n",
"wandb.init(reinit=True, project=\"transformers_sandbox\", entity=\"fastai_community\", \n",
" name=WANDB_NAME, group=GROUP, notes=NOTES, tags=TAGS, config=CONFIG)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" <div>\n",
" <style>\n",
" /* Turns off some styling */\n",
" progress {\n",
" /* gets rid of default border in Firefox and Opera. */\n",
" border: none;\n",
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
" background-size: auto;\n",
" }\n",
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
" background: #F44336;\n",
" }\n",
" </style>\n",
" <progress value='0' class='' max='20' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
" 0.00% [0/20 00:00<00:00]\n",
" </div>\n",
" \n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>perplexity</th>\n",
" <th>bpc</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" </tbody>\n",
"</table><p>\n",
"\n",
" <div>\n",
" <style>\n",
" /* Turns off some styling */\n",
" progress {\n",
" /* gets rid of default border in Firefox and Opera. */\n",
" border: none;\n",
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
" background-size: auto;\n",
" }\n",
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
" background: #F44336;\n",
" }\n",
" </style>\n",
" <progress value='8' class='' max='466' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
" 1.72% [8/466 00:01<01:39 9.5600]\n",
" </div>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"learn.remove_cb(ActivationStats)\n",
"admin_init(learn.model, scales)\n",
"learn.fit_one_cycle(20, 1e-3, cbs=[WandbCallback(log_model=False, log_preds=False)])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Converted 00_core.ipynb.\n",
"Converted 01_layers.ipynb.\n",
"Converted 02_attention.ipynb.\n",
"Converted 03_models.transformer.ipynb.\n",
"Converted 04a_models.reformer.ipynb.\n",
"Converted 04x_models.xtransformer.ipynb.\n",
"Converted 05_tokenizers.ipynb.\n",
"Converted 06_data.ipynb.\n",
"Converted 07_metrics.ipynb.\n",
"Converted 08_optimizers.ipynb.\n",
"Converted 09_tracking.ipynb.\n",
"Converted 10_config.ipynb.\n",
"Converted index.ipynb.\n"
]
}
],
"source": [
"#hide\n",
"from nbdev.export import notebook2script; notebook2script()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.1"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment