joelburget/mixtral-playing-2.ipynb Secret

## mixtral-playing-2.ipynb

      
Display the source blob

    
Display the rendered blob

    
    Raw
  

              mixtral-playing-2.ipynb
            
          
        Loading

      Sorry, something went wrong. Reload?
      Sorry, we cannot display this file.
      Sorry, this file is invalid so it cannot be displayed.
      
          Viewer requires iframe.
      
    
## mixtral-playing.ipynb
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4fb7e0bc-4ef5-40c8-8222-336e83bd6e66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(truncated)"
     ]
    }
   ],
   "source": [
    "%pip install transformers huggingface_hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "be241e96-3bbb-46a4-a4d4-0213eb094d6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(truncated)"
     ]
    }
   ],
   "source": [
    "%pip install git+https://github.com/TransformerLensOrg/TransformerLens.git@mixtral-playing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d7341d8-881c-41c3-8199-ae9590d51a5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from huggingface_hub import login\n",
    "login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cba8adb4-03a4-4061-b62b-18bcc091b8af",
   "metadata": {},
   "outputs": [],
   "source": [
    "import einops\n",
    "model_id = \"mistralai/Mixtral-8x7B-v0.1\"\n",
    "text = \"Hello my name is\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2c3cb338-cf1b-4775-b278-302999164e6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a18a68e32bb40e686ae49dd8854c130",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/19 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained model mistralai/Mixtral-8x7B-v0.1 into HookedTransformer\n"
     ]
    }
   ],
   "source": [
    "from transformer_lens import HookedTransformer\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "tl_model = HookedTransformer.from_pretrained_no_processing(\n",
    "    \"mistralai/Mixtral-8x7B-v0.1\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "617609fe-060e-48b5-9daf-cb45cb6cc5d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hello my name is\\n cum e e e e e e o e e e o e e e e e e e AlbertAltyese  e e e e e e e e2 e e e e e e e e E e Eisenisisisisis'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.generate(\n",
    "    text,\n",
    "    verbose=False,\n",
    "    max_new_tokens=50,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4f3ccaf0-1650-4621-84f4-dbcdef132447",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6523f584ec9b4ceda67a4d6ed74d4af9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/19 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
    "hf_model = AutoModelForCausalLM.from_pretrained(model_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "97cec596-b5fc-48c2-82db-08af7e238a0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello my name is Katie and I am a 20 year old student from the UK. I am currently studying a degree in English Literature and Creative Writing at the University of Winchester. I have always had a passion for writing and I am hoping to pursue\n"
     ]
    }
   ],
   "source": [
    "inputs = tokenizer(text, return_tensors=\"pt\")\n",
    "outputs = hf_model.generate(**inputs, max_new_tokens=50)\n",
    "print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "1df1c967-4dfd-41ef-bab4-dea6072678aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.testing import assert_close\n",
    "import torch\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cfa7d1c3-72e8-45b3-850c-091669f48b1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    einops.rearrange(tl_model.blocks[0].attn.W_Q, \"n m h -> (n h) m\") ==\n",
    "    hf_model.model.layers[0].self_attn.q_proj.weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "83649934-f06b-4f94-8004-59b8d4098589",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([32, 4096, 128]), torch.Size([1024, 4096]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].attn.W_K.shape, hf_model.model.layers[0].self_attn.k_proj.weight.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4fa20cf5-b720-4946-a7e5-e1d2e6277f6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    einops.reduce(\n",
    "        tl_model.blocks[0].attn.W_K, \"(n repeat) m h -> (n h) m\",\n",
    "        'max',\n",
    "        n=tl_model.cfg.n_key_value_heads,\n",
    "        repeat=4) ==\n",
    "    hf_model.model.layers[0].self_attn.k_proj.weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ef6f7ea9-ef0b-4091-8d00-504b481fc59a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    einops.reduce(\n",
    "        tl_model.blocks[0].attn.W_V, \"(n repeat) m h -> (n h) m\",\n",
    "        'max',\n",
    "        n=tl_model.cfg.n_key_value_heads,\n",
    "        repeat=4) ==\n",
    "    hf_model.model.layers[0].self_attn.v_proj.weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "04b8f4be-ce7d-4dc2-acda-d023c721525c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    einops.rearrange(tl_model.blocks[0].attn.W_O, \"n h m -> m (n h)\") ==\n",
    "    hf_model.model.layers[0].self_attn.o_proj.weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1e10ed87-31b5-4c1c-b726-7a3f49fbd136",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.]], requires_grad=True)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].attn.b_Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6caf9d98-adb2-45e7-8357-34288b2156f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(False)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(hf_model.model.layers[0].block_sparse_moe.gate.weight.T == tl_model.blocks[0].mlp.W_gate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "00e9ea5d-74c2-4c2a-8e9d-6fc196cb8fc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.float32, torch.float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].block_sparse_moe.gate.weight.dtype, tl_model.blocks[0].mlp.W_gate.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "44a55507-e639-414a-a297-e68e1c0696f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    tl_model.blocks[0].mlp.experts[0].W_in ==\n",
    "    hf_model.model.layers[0].block_sparse_moe.experts[0].w3.weight.T\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "03944deb-aa8d-46ff-83dd-4f7ee955656c",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tensor = torch.randn((1, 1, 4096,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "eb0109ee-b82a-4ea0-b50b-8e6408647cea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(False)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    hf_model.model.layers[0].block_sparse_moe(test_tensor)[0] ==\n",
    "    tl_model.blocks[0].mlp(test_tensor)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "25ce75bf-706e-4ae8-8f74-bc9c40e88c25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.2663,  0.3429,  0.1977,  ...,  0.3202, -0.2134,  0.4128]]],\n",
       "       grad_fn=<ReshapeAliasBackward0>)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].block_sparse_moe(test_tensor)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c016430e-0a30-426b-bfd0-0b1b423b3ff6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.5243,  0.1930,  0.0679,  ..., -0.4153, -0.0232, -0.2474]]],\n",
       "       grad_fn=<IndexPutBackward0>)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].mlp(test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "46353486-0a3f-4241-9cf5-ed25c7539f71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 4096])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].mlp(test_tensor).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e25ada54-4e3c-42b7-8f35-ba67bfa500e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[False, False, False,  ..., False, False, False]]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].block_sparse_moe(test_tensor)[0] == tl_model.blocks[0].mlp(test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8f3a2865-645d-4441-95fb-32446f866760",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sum(hf_model.model.layers[0].block_sparse_moe(test_tensor)[0] == tl_model.blocks[0].mlp(test_tensor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "c6ef1f5e-bdf0-45e5-9347-6972e91e2f2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHFCAYAAAAUpjivAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABFx0lEQVR4nO3deXgUZb7+/7tNOp09EEK2IYQcBFkCLjAKGRUiJCwTZBkHla8KiFw4IGMEDjO4HIIHQUURBwRRMYjIgKOijsMWJIAMcFhcWMZRxLBJQmRLWELSCfX7w196aLKQdDrdoXi/ritX0k8/XfWppyvh5qmqLothGIYAAABM6jpvFwAAAFCfCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDu46ixcuFAWi8Xx5e/vr+joaCUnJ2v69OnKz8+v8JqMjAxZLBantpKSEj366KOKiYmRj4+PbrrpJknSyZMndd999ykyMlIWi0UDBgzwwFZdvQ4cOOD0flgsFoWGhurGG2/UrFmzVFZW5tJyV6xYoYyMDPcW28CUj9ewYcMqff7ZZ5919Dlw4ICjfdiwYQoODq522Zf/nvj6+qpZs2YaPny4fvrppxrVd+LECU2aNEnt2rVTYGCgQkND1aVLF7322muy2+013cwKPP3ezp07VwsXLvTY+tAAGcBVJjMz05BkZGZmGlu2bDE2btxofPDBB0Z6eroRFhZmhIeHG1lZWU6vOXz4sLFlyxantlmzZhmSjNmzZxubN282du3aZRiGYaSnpxt+fn7G4sWLjS1bthjfffedx7btapSTk2NIMsaOHWts2bLF2LJli7Fy5UrjD3/4gyHJGDdunEvLHTNmjGH2P1GSjJCQECMwMNAoLCx0eu7ixYtGQkKCERoaakgycnJyHM8NHTrUCAoKqnbZl/+erFu3zsjIyDBsNpuRkJBgnD17ttrXf/vtt0ZcXJzRuHFjY+rUqca6deuMf/zjH8Yf/vAHw8fHx+jWrZtx7tw5l7bb0+9t+/btjW7dunlsfWh4zP2XBKZU/kd8+/btFZ47ePCgERcXZ4SEhBh5eXnVLueRRx4xAgICKrT37NnTaNu2rdvqNQzDOH/+vFuX15CUh50ZM2ZUeO6OO+4wYmJiXFrutRJ2HnjgASMgIMB44403nJ5bu3atIckYOXJkncLO5b8nzzzzjCHJWLx4cZWvLS0tNdq1a2eEhYVVGvaXLl1qSDJGjRpVg62siLADT+MwFkylefPmevnll3XmzBnNnz/f0X75YSyLxaK33npLRUVFjmn+8mn/tWvX6ttvv3W0r1+/XtIvh72mTp2qNm3ayGazqWnTpho+fLh+/vlnpxpatGihtLQ0ffTRR7r55pvl7++vKVOmSJLy8vI0atQoNWvWTH5+fkpISNCUKVNUWlrqeH35YaGXXnpJM2fOVEJCgoKDg9W1a1dt3bq1wjb/3//9n/r166cmTZrI399fLVu2VHp6ulOfffv2aciQIYqMjJTNZlPbtm312muvOfW5ePGipk6dqhtuuEEBAQFq1KiROnbsqFdffdWl90KSwsLCZLVaK7QvW7ZMXbt2VVBQkIKDg9WrVy999dVXjueHDRvmqO/SQzEHDhzQ73//e7Vv395pef369ZPFYtHf/vY3R9uXX34pi8Wiv//97462moy/VPv3etWqVbrlllsUEBCgNm3a6O23367VGA0cOLDCa95++2395je/UevWrWu8rJro0qWLJOngwYNV9lm+fLn+9a9/6c9//nOl67/33nuVmpqqBQsWKC8vT5K0fv16p9+XcuX7c/lhpOre2/K2xx57TPPnz1fr1q1ls9nUrl07LV261Gm5lR2alv5z+K58eS1atNDevXu1YcMGx7patGghqX72eTRMvt4uAHC3vn37ysfHRxs3bqyyz5YtW/S///u/ys7O1rp16yRJCQkJ2rJli0aPHq2CggK99957kqR27drp4sWL6t+/v7744gtNnDhRSUlJOnjwoCZPnqzu3btrx44dCggIcCz/yy+/1Lfffqunn35aCQkJCgoKUl5enm699VZdd911+p//+R+1bNlSW7Zs0dSpU3XgwAFlZmY61fjaa6+pTZs2mjVrliTpmWeeUd++fZWTk6OwsDBJ0urVq9WvXz+1bdtWM2fOVPPmzXXgwAGtWbPGsZx//etfSkpKcgTB6OhorV69Wn/84x91/PhxTZ48WZL04osvKiMjQ08//bTuvPNO2e12/fvf/9bp06drNO4XL150hIaCggJ98sknWrVqlf70pz859Zs2bZqefvppDR8+XE8//bRKSko0Y8YM3XHHHdq2bZvatWunZ555RufOndMHH3ygLVu2OF4bExOjnj176oMPPlBubq5iYmJUWlqqDRs2KCAgQFlZWfr9738vSVq7dq18fX3VvXt3Sarx+Nf2vf7mm280fvx4/fnPf1ZUVJTeeustjRgxQtdff73uvPPOGo3diBEj1KNHD3377bdq27atTp8+rY8++khz587ViRMnarSMmvrhhx8kSU2bNq2yT1ZWliRVe77agAEDtGbNGq1fv1733Xdfjddf3Xtb7tNPP1V2draeffZZBQUFae7cubr//vvl6+ure+65p8brkn4Jbvfcc4/CwsI0d+5cSZLNZpNU930eVxFvTy0BtVXdYaxyUVFRToeiJk+eXGHavKpDAd26dTPat2/v1PbXv/7VkGR8+OGHTu3bt283JBlz5851tMXHxxs+Pj4Vpv9HjRplBAcHGwcPHnRqf+mllwxJxt69ew3D+M9hoQ4dOhilpaWOftu2bTMkGX/9618dbS1btjRatmxpFBUVVTkWvXr1Mpo1a2YUFBQ4tT/22GOGv7+/cfLkScMwDCMtLc246aabqlxOVcrrrexr2LBhTttw6NAhw9fX1xg7dqzTMs6cOWNER0cbgwcPdrRVdajjhx9+MCQZixYtMgzDMDZt2mRIMiZOnGgkJCQ4+qWkpBhJSUmOxzUd/9q+1/7+/k7LLCoqMsLDw2t0iEeSMWbMGMf5ORMmTDAMwzBee+01Izg42Dhz5owxY8aMOh3G2rp1q2G3240zZ84Yn332mdG0adMrHubt3bu3Icm4cOFClX1WrlxpSDJeeOEFwzAMIzs725BkZGdnO/Ur3z8yMzMdbdUdxpJkBAQEONVXWlpqtGnTxrj++usdbZX9Tl+63ZeOV1WHsVzd53H14TAWTMkwDLcu77PPPlOjRo3Ur18/lZaWOr5uuukmRUdHV5i679ixY4Xp/88++0zJycmKjY11WkafPn0kSRs2bHDq/9vf/lY+Pj5Oy5T+c/jh+++/1/79+zVixAj5+/tXWveFCxf0+eefa+DAgQoMDHRab9++fXXhwgXHobFbb71V33zzjUaPHq3Vq1ersLCwVmP0+OOPa/v27dq+fbuys7M1bdo0vf/++7r//vsdfVavXq3S0lI99NBDTrX4+/urW7duFcaxMi1btlSLFi20du1aSb/MQnTo0EEPPPCAcnJytH//fhUXF2vTpk3q2bOn43U1Hf/avtc33XSTmjdv7njs7++v1q1bV3uY6HLlV2S9++67Ki0t1YIFCzR48OArXnFVE126dJHValVISIjS0tIUHR2tlStXKioqqk7LLf8dq+xQUl316NHDqT4fHx/de++9+uGHH3TkyBG3raeu+zyuHhzGgumcO3dOJ06cUIcOHdy2zGPHjun06dPy8/Or9Pnjx487Pb50Sv7SZfz973+v9ByWypbRpEkTp8flU+9FRUWS5Dh/pFmzZlXWfeLECZWWlmr27NmaPXt2teudNGmSgoKCtHjxYr3++uvy8fHRnXfeqRdeeEGdO3euch3lmjVr5tSve/fuslgsmjRpklavXq1evXrp2LFjkqRf//rXlS7juutq9v+vHj16aNWqVZJ+OVyVkpKiDh06KCoqSmvXrlWrVq1UVFTkFHZqOv61fa8vf5+kX96r8veppoYPH64pU6Zo2rRp+vLLL6t8v2pr0aJFatu2rXx9fRUVFVXpvnm58vCWk5OjNm3aVNqn/JyYuLg4t9R5qejo6CrbTpw4Ue0+Xxt13edx9SDswHT+8Y9/qKyszHGuhjtERESoSZMmjn9gLxcSEuL0uLL/7UZERKhjx4567rnnKl1GbGxsrWoqP+eiuv/pNm7cWD4+PnrwwQc1ZsyYSvskJCRIknx9fTVu3DiNGzdOp0+f1tq1a/Xkk0+qV69eOnz4sAIDA2tVn/Sf2ahvvvlGvXr1UkREhCTpgw8+UHx8fK2XV65Hjx5asGCBtm3bpv/7v//T008/LUm66667lJWVpYMHDyo4ONhxMq5U8/Gv7XvtLnFxcerZs6emTJmiG264QUlJSW5Zbtu2bWv9D3dKSoreeOMNffzxx/rzn/9caZ+PP/7Y6Zyo8tnF4uJip36Xh8OaKD/pubK28nB56frK/yNQ2/XVxz6PhomwA1M5dOiQJkyYoLCwMI0aNcpty01LS9PSpUtVVlam2267zeVlrFixQi1btlTjxo3rXFPr1q3VsmVLvf322xo3bpzTH/xygYGBSk5O1ldffaWOHTtWOVtxuUaNGumee+7RTz/9pPT0dB04cEDt2rWrdY1ff/21JCkyMlKS1KtXL/n6+mr//v363e9+V+1rL53JuvSEYOmXsGOxWPTMM8/ouuuuc5wI3LNnT/33f/+3Dh48qDvvvNNpFqem4++O99pV48ePV0BAgOMka28ZOHCg2rVrp+eff16DBg2qcEh22bJlWrNmjR599FHHjEv5FU67du1Sr169HH0//fTTCsuv7r2VpM8//1zHjh1zHMoqKyvTsmXL1LJlS8eszqXru3Sm8NKr7y5d35Vm2ty1z6NhIuzgqrVnzx7H+RT5+fn64osvlJmZKR8fHy1fvrzaq01q67777tN7772nvn376vHHH9ett94qq9WqI0eOKDs7W/3799fAgQOrXcazzz6rrKwsJSUl6Y9//KNuuOEGXbhwQQcOHNCKFSv0+uuv13p6/rXXXlO/fv3UpUsXPfHEE2revLkOHTqk1atXO64me/XVV3X77bfrjjvu0B/+8Ae1aNFCZ86c0Q8//KC///3vjqvR+vXrp8TERHXu3FlNmzbVwYMHNWvWLMXHx6tVq1ZXrOXQoUOO83/OnTunLVu2aPr06YqPj9egQYMk/fIP1LPPPqunnnpKP/74o3r37q3GjRvr2LFj2rZtm4KCghyX6ZcfhnzhhRfUp08f+fj4OAJbZGSkEhMTtWbNGiUnJzv+B96zZ0+dPHlSJ0+e1MyZM10af3e8165KTU1VampqjfqWlZXpgw8+qNAeFBTkOA/JVT4+Pvrwww+VkpKirl27avz48eratauKi4v197//XW+88Ya6deuml19+2fGa6Oho9ezZU9OnT1fjxo0VHx+vzz//XB999FGF5Vf33kq/zK7dddddeuaZZxxXY/373/92uvy8b9++Cg8P14gRI/Tss8/K19dXCxcu1OHDhytd39KlS7Vs2TL913/9l/z9/dWhQ4c67/O4inj7DGmgtsqvtij/8vPzMyIjI41u3boZ06ZNM/Lz8yu8pq5XYxmGYdjtduOll14ybrzxRsPf398IDg422rRpY4waNcrYt2+fo198fLzx29/+ttLaf/75Z+OPf/yjkZCQYFitViM8PNzo1KmT8dRTTzk+0ba6D+mTZEyePNmpbcuWLUafPn2MsLAww2azGS1btjSeeOIJpz45OTnGww8/bPzqV78yrFar0bRpUyMpKcmYOnWqo8/LL79sJCUlGREREYafn5/RvHlzY8SIEcaBAwcq3ZZLl63LrsLy9/c3WrdubaSnpxu5ubkVXvPxxx8bycnJRmhoqGGz2Yz4+HjjnnvuMdauXevoU1xcbDzyyCNG06ZNDYvFUuEKmyeeeMKQZDz33HNOy27VqpUhyfGJ2JeqyfgbRt3f627dutXoQ+z0/1+NVZ2qrsa6fMzLv+Lj4w3DqNlVi1dy/Phx489//rPRpk0bxzjceuutxpw5c4ySkpIK/XNzc4177rnHCA8PN8LCwowHHnjA2LFjR4Wrsap7b8vHZO7cuUbLli0Nq9VqtGnTxnjvvfcqrG/btm1GUlKSERQUZPzqV78yJk+ebLz11lsVxuvAgQNGamqqERIS4jRGru7zuPpYDMPNl60AAOAii8WiMWPGaM6cOd4uBSbCpecAAMDUCDsAAMDUOEEZANBgcGYF6gMzOwAAwNQIOwAAwNQIOwAAwNQ4Z0fSxYsXdfToUYWEhNTLTe0AAID7GYahM2fOKDY2ttp76xF2JB09erRebmYHAADq3+HDh6v9BHrCjv5zY7/Dhw8rNDRUkmS327VmzRqlpqZWeZdkuA/j7VmMt+cx5p7FeHueN8a8sLBQcXFxV7xBL2FH/7lDdWhoqFPYCQwMVGhoKL8oHsB4exbj7XmMuWcx3p7nzTG/0ikonKAMAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADAABMzdfbBQDAtSY5+cp9srPrvw7gWsHMDgAAMDXCDgAAMDXCDgAAMDXCDgAAMDXCDgAAMDXCDgAAMDXCDgAAMDU+ZwcAaojPxwGuTszsAAAAU2NmBwDcqKrZHz8/afRoKS3Ns/UAYGYHAACYHGEHAACYGmEHAACYGmEHAACYGmEHAACYGmEHAACYmlfDzrx589SxY0eFhoYqNDRUXbt21cqVKx3PDxs2TBaLxemrS5cuTssoLi7W2LFjFRERoaCgIN199906cuSIpzcFAAA0UF4NO82aNdPzzz+vHTt2aMeOHbrrrrvUv39/7d2719Gnd+/eys3NdXytWLHCaRnp6elavny5li5dqk2bNuns2bNKS0tTWVmZpzcHAAA0QF79UMF+/fo5PX7uuec0b948bd26Ve3bt5ck2Ww2RUdHV/r6goICLViwQO+++6569uwpSVq8eLHi4uK0du1a9erVq343AAAANHgN5hOUy8rK9Le//U3nzp1T165dHe3r169XZGSkGjVqpG7duum5555TZGSkJGnnzp2y2+1KTU119I+NjVViYqI2b95cZdgpLi5WcXGx43FhYaEkyW63y263O36+9DvqF+PtWYy3a/z8XH+t1Wp3+n4lvDV1wz7ued4Y85quy2IYhlHPtVRr9+7d6tq1qy5cuKDg4GAtWbJEffv2lSQtW7ZMwcHBio+PV05Ojp555hmVlpZq586dstlsWrJkiYYPH+4UXCQpNTVVCQkJmj9/fqXrzMjI0JQpUyq0L1myRIGBge7fSAAA4Hbnz5/XkCFDVFBQoNDQ0Cr7eT3slJSU6NChQzp9+rQ+/PBDvfXWW9qwYYPatWtXoW9ubq7i4+O1dOlSDRo0qMqwk5KSopYtW+r111+vdJ2VzezExcXp+PHjjsGy2+3KyspSSkqKrFarG7cYlWG8PYvxdk1d7mtltdo1cmSW3nwzRXb7lcf8s89cXxfYx73BG2NeWFioiIiIK4Ydrx/G8vPz0/XXXy9J6ty5s7Zv365XX3210lmZmJgYxcfHa9++fZKk6OholZSU6NSpU2rcuLGjX35+vpKSkqpcp81mk81mq9ButVorvEGVtaH+MN6exXjXTklJ3Zdht1tVUnLlMedtcQ/2cc/z5JjXdD0N7nN2DMOoMFNT7sSJEzp8+LBiYmIkSZ06dZLValVWVpajT25urvbs2VNt2AEAANcOr87sPPnkk+rTp4/i4uJ05swZLV26VOvXr9eqVat09uxZZWRk6He/+51iYmJ04MABPfnkk4qIiNDAgQMlSWFhYRoxYoTGjx+vJk2aKDw8XBMmTFCHDh0cV2cBAIBrm1fDzrFjx/Tggw8qNzdXYWFh6tixo1atWqWUlBQVFRVp9+7dWrRokU6fPq2YmBglJydr2bJlCgkJcSzjlVdeka+vrwYPHqyioiL16NFDCxculI+Pjxe3DAAANBReDTsLFiyo8rmAgACtXr36isvw9/fX7NmzNXv2bHeWBgAATKLBnbMDAADgToQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgar7eLgAAUFFy8pX7ZGfXfx2AGTCzAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATM2rYWfevHnq2LGjQkNDFRoaqq5du2rlypWO5w3DUEZGhmJjYxUQEKDu3btr7969TssoLi7W2LFjFRERoaCgIN199906cuSIpzcFAAA0UF4NO82aNdPzzz+vHTt2aMeOHbrrrrvUv39/R6B58cUXNXPmTM2ZM0fbt29XdHS0UlJSdObMGccy0tPTtXz5ci1dulSbNm3S2bNnlZaWprKyMm9tFgAAaEC8Gnb69eunvn37qnXr1mrdurWee+45BQcHa+vWrTIMQ7NmzdJTTz2lQYMGKTExUe+8847Onz+vJUuWSJIKCgq0YMECvfzyy+rZs6duvvlmLV68WLt379batWu9uWkAAKCBaDDn7JSVlWnp0qU6d+6cunbtqpycHOXl5Sk1NdXRx2azqVu3btq8ebMkaefOnbLb7U59YmNjlZiY6OgDAACubb7eLmD37t3q2rWrLly4oODgYC1fvlzt2rVzhJWoqCin/lFRUTp48KAkKS8vT35+fmrcuHGFPnl5eVWus7i4WMXFxY7HhYWFkiS73S673e74+dLvqF+Mt2cx3q7x83P9tVar3em7O/D2VY193PO8MeY1XZfXw84NN9ygr7/+WqdPn9aHH36ooUOHasOGDY7nLRaLU3/DMCq0Xe5KfaZPn64pU6ZUaF+zZo0CAwOd2rKysmqyGXATxtuzGO/aGT267ssYOdJ9Y75ihdsWZVrs457nyTE/f/58jfp5Pez4+fnp+uuvlyR17txZ27dv16uvvqo//elPkn6ZvYmJiXH0z8/Pd8z2REdHq6SkRKdOnXKa3cnPz1dSUlKV65w0aZLGjRvneFxYWKi4uDilpqYqNDRU0i9pMSsrSykpKbJare7bYFSK8fYsxts1aWmuv9ZqtWvkyCy9+WaK7Hb3jPlnn7llMabEPu553hjz8iMzV+L1sHM5wzBUXFyshIQERUdHKysrSzfffLMkqaSkRBs2bNALL7wgSerUqZOsVquysrI0ePBgSVJubq727NmjF198scp12Gw22Wy2Cu1Wq7XCG1RZG+oP4+1ZjHftlJTUfRl2u1UlJe4Zc966K2Mf9zxPjnlN1+PVsPPkk0+qT58+iouL05kzZ7R06VKtX79eq1atksViUXp6uqZNm6ZWrVqpVatWmjZtmgIDAzVkyBBJUlhYmEaMGKHx48erSZMmCg8P14QJE9ShQwf17NnTm5sGAAAaCK+GnWPHjunBBx9Ubm6uwsLC1LFjR61atUopKSmSpIkTJ6qoqEijR4/WqVOndNttt2nNmjUKCQlxLOOVV16Rr6+vBg8erKKiIvXo0UMLFy6Uj4+PtzYLAAA0IF4NOwsWLKj2eYvFooyMDGVkZFTZx9/fX7Nnz9bs2bPdXB2Aa0lysrcrqL2a1JydXf91AA1dg/mcHQAAgPpA2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKbW4D5BGQDc7Wq8rByA+zCzAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATM3X2wUAAOpPcvKV+2Rn138dgDcxswMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEzNq2Fn+vTp+vWvf62QkBBFRkZqwIAB+u6775z6DBs2TBaLxemrS5cuTn2Ki4s1duxYRUREKCgoSHfffbeOHDniyU0BAAANlFfDzoYNGzRmzBht3bpVWVlZKi0tVWpqqs6dO+fUr3fv3srNzXV8rVixwun59PR0LV++XEuXLtWmTZt09uxZpaWlqayszJObAwAAGiBfb6581apVTo8zMzMVGRmpnTt36s4773S022w2RUdHV7qMgoICLViwQO+++6569uwpSVq8eLHi4uK0du1a9erVq/42AAAANHheDTuXKygokCSFh4c7ta9fv16RkZFq1KiRunXrpueee06RkZGSpJ07d8putys1NdXRPzY2VomJidq8eXOlYae4uFjFxcWOx4WFhZIku90uu93u+PnS76hfjLdnXWvj7efn7Qokq9Xu9L0hMeNucK3t4w2BN8a8puuyGIZh1HMtNWIYhvr3769Tp07piy++cLQvW7ZMwcHBio+PV05Ojp555hmVlpZq586dstlsWrJkiYYPH+4UXiQpNTVVCQkJmj9/foV1ZWRkaMqUKRXalyxZosDAQPdvHAAAcLvz589ryJAhKigoUGhoaJX9GszMzmOPPaZdu3Zp06ZNTu333nuv4+fExER17txZ8fHx+sc//qFBgwZVuTzDMGSxWCp9btKkSRo3bpzjcWFhoeLi4pSamuoYLLvdrqysLKWkpMhqtdZl01ADjLdnXWvjnZbm7Qp+mdEZOTJLb76ZIru9YY35Z595uwL3u9b28YbAG2NefmTmShpE2Bk7dqw+/fRTbdy4Uc2aNau2b0xMjOLj47Vv3z5JUnR0tEpKSnTq1Ck1btzY0S8/P19JSUmVLsNms8lms1Vot1qtFd6gytpQfxhvz7pWxrukxNsV/IfdblVJScMaczPvAtfKPt6QeHLMa7oer16NZRiGHnvsMX300Udat26dEhISrviaEydO6PDhw4qJiZEkderUSVarVVlZWY4+ubm52rNnT5VhBwAAXDu8OrMzZswYLVmyRJ988olCQkKUl5cnSQoLC1NAQIDOnj2rjIwM/e53v1NMTIwOHDigJ598UhERERo4cKCj74gRIzR+/Hg1adJE4eHhmjBhgjp06OC4OgsAAFy7vBp25s2bJ0nq3r27U3tmZqaGDRsmHx8f7d69W4sWLdLp06cVExOj5ORkLVu2TCEhIY7+r7zyinx9fTV48GAVFRWpR48eWrhwoXx8fDy5OQAAoAHyati50oVgAQEBWr169RWX4+/vr9mzZ2v27NnuKg0AAJgE98YCAACmRtgBAACmRtgBAACmRtgBAACmRtgBAACmRtgBAACm1iBuFwEArkpO9nYFABo6ZnYAAICpEXYAAICpEXYAAICpEXYAAICpuRR2cnJy3F0HAABAvXAp7Fx//fVKTk7W4sWLdeHCBXfXBAAA4DYuhZ1vvvlGN998s8aPH6/o6GiNGjVK27Ztc3dtAAAAdeZS2ElMTNTMmTP1008/KTMzU3l5ebr99tvVvn17zZw5Uz///LO76wQAAHBJnU5Q9vX11cCBA/X+++/rhRde0P79+zVhwgQ1a9ZMDz30kHJzc91VJwAAgEvqFHZ27Nih0aNHKyYmRjNnztSECRO0f/9+rVu3Tj/99JP69+/vrjoBAABc4tLtImbOnKnMzEx999136tu3rxYtWqS+ffvquut+yU4JCQmaP3++2rRp49ZiAQAAasulsDNv3jw9/PDDGj58uKKjoyvt07x5cy1YsKBOxQEAANSVS2Fn3759V+zj5+enoUOHurJ4AAAAt3HpnJ3MzEz97W9/q9D+t7/9Te+8806diwIAAHAXl8LO888/r4iIiArtkZGRmjZtWp2LAgAAcBeXws7BgweVkJBQoT0+Pl6HDh2qc1EAAADu4lLYiYyM1K5duyq0f/PNN2rSpEmdiwIAAHAXl8LOfffdpz/+8Y/Kzs5WWVmZysrKtG7dOj3++OO677773F0jAACAy1y6Gmvq1Kk6ePCgevToIV/fXxZx8eJFPfTQQ5yzAwAAGhSXwo6fn5+WLVum//3f/9U333yjgIAAdejQQfHx8e6uDwAAoE5cCjvlWrdurdatW7urFgAAALdzKeyUlZVp4cKF+vzzz5Wfn6+LFy86Pb9u3Tq3FAcAAFBXLoWdxx9/XAsXLtRvf/tbJSYmymKxuLsuAAAAt3Ap7CxdulTvv/+++vbt6+56AAAA3MqlS8/9/Px0/fXXu7sWAAAAt3Mp7IwfP16vvvqqDMNwdz0AAABu5dJhrE2bNik7O1srV65U+/btZbVanZ7/6KOP3FIcAABAXbkUdho1aqSBAwe6uxYAAAC3cynsZGZmursOAACAeuHSOTuSVFpaqrVr12r+/Pk6c+aMJOno0aM6e/ZsjZcxffp0/frXv1ZISIgiIyM1YMAAfffdd059DMNQRkaGYmNjFRAQoO7du2vv3r1OfYqLizV27FhFREQoKChId999t44cOeLqpgEAABNxKewcPHhQHTp0UP/+/TVmzBj9/PPPkqQXX3xREyZMqPFyNmzYoDFjxmjr1q3KyspSaWmpUlNTde7cOUefF198UTNnztScOXO0fft2RUdHKyUlxRGwJCk9PV3Lly/X0qVLtWnTJp09e1ZpaWkqKytzZfMAAICJuPyhgp07d9Y333yjJk2aONoHDhyoRx55pMbLWbVqldPjzMxMRUZGaufOnbrzzjtlGIZmzZqlp556SoMGDZIkvfPOO4qKitKSJUs0atQoFRQUaMGCBXr33XfVs2dPSdLixYsVFxentWvXqlevXq5sIgAAMAmXr8b65z//KT8/P6f2+Ph4/fTTTy4XU1BQIEkKDw+XJOXk5CgvL0+pqamOPjabTd26ddPmzZs1atQo7dy5U3a73alPbGysEhMTtXnz5krDTnFxsYqLix2PCwsLJUl2u112u93x86XfUb8Yb88y03hf9meowbJa7U7fGxIT7AYVmGkfv1p4Y8xrui6Xws7FixcrPUR05MgRhYSEuLJIGYahcePG6fbbb1diYqIkKS8vT5IUFRXl1DcqKkoHDx509PHz81Pjxo0r9Cl//eWmT5+uKVOmVGhfs2aNAgMDndqysrJc2h64hvH2LDOM9+jR3q6gdkaObHhjvmKFtyuoP2bYx682nhzz8+fP16ifS2EnJSVFs2bN0htvvCFJslgsOnv2rCZPnuzyLSQee+wx7dq1S5s2barw3OX33jIM44r346quz6RJkzRu3DjH48LCQsXFxSk1NVWhoaGSfkmLWVlZSklJqfA5QnA/xtuzzDTeaWnerqBmrFa7Ro7M0ptvpshub1hj/tln3q7A/cy0j18tvDHm5UdmrsSlsPPKK68oOTlZ7dq104ULFzRkyBDt27dPERER+utf/1rr5Y0dO1affvqpNm7cqGbNmjnao6OjJf0yexMTE+Noz8/Pd8z2REdHq6SkRKdOnXKa3cnPz1dSUlKl67PZbLLZbBXarVZrhTeosjbUH8bbs8ww3iUl3q6gdux2q0pKGtaYX+W7QLXMsI9fbTw55jVdj0tXY8XGxurrr7/WhAkTNGrUKN188816/vnn9dVXXykyMrLGyzEMQ4899pg++ugjrVu3TgkJCU7PJyQkKDo62mlKrKSkRBs2bHAEmU6dOslqtTr1yc3N1Z49e6oMOwAA4Nrh0syOJAUEBOjhhx/Www8/7PLKx4wZoyVLluiTTz5RSEiI4xybsLAwBQQEyGKxKD09XdOmTVOrVq3UqlUrTZs2TYGBgRoyZIij74gRIzR+/Hg1adJE4eHhmjBhgjp06OC4OgsAAFy7XAo7ixYtqvb5hx56qEbLmTdvniSpe/fuTu2ZmZkaNmyYJGnixIkqKirS6NGjderUKd12221as2aN04nQr7zyinx9fTV48GAVFRWpR48eWrhwoXx8fGq+UQAAwJRc/pydS9ntdp0/f15+fn4KDAyscdipyV3TLRaLMjIylJGRUWUff39/zZ49W7Nnz67RegEAwLXDpXN2Tp065fR19uxZfffdd7r99ttdOkEZAACgvrh8b6zLtWrVSs8//3yFWR8AAABvclvYkSQfHx8dPXrUnYsEAACoE5fO2fn000+dHhuGodzcXM2ZM0e/+c1v3FIYAACAO7gUdgYMGOD02GKxqGnTprrrrrv08ssvu6MuAAAAt3D53lgAAHNITr5yn+zs+q8DqC9uPWcHAACgoXFpZufSm2heycyZM11ZBQAAgFu4FHa++uorffnllyotLdUNN9wgSfr+++/l4+OjW265xdHvSncmB4Dq1OTwCgBciUthp1+/fgoJCdE777zjuNP4qVOnNHz4cN1xxx0aP368W4sEAABwlUth5+WXX9aaNWscQUeSGjdurKlTpyo1NZWwAwAmw0nMuJq5dIJyYWGhjh07VqE9Pz9fZ86cqXNRAAAA7uJS2Bk4cKCGDx+uDz74QEeOHNGRI0f0wQcfaMSIERo0aJC7awQAAHCZS4exXn/9dU2YMEEPPPCA7Hb7Lwvy9dWIESM0Y8YMtxYIAABQFy6FncDAQM2dO1czZszQ/v37ZRiGrr/+egUFBbm7PgAAgDqp04cK5ubmKjc3V61bt1ZQUJAMw3BXXQAAAG7hUtg5ceKEevToodatW6tv377Kzc2VJD3yyCNciQUAABoUl8LOE088IavVqkOHDikwMNDRfu+992rVqlVuKw4AAKCuXDpnZ82aNVq9erWaNWvm1N6qVSsdPHjQLYUBAAC4g0szO+fOnXOa0Sl3/Phx2Wy2OhcFAADgLi6FnTvvvFOLFi1yPLZYLLp48aJmzJihZG5mAwAAGhCXDmPNmDFD3bt3144dO1RSUqKJEydq7969OnnypP75z3+6u0YAAACXuTSz065dO+3atUu33nqrUlJSdO7cOQ0aNEhfffWVWrZs6e4aAQAAXFbrmR273a7U1FTNnz9fU6ZMqY+aAAAA3KbWMztWq1V79uyRxWKpj3oAAADcyqXDWA899JAWLFjg7loAAADczqUTlEtKSvTWW28pKytLnTt3rnBPrJkzZ7qlOAAAgLqqVdj58ccf1aJFC+3Zs0e33HKLJOn777936sPhLQAA0JDUKuy0atVKubm5ys7OlvTL7SH+8pe/KCoqql6KAwAAqKtanbNz+V3NV65cqXPnzrm1IAAAAHdy6QTlcpeHHwAAgIamVmHHYrFUOCeHc3QAAEBDVqtzdgzD0LBhwxw3+7xw4YIeffTRCldjffTRR+6rEAAAoA5qFXaGDh3q9PiBBx5wazEAAADuVquwk5mZWV91AAAA1Is6naAMAADQ0Hk17GzcuFH9+vVTbGysLBaLPv74Y6fnhw0b5jgpuvyrS5cuTn2Ki4s1duxYRUREKCgoSHfffbeOHDniwa0AAAANmVfDzrlz53TjjTdqzpw5Vfbp3bu3cnNzHV8rVqxwej49PV3Lly/X0qVLtWnTJp09e1ZpaWkqKyur7/IBAMBVwKV7Y7lLnz591KdPn2r72Gw2RUdHV/pcQUGBFixYoHfffVc9e/aUJC1evFhxcXFau3atevXq5faaAQDA1cWrYacm1q9fr8jISDVq1EjdunXTc889p8jISEnSzp07ZbfblZqa6ugfGxurxMREbd68ucqwU1xcrOLiYsfjwsJCSZLdbpfdbnf8fOl31C/G27OulvH28/N2Be5jtdqdvptRQ9qdrpZ93Ey8MeY1XZfFaCAfg2yxWLR8+XINGDDA0bZs2TIFBwcrPj5eOTk5euaZZ1RaWqqdO3fKZrNpyZIlGj58uFNwkaTU1FQlJCRo/vz5la4rIyNDU6ZMqdC+ZMkSBQYGunW7AABA/Th//ryGDBmigoIChYaGVtmvQc/s3HvvvY6fExMT1blzZ8XHx+sf//iHBg0aVOXrDMOo9pOdJ02apHHjxjkeFxYWKi4uTqmpqY7BstvtysrKUkpKiqxWqxu2BtVhvD3rahnvtDRvV+A+VqtdI0dm6c03U2S3N9wxr4vPPvN2Bf9xtezjZuKNMS8/MnMlDTrsXC4mJkbx8fHat2+fJCk6OlolJSU6deqUGjdu7OiXn5+vpKSkKpdjs9kcnwJ9KavVWuENqqwN9Yfx9qyGPt4lJd6uwP3sdqtKShrumNdFQ9yVGvo+bkaeHPOarueq+pydEydO6PDhw4qJiZEkderUSVarVVlZWY4+ubm52rNnT7VhBwAAXDu8OrNz9uxZ/fDDD47HOTk5+vrrrxUeHq7w8HBlZGTod7/7nWJiYnTgwAE9+eSTioiI0MCBAyVJYWFhGjFihMaPH68mTZooPDxcEyZMUIcOHRxXZwEAgGubV8POjh07lJyc7Hhcfh7N0KFDNW/ePO3evVuLFi3S6dOnFRMTo+TkZC1btkwhISGO17zyyivy9fXV4MGDVVRUpB49emjhwoXy8fHx+PYAAICGx6thp3v37qruYrDVq1dfcRn+/v6aPXu2Zs+e7c7SAACASVxV5+wAAADUFmEHAACYGmEHAACYGmEHAACYGmEHAACYGmEHAACYGmEHAACYGmEHAACYGmEHAACYGmEHAACYGmEHAACYGmEHAACYGmEHAACYmlfveg4AMI/k5Cv3yc6u/zqAyzGzAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI3P2QHgFTX5TBYAcAdmdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKl5Nexs3LhR/fr1U2xsrCwWiz7++GOn5w3DUEZGhmJjYxUQEKDu3btr7969Tn2Ki4s1duxYRUREKCgoSHfffbeOHDniwa0AcLnk5Ct/AYCneDXsnDt3TjfeeKPmzJlT6fMvvviiZs6cqTlz5mj79u2Kjo5WSkqKzpw54+iTnp6u5cuXa+nSpdq0aZPOnj2rtLQ0lZWVeWozAABAA+brzZX36dNHffr0qfQ5wzA0a9YsPfXUUxo0aJAk6Z133lFUVJSWLFmiUaNGqaCgQAsWLNC7776rnj17SpIWL16suLg4rV27Vr169fLYtgAAgIbJq2GnOjk5OcrLy1NqaqqjzWazqVu3btq8ebNGjRqlnTt3ym63O/WJjY1VYmKiNm/eXGXYKS4uVnFxseNxYWGhJMlut8tutzt+vvQ76hfj7Vn1Pd5+fvWy2Kua1Wp3+n6t8tSvOH9TPM8bY17TdTXYsJOXlydJioqKcmqPiorSwYMHHX38/PzUuHHjCn3KX1+Z6dOna8qUKRXa16xZo8DAQKe2rKwsl+qHaxhvz6qv8R49ul4WawojR17b+/iKFZ5dH39TPM+TY37+/Pka9WuwYaecxWJxemwYRoW2y12pz6RJkzRu3DjH48LCQsXFxSk1NVWhoaGSfkmLWVlZSklJkdVqrcMWoCYYb8+q7/FOS3P7Iq96VqtdI0dm6c03U2S3X7v7+GefeWY9/E3xPG+MefmRmStpsGEnOjpa0i+zNzExMY72/Px8x2xPdHS0SkpKdOrUKafZnfz8fCUlJVW5bJvNJpvNVqHdarVWeIMqa0P9Ybw9q77Gu6TE7Ys0DbvdqpKSa3cf9/SvN39TPM+TY17T9TTYz9lJSEhQdHS003RYSUmJNmzY4AgynTp1ktVqdeqTm5urPXv2VBt2AADAtcOrMztnz57VDz/84Hick5Ojr7/+WuHh4WrevLnS09M1bdo0tWrVSq1atdK0adMUGBioIUOGSJLCwsI0YsQIjR8/Xk2aNFF4eLgmTJigDh06OK7OAgAA1zavhp0dO3Yo+ZJPFys/j2bo0KFauHChJk6cqKKiIo0ePVqnTp3SbbfdpjVr1igkJMTxmldeeUW+vr4aPHiwioqK1KNHDy1cuFA+Pj4e3x4AANDweDXsdO/eXYZhVPm8xWJRRkaGMjIyquzj7++v2bNna/bs2fVQIQAAuNo12HN2AAAA3IGwAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATI2wAwAATM2rNwIFAFxbkpOv3Cc7u/7rwLWFmR0AAGBqhB0AAGBqhB0AAGBqhB0AAGBqhB0AAGBqhB0AAGBqXHoOoFZqcukwADQkzOwAAABTI+wAAABTI+wAAABTI+wAAABTI+wAAABTI+wAAABTI+wAAABTI+wAAABTI+wAAABTI+wAAABTI+wAAABTI+wAAABTI+wAAABTI+wAAABTI+wAAABTI+wAAABTa9BhJyMjQxaLxekrOjra8bxhGMrIyFBsbKwCAgLUvXt37d2714sVAwCAhqZBhx1Jat++vXJzcx1fu3fvdjz34osvaubMmZozZ462b9+u6OhopaSk6MyZM16sGAAANCS+3i7gSnx9fZ1mc8oZhqFZs2bpqaee0qBBgyRJ77zzjqKiorRkyRKNGjXK06UCANwgOfnKfbKz678OmEeDn9nZt2+fYmNjlZCQoPvuu08//vijJCknJ0d5eXlKTU119LXZbOrWrZs2b97srXIBAEAD06Bndm677TYtWrRIrVu31rFjxzR16lQlJSVp7969ysvLkyRFRUU5vSYqKkoHDx6sdrnFxcUqLi52PC4sLJQk2e122e12x8+Xfkf9Yrw9qy7j7efn7mquDVar3ek76uZKuy5/UzzPG2Ne03VZDMMw6rkWtzl37pxatmypiRMnqkuXLvrNb36jo0ePKiYmxtFn5MiROnz4sFatWlXlcjIyMjRlypQK7UuWLFFgYGC91A4AANzr/PnzGjJkiAoKChQaGlplvwY9s3O5oKAgdejQQfv27dOAAQMkSXl5eU5hJz8/v8Jsz+UmTZqkcePGOR4XFhYqLi5OqampjsGy2+3KyspSSkqKrFar+zcGThhvz6pqvNPSvFiUyVmtdo0cmaU330yR3c4+XleffVb98/xN8TxvjHn5kZkruarCTnFxsb799lvdcccdSkhIUHR0tLKysnTzzTdLkkpKSrRhwwa98MIL1S7HZrPJZrNVaLdarRXeoMraUH8Yb8+6fLxLSrxYzDXCbreqpIR9vK5q+meCvyme58kxr+l6GnTYmTBhgvr166fmzZsrPz9fU6dOVWFhoYYOHSqLxaL09HRNmzZNrVq1UqtWrTRt2jQFBgZqyJAh3i4dAAA0EA067Bw5ckT333+/jh8/rqZNm6pLly7aunWr4uPjJUkTJ05UUVGRRo8erVOnTum2227TmjVrFBIS4uXKAQBAQ9Ggw87SpUurfd5isSgjI0MZGRmeKQgAAFx1Gvzn7AAAANQFYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJhag743FgAAlUlOrv55Pz9p9GjP1IKGj5kdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgatwIFLhGXHrjxPKbJKalSSUl3qsJADyBsAOYwJXuAA0A1zIOYwEAAFMj7AAAAFPjMBYAwLSudF5adrbnaoH3MLMDAABMjbADAABMjcNYQAPHlVYAUDfM7AAAAFNjZgcAgGrUZHaVE50bNsIOAOCaxWHiawOHsQAAgKmZZmZn7ty5mjFjhnJzc9W+fXvNmjVLd9xxh7fLAqrF/yoBoP6ZIuwsW7ZM6enpmjt3rn7zm99o/vz56tOnj/71r3+pefPm3i4PAGBynNfTsJki7MycOVMjRozQI488IkmaNWuWVq9erXnz5mn69Olerg7XKmZtAFzKXX8TCE21d9Wfs1NSUqKdO3cqNTXVqT01NVWbN2/2UlUAAKChuOpndo4fP66ysjJFRUU5tUdFRSkvL6/S1xQXF6u4uNjxuKCgQJJ08uRJ2e12SZLdbtf58+d14sQJWa1Wl+sbPPjKfd5/3+XFm4a7xvtKavJ+uMt1Dfq/Er+Mt3RC111Xf+ONSzHmnmXe8T5xwtsVOCv/u+rra9ewYeeVlnZCpaXOY15f/86dOXNGkmQYRrX9rvqwU85isTg9NgyjQlu56dOna8qUKRXaExIS6qW2K4mI8MpqcY1bs8bbFVx7GHPPMut4N+R/M6oa8/qu+cyZMwoLC6vy+as+7ERERMjHx6fCLE5+fn6F2Z5ykyZN0rhx4xyPL168qJMnT6pJkyaOgFRYWKi4uDgdPnxYoaGh9bcBkMR4exrj7XmMuWcx3p7njTE3DENnzpxRbGxstf2u+rDj5+enTp06KSsrSwMHDnS0Z2VlqX///pW+xmazyWazObU1atSo0r6hoaH8ongQ4+1ZjLfnMeaexXh7nqfHvLoZnXJXfdiRpHHjxunBBx9U586d1bVrV73xxhs6dOiQHn30UW+XBgAAvMwUYefee+/ViRMn9Oyzzyo3N1eJiYlasWKF4uPjvV0aAADwMlOEHUkaPXq0Ro8e7bbl2Ww2TZ48ucLhLtQPxtuzGG/PY8w9i/H2vIY85hbjStdrAQAAXMUa9CeBAAAA1BVhBwAAmBphBwAAmBphBwAAmBphpwbuvvtuNW/eXP7+/oqJidGDDz6oo0ePerssUzpw4IBGjBihhIQEBQQEqGXLlpo8ebJKSkq8XZqpPffcc0pKSlJgYGCVH7AJ182dO1cJCQny9/dXp06d9MUXX3i7JNPauHGj+vXrp9jYWFksFn388cfeLsnUpk+frl//+tcKCQlRZGSkBgwYoO+++87bZVVA2KmB5ORkvf/++/ruu+/04Ycfav/+/brnnnu8XZYp/fvf/9bFixc1f/587d27V6+88opef/11Pfnkk94uzdRKSkr0+9//Xn/4wx+8XYrpLFu2TOnp6Xrqqaf01Vdf6Y477lCfPn106NAhb5dmSufOndONN96oOXPmeLuUa8KGDRs0ZswYbd26VVlZWSotLVVqaqrOnTvn7dKccOm5Cz799FMNGDBAxcXF9XqHbvxixowZmjdvnn788Udvl2J6CxcuVHp6uk6fPu3tUkzjtttu0y233KJ58+Y52tq2basBAwZo+vTpXqzM/CwWi5YvX64BAwZ4u5Rrxs8//6zIyEht2LBBd955p7fLcWBmp5ZOnjyp9957T0lJSQQdDykoKFB4eLi3ywBqraSkRDt37lRqaqpTe2pqqjZv3uylqoD6U1BQIEkN7m82YaeG/vSnPykoKEhNmjTRoUOH9Mknn3i7pGvC/v37NXv2bO5zhqvS8ePHVVZWpqioKKf2qKgo5eXleakqoH4YhqFx48bp9ttvV2JiorfLcXLNhp2MjAxZLJZqv3bs2OHo/9///d/66quvtGbNGvn4+Oihhx4SRwBrrrbjLUlHjx5V79699fvf/16PPPKIlyq/erky5qgfFovF6bFhGBXagKvdY489pl27dumvf/2rt0upwDT3xqqtxx57TPfdd1+1fVq0aOH4OSIiQhEREWrdurXatm2ruLg4bd26VV27dq3nSs2htuN99OhRJScnO+5ij9qr7ZjD/SIiIuTj41NhFic/P7/CbA9wNRs7dqw+/fRTbdy4Uc2aNfN2ORVcs2GnPLy4onxGp7i42J0lmVptxvunn35ScnKyOnXqpMzMTF133TU7AVknddnH4R5+fn7q1KmTsrKyNHDgQEd7VlaW+vfv78XKAPcwDENjx47V8uXLtX79eiUkJHi7pEpds2GnprZt26Zt27bp9ttvV+PGjfXjjz/qf/7nf9SyZUtmderB0aNH1b17dzVv3lwvvfSSfv75Z8dz0dHRXqzM3A4dOqSTJ0/q0KFDKisr09dffy1Juv766xUcHOzd4q5y48aN04MPPqjOnTs7ZioPHTrEeWj15OzZs/rhhx8cj3NycvT1118rPDxczZs392Jl5jRmzBgtWbJEn3zyiUJCQhyzmGFhYQoICPBydZcwUK1du3YZycnJRnh4uGGz2YwWLVoYjz76qHHkyBFvl2ZKmZmZhqRKv1B/hg4dWumYZ2dne7s0U3jttdeM+Ph4w8/Pz7jllluMDRs2eLsk08rOzq50Xx46dKi3SzOlqv5eZ2Zmers0J3zODgAAMDVOhgAAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AHgVRaLRR9//LHj8b///W916dJF/v7+uummm6psA4Ca4nYRAOrFsGHD9M4770iSfH19FR4ero4dO+r+++/XsGHDHPc8y83NVePGjR2vmzx5soKCgvTdd985blVRWRsA1BQzOwDqTe/evZWbm6sDBw5o5cqVSk5O1uOPP660tDSVlpZK+uWeZzabzfGa/fv36/bbb1d8fLyaNGlSZVttlZSU1H2DAFyVCDsA6o3NZlN0dLR+9atf6ZZbbtGTTz6pTz75RCtXrtTChQslOR/Gslgs2rlzp5599llZLBZlZGRU2iZJP/30k+699141btxYTZo0Uf/+/XXgwAHHuocNG6YBAwZo+vTpio2NVevWrWv1updeekkxMTFq0qSJxowZI7vd7uhTXFysiRMnKi4uTjabTa1atdKCBQscz//rX/9S3759FRwcrKioKD344IM6fvx4vYwxgCsj7ADwqLvuuks33nijPvroowrP5ebmqn379ho/frxyc3M1YcKEStvOnz+v5ORkBQcHa+PGjdq0aZOCg4PVu3dvpxmczz//XN9++62ysrL02Wef1fh12dnZ2r9/v7Kzs/XOO+9o4cKFjnAmSQ899JCWLl2qv/zlL/r222/1+uuvOw6v5ebmqlu3brrpppu0Y8cOrVq1SseOHdPgwYPrb1ABVItzdgB4XJs2bbRr164K7dHR0fL19VVwcLCio6MlScHBwRXa3n77bV133XV66623ZLFYJEmZmZlq1KiR1q9fr9TUVElSUFCQ3nrrLfn5+dXqdY0bN9acOXPk4+OjNm3a6Le//a0+//xzjRw5Ut9//73ef/99ZWVlqWfPnpKk//qv/3Jsw7x583TLLbdo2rRpjra3335bcXFx+v777x0zTAA8h7ADwOMMw3CEDVfs3LlTP/zwg0JCQpzaL1y4oP379zsed+jQwRF0avO69u3by8fHx/E4JiZGu3fvliR9/fXX8vHxUbdu3aqsLTs7u9ITqffv30/YAbyAsAPA47799lslJCS4/PqLFy+qU6dOeu+99yo817RpU8fPQUFBLr3OarU6PWexWHTx4kVJUkBAwBVr69evn1544YUKz8XExFT7WgD1g7ADwKPWrVun3bt364knnnB5GbfccouWLVumyMhIhYaG1vvrLtWhQwddvHhRGzZscBzGunwdH374oVq0aCFfX/7EAg0BJygDqDfFxcXKy8vTTz/9pC+//FLTpk1T//79lZaWpoceesjl5f6///f/FBERof79++uLL75QTk6ONmzYoMcff1xHjhxx++su1aJFCw0dOlQPP/ywPv74Y+Xk5Gj9+vV6//33JUljxozRyZMndf/992vbtm368ccftWbNGj388MMqKytzeZsBuI6wA6DerFq1SjExMWrRooV69+6t7Oxs/eUvf9Enn3zidE5MbQUGBmrjxo1q3ry5Bg0apLZt2+rhhx9WUVFRtTM2rr7ucvPmzdM999yj0aNHq02bNho5cqTOnTsnSYqNjdU///lPlZWVqVevXkpMTNTjjz+usLAwxwcpAvAsi2EYhreLAAAAqC/8NwMAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJja/wdNjzTT9mv/HAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "differences = hf_model.model.layers[0].block_sparse_moe(test_tensor)[0] - tl_model.blocks[0].mlp(test_tensor)\n",
    "\n",
    "# Flatten the differences to create a one-dimensional tensor\n",
    "flattened_differences = differences.flatten().cpu().detach().numpy()\n",
    "\n",
    "# Plot the histogram of the differences\n",
    "plt.hist(flattened_differences, bins=50, alpha=0.75, color='blue')\n",
    "plt.title('Differences Between MLP Outputs')\n",
    "plt.xlabel('Difference')\n",
    "plt.ylabel('Frequency')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ac306e1c-9972-466a-8f4a-f3eb56042f53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.2662847638130188"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].block_sparse_moe(test_tensor)[0][0, 0, 0].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e9481397-6e87-435a-a0cf-ef409630d17c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5243168473243713"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].mlp(test_tensor)[0, 0, 0].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "89f55163-fa2e-44f2-b112-7b052a6e85af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MixtralAttention(\n",
       "  (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "  (k_proj): Linear(in_features=4096, out_features=1024, bias=False)\n",
       "  (v_proj): Linear(in_features=4096, out_features=1024, bias=False)\n",
       "  (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "  (rotary_emb): MixtralRotaryEmbedding()\n",
       ")"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].self_attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f0186768-d8f0-4d55-a94c-606c4ba3f7ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[ 0.0730,  1.2900, -0.1474,  ..., -1.2459,  0.1778, -0.1866]]],\n",
       "        grad_fn=<AddBackward0>),)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0](test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "432bb274-b499-44c9-98d1-777d03425daa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.0990,  1.2813, -0.1482,  ..., -1.2642,  0.1893, -0.2015]]],\n",
       "       grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0](test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4a440811-e7f0-4092-b8e7-f7cac80dc84a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[False, False, False,  ..., False, False, False]]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0](test_tensor)[0] == tl_model.blocks[0](test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8ed65bb3-6990-48e5-9ef2-1becd9dfaffc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.17776145040988922, 0.18934544920921326)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0](test_tensor)[0][0, 0, -2].item(), tl_model.blocks[0](test_tensor)[0, 0, -2].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "763f6c2e-b71f-4724-b2f7-f79a9ab29caf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sum(hf_model.model.layers[0](test_tensor)[0] == tl_model.blocks[0](test_tensor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f41aa0eb-6386-476d-ae1a-b5e7e06893aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHFCAYAAAAUpjivAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABFZklEQVR4nO3daXhUVbr28btMKvNAIJBBIAwyJyhgi6AIEQiDIEIrIoqgyMEGkfHY0tiHqMikDDYo2MxqK4gCTogECCgGlcGBwaOCjJoQBGQKJEWy3g+c1GuREEKlKgmb/++6cmGtWnvv56mqyM2qvatsxhgjAAAAi7qurAsAAADwJsIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOyq2FCxfKZrM5fwICAhQdHa3ExERNmDBBmZmZBbZJTk6WzWZzGcvJydHjjz+umJgY+fj46KabbpIkHTt2TL169VKVKlVks9l0zz33lEJXV699+/a5PB82m01hYWG68cYbNX36dOXm5rq135UrVyo5OdmzxZYzNptNTzzxRFmX4VE//PCD+vXrp+rVq8vPz0+RkZHq3LmzPvnkkxLt99VXX9XChQs9U+RlZGVlKTk5WevXry+V46Hs+JZ1AcDlLFiwQPXr15fD4VBmZqY2btyoSZMm6aWXXtKSJUvUrl0759zHHntMHTt2dNl+1qxZeu211zRjxgw1a9ZMISEhkqTnn39ey5cv1/z581W7dm1VrFixVPu6Wg0ZMkS9e/eWJP3xxx/64IMPNHz4cB08eFBTpky54v2tXLlSr7zyiuUDj5UsW7ZMvXv3Vq1atfTPf/5T9erV0+HDh7VgwQJ17txZ//3f/63Jkye7te9XX31VkZGR6tevn2eLLkRWVpaeffZZSVKbNm28fjyUHcIOyr34+HjdfPPNztt//etfNXz4cN1+++3q0aOHfv75Z0VFRUmSqlatqqpVq7psv2PHDgUGBhb4l/WOHTtUu3ZtPfjggx6r9ezZswoMDPTY/sqj6tWr69Zbb3Xe7tixo3bs2KG3337brbCD8icrK0tBQUGF3rdnzx716dNHCQkJWr9+vYKDg5333Xffffrb3/6mF198UU2bNlWvXr1Kq2SgSLyNhatS9erVNWXKFJ06dUqvvfaac/zit7FsNpvmzp2rs2fPOt96yX97bM2aNfrhhx+c4/lL2Tk5ORo3bpzq168vf39/Va5cWY888oiOHDniUkONGjXUpUsXLVu2TE2aNFFAQIDzX4kZGRkaOHCgqlatKj8/P9WsWVPPPvuszp8/79w+/22hl156SVOnTlXNmjUVEhKiFi1a6MsvvyzQ81dffaWuXbuqUqVKCggIUO3atTVs2DCXOT///LN69+6tKlWqyN/fXw0aNNArr7ziMicvL0/jxo1TvXr1FBgYqAoVKqhx48Z6+eWX3XouJCk8PFx2u73A+JIlS9SiRQsFBwcrJCREHTp00DfffOO8v1+/fs76/vz22L59+3TfffepUaNGLvvr2rWrbDabli5d6hzbtm2bbDabPvzwQ+dYcR5/6cqf61WrVqlp06YKDAxU/fr1NX/+fLcfs8Ieq6SkJMXExCgwMFANGjTQ008/rTNnzjjnvPHGG7LZbNq0aVOB7Z977jnZ7Xb99ttvzrE1a9aobdu2CgsLU1BQkG677TatXbvWZbv835lt27bp3nvvVUREhGrXrn3JOqdNm6asrCzNmDHDJejkmzJliipUqKAXXnihwDEulv+7uG/fPkkXHuedO3dqw4YNztdCjRo1JEnr16+XzWbTm2++qREjRig6OlqBgYFq3bq1y2tKurBKU9hKTb9+/Zz727dvnypXrixJevbZZ53Hy19ROnLkiP7rv/5L1apVc742brvtNq1Zs+aSjw3KMQOUUwsWLDCSzObNmwu9//Tp08bHx8e0bdvWOTZ27Fjz55f1pk2bTOfOnU1gYKDZtGmT2bRpk8nIyDCbNm0yTZo0MbVq1XKOnzhxwuTm5pqOHTua4OBg8+yzz5qUlBQzd+5cc/3115uGDRuarKws577j4uJMTEyMqVWrlpk/f75JTU01X3/9tUlPTzfVqlUzcXFx5rXXXjNr1qwxzz//vPH39zf9+vVzbr93714jydSoUcN07NjRrFixwqxYscIkJCSYiIgI88cffzjnrlq1ytjtdtO4cWOzcOFCs27dOjN//nzTq1cv55ydO3ea8PBwk5CQYF5//XWzevVqM3LkSHPdddeZ5ORk57wJEyYYHx8fM3bsWLN27VqzatUqM336dJc5hcmvd9KkScbhcBiHw2F+//13M2/ePOPr62vGjBnjMv+FF14wNpvNPProo+ajjz4yy5YtMy1atDDBwcFm586dxhhjdu/ebe69914jyfk8bNq0yZw7d87Mnj3bSDK//fabMcYYh8NhQkNDTWBgoBkwYIDzOJMmTTK+vr7m5MmTxhhT7Mf/Sp/rqlWrmoYNG5rXX3/dfPrpp+a+++4zksyGDRuKfNyMMUaSGTx4cJFznn/+eTNt2jTz8ccfm/Xr15vZs2ebmjVrmsTEROec7OxsEx0dbR588EGXbR0Oh4mNjTX33Xefc+yNN94wNpvN3HPPPWbZsmXmww8/NF26dDE+Pj5mzZo1znn5vzNxcXHm73//u0lJSTErVqy4ZJ1169Y1UVFRRfbSs2dPI8mkp6e7HONi+b/je/fuNcYYs23bNlOrVi3TpEkT52th27ZtxhhjUlNTjSRTrVo1061bN/Phhx+aN99809xwww0mLCzM7Nmzx7nf1q1bm9atWxc4Xt++fU1cXJwxxphz586ZVatWGUmmf//+zuPt3r3bGGNMhw4dTOXKlc2///1vs379erNixQrzP//zP2bx4sVF9o7yibCDcutyYccYY6KiokyDBg2ctwv7n2rfvn1NcHBwgW1bt25tGjVq5DL29ttvG0nmvffecxnfvHmzkWReffVV51hcXJzx8fExP/74o8vcgQMHmpCQELN//36X8ZdeeslIcv5Fnx8eEhISzPnz553zvv76ayPJvP32286x2rVrm9q1a5uzZ89e8rHo0KGDqVq1qjlx4oTL+BNPPGECAgLMsWPHjDHGdOnSxdx0002X3M+l5Ndb2E+/fv1cejhw4IDx9fU1Q4YMcdnHqVOnTHR0tOnZs6dzbPDgwYX+Rbh7924jybz++uvGGGM2btxoJJmnnnrK1KxZ0zmvffv2pmXLls7bxX38r/S5DggIcNnn2bNnTcWKFc3AgQMv+9gVJ+z8WV5ennE4HGbDhg1Gkvnuu++c940dO9b4+fmZw4cPO8eWLFniErzOnDljKlasaLp27eqy39zcXHPjjTeaW265xWV/ksz//M//FKu2gIAAc+uttxY55+9//7uRZL766iuXY1zs4rBjjDGNGjUqNKjkh52mTZuavLw85/i+ffuM3W43jz32mHOsOGHHGGOOHDliJJmxY8cWmBsSEmKGDRtWZJ+4evA2Fq5qxhiP7u+jjz5ShQoV1LVrV50/f975c9NNNyk6OrrAVRuNGzdW3bp1C+wjMTFRsbGxLvvo1KmTJGnDhg0u8++66y75+Pi47FOS9u/fL0n66aeftGfPHvXv318BAQGF1n3u3DmtXbtW3bt3V1BQkMtxO3furHPnzjnfGrvlllv03XffadCgQfr000918uTJK3qMhg4dqs2bN2vz5s1KTU3V+PHj9c477+iBBx5wzvn00091/vx5Pfzwwy61BAQEqHXr1sW6+qV27dqqUaOG822DlJQUJSQk6KGHHtLevXu1Z88eZWdna+PGjS4nqRf38b/S5/qmm25S9erVnbcDAgJUt25d5/NUUr/88ot69+6t6Oho+fj4yG63q3Xr1pIuXPmU729/+5skac6cOc6xmTNnKiEhQXfccYckKS0tTceOHVPfvn1desvLy1PHjh21efNml7fHpAvnwnlK/u9lYW9dlVTv3r1d9hsXF6eWLVsqNTXVo8e55ZZbtHDhQo0bN05ffvmlHA6HR/eP0sUJyrhqnTlzRkePHlVCQoLH9nn48GH98ccf8vPzK/T+33//3eV2TExMofv48MMPCz2HpbB9VKpUyeW2v7+/pAsnO0tynj9y8YnXf3b06FGdP39eM2bM0IwZM4o87ujRoxUcHKw333xTs2fPlo+Pj+644w5NmjTJ5UTwS6latarLvDZt2shms2n06NH69NNP1aFDBx0+fFiS9Je//KXQfVx3XfH+ndW2bVutWrVK0oXzT9q3b6+EhARFRUVpzZo1qlOnjs6ePesSdor7+F/pc33x8yRdeK7yn6eSOH36tFq1aqWAgACNGzdOdevWVVBQkA4ePKgePXq4HCMqKkr333+/XnvtNT399NPauXOnPv/8c5dz1/If/3vvvfeSxzx27JjLOTeFvZYLU716de3du7fIOfnn4FSrVq1Y+7wS0dHRhY599913Hj3OkiVLNG7cOM2dO1f//Oc/FRISou7du2vy5MmF1oDyjbCDq9bHH3+s3Nxcj14yGhkZqUqVKjn/gr1YaGioy+3C/uUaGRmpxo0bu5yg+WexsbFXVFP+SZSHDh265JyIiAj5+PioT58+Gjx4cKFzatasKUny9fXViBEjNGLECP3xxx9as2aN/vGPf6hDhw46ePDgJa/CKUr+atR3332nDh06KDIyUpL07rvvKi4u7or3l69t27aaN2+evv76a3311Vd65plnJEl33nmnUlJStH//foWEhLhcHVbcx/9Kn2tvWrdunX777TetX7/euZojXbi0vzBDhw7VG2+8offff1+rVq1ShQoVXK4qzH/8Z8yY4fLY/Fn+FYz5irsK0759e73yyiv68ssvC913VlaWUlJSFB8f7wwF+SuS2dnZzjAvFQyUxZGRkVHo2J/DaEBAgE6cOFFg3pUcLzIyUtOnT9f06dN14MABffDBB3r66aeVmZl5ydcMyi/CDq5KBw4c0KhRoxQeHq6BAwd6bL9dunTR4sWLlZubq+bNm7u9j5UrV6p27dqKiIgocU1169ZV7dq1NX/+fI0YMcLlL4t8QUFBSkxM1DfffKPGjRtfcrXiYhUqVNC9996rX3/9VcOGDdO+ffvUsGHDK67x22+/lSRVqVJFktShQwf5+vpqz549l3175M8rWRdftt+2bVvZbDb985//1HXXXed8m6Zdu3b67//+b+3fv1933HGHyypOcR9/TzzXnpIfNC5+bv+8WvNnzZo1U8uWLTVp0iTt2LFD//Vf/+WySnPbbbepQoUK2rVrl8c/zHD48OGaP3++hgwZUuDSc0kaNWqUjh8/rlmzZjnH8q+A+v77711W+/58BV2+y62Wvf322xoxYoTzMdu/f7/S0tL08MMPuxxv6dKlLuHq6NGjSktLU1hYmMuxJF12da569ep64okntHbtWn3xxRdFzkX5RNhBubdjxw7nOQeZmZn6/PPPtWDBAvn4+Gj58uXOlQ9P6NWrl/7zn/+oc+fOGjp0qG655RbZ7XYdOnRIqamp6tatm7p3717kPp577jmlpKSoZcuWevLJJ1WvXj2dO3dO+/bt08qVKzV79uwi35IqzCuvvKKuXbvq1ltv1fDhw1W9enUdOHBAn376qf7zn/9Ikl5++WXdfvvtatWqlf72t7+pRo0aOnXqlHbv3q0PP/xQ69atk3Th8u38zy6qXLmy9u/fr+nTpysuLk516tS5bC0HDhxwnv9z5swZbdq0SRMmTFBcXJx69Ogh6cJfNs8995zGjBmjX375RR07dlRERIQOHz6sr7/+WsHBwc7L9PPfhpw0aZI6deokHx8fZ2CrUqWK4uPjtXr1aiUmJjpXndq1a6djx47p2LFjmjp1qluPvyee6yuxZ88evfvuuwXGGzZsqJYtWyoiIkKPP/64xo4dK7vdrv/85z9FvjUzdOhQ3X///bLZbBo0aJDLfSEhIZoxY4b69u2rY8eO6d5771WVKlV05MgRfffddzpy5IhLGLkStWvX1htvvKEHH3xQf/nLXzRixAjnhwrOnz9fn3zyiUaNGqX777/fuU3nzp1VsWJF9e/fX88995x8fX21cOFCHTx4sMD+ExIStHjxYi1ZskS1atVSQECAy1vVmZmZ6t69uwYMGKATJ05o7NixCggI0OjRo51z+vTpo9dee00PPfSQBgwYoKNHj2ry5MkuQUe6sHoXFxen999/X23btlXFihUVGRmpiIgIJSYmqnfv3qpfv75CQ0O1efNmrVq1yvkax1WmrM+QBi4l/0qN/B8/Pz9TpUoV07p1azN+/HiTmZlZYJuSXo1lzIXLeF966SVz4403moCAABMSEmLq169vBg4caH7++WfnvLi4OHPXXXcVWvuRI0fMk08+aWrWrGnsdrupWLGiadasmRkzZow5ffq0Meb/X9304osvFthehVwhsmnTJtOpUycTHh5u/P39Te3atc3w4cNd5uzdu9c8+uij5vrrrzd2u91UrlzZtGzZ0owbN845Z8qUKaZly5YmMjLS+Pn5merVq5v+/fubffv2FdrLn/eti67CCggIMHXr1jXDhg1zXmb8ZytWrDCJiYkmLCzM+Pv7m7i4OHPvvfe6XPqcnZ1tHnvsMVO5cmVjs9kKXJ0zfPhwI8m88MILLvuuU6eOkWS+//77AsctzuNvTMmf60td9XOxix+3P//kP89paWmmRYsWJigoyFSuXNk89thjZtu2bUaSWbBgQYF9ZmdnG39/f9OxY8dLHnfDhg3mrrvuMhUrVjR2u91cf/315q677jJLly51zsn/nTly5Mhl+/iznTt3mr59+5qqVas6H+OOHTuajz/+uND5X3/9tWnZsqUJDg42119/vRk7dqyZO3duged73759JikpyYSGhjoviTfm/1+N9cYbb5gnn3zSVK5c2fj7+5tWrVqZLVu2FDjeokWLTIMGDUxAQIBp2LChWbJkSYGrsYwxZs2aNaZJkybG39/fSDJ9+/Y1586dM48//rhp3LixCQsLM4GBgaZevXpm7Nix5syZM1f0OKF8sBnj4ctZAABe9+GHH+ruu+/Wxx9/rM6dO5d1OV63fv16JSYmaunSpUWeeA0UhrexAOAqsmvXLu3fv18jR47UTTfd5LykHsCl8Tk7AHAVGTRokO6++25FRETo7bff9spn2QBWw9tYAADA0ljZAQAAlkbYAQAAlkbYAQAAlsbVWJLy8vL022+/KTQ0lJP9AAC4ShhjdOrUKcXGxhb5nXuEHUm//fabV76wDgAAeN/BgweL/GR6wo7+/xf+HTx4sMDHiV+tHA6HVq9eraSkpEt++7OV0K+1XUv9Xku9SvRrdd7u9+TJk6pWrdplv7iXsKP//yV8YWFhlgo7QUFBCgsLu2Z+oejXuq6lfq+lXiX6tbrS6vdyp6BwgjIAALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA037IuAACuNYmJl77Pz08aNKj0agGuBazsAAAASyPsAAAASyPsAAAASyPsAAAAS+MEZQAoh7p0kXJyip6Tmlo6tQBXO1Z2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApXHpOQBcpYr6jq18XJ4OsLIDAAAsjrADAAAsjbADAAAsjbADAAAsjbADAAAsjbADAAAsjbADAAAsjbADAAAsjbADAAAsrdyEnQkTJshms2nYsGHOMWOMkpOTFRsbq8DAQLVp00Y7d+502S47O1tDhgxRZGSkgoODdffdd+vQoUOlXD0AACivykXY2bx5s/7973+rcePGLuOTJ0/W1KlTNXPmTG3evFnR0dFq3769Tp065ZwzbNgwLV++XIsXL9bGjRt1+vRpdenSRbm5uaXdBgAAKIfKPOycPn1aDz74oObMmaOIiAjnuDFG06dP15gxY9SjRw/Fx8dr0aJFysrK0ltvvSVJOnHihObNm6cpU6aoXbt2atKkid58801t375da9asKauWAABAOVLmYWfw4MG666671K5dO5fxvXv3KiMjQ0lJSc4xf39/tW7dWmlpaZKkrVu3yuFwuMyJjY1VfHy8cw4AALi2lem3ni9evFjbtm3T5s2bC9yXkZEhSYqKinIZj4qK0v79+51z/Pz8XFaE8ufkb1+Y7OxsZWdnO2+fPHlSkuRwOORwONxrppzJ78Mq/VwO/Vqb1fr187v0fXa7w+XPkirvD5nVntvLoV/v7P9yyizsHDx4UEOHDtXq1asVEBBwyXk2m83ltjGmwNjFLjdnwoQJevbZZwuMr169WkFBQZep/OqSkpJS1iWUKvq1Nqv0O2jQ5ecMGOCZXleu9MhuvM4qz21x0a9nZGVlFWtemYWdrVu3KjMzU82aNXOO5ebm6rPPPtPMmTP1448/SrqwehMTE+Ock5mZ6VztiY6OVk5Ojo4fP+6yupOZmamWLVte8tijR4/WiBEjnLdPnjypatWqKSkpSWFhYR7rsSw5HA6lpKSoffv2stvtZV2O19GvtVmt3y5dLn2f3e7QgAEpmjOnvRyOkvf60Ucl3oVXWe25vRz69az8d2Yup8zCTtu2bbV9+3aXsUceeUT169fX3//+d9WqVUvR0dFKSUlRkyZNJEk5OTnasGGDJk2aJElq1qyZ7Ha7UlJS1LNnT0lSenq6duzYocmTJ1/y2P7+/vL39y8wbrfbLffis2JPRaFfa7NKvzk5l5/jcNiVk1PyXq+Wh8sqz21x0a/n9lscZRZ2QkNDFR8f7zIWHBysSpUqOceHDRum8ePHq06dOqpTp47Gjx+voKAg9e7dW5IUHh6u/v37a+TIkapUqZIqVqyoUaNGKSEhocAJzwAA4NpUpicoX85TTz2ls2fPatCgQTp+/LiaN2+u1atXKzQ01Dln2rRp8vX1Vc+ePXX27Fm1bdtWCxculI+PTxlWDgAAyotyFXbWr1/vcttmsyk5OVnJycmX3CYgIEAzZszQjBkzvFscAAC4KpX55+wAAAB4E2EHAABYGmEHAABYGmEHAABYGmEHAABYGmEHAABYGmEHAABYGmEHAABYGmEHAABYGmEHAABYGmEHAABYGmEHAABYGmEHAABYGmEHAABYGmEHAABYmm9ZFwAAVpKYWNYVALgYKzsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSfMu6AACA9yQmXn5Oaqr36wDKEis7AADA0ljZAYBiKs4qCYDyh5UdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaWUadmbNmqXGjRsrLCxMYWFhatGihT755BPn/cYYJScnKzY2VoGBgWrTpo127tzpso/s7GwNGTJEkZGRCg4O1t13361Dhw6VdisAAKCcKtOwU7VqVU2cOFFbtmzRli1bdOedd6pbt27OQDN58mRNnTpVM2fO1ObNmxUdHa327dvr1KlTzn0MGzZMy5cv1+LFi7Vx40adPn1aXbp0UW5ublm1BQAAypEyDTtdu3ZV586dVbduXdWtW1cvvPCCQkJC9OWXX8oYo+nTp2vMmDHq0aOH4uPjtWjRImVlZemtt96SJJ04cULz5s3TlClT1K5dOzVp0kRvvvmmtm/frjVr1pRlawAAoJzwLesC8uXm5mrp0qU6c+aMWrRoob179yojI0NJSUnOOf7+/mrdurXS0tI0cOBAbd26VQ6Hw2VObGys4uPjlZaWpg4dOhR6rOzsbGVnZztvnzx5UpLkcDjkcDi81GHpyu/DKv1cDv1aW3np18/P+8ew2x0uf5aGsnxYy8tzW1ro1zv7v5wyDzvbt29XixYtdO7cOYWEhGj58uVq2LCh0tLSJElRUVEu86OiorR//35JUkZGhvz8/BQREVFgTkZGxiWPOWHCBD377LMFxlevXq2goKCStlSupKSklHUJpYp+ra2s+x00qPSONWBA6fW6cmWpHeqSyvq5LW306xlZWVnFmlfmYadevXr69ttv9ccff+i9995T3759tWHDBuf9NpvNZb4xpsDYxS43Z/To0RoxYoTz9smTJ1WtWjUlJSUpLCzMzU7KF4fDoZSUFLVv3152u72sy/E6+rW28tJvly7eP4bd7tCAASmaM6e9HI7S6fWjj0rlMIUqL89taaFfz8p/Z+Zyyjzs+Pn56YYbbpAk3Xzzzdq8ebNefvll/f3vf5d0YfUmJibGOT8zM9O52hMdHa2cnBwdP37cZXUnMzNTLVu2vOQx/f395e/vX2Dcbrdb7sVnxZ6KQr/WVtb95uSU3rEcDrtyckqn1/LwEirr57a00a/n9lsc5e5zdowxys7OVs2aNRUdHe2y9JWTk6MNGzY4g0yzZs1kt9td5qSnp2vHjh1Fhh0AAHDtKNOVnX/84x/q1KmTqlWrplOnTmnx4sVav369Vq1aJZvNpmHDhmn8+PGqU6eO6tSpo/HjxysoKEi9e/eWJIWHh6t///4aOXKkKlWqpIoVK2rUqFFKSEhQu3btyrI1AABQTpRp2Dl8+LD69Omj9PR0hYeHq3Hjxlq1apXat28vSXrqqad09uxZDRo0SMePH1fz5s21evVqhYaGOvcxbdo0+fr6qmfPnjp79qzatm2rhQsXysfHp6zaAgAA5UiZhp158+YVeb/NZlNycrKSk5MvOScgIEAzZszQjBkzPFwdAACwgnJ3zg4AAIAnEXYAAIClEXYAAIClEXYAAICllfmHCgJAeZCYWNYVAPAWVnYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAICl8QnKAHCNK86nR6emer8OwFtY2QEAAJZG2AEAAJZG2AEAAJZG2AEAAJbmVtjZu3evp+sAAADwCrfCzg033KDExES9+eabOnfunKdrAgAA8Bi3ws53332nJk2aaOTIkYqOjtbAgQP19ddfe7o2AACAEnMr7MTHx2vq1Kn69ddftWDBAmVkZOj2229Xo0aNNHXqVB05csTTdQIAALilRCco+/r6qnv37nrnnXc0adIk7dmzR6NGjVLVqlX18MMPKz093VN1AgAAuKVEYWfLli0aNGiQYmJiNHXqVI0aNUp79uzRunXr9Ouvv6pbt26eqhMAAMAtbn1dxNSpU7VgwQL9+OOP6ty5s15//XV17txZ1113ITvVrFlTr732murXr+/RYgEAAK6UW2Fn1qxZevTRR/XII48oOjq60DnVq1fXvHnzSlQcAABASbkVdn7++efLzvHz81Pfvn3d2T0AAIDHuHXOzoIFC7R06dIC40uXLtWiRYtKXBQAAICnuBV2Jk6cqMjIyALjVapU0fjx40tcFAAAgKe4FXb279+vmjVrFhiPi4vTgQMHSlwUAACAp7gVdqpUqaLvv/++wPh3332nSpUqlbgoAAAAT3Er7PTq1UtPPvmkUlNTlZubq9zcXK1bt05Dhw5Vr169PF0jAACA29y6GmvcuHHav3+/2rZtK1/fC7vIy8vTww8/zDk7AACgXHEr7Pj5+WnJkiV6/vnn9d133ykwMFAJCQmKi4vzdH0AAAAl4lbYyVe3bl3VrVvXU7UAAAB4nFthJzc3VwsXLtTatWuVmZmpvLw8l/vXrVvnkeIAAABKyq2wM3ToUC1cuFB33XWX4uPjZbPZPF0XAACAR7gVdhYvXqx33nlHnTt39nQ9AAAAHuXWped+fn664YYbPF0LAACAx7kVdkaOHKmXX35ZxhhP1wMAAOBRbr2NtXHjRqWmpuqTTz5Ro0aNZLfbXe5ftmyZR4oDAAAoKbfCToUKFdS9e3dP1wIAAOBxboWdBQsWeLoOAAAAr3DrnB1JOn/+vNasWaPXXntNp06dkiT99ttvOn36tMeKAwAAKCm3Vnb279+vjh076sCBA8rOzlb79u0VGhqqyZMn69y5c5o9e7an6wQAAHCLWys7Q4cO1c0336zjx48rMDDQOd69e3etXbvWY8UBAACUlNtXY33xxRfy8/NzGY+Li9Ovv/7qkcIAAAA8wa2Vnby8POXm5hYYP3TokEJDQ0tcFAAAgKe4FXbat2+v6dOnO2/bbDadPn1aY8eO5SskAABAueLW21jTpk1TYmKiGjZsqHPnzql37976+eefFRkZqbffftvTNQIAALjNrbATGxurb7/9Vm+//ba2bdumvLw89e/fXw8++KDLCcsAAABlza2wI0mBgYF69NFH9eijj3qyHgAAAI9yK+y8/vrrRd7/8MMPu1UMAACAp7kVdoYOHepy2+FwKCsrS35+fgoKCiLsAACAcsOtq7GOHz/u8nP69Gn9+OOPuv322zlBGQAAlCtufzfWxerUqaOJEycWWPUBAAAoSx4LO5Lk4+Oj3377zZO7BAAAKBG3ztn54IMPXG4bY5Senq6ZM2fqtttu80hhAAAAnuBW2LnnnntcbttsNlWuXFl33nmnpkyZ4om6AAAAPMKtsJOXl+fpOgAAALzCo+fsAAAAlDdureyMGDGi2HOnTp3qziEAAAA8wq2w880332jbtm06f/686tWrJ0n66aef5OPjo6ZNmzrn2Ww2z1QJAADgJrfCTteuXRUaGqpFixYpIiJC0oUPGnzkkUfUqlUrjRw50qNFAgAAuMutc3amTJmiCRMmOIOOJEVERGjcuHFcjQUAAMoVt8LOyZMndfjw4QLjmZmZOnXqVImLAgAA8BS3wk737t31yCOP6N1339WhQ4d06NAhvfvuu+rfv7969Ojh6RoBAADc5tY5O7Nnz9aoUaP00EMPyeFwXNiRr6/69++vF1980aMFAkBJJSaWdQUAypJbYScoKEivvvqqXnzxRe3Zs0fGGN1www0KDg72dH0AgHKgOIExNdX7dQDuKNGHCqanpys9PV1169ZVcHCwjDFXtP2ECRP0l7/8RaGhoapSpYruuece/fjjjy5zjDFKTk5WbGysAgMD1aZNG+3cudNlTnZ2toYMGaLIyEgFBwfr7rvv1qFDh0rSGgAAsAi3ws7Ro0fVtm1b1a1bV507d1Z6erok6bHHHruiy843bNigwYMH68svv1RKSorOnz+vpKQknTlzxjln8uTJmjp1qmbOnKnNmzcrOjpa7du3dzkRetiwYVq+fLkWL16sjRs36vTp0+rSpYtyc3PdaQ8AAFiIW2Fn+PDhstvtOnDggIKCgpzj999/v1atWlXs/axatUr9+vVTo0aNdOONN2rBggU6cOCAtm7dKunCqs706dM1ZswY9ejRQ/Hx8Vq0aJGysrL01ltvSZJOnDihefPmacqUKWrXrp2aNGmiN998U9u3b9eaNWvcaQ8AAFiIW+fsrF69Wp9++qmqVq3qMl6nTh3t37/f7WJOnDghSapYsaIkae/evcrIyFBSUpJzjr+/v1q3bq20tDQNHDhQW7dulcPhcJkTGxur+Ph4paWlqUOHDgWOk52drezsbOftkydPSpIcDofzhOurXX4fVunncujX2krar5+fJ6vxLrvd4fLn1cSdp4fXsrV5u9/i7tetsHPmzBmXFZ18v//+u/z9/d3ZpYwxGjFihG6//XbFx8dLkjIyMiRJUVFRLnOjoqKcoSojI0N+fn4uH3CYPyd/+4tNmDBBzz77bIHx1atXF9rX1SwlJaWsSyhV9Gtt7vY7aJCHCykFAwZcfc/typXub8tr2dq81W9WVlax5rkVdu644w69/vrrev755yVd+A6svLw8vfjii0p08xrPJ554Qt9//702btxY4L6Lv2PLGHPZ790qas7o0aNdvsz05MmTqlatmpKSkhQWFuZG9eWPw+FQSkqK2rdvL7vdXtbleB39WltJ++3SxQtFeYnd7tCAASmaM6e9HI6r67n96KMr34bXsrV5u9/8d2Yux62w8+KLL6pNmzbasmWLcnJy9NRTT2nnzp06duyYvvjiiyve35AhQ/TBBx/os88+c3lrLDo6WtKF1ZuYmBjneGZmpnO1Jzo6Wjk5OTp+/LjL6k5mZqZatmxZ6PH8/f0LXYGy2+2We/FZsaei0K+1udtvTo4XivEyh8OunJyr67ktyUuR17K1eavf4u7TrROUGzZsqO+//1633HKL2rdvrzNnzqhHjx765ptvVLt27WLvxxijJ554QsuWLdO6detUs2ZNl/tr1qyp6Ohol+WvnJwcbdiwwRlkmjVrJrvd7jInPT1dO3bsuGTYAQAA144rXtnJPxn4tddeK/S8lysxePBgvfXWW3r//fcVGhrqPMcmPDxcgYGBstlsGjZsmMaPH686deqoTp06Gj9+vIKCgtS7d2/n3P79+2vkyJGqVKmSKlasqFGjRikhIUHt2rUrUX0AAODqd8Vhx263a8eOHZc9Z6Y4Zs2aJUlq06aNy/iCBQvUr18/SdJTTz2ls2fPatCgQTp+/LiaN2+u1atXKzQ01Dl/2rRp8vX1Vc+ePXX27Fm1bdtWCxculI+PT4lrBAAAVze3ztl5+OGHNW/ePE2cOLFEBy/OJy7bbDYlJycrOTn5knMCAgI0Y8YMzZgxo0T1AAAA63Er7OTk5Gju3LlKSUnRzTffXOA7saZOneqR4gAAAErqisLOL7/8oho1amjHjh1q2rSpJOmnn35ymeOJt7cAAAA85YrCTp06dZSenq7U//tq2/vvv1//+te/CnzoHwAAQHlxRZeeX3yOzSeffOLypZ0AAADljVufs5OvOCcYAwAAlKUrCjs2m63AOTmcowMAAMqzKzpnxxijfv36Ob9q4dy5c3r88ccLXI21bNkyz1UIAABQAlcUdvr27ety+6GHHvJoMQAAAJ52RWFnwYIF3qoDAADAK0p0gjIAAEB5R9gBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACW5lvWBQBASSQmlnUFAMo7VnYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAICl8d1YAACPKM73lKWmer8O4GKs7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEvzLesCAADXjsRE19t+ftKgQVKXLlJOzoWx1NTSrwvWRtgBUG7l/8VY2F+IAFBcZfo21meffaauXbsqNjZWNptNK1ascLnfGKPk5GTFxsYqMDBQbdq00c6dO13mZGdna8iQIYqMjFRwcLDuvvtuHTp0qBS7AAAA5VmZhp0zZ87oxhtv1MyZMwu9f/LkyZo6dapmzpypzZs3Kzo6Wu3bt9epU6ecc4YNG6bly5dr8eLF2rhxo06fPq0uXbooNze3tNoAAADlWJm+jdWpUyd16tSp0PuMMZo+fbrGjBmjHj16SJIWLVqkqKgovfXWWxo4cKBOnDihefPm6Y033lC7du0kSW+++aaqVaumNWvWqEOHDqXWCwAAKJ/K7Tk7e/fuVUZGhpKSkpxj/v7+at26tdLS0jRw4EBt3bpVDofDZU5sbKzi4+OVlpZ2ybCTnZ2t7Oxs5+2TJ09KkhwOhxwOh5c6Kl35fViln8uhX2vy87vwp93ucPnTyq6lXqXC+7Xyy/pa+d3N5+1+i7vfcht2MjIyJElRUVEu41FRUdq/f79zjp+fnyIiIgrMyd++MBMmTNCzzz5bYHz16tUKCgoqaenlSkpKSlmXUKro11oGDXK9PWCAtfv9s2upV8m135Ury7CQUmL1392LeavfrKysYs0rt2Enn81mc7ltjCkwdrHLzRk9erRGjBjhvH3y5ElVq1ZNSUlJCgsLK1nB5YTD4VBKSorat28vu91e1uV4Hf1aU5cuF/602x0aMCBFc+a0l8Nh3X6la6tXqfB+P/qojIvyomvldzeft/vNf2fmcspt2ImOjpZ0YfUmJibGOZ6Zmelc7YmOjlZOTo6OHz/usrqTmZmpli1bXnLf/v7+8vf3LzBut9st9+KzYk9FoV9rufgyc4fDrpwc6/b7Z9dSr5JrvxZ+STtZ/Xf3Yt7qt7j7LLefoFyzZk1FR0e7LH3l5ORow4YNziDTrFkz2e12lznp6enasWNHkWEHAABcO8p0Zef06dPavXu38/bevXv17bffqmLFiqpevbqGDRum8ePHq06dOqpTp47Gjx+voKAg9e7dW5IUHh6u/v37a+TIkapUqZIqVqyoUaNGKSEhwXl1FgAAuLaVadjZsmWLEv/02eH559H07dtXCxcu1FNPPaWzZ89q0KBBOn78uJo3b67Vq1crNDTUuc20adPk6+urnj176uzZs2rbtq0WLlwoHx+fUu8HAACUP2Uadtq0aSNjzCXvt9lsSk5OVnJy8iXnBAQEaMaMGZoxY4YXKgQAAFe7cnvODgAAgCcQdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKX5lnUBAK5NiYllXQGAawUrOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNL4nB0AQLlSnM9gSk31fh2wDlZ2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApfmWdQEAAFypxMTLz0lN9X4duDqwsgMAACyNsAMAACyNsAMAACyNsAMAACyNsAMAACyNq7EAeFxxrpQBgNLCyg4AALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA0wg4AALA0Lj0HcEW4rBzA1YaVHQAAYGms7AAALKk4q5Cpqd6vA2WPlR0AAGBprOwAAK5ZrP5cG1jZAQAAlkbYAQAAlsbbWACcuKwcgBWxsgMAACyNsAMAACyNsAMAACzNMmHn1VdfVc2aNRUQEKBmzZrp888/L+uSAABAOWCJsLNkyRINGzZMY8aM0TfffKNWrVqpU6dOOnDgQFmXBgAAypglrsaaOnWq+vfvr8cee0ySNH36dH366aeaNWuWJkyYUMbVASVT2BVSfn7SoEFSly5STg4fegYARbnqw05OTo62bt2qp59+2mU8KSlJaWlpZVQVAMAqPPmRDBf/Q+VixfmHC5/6fOWu+rDz+++/Kzc3V1FRUS7jUVFRysjIKHSb7OxsZWdnO2+fOHFCknTs2DE5HA6P1tez5+XnvPOORw8pSXI4HMrKytLRo0dlt9s9f4Byprj9ltXzURLXFfpm84V+paO67jq7jh69/H6K03vhxyoPXPu1tmupV4l+XRXnd7k4v6fF2Y+nFPX/Fl9fh/r1897fRadOnZIkGWOKnmiucr/++quRZNLS0lzGx40bZ+rVq1foNmPHjjWS+OGHH3744YcfC/wcPHiwyKxw1a/sREZGysfHp8AqTmZmZoHVnnyjR4/WiBEjnLfz8vJ07NgxVapUSTabzav1lpaTJ0+qWrVqOnjwoMLCwsq6HK+jX2u7lvq9lnqV6NfqvN2vMUanTp1SbGxskfOu+rDj5+enZs2aKSUlRd27d3eOp6SkqFu3boVu4+/vL39/f5exChUqeLPMMhMWFnZN/ELlo19ru5b6vZZ6lejX6rzZb3h4+GXnXPVhR5JGjBihPn366Oabb1aLFi3073//WwcOHNDjjz9e1qUBAIAyZomwc//99+vo0aN67rnnlJ6ervj4eK1cuVJxcXFlXRoAAChjlgg7kjRo0CANGjSorMsoN/z9/TV27NgCb9dZFf1a27XU77XUq0S/Vlde+rUZc7nrtQAAAK5e5fZTNQAAADyBsAMAACyNsAMAACyNsAMAACyNsHOVOn78uPr06aPw8HCFh4erT58++uOPP4rcxhij5ORkxcbGKjAwUG3atNHOnTtd5mRkZKhPnz6Kjo5WcHCwmjZtqnfffdeLnRSPt/qVpE2bNunOO+9UcHCwKlSooDZt2ujs2bNe6qR4vNlv/txOnTrJZrNpxYoVnm/gCnmj32PHjmnIkCGqV6+egoKCVL16dT355JPO78IrTa+++qpq1qypgIAANWvWTJ9//nmR8zds2KBmzZopICBAtWrV0uzZswvMee+999SwYUP5+/urYcOGWr58ubfKv2Ke7nfOnDlq1aqVIiIiFBERoXbt2unrr7/2ZgtXxBvPb77FixfLZrPpnnvu8XDV7vNGv3/88YcGDx6smJgYBQQEqEGDBlq5cqXnii7xl1OhTHTs2NHEx8ebtLQ0k5aWZuLj402XLl2K3GbixIkmNDTUvPfee2b79u3m/vvvNzExMebkyZPOOe3atTN/+ctfzFdffWX27Nljnn/+eXPdddeZbdu2ebulInmr37S0NBMWFmYmTJhgduzYYX766SezdOlSc+7cOW+3VCRv9Ztv6tSpplOnTkaSWb58uZe6KD5v9Lt9+3bTo0cP88EHH5jdu3ebtWvXmjp16pi//vWvpdGS0+LFi43dbjdz5swxu3btMkOHDjXBwcFm//79hc7/5ZdfTFBQkBk6dKjZtWuXmTNnjrHb7ebdd991zklLSzM+Pj5m/Pjx5ocffjDjx483vr6+5ssvvyytti7JG/327t3bvPLKK+abb74xP/zwg3nkkUdMeHi4OXToUGm1dUne6Dffvn37zPXXX29atWplunXr5uVOiscb/WZnZ5ubb77ZdO7c2WzcuNHs27fPfP755+bbb7/1WN2EnavQrl27jCSX/7Ft2rTJSDL/+7//W+g2eXl5Jjo62kycONE5du7cORMeHm5mz57tHAsODjavv/66y7YVK1Y0c+fO9XAXxefNfps3b26eeeYZ7xXvBm/2a4wx3377ralatapJT08vF2HH2/3+2TvvvGP8/PyMw+HwXAOXccstt5jHH3/cZax+/frm6aefLnT+U089ZerXr+8yNnDgQHPrrbc6b/fs2dN07NjRZU6HDh1Mr169PFS1+7zR78XOnz9vQkNDzaJFi0pecAl5q9/z58+b2267zcydO9f07du33IQdb/Q7a9YsU6tWLZOTk+P5gv8Pb2NdhTZt2qTw8HA1b97cOXbrrbcqPDxcaWlphW6zd+9eZWRkKCkpyTnm7++v1q1bu2xz++23a8mSJTp27Jjy8vK0ePFiZWdnq02bNl7r53K81W9mZqa++uorValSRS1btlRUVJRat26tjRs3erehy/Dm85uVlaUHHnhAM2fOVHR0tPeauALe7PdiJ06cUFhYmHx9S+fzVHNycrR161aXOiUpKSnpknVu2rSpwPwOHTpoy5YtcjgcRc4pqvfS4K1+L5aVlSWHw6GKFSt6pnA3ebPf5557TpUrV1b//v09X7ibvNXvBx98oBYtWmjw4MGKiopSfHy8xo8fr9zcXI/VTti5CmVkZKhKlSoFxqtUqVLg29//vI2kAt8EHxUV5bLNkiVLdP78eVWqVEn+/v4aOHCgli9frtq1a3uwgyvjrX5/+eUXSVJycrIGDBigVatWqWnTpmrbtq1+/vlnT7ZwRbz5/A4fPlwtW7a85JfklgVv9vtnR48e1fPPP6+BAweWsOLi+/3335Wbm3tFdWZkZBQ6//z58/r999+LnHOpfZYWb/V7saefflrXX3+92rVr55nC3eStfr/44gvNmzdPc+bM8U7hbvJWv7/88oveffdd5ebmauXKlXrmmWc0ZcoUvfDCCx6rnbBTjiQnJ8tmsxX5s2XLFkmSzWYrsL0xptDxP7v4/ou3eeaZZ3T8+HGtWbNGW7Zs0YgRI3Tfffdp+/btHujQVVn3m5eXJ0kaOHCgHnnkETVp0kTTpk1TvXr1NH/+fE+06KKs+/3ggw+0bt06TZ8+3TMNXUZZ9/tnJ0+e1F133aWGDRtq7NixJejKPcWts6j5F49f6T5Lkzf6zTd58mS9/fbbWrZsmQICAjxQbcl5st9Tp07poYce0pw5cxQZGen5Yj3A089vXl6eqlSpon//+99q1qyZevXqpTFjxmjWrFkeq9ky341lBU888YR69epV5JwaNWro+++/1+HDhwvcd+TIkQIJOl/+WxYZGRmKiYlxjmdmZjq32bNnj2bOnKkdO3aoUaNGkqQbb7xRn3/+uV555ZUirxhwR1n3mz/esGFDl20bNGigAwcOFL+RYirrftetW6c9e/aoQoUKLtv+9a9/VatWrbR+/for6ObyyrrffKdOnVLHjh0VEhKi5cuXy263X2krbouMjJSPj0+Bf/UWVme+6OjoQuf7+vqqUqVKRc651D5Li7f6zffSSy9p/PjxWrNmjRo3buzZ4t3gjX537typffv2qWvXrs778/9h5uvrqx9//LHMVtq99fzGxMTIbrfLx8fHOadBgwbKyMhQTk6O/Pz8Sl68184Ggtfkn9D51VdfOce+/PLLYp3QOWnSJOdYdna2ywmd33//vZFkdu3a5bJtUlKSGTBggBc6KR5v9ZuXl2diY2MLnKB80003mdGjR3uhk+LxVr/p6elm+/btLj+SzMsvv2x++eUX7zZVBG/1a4wxJ06cMLfeeqtp3bq1OXPmjPeaKMItt9xi/va3v7mMNWjQoMgTOhs0aOAy9vjjjxc4QblTp04uczp27FhuTlD2dL/GGDN58mQTFhZmNm3a5NmCS8jT/Z49e7bA72m3bt3MnXfeabZv326ys7O900gxeeP5HT16tImLizO5ubnOsenTp5uYmBiP1U3YuUp17NjRNG7c2GzatMls2rTJJCQkFLhUt169embZsmXO2xMnTjTh4eFm2bJlZvv27eaBBx5wuVQ3JyfH3HDDDaZVq1bmq6++Mrt37zYvvfSSsdls5uOPPy7V/i7mjX6NMWbatGkmLCzMLF261Pz888/mmWeeMQEBAWb37t2l1lthvNXvxVQOrsYyxjv9njx50jRv3twkJCSY3bt3m/T0dOfP+fPnS623/Et1582bZ3bt2mWGDRtmgoODzb59+4wxxjz99NOmT58+zvn5l+oOHz7c7Nq1y8ybN6/ApbpffPGF8fHxMRMnTjQ//PCDmThxYrm79NyT/U6aNMn4+fmZd9991+V5PHXqVKn3dzFv9Hux8nQ1ljf6PXDggAkJCTFPPPGE+fHHH81HH31kqlSpYsaNG+exugk7V6mjR4+aBx980ISGhprQ0FDz4IMPmuPHj7vMkWQWLFjgvJ2Xl2fGjh1roqOjjb+/v7njjjvM9u3bXbb56aefTI8ePUyVKlVMUFCQady4cYFL0cuCt/o1xpgJEyaYqlWrmqCgINOiRQvz+eefe7mby/NmvxfvozyEHW/0m5qaaiQV+rN3797Saez/vPLKKyYuLs74+fmZpk2bmg0bNjjv69u3r2ndurXL/PXr15smTZoYPz8/U6NGDTNr1qwC+1y6dKmpV6+esdvtpn79+ua9997zdhvF5ul+4+LiCn0ex44dWwrdXJ43nt8/K09hxxjv9JuWlmaaN29u/P39Ta1atcwLL7zg0X+U2Iz5vzOFAAAALIirsQAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgCUKZvNphUrVjhv/+///q9uvfVWBQQE6KabbrrkGAAUF18ECsAr+vXrp0WLFkm68AWGFStWVOPGjfXAAw+oX79+uu66C//WSk9PV0REhHO7sWPHKjg4WD/++KNCQkIuOQYAxcXKDgCv6dixo9LT07Vv3z598sknSkxM1NChQ9WlSxedP39e0oVvRfb393dus2fPHt1+++2Ki4tzfityYWNXKicnp+QNAbgqEXYAeI2/v7+io6N1/fXXq2nTpvrHP/6h999/X5988okWLlwoyfVtLJvNpq1bt+q5556TzWZTcnJyoWOS9Ouvv+r+++9XRESEKlWqpG7dumnfvn3OY/fr10/33HOPJkyYoNjYWNWtW/eKtnvppZcUExOjSpUqafDgwXI4HM452dnZeuqpp1StWjX5+/urTp06mjdvnvP+Xbt2qXPnzgoJCVFUVJT69Omj33//3SuPMYDLI+wAKFV33nmnbrzxRi1btqzAfenp6WrUqJFGjhyp9PR0jRo1qtCxrKwsJSYmKiQkRJ999pk2btyokJAQdezY0WUFZ+3atfrhhx+UkpKijz76qNjbpaamas+ePUpNTdWiRYu0cOFCZziTpIcffliLFy/Wv/71L/3www+aPXu28+219PR0tW7dWjfddJO2bNmiVatW6fDhw+rZs6f3HlQAReKcHQClrn79+vr+++8LjEdHR8vX11chISGKjo6WJIWEhBQYmz9/vq677jrNnTtXNptNkrRgwQJVqFBB69evV1JSkiQpODhYc+fOlZ+f3xVtFxERoZkzZ8rHx0f169fXXXfdpbVr12rAgAH66aef9M477yglJUXt2rWTJNWqVcvZw6xZs9S0aVONHz/eOTZ//nxVq1ZNP/30k3OFCUDpIewAKHXGGGfYcMfWrVu1e/duhYaGuoyfO3dOe/bscd5OSEhwBp0r2a5Ro0by8fFx3o6JidH27dslSd9++618fHzUunXrS9aWmppa6InUe/bsIewAZYCwA6DU/fDDD6pZs6bb2+fl5alZs2b6z3/+U+C+ypUrO/87ODjYre3sdrvLfTabTXl5eZKkwMDAy9bWtWtXTZo0qcB9MTExRW4LwDsIOwBK1bp167R9+3YNHz7c7X00bdpUS5YsUZUqVRQWFub17f4sISFBeXl52rBhg/NtrIuP8d5776lGjRry9eV/sUB5wAnKALwmOztbGRkZ+vXXX7Vt2zaNHz9e3bp1U5cuXfTwww+7vd8HH3xQkZGR6tatmz7//HPt3btXGzZs0NChQ3Xo0CGPb/dnNWrUUN++ffXoo49qxYoV2rt3r9avX6933nlHkjR48GAdO3ZMDzzwgL7++mv98ssvWr16tR599FHl5ua63TMA9xF2AHjNqlWrFBMToxo1aqhjx45KTU3Vv/71L73//vsu58RcqaCgIH322WeqXr26evTooQYNGujRRx/V2bNni1yxcXe7i82aNUv33nuvBg0apPr162vAgAE6c+aMJCk2NlZffPGFcnNz1aFDB8XHx2vo0KEKDw93fpAigNJlM8aYsi4CAADAW/hnBgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsLT/B6Q2y3L6qVBQAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "differences = hf_model.model.layers[0](test_tensor)[0] - tl_model.blocks[0](test_tensor)\n",
    "\n",
    "# Flatten the differences to create a one-dimensional tensor\n",
    "flattened_differences = differences.flatten().cpu().detach().numpy()\n",
    "\n",
    "# Plot the histogram of the differences\n",
    "plt.hist(flattened_differences, bins=50, alpha=0.75, color='blue')\n",
    "plt.title('Differences Between Layer Outputs')\n",
    "plt.xlabel('Difference')\n",
    "plt.ylabel('Frequency')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5172efa2-0066-4ae0-a6a2-530d815b053b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.1874,  0.0542, -0.1336,  ..., -0.1576, -0.1348,  0.0680]]],\n",
       "       grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].attn.forward(test_tensor, test_tensor, test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "92781a06-e16d-43f9-be4c-3ef04b3d4b08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.1874,  0.0542, -0.1336,  ..., -0.1576, -0.1348,  0.0680]]],\n",
       "       grad_fn=<UnsafeViewBackward0>)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].self_attn.forward(test_tensor)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "943cd506-2bb8-45bf-afc7-7f6b4f8043f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[False, False, False,  ..., False, False, False]]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(tl_model.blocks[0].attn.forward(test_tensor, test_tensor, test_tensor) == \n",
    " hf_model.model.layers[0].self_attn.forward(test_tensor)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "57ffc181-abed-4784-86eb-6e6b4f174bc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(247)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sum(tl_model.blocks[0].attn.forward(test_tensor, test_tensor, test_tensor) == \n",
    " hf_model.model.layers[0].self_attn.forward(test_tensor)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "7427fd15-3029-45c3-9d12-64c80f1048f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHFCAYAAAAUpjivAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABJ0UlEQVR4nO3deVxU9f4/8NcIw7AIyiIMKCISagYqYbkWEIIbalqpae55LcxE9OpV7/cblkFiooU3zULUXGjVzK8Lg6Jm6BUxc6mraYgbiAvJIg4jfH5/+ONcR/ZxYOD0ej4ePPR85nPOeZ/PDPDiM+fMUQghBIiIiIhkqpmpCyAiIiKqTww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtUb9atWweFQiF9WVpaQq1WIygoCDExMcjNza2wTlRUFBQKhV5bSUkJ3njjDbi6usLMzAzdunUDANy+fRujR4+Gs7MzFAoFXnzxxQY4qqbr4sWLes+HQqGAnZ0dunbtihUrVqC0tNSg7e7cuRNRUVHGLbaRunnzJlQqFRQKBY4dO1Zpn+joaGzbtq1C+6+//oqoqChcvHixfousoY79+/dDoVBg//79DVLHo4QQ2Lx5M1544QXY29tDpVKhffv2mD59Oi5fvmzwdq9du4aoqCicOHHCeMVWIy0tDVFRUfjzzz8bZH/0mARRPUlMTBQARGJiojh8+LA4ePCg+Oabb0RERIRo0aKFcHBwEBqNRm+dy5cvi8OHD+u1rVixQgAQ8fHxIi0tTZw8eVIIIURERISwsLAQGzduFIcPHxZnz55tsGNrijIzMwUAMWPGDHH48GFx+PBhsWvXLvHmm28KACIyMtKg7U6fPl38VX6UxMXFCQACgHjjjTcq7WNjYyMmTJhQof3rr78WAERqamr9FllDHXfu3BGHDx8Wd+7caZA6HlZaWipGjRolAIhXX31VbNu2TaSmpoqPPvpItGnTRrRs2VIcOnTIoG2np6dLP28awtKlSwUAkZmZ2SD7o8djbqKMRX8hPj4+6N69u7T80ksvYdasWejbty9GjBiB33//HS4uLgCANm3aoE2bNnrrnz59GlZWVnjrrbcqtHt5eWHs2LFGq7W4uBhWVlZG215j1LZtW/Ts2VNaHjBgAE6fPo0tW7Zg2bJlJqys8Vu7di2cnZ3h4eGBLVu2IC4ursm9Xuzs7PSe/4a0ZMkSfPnll/jggw8wb948qT0wMBCjRo1Cjx498NJLL+E///kPWrZsaZIaSaZMnbZIvspndtLT0yt9/KuvvhIAxKJFi6S2d955R2+WAP//r+iHv8q3++hX+V/MWq1WvPfee6Jjx47CwsJCODk5iYkTJ4rc3Fy9/Xt4eIjBgweLb7/9VnTr1k2oVCoxb948IYQQ2dnZ4m9/+5to3bq1UCqVol27diIqKkrodDpp/fKZkqVLl4ply5aJdu3aCRsbG9GzZ88Ks1NCCHHkyBERFhYmHBwchEqlEu3btxczZ87U63Pu3Dnx6quvilatWgkLCwvRqVMnsXLlSr0+paWl4r333hMdOnQQlpaWokWLFsLX11esWLGi2ufj4XofFRYWJtq2bVuhPSkpSfTs2VNYW1sLGxsbERoaKo4fPy49PmHChEqfi8zMTPHyyy+Lzp07V9gPAPHVV19JbRkZGQKA2L59u9RWm/EXou7P9a5du4Sfn5+wtLQUHTt2FAkJCdWO2cOOHDkiAIjZs2eLNWvWCADiiy++0OtT2VgEBARU+Zp9eBZCo9GIF154Qdja2gorKyvRu3dvkZKSorf98u+P06dPi9GjRws7Ozvh7OwsJk2aJP78888a6xBCiNTU1EpnmL7//nvRs2dPYWVlJZo3by769esn0tLSDNp/ZbRarbC3txdPPvmkKCsrq7TP5s2bBQDx4YcfSm0eHh6VzlAFBARUOKZHv9555x0hxIPXqY2NjTh9+rR44YUXhLW1tXBychLTp08XRUVF0jbLv0cqmx16eHvl41DVz6C9e/eKgIAA4eDgICwtLYW7u7sYMWKE3r6oYTHsUL2pKewUFhYKMzMzERwcLLU9GnYOHz4sBg0aJKysrKS3XnJycsThw4eFn5+faN++vdR+584dUVpaKgYMGCBsbGzEokWLhEajEZ9//rlo3bq16Ny5s7h79660bQ8PD+Hq6irat28v1q5dK1JTU8XRo0dFdna2cHd3Fx4eHuLTTz8VKSkp4r333hMqlUpMnDhRWr/8B2O7du3EgAEDxLZt28S2bduEr6+vsLe31/vhv3v3bqFUKkWXLl3EunXrxL59+8TatWvF6NGjpT5nzpyRgsuGDRtEcnKymD17tmjWrJmIioqS+sXExAgzMzPxzjvviL1794rdu3eLFStW6PWpTHm9S5YsETqdTuh0OnHz5k2RkJAgzM3NxcKFC/X6v//++0KhUIjJkyeLHTt2iO+++0706tVL2NjYiDNnzgghhDh//rx4+eWXBQDpeTh8+LC4d++eWL16tQAgrl27JoQQQqfTSb/Ip06dKu1nyZIlwtzcXOTn5wshRK3Hv67PdZs2bUTnzp3Fhg0bxJ49e8Qrr7wiAIgDBw5UO27lpk6dKgCIM2fOiPz8fGFtbS0CAwP1+hw+fFhYWVmJQYMGSWNx5swZkZubK6KjowUA8a9//Ut6rDyUffHFF0KhUIgXX3xRfPfdd+KHH34QYWFhwszMTC/wlH9/dOzYUfzv//6v0Gg0Ii4uTqhUKjFp0qQa6xCi8rCzadMmAUCEhoaKbdu2iS+//FL4+/sLCwsL8eOPP9Z5/5VJS0sTAKQ/KCpTUFAgmjVrJvr37y+11Sbs3LlzR/p5889//lM65suXLwshHoQdCwsL0bZtW/H++++L5ORkERUVJczNzUVYWJi0zdqGncuXL4sZM2YIAOK7777T+xmUmZkpLC0tRUhIiNi2bZvYv3+/2LRpkxg3bpzIy8urdoyo/jDsUL2pKewIIYSLi4t48sknpeVHw44Q//2r7FEBAQHiqaee0mvbsmWLACC+/fZbvfby9/M/+eQTqc3Dw0OYmZlVONdn2rRponnz5iIrK0uv/cMPP5R+2Qnx3x+Mvr6+4v79+1K/o0ePCgBiy5YtUpuXl5fw8vISxcXFVY5F//79RZs2bSqcS/HWW28JS0tLcfv2bSHEg9mRbt26VbmdqpTXW9nXxIkT9Y7h0qVLwtzcXMyYMUNvGwUFBUKtVouRI0dKbVWds3P+/HkBQGzYsEEIIcShQ4cEADF37lzh6ekp9QsJCRG9e/eWlms7/nV9ri0tLfW2WVxcLBwcHMS0adNqHLuioiJhZ2cnevbsKbVNmDBBKBQKcf78eb2+dT1np6ioSDg4OIghQ4botZeWloquXbuKZ599Vmor//6IjY3V6xseHi4sLS31ZkyqquPRsFNaWirc3NyEr6+vKC0tlfoVFBQIZ2dnveemLvt/VFJSkgAgVq9eXWUfISr+TKhN2BGi+nN2ymcgP/roI732999/XwCQzhOqbdgRoupzdr755hsBQJw4caLa46SGxauxyKSEEEbd3o4dO9CyZUsMGTIE9+/fl766desGtVpd4QqULl26oEOHDhW2ERQUBDc3N71tDBw4EABw4MABvf6DBw+GmZmZ3jYBICsrCwBw7tw5XLhwAVOmTIGlpWWldd+7dw979+7F8OHDYW1trbffQYMG4d69ezhy5AgA4Nlnn8Uvv/yC8PBw7NmzB/n5+XUao5kzZyI9PR3p6elITU1FdHQ0vvrqK7z66qtSnz179uD+/fsYP368Xi2WlpYICAio1ZU8Xl5eaNeuHVJSUgAAGo0Gvr6+eO2115CZmYkLFy5Aq9Xi0KFD6Nevn7Rebce/rs91t27d0LZtW2nZ0tISHTp0kJ6n6nz11VfIz8/H5MmTpbbJkydDCIHExMQa169OWloabt++jQkTJugdR1lZGQYMGID09HQUFRXprTN06FC95S5duuDevXuVXuFYk7Nnz+LatWsYN24cmjX776+E5s2b46WXXsKRI0dw9+7detv/o4QQFa7INJZHz+8bM2YMACA1NdVo++jWrRssLCzwt7/9DevXr8cff/xhtG2T4XiCMplMUVERbt26BV9fX6Nt8/r16/jzzz9hYWFR6eM3b97UW3Z1da10Gz/88AOUSmWttuHo6Ki3rFKpADw42RkAbty4AQAVTrx+2K1bt3D//n3Ex8cjPj6+2v3Onz8fNjY22LhxI1avXg0zMzM8//zzWLJkid6J4FVp06aNXr/AwEAoFArMnz8fe/bsQf/+/XH9+nUAwDPPPFPpNh7+pVid4OBg7N69GwCQkpKCkJAQ+Pr6wsXFBSkpKfD29kZxcbFe2Knt+Nf1uX70eQIePFflz1N1EhISYGlpiQEDBkiXGnfp0gXt2rXDunXrsGjRIr3AWxflY/3yyy9X2ef27duwsbGRlmt6zdXFrVu3AFT+veDm5oaysjLk5eXB2tr6sfZfHjQzMzOr7FNUVISbN2/Cz8+v9gdQS+bm5hXqVqvVAP47Bsbg5eWFlJQUxMbGYvr06SgqKkL79u3x9ttvY+bMmUbbD9UNww6ZzP/93/+htLQUgYGBRtumk5MTHB0dpV+wj7K1tdVbruwvSCcnJ3Tp0gXvv/9+pdtwc3OrU02tWrUCAFy5cqXKPvb29jAzM8O4ceMwffr0Svt4enoCePBDOzIyEpGRkfjzzz+RkpKCBQsWoH///rh8+bLeL6XaKp+N+uWXX9C/f384OTkBAL755ht4eHjUeXvlgoODkZCQgKNHj+Lf//43/vnPfwIAXnjhBWg0GmRlZaF58+Z6VwfVdvzr+lwb6ty5czh06BAA6M0MPWzPnj0YNGiQQdsvH+v4+Pgqr5Iqv1qxPpQHgOzs7AqPXbt2Dc2aNYO9vf1j78ff3x/29vbYvn07YmJiKv3e2759O8rKyhASEiK1WVpaQqvVVuh78+ZNaexq4/79+7h165Ze4MnJyQHw3zEon3l9dH91DUPPPfccnnvuOZSWluLYsWOIj49HREQEXFxcMHr06Dpti4yDYYdM4tKlS5gzZw5atGiBadOmGW27YWFhSEpKQmlpKXr06GHwNnbu3AkvLy+j/JDv0KEDvLy8sHbtWkRGRkp/BT/M2toaQUFB+Pnnn9GlS5cqZyse1bJlS7z88su4evUqIiIicPHiRXTu3LnONZZ/EJuzszMAoH///jA3N8eFCxfw0ksvVbvuw3/VP3oZdnBwMBQKBf7nf/4HzZo1w/PPPw8A6NevH/7+978jKysLzz//vN4sTm3H3xjPdW0kJCQAAD777DM88cQTeo8VFxdj2LBhWLt2rRR2qpotqmr2o0+fPmjZsiV+/fXXCh+v8DhqO2vVsWNHtG7dGps3b8acOXOkEFJUVIRvv/0WvXr1MihAP8rCwgJ///vfsWDBAixduhRz587Vezw3Nxfz58+Hi4sLXn/9dam9Xbt2OHnypF7fc+fO4ezZs3phpzazS5s2bcLbb78tLW/evBkApD+4XFxcYGlpWWF/33//fYVt1WZ/ZmZm6NGjBzp16oRNmzbh+PHjDDsmwrBD9e706dPSeQi5ubn48ccfkZiYCDMzM2zdulWa+TCG0aNHY9OmTRg0aBBmzpyJZ599FkqlEleuXEFqaiqGDRuG4cOHV7uNd999FxqNBr1798bbb7+Njh074t69e7h48SJ27tyJ1atXV/uWVGX+9a9/YciQIejZsydmzZqFtm3b4tKlS9izZw82bdoEAPjoo4/Qt29fPPfcc3jzzTfRrl07FBQU4Pz58/jhhx+wb98+AMCQIUOkzy5q1aoVsrKysGLFCnh4eMDb27vGWi5duiSd/1NUVITDhw8jJiYGHh4eGDFiBIAHv2DeffddLFy4EH/88QcGDBgAe3t7XL9+HUePHoWNjQ0WLVoEANLbkEuWLMHAgQNhZmYmBTZnZ2f4+PggOTkZQUFB0i/Nfv364fbt27h9+zbi4uIMGn9jPNc1uX//PjZs2IAnn3xS7xfww4YMGYLt27fjxo0baNWqFXx9fbF//3788MMPcHV1ha2tLTp27AgfHx8AwJo1a2BrawtLS0t4enrC0dER8fHxmDBhAm7fvo2XX34Zzs7OuHHjBn755RfcuHEDq1atqnPtVdXxqGbNmiE2NhZjx45FWFgYpk2bBq1Wi6VLl+LPP//EBx98UOd9V2XevHn45ZdfpH9HjRqFFi1a4OTJk1i6dCkKCgqwY8cOtGjRQlpn3LhxeO211xAeHo6XXnoJWVlZiI2NrfBzw8vLC1ZWVti0aROefPJJNG/eHG5ubtJMoIWFBZYtW4bCwkI888wzSEtLw+LFizFw4ED07dsXwIOZ3tdeew1r166Fl5cXunbtiqNHj0qh6NHxBR58306YMAFKpRIdO3bEpk2bsG/fPgwePBht27bFvXv3sHbtWgDQe7uWGpiJT5AmGXv0s0UsLCyEs7OzCAgIENHR0RU+C0WIx78aS4gHlzh/+OGHomvXrsLS0lI0b95cdOrUSUybNk38/vvvUr/yz16pzI0bN8Tbb78tPD09hVKpFA4ODsLf318sXLhQFBYWCiGq/9waPHLlhhAPLgceOHCgaNGihVCpVMLLy0vMmjVLr09mZqaYPHmy9PkyrVq1Er179xaLFy+W+ixbtkz07t1bODk5SZfTTpkyRVy8eLHSY3l423jkKixLS0vRoUMHERERIbKzsyuss23bNhEUFCTs7OyESqUSHh4e4uWXX9a7HFqr1YrXX39dtGrVSigUigpXqMyaNUsAEO+//77etr29vQUA6ROxH1ab8Rfi8Z/rR6/oqez4AVT7GUa7d+8WAMSyZcuEEEKcOHFC9OnTR1hbW+t9vo0QDz4N3NPTU5iZmVW46ufAgQNi8ODBwsHBQSiVStG6dWsxePBg8fXXX0t9yr8/bty4oVdD+ffaw+NeVR1Vfc7Otm3bRI8ePYSlpaWwsbERwcHB4qefftLrU5f9V6WsrExs2rRJBAYGipYtWwoLCwvh6ekp3nzzzQpX4JX3j42NFe3btxeWlpaie/fuYt++fZU+d1u2bBGdOnUSSqWy0s/ZOXnypAgMDBRWVlbCwcFBvPnmm3qvJyEeXMb++uuvCxcXF2FjYyOGDBkiLl68WOn39Pz584Wbm5to1qyZNKaHDx8Ww4cPFx4eHkKlUglHR0cREBCg9zlS1PAUQhj5chgiIqJGZOLEifjmm29QWFho6lLIRHjpOREREckaww4RERHJGt/GIiIiIlnjzA4RERHJGsMOERERyRrDDhEREckaP1QQQFlZGa5duwZbW9t6uwEdERERGZcQAgUFBXBzc6v2nn0MO3hw/xd3d3dTl0FEREQGuHz5crWfbM+wg//eMPDy5cuws7Nr0H3rdDokJycjNDS0yrs8U+1xPI2L42lcHE/j4VgaV1Mdz/z8fLi7u9d441+GHfz3ztd2dnYmCTvW1taws7NrUi+wxorjaVwcT+PieBoPx9K4mvp41nQKCk9QJiIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIlkzedi5evUqXnvtNTg6OsLa2hrdunVDRkaG9LgQAlFRUXBzc4OVlRUCAwNx5swZvW1otVrMmDEDTk5OsLGxwdChQ3HlypWGPhQiIiJqhEwadvLy8tCnTx8olUrs2rULv/76K5YtW4aWLVtKfWJjYxEXF4eVK1ciPT0darUaISEhKCgokPpERERg69atSEpKwqFDh1BYWIiwsDCUlpaa4KiIiIioMTHpJygvWbIE7u7uSExMlNratWsn/V8IgRUrVmDhwoUYMWIEAGD9+vVwcXHB5s2bMW3aNNy5cwcJCQn44osv0K9fPwDAxo0b4e7ujpSUFPTv379Bj4mIiIgaF5PO7Gzfvh3du3fHK6+8AmdnZ/j5+eGzzz6THs/MzEROTg5CQ0OlNpVKhYCAAKSlpQEAMjIyoNPp9Pq4ubnBx8dH6kNERER/XSad2fnjjz+watUqREZGYsGCBTh69CjefvttqFQqjB8/Hjk5OQAAFxcXvfVcXFyQlZUFAMjJyYGFhQXs7e0r9Clf/1FarRZarVZazs/PB/Dg3iA6nc5ox1cb5ftr6P3KFcfTuDiexsXxNB6OpXE11fGsbb0mDTtlZWXo3r07oqOjAQB+fn44c+YMVq1ahfHjx0v9Hr3BlxCixpt+VdcnJiYGixYtqtCenJwMa2vruh6GUWg0GpPsV644nsbF8TQujqfxcCyNq6mN5927d2vVz6Rhx9XVFZ07d9Zre/LJJ/Htt98CANRqNYAHszeurq5Sn9zcXGm2R61Wo6SkBHl5eXqzO7m5uejdu3el+50/fz4iIyOl5fJbxIeGhprkrucajQYhISFN8k6zjQ3H07g4nsbF8TQejqVxNdXxLH9npiYmDTt9+vTB2bNn9drOnTsHDw8PAICnpyfUajU0Gg38/PwAACUlJThw4ACWLFkCAPD394dSqYRGo8HIkSMBANnZ2Th9+jRiY2Mr3a9KpYJKparQrlQqTfYkm3LfcsTxNC6Op3FxPI2HY2lcTW08a1urScPOrFmz0Lt3b0RHR2PkyJE4evQo1qxZgzVr1gB48PZVREQEoqOj4e3tDW9vb0RHR8Pa2hpjxowBALRo0QJTpkzB7Nmz4ejoCAcHB8yZMwe+vr7S1VlEZDxBQTX3SU2t/zqIiGrLpGHnmWeewdatWzF//ny8++678PT0xIoVKzB27Fipz9y5c1FcXIzw8HDk5eWhR48eSE5Ohq2trdRn+fLlMDc3x8iRI1FcXIzg4GCsW7cOZmZmpjgsIiIiakRMGnYAICwsDGFhYVU+rlAoEBUVhaioqCr7WFpaIj4+HvHx8fVQIRERETVlJr9dBBEREVF9YtghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZM2nYiYqKgkKh0PtSq9XS40IIREVFwc3NDVZWVggMDMSZM2f0tqHVajFjxgw4OTnBxsYGQ4cOxZUrVxr6UIgavaCgmr+IiOTI5DM7Tz31FLKzs6WvU6dOSY/FxsYiLi4OK1euRHp6OtRqNUJCQlBQUCD1iYiIwNatW5GUlIRDhw6hsLAQYWFhKC0tNcXhEBERUSNjbvICzM31ZnPKCSGwYsUKLFy4ECNGjAAArF+/Hi4uLti8eTOmTZuGO3fuICEhAV988QX69esHANi4cSPc3d2RkpKC/v37N+ixEBERUeNj8rDz+++/w83NDSqVCj169EB0dDTat2+PzMxM5OTkIDQ0VOqrUqkQEBCAtLQ0TJs2DRkZGdDpdHp93Nzc4OPjg7S0tCrDjlarhVarlZbz8/MBADqdDjqdrp6OtHLl+2vo/coVx7NqFhY193l02CobT0O2Qw/w9Wk8HEvjaqrjWdt6FUIIUc+1VGnXrl24e/cuOnTogOvXr2Px4sX4z3/+gzNnzuDs2bPo06cPrl69Cjc3N2mdv/3tb8jKysKePXuwefNmTJo0SS+4AEBoaCg8PT3x6aefVrrfqKgoLFq0qEL75s2bYW1tbdyDJCIionpx9+5djBkzBnfu3IGdnV2V/Uw6szNw4EDp/76+vujVqxe8vLywfv169OzZEwCgUCj01hFCVGh7VE195s+fj8jISGk5Pz8f7u7uCA0NrXaw6oNOp4NGo0FISAiUSmWD7luOOJ5VCwuruc+OHfrLlY2nIduhB/j6NB6OpXE11fEsf2emJiZ/G+thNjY28PX1xe+//44XX3wRAJCTkwNXV1epT25uLlxcXAAAarUaJSUlyMvLg729vV6f3r17V7kflUoFlUpVoV2pVJrsSTblvuWI41lRSUnNfaoasofH83G2Qw/w9Wk8HEvjamrjWdtaTX411sO0Wi1+++03uLq6wtPTE2q1GhqNRnq8pKQEBw4ckIKMv78/lEqlXp/s7GycPn262rBDREREfx0mndmZM2cOhgwZgrZt2yI3NxeLFy9Gfn4+JkyYAIVCgYiICERHR8Pb2xve3t6Ijo6GtbU1xowZAwBo0aIFpkyZgtmzZ8PR0REODg6YM2cOfH19pauziIiI6K/NpGHnypUrePXVV3Hz5k20atUKPXv2xJEjR+Dh4QEAmDt3LoqLixEeHo68vDz06NEDycnJsLW1lbaxfPlymJubY+TIkSguLkZwcDDWrVsHMzMzUx0WERERNSImDTtJSUnVPq5QKBAVFYWoqKgq+1haWiI+Ph7x8fFGro6IiIjkoFGds0NERERkbAw7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQka43q3lhEZFpBQfrLFhZAePiDm3/W5p5YRESNEWd2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYaTdiJiYmBQqFARESE1CaEQFRUFNzc3GBlZYXAwECcOXNGbz2tVosZM2bAyckJNjY2GDp0KK5cudLA1RMREVFj1SjCTnp6OtasWYMuXbrotcfGxiIuLg4rV65Eeno61Go1QkJCUFBQIPWJiIjA1q1bkZSUhEOHDqGwsBBhYWEoLS1t6MMgIiKiRsjkYaewsBBjx47FZ599Bnt7e6ldCIEVK1Zg4cKFGDFiBHx8fLB+/XrcvXsXmzdvBgDcuXMHCQkJWLZsGfr16wc/Pz9s3LgRp06dQkpKiqkOiYiIiBoRc1MXMH36dAwePBj9+vXD4sWLpfbMzEzk5OQgNDRUalOpVAgICEBaWhqmTZuGjIwM6HQ6vT5ubm7w8fFBWloa+vfvX+k+tVottFqttJyfnw8A0Ol00Ol0xj7EapXvr6H3K1ccz6pZWNR9HaVSp/dvbXH4K8fXp/FwLI2rqY5nbes1adhJSkrC8ePHkZ6eXuGxnJwcAICLi4teu4uLC7KysqQ+FhYWejNC5X3K169MTEwMFi1aVKE9OTkZ1tbWdT4OY9BoNCbZr1xxPCsKDzd83alT6zaeO3cavq+/Ar4+jYdjaVxNbTzv3r1bq34mCzuXL1/GzJkzkZycDEtLyyr7KRQKvWUhRIW2R9XUZ/78+YiMjJSW8/Pz4e7ujtDQUNjZ2dXyCIxDp9NBo9EgJCQESqWyQfctRxzPqoWF1X0dpVKHqVM1+OyzEOh0tR/PHTvqvq+/Ar4+jYdjaVxNdTzL35mpicnCTkZGBnJzc+Hv7y+1lZaW4uDBg1i5ciXOnj0L4MHsjaurq9QnNzdXmu1Rq9UoKSlBXl6e3uxObm4uevfuXeW+VSoVVCpVhXalUmmyJ9mU+5YjjmdFJSWGr6vTKVFSUvvx5NBXj69P4+FYGldTG8/a1mqyE5SDg4Nx6tQpnDhxQvrq3r07xo4dixMnTqB9+/ZQq9V6U2olJSU4cOCAFGT8/f2hVCr1+mRnZ+P06dPVhh0iIiL66zDZzI6trS18fHz02mxsbODo6Ci1R0REIDo6Gt7e3vD29kZ0dDSsra0xZswYAECLFi0wZcoUzJ49G46OjnBwcMCcOXPg6+uLfv36NfgxERERUeNj8quxqjN37lwUFxcjPDwceXl56NGjB5KTk2Frayv1Wb58OczNzTFy5EgUFxcjODgY69atg5mZmQkrJyIiosaiUYWd/fv36y0rFApERUUhKiqqynUsLS0RHx+P+Pj4+i2OiIiImiSTf6ggERERUX1i2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWTMo7GRmZhq7DiIiIqJ6YVDYeeKJJxAUFISNGzfi3r17xq6JiIiIyGgMCju//PIL/Pz8MHv2bKjVakybNg1Hjx41dm1EREREj82gsOPj44O4uDhcvXoViYmJyMnJQd++ffHUU08hLi4ON27cMHadRERERAYxf6yVzc0xfPhwDBo0CJ988gnmz5+POXPmYP78+Rg1ahSWLFkCV1dXY9VK9JcUFFRzn9TU+q+DiKipeqyrsY4dO4bw8HC4uroiLi4Oc+bMwYULF7Bv3z5cvXoVw4YNM1adRERERAYxaGYnLi4OiYmJOHv2LAYNGoQNGzZg0KBBaNbsQXby9PTEp59+ik6dOhm1WCIiIqK6MijsrFq1CpMnT8akSZOgVqsr7dO2bVskJCQ8VnFEREREj8ugsPP777/X2MfCwgITJkwwZPNE1MTxPCMiakwMOmcnMTERX3/9dYX2r7/+GuvXr3/sooiIiIiMxaCw88EHH8DJyalCu7OzM6Kjox+7KCIiIiJjMSjsZGVlwdPTs0K7h4cHLl269NhFERERERmLQWHH2dkZJ0+erND+yy+/wNHR8bGLIiIiIjIWg8LO6NGj8fbbbyM1NRWlpaUoLS3Fvn37MHPmTIwePdrYNRIREREZzKCrsRYvXoysrCwEBwfD3PzBJsrKyjB+/Hies0NERESNikFhx8LCAl9++SXee+89/PLLL7CysoKvry88PDyMXR8RERHRY3mse2N16NABHTp0MFYtREREREZnUNgpLS3FunXrsHfvXuTm5qKsrEzv8X379hmlOCIiIqLHZVDYmTlzJtatW4fBgwfDx8cHCoXC2HURERERGYVBYScpKQlfffUVBg0aZOx6iIiIiIzKoEvPLSws8MQTTxi7FiIiIiKjMyjszJ49Gx999BGEEMauh4iIiMioDHob69ChQ0hNTcWuXbvw1FNPQalU6j3+3XffGaU4Iqqd2txlnIjor8qgsNOyZUsMHz7c2LUQERERGZ1BYScxMdHYdRARERHVC4PO2QGA+/fvIyUlBZ9++ikKCgoAANeuXUNhYaHRiiMiIiJ6XAbN7GRlZWHAgAG4dOkStFotQkJCYGtri9jYWNy7dw+rV682dp1EREREBjFoZmfmzJno3r078vLyYGVlJbUPHz4ce/fuNVpxRERERI/L4KuxfvrpJ1hYWOi1e3h44OrVq0YpjIiIiMgYDJrZKSsrQ2lpaYX2K1euwNbW9rGLIiIiIjIWg8JOSEgIVqxYIS0rFAoUFhbinXfe4S0kiIiIqFEx6G2s5cuXIygoCJ07d8a9e/cwZswY/P7773BycsKWLVuMXSMRERGRwQwKO25ubjhx4gS2bNmC48ePo6ysDFOmTMHYsWP1TlgmIiIiMjWDwg4AWFlZYfLkyZg8ebIx6yEiIiIyKoPCzoYNG6p9fPz48QYVQ0RERGRsBoWdmTNn6i3rdDrcvXsXFhYWsLa2ZtghIiKiRsOgq7Hy8vL0vgoLC3H27Fn07du3Ticor1q1Cl26dIGdnR3s7OzQq1cv7Nq1S3pcCIGoqCi4ubnBysoKgYGBOHPmjN42tFotZsyYAScnJ9jY2GDo0KG4cuWKIYdFREREMmTwvbEe5e3tjQ8++KDCrE912rRpgw8++ADHjh3DsWPH8MILL2DYsGFSoImNjUVcXBxWrlyJ9PR0qNVqhISESPfiAoCIiAhs3boVSUlJOHToEAoLCxEWFlbp5wARERHRX4/Rwg4AmJmZ4dq1a7XuP2TIEAwaNAgdOnRAhw4d8P7776N58+Y4cuQIhBBYsWIFFi5ciBEjRsDHxwfr16/H3bt3sXnzZgDAnTt3kJCQgGXLlqFfv37w8/PDxo0bcerUKaSkpBjz0IiIiKiJMuicne3bt+stCyGQnZ2NlStXok+fPgYVUlpaiq+//hpFRUXo1asXMjMzkZOTg9DQUKmPSqVCQEAA0tLSMG3aNGRkZECn0+n1cXNzg4+PD9LS0tC/f/9K96XVaqHVaqXl/Px8AA/OPdLpdAbVb6jy/TX0fuVKjuP5yF1ZGpRSqdP715hk9BTVmhxfn6bCsTSupjqeta3XoLDz4osv6i0rFAq0atUKL7zwApYtW1anbZ06dQq9evXCvXv30Lx5c2zduhWdO3dGWloaAMDFxUWvv4uLC7KysgAAOTk5sLCwgL29fYU+OTk5Ve4zJiYGixYtqtCenJwMa2vrOtVvLBqNxiT7lSs5jWd4uKkrAKZONf547txp9E02GXJ6fZoax9K4mtp43r17t1b9DAo7ZWVlhqxWqY4dO+LEiRP4888/8e2332LChAk4cOCA9LhCodDrL4So0PaomvrMnz8fkZGR0nJ+fj7c3d0RGhoKOzs7A4/EMDqdDhqNBiEhIVAqlQ26bzmS43iGhZlu30qlDlOnavDZZyHQ6Yw7njt2GHVzTYIcX5+mwrE0rqY6nuXvzNTE4A8VNBYLCws88cQTAIDu3bsjPT0dH330EebNmwfgweyNq6ur1D83N1ea7VGr1SgpKUFeXp7e7E5ubi569+5d5T5VKhVUKlWFdqVSabIn2ZT7liM5jWdJiakrAHQ6JUpKjDueMnl6DCKn16epcSyNq6mNZ21rNSjsPDwrUpO4uLg6bVsIAa1WC09PT6jVamg0Gvj5+QEASkpKcODAASxZsgQA4O/vD6VSCY1Gg5EjRwIAsrOzcfr0acTGxtZpv0RERCRPBoWdn3/+GcePH8f9+/fRsWNHAMC5c+dgZmaGp59+WupX09tNCxYswMCBA+Hu7o6CggIkJSVh//792L17NxQKBSIiIhAdHQ1vb294e3sjOjoa1tbWGDNmDACgRYsWmDJlCmbPng1HR0c4ODhgzpw58PX1Rb9+/Qw5NCIiIpIZg8LOkCFDYGtri/Xr10tvH+Xl5WHSpEl47rnnMHv27Fpt5/r16xg3bhyys7PRokULdOnSBbt370ZISAgAYO7cuSguLkZ4eDjy8vLQo0cPJCcnw9bWVtrG8uXLYW5ujpEjR6K4uBjBwcFYt24dzMzMDDk0IiIikhmDws6yZcuQnJysd56Mvb09Fi9ejNDQ0FqHnYSEhGofVygUiIqKQlRUVJV9LC0tER8fj/j4+Frtk4iIiP5aDPpQwfz8fFy/fr1Ce25urt6nGxMRERGZmkFhZ/jw4Zg0aRK++eYbXLlyBVeuXME333yDKVOmYMSIEcaukYiIiMhgBr2NtXr1asyZMwevvfaa9OmF5ubmmDJlCpYuXWrUAomIiIgeh0Fhx9raGp988gmWLl2KCxcuQAiBJ554AjY2Nsauj4iIiOixPNaNQLOzs5GdnY0OHTrAxsYGQghj1UVERERkFAaFnVu3biE4OBgdOnTAoEGDkJ2dDQB4/fXXa30lFhEREVFDMOhtrFmzZkGpVOLSpUt48sknpfZRo0Zh1qxZdb4ZKBFRZYKCau6Tmlr/dRBR02ZQ2ElOTsaePXvQpk0bvXZvb2/pjuREREREjYFBb2MVFRXB2tq6QvvNmzcrvcEmERERkakYFHaef/55bNiwQVpWKBQoKyvD0qVLEVSbeWciIiKiBmLQ21hLly5FYGAgjh07hpKSEsydOxdnzpzB7du38dNPPxm7RiKSIf5dREQNxaCZnc6dO+PkyZN49tlnERISgqKiIowYMQI///wzvLy8jF0jERERkcHqPLOj0+kQGhqKTz/9FIsWLaqPmoiIiIiMps4zO0qlEqdPn4ZCoaiPeoiIiIiMyqC3scaPH4+EhARj10JERERkdAadoFxSUoLPP/8cGo0G3bt3r3BPrLi4OKMUR0RERPS46hR2/vjjD7Rr1w6nT5/G008/DQA4d+6cXh++vUVERESNSZ3Cjre3N7Kzs5H6/z+ffdSoUfj444/h4uJSL8URERERPa46nbPz6F3Nd+3ahaKiIqMWRERERGRMBp2gXO7R8ENERETU2NQp7CgUigrn5PAcHSIiImrM6nTOjhACEydOlG72ee/ePbzxxhsVrsb67rvvjFchERER0WOoU9iZMGGC3vJrr71m1GKIiIiIjK1OYScxMbG+6iAiIiKqF491gjIRERFRY8ewQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyZm7qAoj+yoKCTF0BEZH8cWaHiIiIZI1hh4iIiGTNpGEnJiYGzzzzDGxtbeHs7IwXX3wRZ8+e1esjhEBUVBTc3NxgZWWFwMBAnDlzRq+PVqvFjBkz4OTkBBsbGwwdOhRXrlxpyEMhIiKiRsqkYefAgQOYPn06jhw5Ao1Gg/v37yM0NBRFRUVSn9jYWMTFxWHlypVIT0+HWq1GSEgICgoKpD4RERHYunUrkpKScOjQIRQWFiIsLAylpaWmOCwiIiJqREx6gvLu3bv1lhMTE+Hs7IyMjAw8//zzEEJgxYoVWLhwIUaMGAEAWL9+PVxcXLB582ZMmzYNd+7cQUJCAr744gv069cPALBx40a4u7sjJSUF/fv3b/DjIiIiosajUV2NdefOHQCAg4MDACAzMxM5OTkIDQ2V+qhUKgQEBCAtLQ3Tpk1DRkYGdDqdXh83Nzf4+PggLS2t0rCj1Wqh1Wql5fz8fACATqeDTqerl2OrSvn+Gnq/ctXUxtPCwtQVVE+p1On92xg1kacaQNN7fTZmHEvjaqrjWdt6G03YEUIgMjISffv2hY+PDwAgJycHAODi4qLX18XFBVlZWVIfCwsL2NvbV+hTvv6jYmJisGjRogrtycnJsLa2fuxjMYRGozHJfuWqqYxneLipK6idqVMb73ju3GnqCuquqbw+mwKOpXE1tfG8e/durfo1mrDz1ltv4eTJkzh06FCFxxQKhd6yEKJC26Oq6zN//nxERkZKy/n5+XB3d0doaCjs7OwMqN5wOp0OGo0GISEhUCqVDbpvOWpq4xkWZuoKqqdU6jB1qgaffRYCna5xjueOHaauoPaa2uuzMeNYGldTHc/yd2Zq0ijCzowZM7B9+3YcPHgQbdq0kdrVajWAB7M3rq6uUntubq4026NWq1FSUoK8vDy92Z3c3Fz07t270v2pVCqoVKoK7Uql0mRPsin3LUdNZTxLSkxdQe3odEqUlDTO8WwCT3MFTeX12RRwLI2rqY1nbWs16dVYQgi89dZb+O6777Bv3z54enrqPe7p6Qm1Wq03rVZSUoIDBw5IQcbf3x9KpVKvT3Z2Nk6fPl1l2CEiIqK/DpPO7EyfPh2bN2/G999/D1tbW+kcmxYtWsDKygoKhQIRERGIjo6Gt7c3vL29ER0dDWtra4wZM0bqO2XKFMyePRuOjo5wcHDAnDlz4OvrK12dRURERH9dJg07q1atAgAEBgbqtScmJmLixIkAgLlz56K4uBjh4eHIy8tDjx49kJycDFtbW6n/8uXLYW5ujpEjR6K4uBjBwcFYt24dzMzMGupQiIiIqJEyadgRQtTYR6FQICoqClFRUVX2sbS0RHx8POLj441YHREREckB741FREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREsmZu6gKI5CooyNQVEBERwJkdIiIikjmGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1c1MXQET0OIKCau6Tmlr/dRBR48WZHSIiIpI1hh0iIiKSNYYdIiIikjWes0NEssfzeoj+2jizQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREsmbSsHPw4EEMGTIEbm5uUCgU2LZtm97jQghERUXBzc0NVlZWCAwMxJkzZ/T6aLVazJgxA05OTrCxscHQoUNx5cqVBjwKIiIiasxMGnaKiorQtWtXrFy5stLHY2NjERcXh5UrVyI9PR1qtRohISEoKCiQ+kRERGDr1q1ISkrCoUOHUFhYiLCwMJSWljbUYRAREVEjZtJ7Yw0cOBADBw6s9DEhBFasWIGFCxdixIgRAID169fDxcUFmzdvxrRp03Dnzh0kJCTgiy++QL9+/QAAGzduhLu7O1JSUtC/f/8GOxYiIiJqnBrtOTuZmZnIyclBaGio1KZSqRAQEIC0tDQAQEZGBnQ6nV4fNzc3+Pj4SH2IiIjor63R3vU8JycHAODi4qLX7uLigqysLKmPhYUF7O3tK/QpX78yWq0WWq1WWs7PzwcA6HQ66HQ6o9RfW+X7a+j9ylVjGk8LC1NX8PiUSp3ev3LWEC+ZxvT6bOo4lsbVVMeztvU22rBTTqFQ6C0LISq0PaqmPjExMVi0aFGF9uTkZFhbWxtW6GPSaDQm2a9cNYbxDA83dQXGM3Wq6cezvu3c2XD7agyvT7ngWBpXUxvPu3fv1qpfow07arUawIPZG1dXV6k9NzdXmu1Rq9UoKSlBXl6e3uxObm4uevfuXeW258+fj8jISGk5Pz8f7u7uCA0NhZ2dnbEPpVo6nQ4ajQYhISFQKpUNum85akzjGRZm0t0bhVKpw9SpGnz2WQh0Onm/PnfsqP99NKbXZ1PHsTSupjqe5e/M1KTRhh1PT0+o1WpoNBr4+fkBAEpKSnDgwAEsWbIEAODv7w+lUgmNRoORI0cCALKzs3H69GnExsZWuW2VSgWVSlWhXalUmuxJNuW+5agxjGdJiUl3b1Q6nRIlJfJ+fTbky6UxvD7lgmNpXE1tPGtbq0nDTmFhIc6fPy8tZ2Zm4sSJE3BwcEDbtm0RERGB6OhoeHt7w9vbG9HR0bC2tsaYMWMAAC1atMCUKVMwe/ZsODo6wsHBAXPmzIGvr690dRYRERH9tZk07Bw7dgxBQUHScvlbSxMmTMC6deswd+5cFBcXIzw8HHl5eejRoweSk5Nha2srrbN8+XKYm5tj5MiRKC4uRnBwMNatWwczM7MGPx4iIiJqfEwadgIDAyGEqPJxhUKBqKgoREVFVdnH0tIS8fHxiI+Pr4cKiYiIqKlrtJ+zQ0RERGQMDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRr5qYugKgpCgoydQVkbLV5TlNT678OIjI+zuwQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkazx3lhEj+B9r4iI5IUzO0RERCRrDDtEREQka3wbi/5S+BYVEdFfD2d2iIiISNYYdoiIiEjW+DYWEVEt1eZt0NTU+q+DiOqGMztEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQka7xdBBFRAwsLA0pKqu/D204QGQ/DDjUJvCcREREZimGHZCMoCLCwAMLDa/eXM1F9qC6Yl78+iahhyeacnU8++QSenp6wtLSEv78/fvzxR1OXRERERI2ALMLOl19+iYiICCxcuBA///wznnvuOQwcOBCXLl0ydWlERERkYrJ4GysuLg5TpkzB66+/DgBYsWIF9uzZg1WrViEmJsbE1VFNanM+DhERkaGafNgpKSlBRkYG/vGPf+i1h4aGIi0tzURVERHVP564T1Q7TT7s3Lx5E6WlpXBxcdFrd3FxQU5OTqXraLVaaLVaafnOnTsAgNu3b0On0xm1vpEjq3/c3FyHiRPv4tatW1AqlUbdd32r6dhqq5lR30zV4e7duwBuoVmzpjWejRPH07hqP57BwTVvrTbfO7XZzldf1dynNt/vtdmOseh0D8ayKf7sbIweZzxN+dooKCgAAAghqu3X5MNOOYVCobcshKjQVi4mJgaLFi2q0O7p6VkvtdUkOdkku5UtjqdxcTyNqzGOp5NT49oOyU99vzYKCgrQokWLKh9v8mHHyckJZmZmFWZxcnNzK8z2lJs/fz4iIyOl5bKyMty+fRuOjo5VBqT6kp+fD3d3d1y+fBl2dnYNum854ngaF8fTuDiexsOxNK6mOp5CCBQUFMDNza3afk0+7FhYWMDf3x8ajQbDhw+X2jUaDYYNG1bpOiqVCiqVSq+tZcuW9Vlmjezs7JrUC6yx43gaF8fTuDiexsOxNK6mOJ7VzeiUa/JhBwAiIyMxbtw4dO/eHb169cKaNWtw6dIlvPHGG6YujYiIiExMFmFn1KhRuHXrFt59911kZ2fDx8cHO3fuhIeHh6lLIyIiIhOTRdgBgPDwcIQ3wc9hV6lUeOeddyq8rUaG4XgaF8fTuDiexsOxNC65j6dC1HS9FhEREVETJovbRRARERFVhWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGnEdJqtejWrRsUCgVOnDhh6nKapIsXL2LKlCnw9PSElZUVvLy88M4776CkpMTUpTUZn3zyCTw9PWFpaQl/f3/8+OOPpi6pSYqJicEzzzwDW1tbODs748UXX8TZs2dNXZZsxMTEQKFQICIiwtSlNFlXr17Fa6+9BkdHR1hbW6Nbt27IyMgwdVlGxbDTCM2dO7fGj76m6v3nP/9BWVkZPv30U5w5cwbLly/H6tWrsWDBAlOX1iR8+eWXiIiIwMKFC/Hzzz/jueeew8CBA3Hp0iVTl9bkHDhwANOnT8eRI0eg0Whw//59hIaGoqioyNSlNXnp6elYs2YNunTpYupSmqy8vDz06dMHSqUSu3btwq+//oply5aZ/K4CRieoUdm5c6fo1KmTOHPmjAAgfv75Z1OXJBuxsbHC09PT1GU0Cc8++6x444039No6deok/vGPf5ioIvnIzc0VAMSBAwdMXUqTVlBQILy9vYVGoxEBAQFi5syZpi6pSZo3b57o27evqcuod5zZaUSuX7+OqVOn4osvvoC1tbWpy5GdO3fuwMHBwdRlNHolJSXIyMhAaGioXntoaCjS0tJMVJV83LlzBwD4WnxM06dPx+DBg9GvXz9Tl9Kkbd++Hd27d8crr7wCZ2dn+Pn54bPPPjN1WUbHsNNICCEwceJEvPHGG+jevbupy5GdCxcuID4+nvdLq4WbN2+itLQULi4ueu0uLi7IyckxUVXyIIRAZGQk+vbtCx8fH1OX02QlJSXh+PHjiImJMXUpTd4ff/yBVatWwdvbG3v27MEbb7yBt99+Gxs2bDB1aUbFsFPPoqKioFAoqv06duwY4uPjkZ+fj/nz55u65EattuP5sGvXrmHAgAF45ZVX8Prrr5uo8qZHoVDoLQshKrRR3bz11ls4efIktmzZYupSmqzLly9j5syZ2LhxIywtLU1dTpNXVlaGp59+GtHR0fDz88O0adMwdepUrFq1ytSlGZVs7o3VWL311lsYPXp0tX3atWuHxYsX48iRIxXuS9K9e3eMHTsW69evr88ym4zajme5a9euISgoCL169cKaNWvquTp5cHJygpmZWYVZnNzc3AqzPVR7M2bMwPbt23Hw4EG0adPG1OU0WRkZGcjNzYW/v7/UVlpaioMHD2LlypXQarUwMzMzYYVNi6urKzp37qzX9uSTT+Lbb781UUX1g2Gnnjk5OcHJyanGfh9//DEWL14sLV+7dg39+/fHl19+iR49etRniU1KbccTeHA5ZVBQEPz9/ZGYmIhmzTiRWRsWFhbw9/eHRqPB8OHDpXaNRoNhw4aZsLKmSQiBGTNmYOvWrdi/fz88PT1NXVKTFhwcjFOnTum1TZo0CZ06dcK8efMYdOqoT58+FT4K4dy5c/Dw8DBRRfWDYaeRaNu2rd5y8+bNAQBeXl78K9AA165dQ2BgINq2bYsPP/wQN27ckB5Tq9UmrKxpiIyMxLhx49C9e3dpVuzSpUs858kA06dPx+bNm/H999/D1tZWmjFr0aIFrKysTFxd02Nra1vhfCcbGxs4OjryPCgDzJo1C71790Z0dDRGjhyJo0ePYs2aNbKbCWfYIVlKTk7G+fPncf78+QphUQhhoqqajlGjRuHWrVt49913kZ2dDR8fH+zcuVN2f+01hPJzHwIDA/XaExMTMXHixIYviOghzzzzDLZu3Yr58+fj3XffhaenJ1asWIGxY8eaujSjUgj+5CciIiIZ40kMREREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RkUgqFAtu2bZOW//Of/6Bnz56wtLREt27dqmwjItM4ePAghgwZAjc3twrfv/WhXbt2ld70efr06bXeBsMOEdWLiRMnSj+UlEolXFxcEBISgrVr16KsrEzql52djYEDB0rL77zzDmxsbHD27Fns3bu3yjYiMo2ioiJ07doVK1eubJD9paenIzs7W/rSaDQAgFdeeaXW22DYIaJ6M2DAAGRnZ+PixYvYtWsXgoKCMHPmTISFheH+/fsAHtyrTKVSSetcuHABffv2hYeHBxwdHatsq6uSkpLHPyAiwsCBA7F48WKMGDGi0sdLSkowd+5ctG7dGjY2NujRowf2799v8P5atWoFtVotfe3YsQNeXl4ICAio9TYYdoio3qhUKqjVarRu3RpPP/00FixYgO+//x67du3CunXrAOi/jaVQKJCRkYF3330XCoUCUVFRlbYBD+5qP2rUKNjb28PR0RHDhg3DxYsXpX1PnDgRL774ImJiYuDm5oYOHTrUab0PP/wQrq6ucHR0xPTp06HT6aQ+Wq0Wc+fOhbu7O1QqFby9vZGQkCA9/uuvv2LQoEFo3rw5XFxcMG7cONy8ebNexpiosZk0aRJ++uknJCUl4eTJk3jllVcwYMAA/P7774+97ZKSEmzcuBGTJ0+GQqGo9XoMO0TUoF544QV07doV3333XYXHsrOz8dRTT2H27NnIzs7GnDlzKm27e/cugoKC0Lx5cxw8eBCHDh1C8+bNMWDAAL0ZnL179+K3336DRqPBjh07ar1eamoqLly4gNTUVKxfvx7r1q2TwhkAjB8/HklJSfj444/x22+/YfXq1WjevLl0DAEBAejWrRuOHTuG3bt34/r16xg5cmT9DSpRI3HhwgVs2bIFX3/9NZ577jl4eXlhzpw56Nu3LxITEx97+9u2bcOff/5Z55vo8q7nRNTgOnXqhJMnT1ZoV6vVMDc3R/PmzaFWqwEAzZs3r9C2du1aNGvWDJ9//rn0111iYiJatmyJ/fv3IzQ0FABgY2ODzz//HBYWFnVaz97eHitXroSZmRk6deqEwYMHY+/evZg6dSrOnTuHr776ChqNBv369QMAtG/fXjqGVatW4emnn0Z0dLTUtnbtWri7u+PcuXPSDBORHB0/fhxCiAqvc61WK70FffHiRXh6ela7nenTp1d6TlBCQgIGDhwINze3OtXFsENEDU4IUacp6EdlZGTg/PnzsLW11Wu/d+8eLly4IC37+vpKQacu6z311FMwMzOTll1dXXHq1CkAwIkTJ2BmZlbl+QIZGRlITU2VZnoeduHCBYYdkrWysjKYmZkhIyND73sIgPQ90bp1a/z222/Vbsfe3r5CW1ZWFlJSUiqdFa4Jww4RNbjffvutxr/sqlNWVgZ/f39s2rSpwmOtWrWS/m9jY2PQekqlUu8xhUIhXUFmZWVVY21DhgzBkiVLKjzm6upa7bpETZ2fnx9KS0uRm5uL5557rtI+SqUSnTp1qvO2ExMT4ezsjMGDB9d5XYYdImpQ+/btw6lTpzBr1iyDt/H000/jyy+/hLOzM+zs7Op9vYf5+vqirKwMBw4ckN7GenQf3377Ldq1awdzc/6IJfkpLCzE+fPnpeXMzEycOHECDg4O6NChA8aOHYvx48dj2bJl8PPzw82bN7Fv3z74+vpi0KBBBu2zrKwMiYmJmDBhgkHfVzxBmYjqjVarRU5ODq5evYrjx48jOjoaw4YNQ1hYGMaPH2/wdseOHQsnJycMGzYMP/74IzIzM3HgwAHMnDkTV65cMfp6D2vXrh0mTJiAyZMnY9u2bcjMzMT+/fvx1VdfAXhwrsHt27fx6quv4ujRo/jjjz+QnJyMyZMno7S01OBjJmosjh07Bj8/P/j5+QEAIiMj4efnh//93/8F8GAGZvz48Zg9ezY6duyIoUOH4t///jfc3d0N3mdKSgouXbqEyZMnG7Q+/+wgonqze/duuLq6wtzcHPb29ujatSs+/vhjTJgwAc2aGf63lrW1NQ4ePIh58+ZhxIgRKCgoQOvWrREcHFztjI2h6z1q1apVWLBgAcLDw3Hr1i20bdsWCxYsAAC4ubnhp59+wrx589C/f39otVp4eHhgwIABj3XMRI1FYGAghBBVPq5UKrFo0SIsWrTIaPsMDQ2tdp81UYjHWZuIiIiokeOfGURERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGv/D9zab+oKUczdAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "differences = tl_model.blocks[0].attn.forward(test_tensor, test_tensor, test_tensor) - hf_model.model.layers[0].self_attn.forward(test_tensor)[0]\n",
    "\n",
    "# Flatten the differences to create a one-dimensional tensor\n",
    "flattened_differences = differences.flatten().cpu().detach().numpy()\n",
    "\n",
    "# Plot the histogram of the differences\n",
    "plt.hist(flattened_differences, bins=50, alpha=0.75, color='blue')\n",
    "plt.title('Differences Between Attention Outputs')\n",
    "plt.xlabel('Difference')\n",
    "plt.ylabel('Frequency')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}