joelburget/mixtral-playing-2.ipynb Secret

## mixtral-playing-2.ipynb
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4fb7e0bc-4ef5-40c8-8222-336e83bd6e66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
        "(truncated)"
     ]
    }
   ],
   "source": [
    "%pip install transformers huggingface_hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "be241e96-3bbb-46a4-a4d4-0213eb094d6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(truncated)"
     ]
    }
   ],
   "source": [
    "%pip install git+https://github.com/joelburget/TransformerLens.git@mixtral-playing-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d7341d8-881c-41c3-8199-ae9590d51a5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from huggingface_hub import login\n",
    "login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cba8adb4-03a4-4061-b62b-18bcc091b8af",
   "metadata": {},
   "outputs": [],
   "source": [
    "import einops\n",
    "model_id = \"mistralai/Mixtral-8x7B-v0.1\"\n",
    "text = \"Hello my name is\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2c3cb338-cf1b-4775-b278-302999164e6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb2d9b164be34f45a70d46b22c9e0ea7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/19 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained model mistralai/Mixtral-8x7B-v0.1 into HookedTransformer\n"
     ]
    }
   ],
   "source": [
    "from transformer_lens import HookedTransformer\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "tl_model = HookedTransformer.from_pretrained_no_processing(\n",
    "    \"mistralai/Mixtral-8x7B-v0.1\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "617609fe-060e-48b5-9daf-cb45cb6cc5d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hello my name is musackA. is nexting with ürs… Muhammad, U. M M, M. I * J Years Federation N Fish vADY/A (5 /-AZakyB7.’ for 5A. },'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.generate(\n",
    "    text,\n",
    "    verbose=False,\n",
    "    max_new_tokens=50,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4f3ccaf0-1650-4621-84f4-dbcdef132447",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "043f4bf2d1ec48af85a1a05e5a66c169",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/19 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
    "hf_model = AutoModelForCausalLM.from_pretrained(model_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "97cec596-b5fc-48c2-82db-08af7e238a0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello my name is Katie and I am a 20 year old student from the UK. I am currently studying a degree in English Literature and Creative Writing at the University of Winchester. I have always had a passion for writing and I am hoping to pursue\n"
     ]
    }
   ],
   "source": [
    "inputs = tokenizer(text, return_tensors=\"pt\")\n",
    "outputs = hf_model.generate(**inputs, max_new_tokens=50)\n",
    "print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1df1c967-4dfd-41ef-bab4-dea6072678aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.testing import assert_close\n",
    "import torch\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cfa7d1c3-72e8-45b3-850c-091669f48b1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    einops.rearrange(tl_model.blocks[0].attn.W_Q, \"n m h -> (n h) m\") ==\n",
    "    hf_model.model.layers[0].self_attn.q_proj.weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "83649934-f06b-4f94-8004-59b8d4098589",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([32, 4096, 128]), torch.Size([1024, 4096]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].attn.W_K.shape, hf_model.model.layers[0].self_attn.k_proj.weight.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4fa20cf5-b720-4946-a7e5-e1d2e6277f6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    einops.reduce(\n",
    "        tl_model.blocks[0].attn.W_K, \"(n repeat) m h -> (n h) m\",\n",
    "        'max',\n",
    "        n=tl_model.cfg.n_key_value_heads,\n",
    "        repeat=4) ==\n",
    "    hf_model.model.layers[0].self_attn.k_proj.weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ef6f7ea9-ef0b-4091-8d00-504b481fc59a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    einops.reduce(\n",
    "        tl_model.blocks[0].attn.W_V, \"(n repeat) m h -> (n h) m\",\n",
    "        'max',\n",
    "        n=tl_model.cfg.n_key_value_heads,\n",
    "        repeat=4) ==\n",
    "    hf_model.model.layers[0].self_attn.v_proj.weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "04b8f4be-ce7d-4dc2-acda-d023c721525c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    einops.rearrange(tl_model.blocks[0].attn.W_O, \"n h m -> m (n h)\") ==\n",
    "    hf_model.model.layers[0].self_attn.o_proj.weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1e10ed87-31b5-4c1c-b726-7a3f49fbd136",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.]], requires_grad=True)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].attn.b_Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6caf9d98-adb2-45e7-8357-34288b2156f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(False)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(hf_model.model.layers[0].block_sparse_moe.gate.weight.T == tl_model.blocks[0].mlp.W_gate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "00e9ea5d-74c2-4c2a-8e9d-6fc196cb8fc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.float32, torch.float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].block_sparse_moe.gate.weight.dtype, tl_model.blocks[0].mlp.W_gate.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "44a55507-e639-414a-a297-e68e1c0696f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    tl_model.blocks[0].mlp.experts[0].W_in ==\n",
    "    hf_model.model.layers[0].block_sparse_moe.experts[0].w3.weight.T\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "03944deb-aa8d-46ff-83dd-4f7ee955656c",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tensor = torch.randn((1, 1, 4096,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "eb0109ee-b82a-4ea0-b50b-8e6408647cea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(False)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    hf_model.model.layers[0].block_sparse_moe(test_tensor)[0] ==\n",
    "    tl_model.blocks[0].mlp(test_tensor)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "25ce75bf-706e-4ae8-8f74-bc9c40e88c25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.4516,  0.1519, -0.0053,  ...,  0.4323, -0.1565, -0.3980]]],\n",
       "       grad_fn=<ReshapeAliasBackward0>)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].block_sparse_moe(test_tensor)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c016430e-0a30-426b-bfd0-0b1b423b3ff6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.2893, -0.1216,  0.1572,  ..., -0.1228, -0.2185,  0.6155]]],\n",
       "       grad_fn=<IndexPutBackward0>)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].mlp(test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "46353486-0a3f-4241-9cf5-ed25c7539f71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 4096])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].mlp(test_tensor).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e25ada54-4e3c-42b7-8f35-ba67bfa500e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[False, False, False,  ..., False, False, False]]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].block_sparse_moe(test_tensor)[0] == tl_model.blocks[0].mlp(test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8f3a2865-645d-4441-95fb-32446f866760",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sum(hf_model.model.layers[0].block_sparse_moe(test_tensor)[0] == tl_model.blocks[0].mlp(test_tensor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c6ef1f5e-bdf0-45e5-9347-6972e91e2f2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHFCAYAAAAUpjivAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABJQklEQVR4nO3deXxTdb7/8Xds03ShrZTSbSilgyBLwQVGgVGgQstikUVFZVTAygMHZKjAZcTlWrwIigo4ILghi4jFUXFhBFqkgAxwBdyAcVCxbNLakaVlKWlazu8Pf80ldKEtSdMeXs/How/IN998z+d8k4Y333NOYjEMwxAAAIBJXeHtAgAAADyJsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsAMAAEyNsIMGZ/HixbJYLM4ff39/RUVFKTExUTNmzFB+fn65x6Snp8tisbi0FRcX66GHHlJ0dLR8fHx07bXXSpKOHTumu+++WxEREbJYLBo0aFAd7FXDtX//fpfnw2KxKCQkRNdcc43mzJmj0tLSWo376aefKj093b3F1jNl8zVixIgK73/66aedffbv3+9sHzFihBo1alTl2Bf+nvj6+qpZs2YaOXKkfv7552rVd/ToUU2ZMkXt2rVTYGCgQkJC1KVLF7388styOBzV3c1y6vq5nT9/vhYvXlxn20M9ZAANzKJFiwxJxqJFi4ytW7camzZtMt577z0jLS3NCA0NNcLCwoysrCyXxxw6dMjYunWrS9ucOXMMScbcuXONLVu2GN9++61hGIaRlpZm+Pn5GcuWLTO2bt1q7N27t872rSHKyckxJBnjxo0ztm7damzdutVYvXq18ec//9mQZEyYMKFW444dO9Yw+1uUJCM4ONgIDAw0CgsLXe47d+6cER8fb4SEhBiSjJycHOd9w4cPN4KCgqoc+8Lfk/Xr1xvp6emGzWYz4uPjjVOnTlX5+O+++86IjY01GjdubEybNs1Yv3698Y9//MP485//bPj4+Bg9evQwTp8+Xav9ruvntn379kaPHj3qbHuof8z9TgJTKnsT3759e7n7Dhw4YMTGxhrBwcFGXl5eleM8+OCDRkBAQLn23r17G23btnVbvYZhGGfOnHHrePVJWdh5/vnny9138803G9HR0bUa93IJO/fee68REBBgvPbaay73rVu3zpBkjBo16pLCzoW/J08++aQhyVi2bFmljy0pKTHatWtnhIaGVhj2MzIyDEnG6NGjq7GX5RF2UNc4jAVTad68uV588UWdPHlSr776qrP9wsNYFotFb7zxhoqKipzL/GXL/uvWrdN3333nbN+wYYOk3w57TZs2TW3atJHNZlPTpk01cuRI/ec//3GpoUWLFkpJSdEHH3yg6667Tv7+/po6daokKS8vT6NHj1azZs3k5+en+Ph4TZ06VSUlJc7Hlx0WeuGFFzRr1izFx8erUaNG6tq1q7Zt21Zun//3f/9XAwYMUJMmTeTv76+WLVsqLS3Npc8PP/ygYcOGKSIiQjabTW3bttXLL7/s0ufcuXOaNm2arr76agUEBOjKK69Ux44d9dJLL9XquZCk0NBQWa3Wcu0rVqxQ165dFRQUpEaNGqlPnz766quvnPePGDHCWd/5h2L279+vO++8U+3bt3cZb8CAAbJYLPr73//ubPvyyy9lsVj0ySefONuqM/9SzZ/rNWvW6Prrr1dAQIDatGmjN998s0ZzNHjw4HKPefPNN/XHP/5RrVu3rvZY1dGlSxdJ0oEDByrts3LlSv3rX//So48+WuH277rrLiUnJ2vhwoXKy8uTJG3YsMHl96VM2eu57DBSVc9tWdvDDz+sV199Va1bt5bNZlO7du2UkZHhMm5Fh6al/zt8VzZeixYttGfPHm3cuNG5rRYtWkjyzGse9ZOvtwsA3K1///7y8fHRpk2bKu2zdetW/c///I+ys7O1fv16SVJ8fLy2bt2qMWPGqKCgQG+//bYkqV27djp37pwGDhyozz//XJMnT1a3bt104MABPfXUU+rZs6d27NihgIAA5/hffvmlvvvuOz3xxBOKj49XUFCQ8vLydMMNN+iKK67Qf//3f6tly5baunWrpk2bpv3792vRokUuNb788stq06aN5syZI0l68skn1b9/f+Xk5Cg0NFSStHbtWg0YMEBt27bVrFmz1Lx5c+3fv1+ZmZnOcf71r3+pW7duziAYFRWltWvX6i9/+Yt+/fVXPfXUU5KkmTNnKj09XU888YS6d+8uh8Ohf//73zpx4kS15v3cuXPO0FBQUKCPPvpIa9as0V//+leXftOnT9cTTzyhkSNH6oknnlBxcbGef/553Xzzzfriiy/Url07Pfnkkzp9+rTee+89bd261fnY6Oho9e7dW++9955yc3MVHR2tkpISbdy4UQEBAcrKytKdd94pSVq3bp18fX3Vs2dPSar2/Nf0uf7mm280ceJEPfroo4qMjNQbb7yh1NRUXXXVVerevXu15i41NVW9evXSd999p7Zt2+rEiRP64IMPNH/+fB09erRaY1TXjz/+KElq2rRppX2ysrIkqcrz1QYNGqTMzExt2LBBd999d7W3X9VzW+bjjz9Wdna2nn76aQUFBWn+/Pm655575OvrqzvuuKPa25J+C2533HGHQkNDNX/+fEmSzWaTdOmveTQg3l5aAmqqqsNYZSIjI10ORT311FPlls0rOxTQo0cPo3379i5t77zzjiHJeP/9913at2/fbkgy5s+f72yLi4szfHx8yi3/jx492mjUqJFx4MABl/YXXnjBkGTs2bPHMIz/OyzUoUMHo6SkxNnviy++MCQZ77zzjrOtZcuWRsuWLY2ioqJK56JPnz5Gs2bNjIKCApf2hx9+2PD39zeOHTtmGIZhpKSkGNdee22l41SmrN6KfkaMGOGyDwcPHjR8fX2NcePGuYxx8uRJIyoqyhg6dKizrbJDHT/++KMhyVi6dKlhGIaxefNmQ5IxefJkIz4+3tkvKSnJ6Natm/N2dee/ps+1v7+/y5hFRUVGWFhYtQ7xSDLGjh3rPD9n0qRJhmEYxssvv2w0atTIOHnypPH8889f0mGsbdu2GQ6Hwzh58qSxatUqo2nTphc9zNu3b19DknH27NlK+6xevdqQZDz33HOGYRhGdna2IcnIzs526Vf2+li0aJGzrarDWJKMgIAAl/pKSkqMNm3aGFdddZWzraLf6fP3+/z5quwwVm1f82h4OIwFUzIMw63jrVq1SldeeaUGDBigkpIS58+1116rqKiockv3HTt2LLf8v2rVKiUmJiomJsZljH79+kmSNm7c6NL/1ltvlY+Pj8uY0v8dfvj++++1b98+paamyt/fv8K6z549q88++0yDBw9WYGCgy3b79++vs2fPOg+N3XDDDfrmm280ZswYrV27VoWFhTWao/Hjx2v79u3avn27srOzNX36dL377ru65557nH3Wrl2rkpIS3X///S61+Pv7q0ePHuXmsSItW7ZUixYttG7dOkm/rUJ06NBB9957r3JycrRv3z7Z7XZt3rxZvXv3dj6uuvNf0+f62muvVfPmzZ23/f391bp16yoPE12o7Iqst956SyUlJVq4cKGGDh160SuuqqNLly6yWq0KDg5WSkqKoqKitHr1akVGRl7SuGW/YxUdSrpUvXr1cqnPx8dHd911l3788UcdPnzYbdu51Nc8Gg4OY8F0Tp8+raNHj6pDhw5uG/OXX37RiRMn5OfnV+H9v/76q8vt85fkzx/jk08+qfAclorGaNKkicvtsqX3oqIiSXKeP9KsWbNK6z569KhKSko0d+5czZ07t8rtTpkyRUFBQVq2bJleeeUV+fj4qHv37nruuefUuXPnSrdRplmzZi79evbsKYvFoilTpmjt2rXq06ePfvnlF0nSH/7whwrHuOKK6v3/q1evXlqzZo2k3w5XJSUlqUOHDoqMjNS6devUqlUrFRUVuYSd6s5/TZ/rC58n6bfnqux5qq6RI0dq6tSpmj59ur788stKn6+aWrp0qdq2bStfX19FRkZW+Nq8UFl4y8nJUZs2bSrsU3ZOTGxsrFvqPF9UVFSlbUePHq3yNV8Tl/qaR8NB2IHp/OMf/1BpaanzXA13CA8PV5MmTZz/wF4oODjY5XZF/9sNDw9Xx44d9cwzz1Q4RkxMTI1qKjvnoqr/6TZu3Fg+Pj667777NHbs2Ar7xMfHS5J8fX01YcIETZgwQSdOnNC6dev02GOPqU+fPjp06JACAwNrVJ/0f6tR33zzjfr06aPw8HBJ0nvvvae4uLgaj1emV69eWrhwob744gv97//+r5544glJ0i233KKsrCwdOHBAjRo1cp6MK1V//mv6XLtLbGysevfuralTp+rqq69Wt27d3DJu27Zta/wPd1JSkl577TV9+OGHevTRRyvs8+GHH7qcE1W2umi32136XRgOq6PspOeK2srC5fnbK/uPQE2354nXPOonwg5M5eDBg5o0aZJCQ0M1evRot42bkpKijIwMlZaW6sYbb6z1GJ9++qlatmypxo0bX3JNrVu3VsuWLfXmm29qwoQJLm/4ZQIDA5WYmKivvvpKHTt2rHS14kJXXnml7rjjDv38889KS0vT/v371a5duxrX+PXXX0uSIiIiJEl9+vSRr6+v9u3bp9tvv73Kx56/knX+CcHSb2HHYrHoySef1BVXXOE8Ebh37976r//6Lx04cEDdu3d3WcWp7vy747murYkTJyogIMB5krW3DB48WO3atdOzzz6rIUOGlDsku2LFCmVmZuqhhx5yrriUXeH07bffqk+fPs6+H3/8cbnxq3puJemzzz7TL7/84jyUVVpaqhUrVqhly5bOVZ3zt3f+SuH5V9+dv72LrbS56zWP+omwgwZr9+7dzvMp8vPz9fnnn2vRokXy8fHRypUrq7zapKbuvvtuvf322+rfv7/Gjx+vG264QVarVYcPH1Z2drYGDhyowYMHVznG008/raysLHXr1k1/+ctfdPXVV+vs2bPav3+/Pv30U73yyis1Xp5/+eWXNWDAAHXp0kWPPPKImjdvroMHD2rt2rXOq8leeukl3XTTTbr55pv15z//WS1atNDJkyf1448/6pNPPnFejTZgwAAlJCSoc+fOatq0qQ4cOKA5c+YoLi5OrVq1umgtBw8edJ7/c/r0aW3dulUzZsxQXFychgwZIum3f6CefvppPf744/rpp5/Ut29fNW7cWL/88ou++OILBQUFOS/TLzsM+dxzz6lfv37y8fFxBraIiAglJCQoMzNTiYmJzv+B9+7dW8eOHdOxY8c0a9asWs2/O57r2kpOTlZycnK1+paWluq9994r1x4UFOQ8D6m2fHx89P777yspKUldu3bVxIkT1bVrV9ntdn3yySd67bXX1KNHD7344ovOx0RFRal3796aMWOGGjdurLi4OH322Wf64IMPyo1f1XMr/ba6dsstt+jJJ590Xo3173//2+Xy8/79+yssLEypqal6+umn5evrq8WLF+vQoUMVbi8jI0MrVqzQ73//e/n7+6tDhw6X/JpHA+LtM6SBmiq72qLsx8/Pz4iIiDB69OhhTJ8+3cjPzy/3mEu9GsswDMPhcBgvvPCCcc011xj+/v5Go0aNjDZt2hijR482fvjhB2e/uLg449Zbb62w9v/85z/GX/7yFyM+Pt6wWq1GWFiY0alTJ+Pxxx93fqJtVR/SJ8l46qmnXNq2bt1q9OvXzwgNDTVsNpvRsmVL45FHHnHpk5OTYzzwwAPG7373O8NqtRpNmzY1unXrZkybNs3Z58UXXzS6detmhIeHG35+fkbz5s2N1NRUY//+/RXuy/lj64KrsPz9/Y3WrVsbaWlpRm5ubrnHfPjhh0ZiYqIREhJi2Gw2Iy4uzrjjjjuMdevWOfvY7XbjwQcfNJo2bWpYLJZyV9g88sgjhiTjmWeecRm7VatWhiTnJ2KfrzrzbxiX/lz36NGjWh9ip/9/NVZVKrsa68I5L/uJi4szDKN6Vy1ezK+//mo8+uijRps2bZzzcMMNNxjz5s0ziouLy/XPzc017rjjDiMsLMwIDQ017r33XmPHjh3lrsaq6rktm5P58+cbLVu2NKxWq9GmTRvj7bffLre9L774wujWrZsRFBRk/O53vzOeeuop44033ig3X/v37zeSk5ON4OBglzmq7WseDY/FMNx82QoAALVksVg0duxYzZs3z9ulwES49BwAAJgaYQcAAJgaJygDAOoNzqyAJ7CyAwAATI2wAwAATI2wAwAATI1zdiSdO3dOR44cUXBwsEe+1A4AALifYRg6efKkYmJiqvxuPcKOpCNHjnjky+wAAIDnHTp0qMpPoCfs6P++2O/QoUMKCQlx27gOh0OZmZlKTk6u9JuWcemYZ89jjj2POa4bzLPn1eUcFxYWKjY29qJf0EvY0f99Q3VISIjbw05gYKBCQkL4pfIg5tnzmGPPY47rBvPsed6Y44udgsIJygAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNR8vV0AAKC8xMSL98nO9nwdgBmwsgMAAEyt3oSdGTNmyGKxKC0tzdlmGIbS09MVExOjgIAA9ezZU3v27HF5nN1u17hx4xQeHq6goCDddtttOnz4cB1XDwAA6qt6EXa2b9+u1157TR07dnRpnzlzpmbNmqV58+Zp+/btioqKUlJSkk6ePOnsk5aWppUrVyojI0ObN2/WqVOnlJKSotLS0rreDQAAUA95PeycOnVKf/rTn/T666+rcePGznbDMDRnzhw9/vjjGjJkiBISErRkyRKdOXNGy5cvlyQVFBRo4cKFevHFF9W7d29dd911WrZsmXbt2qV169Z5a5cAAEA94vWwM3bsWN16663q3bu3S3tOTo7y8vKUnJzsbLPZbOrRo4e2bNkiSdq5c6ccDodLn5iYGCUkJDj7AACAy5tXr8bKyMjQl19+qe3bt5e7Ly8vT5IUGRnp0h4ZGakDBw44+/j5+bmsCJX1KXt8Rex2u+x2u/N2YWGhJMnhcMjhcNRuZypQNpY7x0R5zLPnMceed+Ec+/lV5zGerMiceC17Xl3OcXW34bWwc+jQIY0fP16ZmZny9/evtJ/FYnG5bRhGubYLXazPjBkzNHXq1HLtmZmZCgwMvEjlNZeVleX2MVEe8+x5zLHnlc3xmDEX7/vppx4uxsR4LXteXczxmTNnqtXPa2Fn586dys/PV6dOnZxtpaWl2rRpk+bNm6e9e/dK+m31Jjo62tknPz/fudoTFRWl4uJiHT9+3GV1Jz8/X926dat021OmTNGECROctwsLCxUbG6vk5GSFhIS4bR8dDoeysrKUlJQkq9XqtnHhinn2PObY8y6c45SUiz9m1SrP12U2vJY9ry7nuOzIzMV4Lez06tVLu3btcmkbOXKk2rRpo7/+9a/6/e9/r6ioKGVlZem6666TJBUXF2vjxo167rnnJEmdOnWS1WpVVlaWhg4dKknKzc3V7t27NXPmzEq3bbPZZLPZyrVbrVaPPDGeGheumGfPY449r2yOi4ur09fz9ZgVr2XPq4s5ru74Xgs7wcHBSkhIcGkLCgpSkyZNnO1paWmaPn26WrVqpVatWmn69OkKDAzUsGHDJEmhoaFKTU3VxIkT1aRJE4WFhWnSpEnq0KFDuROeAQDA5alef13E5MmTVVRUpDFjxuj48eO68cYblZmZqeDgYGef2bNny9fXV0OHDlVRUZF69eqlxYsXy8fHx4uVAwCA+qJehZ0NGza43LZYLEpPT1d6enqlj/H399fcuXM1d+5czxYHAAAaJK9/zg4AAIAnEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpEXYAAICpeTXsLFiwQB07dlRISIhCQkLUtWtXrV692nn/iBEjZLFYXH66dOniMobdbte4ceMUHh6uoKAg3XbbbTp8+HBd7woAAKinvBp2mjVrpmeffVY7duzQjh07dMstt2jgwIHas2ePs0/fvn2Vm5vr/Pn0009dxkhLS9PKlSuVkZGhzZs369SpU0pJSVFpaWld7w4AAKiHfL258QEDBrjcfuaZZ7RgwQJt27ZN7du3lyTZbDZFRUVV+PiCggItXLhQb731lnr37i1JWrZsmWJjY7Vu3Tr16dPHszsAALWQmFi+zc9PGjNGSkmRiovrvibAzOrNOTulpaXKyMjQ6dOn1bVrV2f7hg0bFBERodatW2vUqFHKz8933rdz5045HA4lJyc722JiYpSQkKAtW7bUaf0AAKB+8urKjiTt2rVLXbt21dmzZ9WoUSOtXLlS7dq1kyT169dPd955p+Li4pSTk6Mnn3xSt9xyi3bu3Cmbzaa8vDz5+fmpcePGLmNGRkYqLy+v0m3a7XbZ7Xbn7cLCQkmSw+GQw+Fw276VjeXOMVEe8+x5zLF7+fmVb7NaHS5/VgdPR83xWva8upzj6m7DYhiG4eFaqlRcXKyDBw/qxIkTev/99/XGG29o48aNzsBzvtzcXMXFxSkjI0NDhgzR8uXLNXLkSJfgIklJSUlq2bKlXnnllQq3mZ6erqlTp5ZrX758uQIDA92zYwAAwKPOnDmjYcOGqaCgQCEhIZX28/rKjp+fn6666ipJUufOnbV9+3a99NJLevXVV8v1jY6OVlxcnH744QdJUlRUlIqLi3X8+HGX1Z38/Hx169at0m1OmTJFEyZMcN4uLCxUbGyskpOTq5ysmnI4HMrKylJSUpKsVqvbxoUr5tnzmGP3Skkp32a1OjRqVJZefz1JDkf15njVKjcXdhngtex5dTnHZUdmLsbrYedChmGUW6kpc/ToUR06dEjR0dGSpE6dOslqtSorK0tDhw6V9Nvqz+7duzVz5sxKt2Gz2WSz2cq1W61WjzwxnhoXrphnz2OO3aOqE5AdDquKi6s3xzwVtcdr2fPqYo6rO75Xw85jjz2mfv36KTY2VidPnlRGRoY2bNigNWvW6NSpU0pPT9ftt9+u6Oho7d+/X4899pjCw8M1ePBgSVJoaKhSU1M1ceJENWnSRGFhYZo0aZI6dOjgvDoLAABc3rwadn755Rfdd999ys3NVWhoqDp27Kg1a9YoKSlJRUVF2rVrl5YuXaoTJ04oOjpaiYmJWrFihYKDg51jzJ49W76+vho6dKiKiorUq1cvLV68WD4+Pl7cMwAAUF94NewsXLiw0vsCAgK0du3ai47h7++vuXPnau7cue4sDQAAmES9O2cHABqyij4wEIB31ZsPFQQAAPAEwg4AADA1wg4AADA1wg4AADA1wg4AADA1wg4AADA1wg4AADA1wg4AADA1wg4AADA1wg4AADA1wg4AADA1wg4AADA1wg4AADA1wg4AADA1wg4AADA1wg4AADA1X28XAAANRWKitysAUBus7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFPz9XYBAIDaSUy8eJ/sbM/XAdR3Xl3ZWbBggTp27KiQkBCFhISoa9euWr16tfN+wzCUnp6umJgYBQQEqGfPntqzZ4/LGHa7XePGjVN4eLiCgoJ022236fDhw3W9KwAAoJ7yathp1qyZnn32We3YsUM7duzQLbfcooEDBzoDzcyZMzVr1izNmzdP27dvV1RUlJKSknTy5EnnGGlpaVq5cqUyMjK0efNmnTp1SikpKSotLfXWbgEAgHrEq2FnwIAB6t+/v1q3bq3WrVvrmWeeUaNGjbRt2zYZhqE5c+bo8ccf15AhQ5SQkKAlS5bozJkzWr58uSSpoKBACxcu1IsvvqjevXvruuuu07Jly7Rr1y6tW7fOm7sGAADqiXpzzk5paan+/ve/6/Tp0+ratatycnKUl5en5ORkZx+bzaYePXpoy5YtGj16tHbu3CmHw+HSJyYmRgkJCdqyZYv69OlT4bbsdrvsdrvzdmFhoSTJ4XDI4XC4bZ/KxnLnmCiPefY85vg3fn6eG9tqdbj86S6X+VNWDq9lz6vLOa7uNrwednbt2qWuXbvq7NmzatSokVauXKl27dppy5YtkqTIyEiX/pGRkTpw4IAkKS8vT35+fmrcuHG5Pnl5eZVuc8aMGZo6dWq59szMTAUGBl7qLpWTlZXl9jFRHvPseZf7HI8Z4/ltjBrl3jn+9FO3Dmcal/truS7UxRyfOXOmWv28Hnauvvpqff311zpx4oTef/99DR8+XBs3bnTeb7FYXPobhlGu7UIX6zNlyhRNmDDBebuwsFCxsbFKTk5WSEhILfekPIfDoaysLCUlJclqtbptXLhinj2POf5NSornxrZaHRo1Kkuvv54kh8N9c7xqlduGMgVey55Xl3NcdmTmYrwedvz8/HTVVVdJkjp37qzt27frpZde0l//+ldJv63eREdHO/vn5+c7V3uioqJUXFys48ePu6zu5Ofnq1u3bpVu02azyWazlWu3Wq0eeWI8NS5cMc+ed7nPcXGx57fhcFhVXOy+Ob6Mn64qXe6v5bpQF3Nc3fHr3YcKGoYhu92u+Ph4RUVFuSyDFRcXa+PGjc4g06lTJ1mtVpc+ubm52r17d5VhBwAAXD68urLz2GOPqV+/foqNjdXJkyeVkZGhDRs2aM2aNbJYLEpLS9P06dPVqlUrtWrVStOnT1dgYKCGDRsmSQoNDVVqaqomTpyoJk2aKCwsTJMmTVKHDh3Uu3dvb+4aAACoJ7wadn755Rfdd999ys3NVWhoqDp27Kg1a9YoKSlJkjR58mQVFRVpzJgxOn78uG688UZlZmYqODjYOcbs2bPl6+uroUOHqqioSL169dLixYvl4+Pjrd0CAAD1iFfDzsKFC6u832KxKD09Xenp6ZX28ff319y5czV37lw3VwcAAMyg3p2zAwAA4E6EHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGq+3i4AAOA5iYkX75Od7fk6AG9iZQcAAJgaKzsAoOqtgABomFjZAQAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApubVsDNjxgz94Q9/UHBwsCIiIjRo0CDt3bvXpc+IESNksVhcfrp06eLSx263a9y4cQoPD1dQUJBuu+02HT58uC53BQAA1FNeDTsbN27U2LFjtW3bNmVlZamkpETJyck6ffq0S7++ffsqNzfX+fPpp5+63J+WlqaVK1cqIyNDmzdv1qlTp5SSkqLS0tK63B0AAFAP1epDBXNychQfH3/JG1+zZo3L7UWLFikiIkI7d+5U9+7dne02m01RUVEVjlFQUKCFCxfqrbfeUu/evSVJy5YtU2xsrNatW6c+ffpccp0AAKDhqlXYueqqq9S9e3elpqbqjjvukL+/v1uKKSgokCSFhYW5tG/YsEERERG68sor1aNHDz3zzDOKiIiQJO3cuVMOh0PJycnO/jExMUpISNCWLVsqDDt2u112u915u7CwUJLkcDjkcDjcsi9l453/JzyDefa8y2GO/fy8u32r1eHyZ10y8dNazuXwWva2upzj6m7DYhiGUdPBd+/erTfffFNvv/227Ha77rrrLqWmpuqGG26ocaFlDMPQwIEDdfz4cX3++efO9hUrVqhRo0aKi4tTTk6OnnzySZWUlGjnzp2y2Wxavny5Ro4c6RJeJCk5OVnx8fF69dVXy20rPT1dU6dOLde+fPlyBQYG1nofAABA3Tlz5oyGDRumgoIChYSEVNqvVmGnTElJiT755BMtXrxYq1evVqtWrZSamqr77rtPTZs2rdFYY8eO1T/+8Q9t3rxZzZo1q7Rfbm6u4uLilJGRoSFDhlQadpKSktSyZUu98sor5caoaGUnNjZWv/76a5WTVVMOh0NZWVlKSkqS1Wp127hwxTx73uUwxykp3t2+1erQqFFZev31JDkcdTvHq1bV6ea86nJ4LXtbXc5xYWGhwsPDLxp2LumLQH19fTV48GD1799f8+fP15QpUzRp0iRNmTJFd911l5577jlFR0dfdJxx48bp448/1qZNm6oMOpIUHR2tuLg4/fDDD5KkqKgoFRcX6/jx42rcuLGzX35+vrp161bhGDabTTabrVy71Wr1yBPjqXHhinn2PDPPcXGxtyv4jcNhVXFx3c7xeWcBVCo72/N11CUzv5bri7qY4+qOf0lXY+3YsUNjxoxRdHS0Zs2apUmTJmnfvn1av369fv75Zw0cOLDKxxuGoYcfflgffPCB1q9fX62Tno8ePapDhw45Q1SnTp1ktVqVlZXl7JObm6vdu3dXGnYAAMDlo1YrO7NmzdKiRYu0d+9e9e/fX0uXLlX//v11xRW/Zaeyc2XatGlT5Thjx47V8uXL9dFHHyk4OFh5eXmSpNDQUAUEBOjUqVNKT0/X7bffrujoaO3fv1+PPfaYwsPDNXjwYGff1NRUTZw4UU2aNFFYWJgmTZqkDh06OK/OAgAAl69ahZ0FCxbogQce0MiRIyu9JLx58+ZauHDhRceRpJ49e7q0L1q0SCNGjJCPj4927dqlpUuX6sSJE4qOjlZiYqJWrFih4OBgZ//Zs2fL19dXQ4cOVVFRkXr16qXFixfLx8enNrsHAABMpFZhp+x8mar4+flp+PDhVfa52LnRAQEBWrt27UW35e/vr7lz52ru3LkX7QsAAC4vtTpnZ9GiRfr73/9erv3vf/+7lixZcslFAQAAuEutws6zzz6r8PDwcu0RERGaPn36JRcFAADgLrUKOwcOHKjwyqm4uDgdPHjwkosCAABwl1qFnYiICH377bfl2r/55hs1adLkkosCAABwl1qFnbvvvlt/+ctflJ2drdLSUpWWlmr9+vUaP3687r77bnfXCAAAUGu1uhpr2rRpOnDggHr16iVf39+GOHfunO6//37O2QEAAPVKrcKOn5+fVqxYof/5n//RN998o4CAAHXo0EFxcXHurg8AAOCSXNJ3Y7Vu3VqtW7d2Vy0AAABuV6uwU1paqsWLF+uzzz5Tfn6+zp0753L/+vXr3VIcAADApapV2Bk/frwWL16sW2+9VQkJCbJYLO6uCwAAwC1qFXYyMjL07rvvqn///u6uBwAAwK1qdem5n5+frrrqKnfXAgAA4Ha1CjsTJ07USy+9dNEv8gQAAPC2Wh3G2rx5s7Kzs7V69Wq1b99eVqvV5f4PPvjALcUBAABcqlqFnSuvvFKDBw92dy0AAABuV6uws2jRInfXAQAek5jo7QoAeFOtztmRpJKSEq1bt06vvvqqTp48KUk6cuSITp065bbiAAAALlWtVnYOHDigvn376uDBg7Lb7UpKSlJwcLBmzpyps2fP6pVXXnF3nQAAALVSq5Wd8ePHq3Pnzjp+/LgCAgKc7YMHD9Znn33mtuIAAAAuVa2vxvrnP/8pPz8/l/a4uDj9/PPPbikMAADAHWq1snPu3DmVlpaWaz98+LCCg4MvuSgAAAB3qVXYSUpK0pw5c5y3LRaLTp06paeeeoqvkAAAAPVKrQ5jzZ49W4mJiWrXrp3Onj2rYcOG6YcfflB4eLjeeecdd9cIAABQa7UKOzExMfr666/1zjvv6Msvv9S5c+eUmpqqP/3pTy4nLAMAAHhbrcKOJAUEBOiBBx7QAw884M56AAAA3KpWYWfp0qVV3n///ffXqhgAAAB3q1XYGT9+vMtth8OhM2fOyM/PT4GBgYQdAABQb9Tqaqzjx4+7/Jw6dUp79+7VTTfdxAnKAACgXqn1d2NdqFWrVnr22WfLrfoAAAB4k9vCjiT5+PjoyJEj7hwSAADgktTqnJ2PP/7Y5bZhGMrNzdW8efP0xz/+0S2FAQAAuEOtws6gQYNcblssFjVt2lS33HKLXnzxRXfUBQAA4Ba1/m6s839KS0uVl5en5cuXKzo6utrjzJgxQ3/4wx8UHBysiIgIDRo0SHv37nXpYxiG0tPTFRMTo4CAAPXs2VN79uxx6WO32zVu3DiFh4crKChIt912mw4fPlybXQMAACbj1nN2amrjxo0aO3astm3bpqysLJWUlCg5OVmnT5929pk5c6ZmzZqlefPmafv27YqKilJSUpJOnjzp7JOWlqaVK1cqIyNDmzdv1qlTp5SSklLhl5UCAIDLS60OY02YMKHafWfNmlXpfWvWrHG5vWjRIkVERGjnzp3q3r27DMPQnDlz9Pjjj2vIkCGSpCVLligyMlLLly/X6NGjVVBQoIULF+qtt95S7969JUnLli1TbGys1q1bpz59+tRiDwEAgFnUKux89dVX+vLLL1VSUqKrr75akvT999/Lx8dH119/vbOfxWKp0bgFBQWSpLCwMElSTk6O8vLylJyc7Oxjs9nUo0cPbdmyRaNHj9bOnTvlcDhc+sTExCghIUFbtmypMOzY7XbZ7Xbn7cLCQkm/fTiiw+GoUc1VKRvLnWOiPObZ8xr6HPv5ebuCi7NaHS5/1jcN9Kkvp6G/lhuCupzj6m6jVmFnwIABCg4O1pIlS9S4cWNJv33Q4MiRI3XzzTdr4sSJNR7TMAxNmDBBN910kxISEiRJeXl5kqTIyEiXvpGRkTpw4ICzj5+fn7OO8/uUPf5CM2bM0NSpU8u1Z2ZmKjAwsMa1X0xWVpbbx0R5zLPnNdQ5HjPG2xVU36hR9XOOP/3U2xW4V0N9LTckdTHHZ86cqVa/WoWdF198UZmZmS4Bo3Hjxpo2bZqSk5NrFXYefvhhffvtt9q8eXO5+y5cITIM46KrRlX1mTJlisuhuMLCQsXGxio5OVkhISE1rr0yDodDWVlZSkpKktVqddu4cMU8e15Dn+OUFG9XcHFWq0OjRmXp9deT5HA0vDmWpFWrvF3BxTX013JDUJdzXHZk5mJqFXYKCwv1yy+/qH379i7t+fn5LicOV9e4ceP08ccfa9OmTWrWrJmzPSoqStJvqzfnX+WVn5/vXO2JiopScXGxjh8/7hK+8vPz1a1btwq3Z7PZZLPZyrVbrVaPPDGeGheumGfPa6hzXFzs7Qqqz+Gwqri44c2xJDWkl0ZDfS03JHUxx9Udv1ZXYw0ePFgjR47Ue++9p8OHD+vw4cN67733lJqa6jyRuDoMw9DDDz+sDz74QOvXr1d8fLzL/fHx8YqKinJZCisuLtbGjRudQaZTp06yWq0ufXJzc7V79+5Kww4AALh81Gpl55VXXtGkSZN07733Ok8O8vX1VWpqqp5//vlqjzN27FgtX75cH330kYKDg53n2ISGhiogIEAWi0VpaWmaPn26WrVqpVatWmn69OkKDAzUsGHDnH1TU1M1ceJENWnSRGFhYZo0aZI6dOjgvDoLAABcvmoVdgIDAzV//nw9//zz2rdvnwzD0FVXXaWgoKAajbNgwQJJUs+ePV3aFy1apBEjRkiSJk+erKKiIo0ZM0bHjx/XjTfeqMzMTAUHBzv7z549W76+vho6dKiKiorUq1cvLV68WD4+PrXZPQAAYCK1CjtlcnNzlZubq+7duysgIKBaJw6fzzCMi/axWCxKT09Xenp6pX38/f01d+5czZ07t9rbBgAAl4danbNz9OhR9erVS61bt1b//v2Vm5srSXrwwQdrdSUWAACAp9Qq7DzyyCOyWq06ePCgy+fS3HXXXeU+FRkAAMCbanUYKzMzU2vXrnW5TFySWrVq5fywPwAAgPqgVis7p0+frvCThn/99dcKP78GAADAW2oVdrp3766lS5c6b1ssFp07d07PP/+8EhMT3VYcAADAparVYaznn39ePXv21I4dO1RcXKzJkydrz549OnbsmP75z3+6u0YAAIBaq9XKTrt27fTtt9/qhhtuUFJSkk6fPq0hQ4boq6++UsuWLd1dIwAAQK3VeGXH4XAoOTlZr776aoXfHA4AAFCf1Hhlx2q1avfu3TX68EAAAABvqdVhrPvvv18LFy50dy0AAABuV6sTlIuLi/XGG28oKytLnTt3LvedWLNmzXJLcQAAAJeqRmHnp59+UosWLbR7925df/31kqTvv//epQ+HtwAAQH1So7DTqlUr5ebmKjs7W9JvXw/xt7/9TZGRkR4pDgAA4FLV6JydC7+lfPXq1Tp9+rRbCwIAAHCnWp2gXObC8AMAAFDf1CjsWCyWcufkcI4OAACoz2p0zo5hGBoxYoTzyz7Pnj2rhx56qNzVWB988IH7KgQAALgENQo7w4cPd7l97733urUYAAAAd6tR2Fm0aJGn6gCAWklM9HYFAOq7SzpBGQAAoL4j7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFPzatjZtGmTBgwYoJiYGFksFn344Ycu948YMUIWi8Xlp0uXLi597Ha7xo0bp/DwcAUFBem2227T4cOH63AvAABAfebrzY2fPn1a11xzjUaOHKnbb7+9wj59+/bVokWLnLf9/Pxc7k9LS9Mnn3yijIwMNWnSRBMnTlRKSop27twpHx8fj9YPwLMSE71dAQAz8GrY6devn/r161dlH5vNpqioqArvKygo0MKFC/XWW2+pd+/ekqRly5YpNjZW69atU58+fdxeMwCgYtUJp9nZnq8DuJBXw051bNiwQREREbryyivVo0cPPfPMM4qIiJAk7dy5Uw6HQ8nJyc7+MTExSkhI0JYtWyoNO3a7XXa73Xm7sLBQkuRwOORwONxWe9lY7hwT5THPnuetOb5gIdfUrFaHy59m5e1fU94vPK8u57i627AYhmF4uJZqsVgsWrlypQYNGuRsW7FihRo1aqS4uDjl5OToySefVElJiXbu3Cmbzably5dr5MiRLsFFkpKTkxUfH69XX321wm2lp6dr6tSp5dqXL1+uwMBAt+4XAADwjDNnzmjYsGEqKChQSEhIpf3q9crOXXfd5fx7QkKCOnfurLi4OP3jH//QkCFDKn2cYRiyWCyV3j9lyhRNmDDBebuwsFCxsbFKTk6ucrJqyuFwKCsrS0lJSbJarW4bF66YZ8/z1hynpNTZprzOanVo1Kgsvf56khwO876OV63y7vZ5v/C8upzjsiMzF1Ovw86FoqOjFRcXpx9++EGSFBUVpeLiYh0/flyNGzd29svPz1e3bt0qHcdms8lms5Vrt1qtHnliPDUuXDHPnlfXc1xcXGebqjccDquKi837Oq4vv6K8X3heXcxxdcdvUJ+zc/ToUR06dEjR0dGSpE6dOslqtSorK8vZJzc3V7t3764y7AAAgMuHV1d2Tp06pR9//NF5OycnR19//bXCwsIUFham9PR03X777YqOjtb+/fv12GOPKTw8XIMHD5YkhYaGKjU1VRMnTlSTJk0UFhamSZMmqUOHDs6rswAAwOXNq2Fnx44dSjzvWsWy82iGDx+uBQsWaNeuXVq6dKlOnDih6OhoJSYmasWKFQoODnY+Zvbs2fL19dXQoUNVVFSkXr16afHixXzGDgAAkOTlsNOzZ09VdTHY2rVrLzqGv7+/5s6dq7lz57qzNAAAYBIN6pwdAACAmiLsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAU/P1dgEAgMtHYuLF+2Rne74OXF5Y2QEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKbm1bCzadMmDRgwQDExMbJYLPrwww9d7jcMQ+np6YqJiVFAQIB69uypPXv2uPSx2+0aN26cwsPDFRQUpNtuu02HDx+uw70AAAD1mVfDzunTp3XNNddo3rx5Fd4/c+ZMzZo1S/PmzdP27dsVFRWlpKQknTx50tknLS1NK1euVEZGhjZv3qxTp04pJSVFpaWldbUbAACgHvP15sb79eunfv36VXifYRiaM2eOHn/8cQ0ZMkSStGTJEkVGRmr58uUaPXq0CgoKtHDhQr311lvq3bu3JGnZsmWKjY3VunXr1KdPnzrbFwA1k5jo7QoAXC68GnaqkpOTo7y8PCUnJzvbbDabevTooS1btmj06NHauXOnHA6HS5+YmBglJCRoy5YtlYYdu90uu93uvF1YWChJcjgccjgcbtuHsrHcOSbKY549zxNz7OfntqFMwWp1uPx5OfPkrzLvF55Xl3Nc3W3U27CTl5cnSYqMjHRpj4yM1IEDB5x9/Pz81Lhx43J9yh5fkRkzZmjq1Knl2jMzMxUYGHippZeTlZXl9jFRHvPsee6c4zFj3DaUqYwaxev40089vw3eLzyvLub4zJkz1epXb8NOGYvF4nLbMIxybRe6WJ8pU6ZowoQJztuFhYWKjY1VcnKyQkJCLq3g8zgcDmVlZSkpKUlWq9Vt48IV8+x5npjjlBS3DGMaVqtDo0Zl6fXXk+RwXN6v41WrPDc27xeeV5dzXHZk5mLqbdiJioqS9NvqTXR0tLM9Pz/fudoTFRWl4uJiHT9+3GV1Jz8/X926dat0bJvNJpvNVq7darV65Inx1LhwxTx7njvnuLjYLcOYjsNhVXHx5f06rotfY94vPK8u5ri649fbz9mJj49XVFSUyzJYcXGxNm7c6AwynTp1ktVqdemTm5ur3bt3Vxl2AADA5cOrKzunTp3Sjz/+6Lydk5Ojr7/+WmFhYWrevLnS0tI0ffp0tWrVSq1atdL06dMVGBioYcOGSZJCQ0OVmpqqiRMnqkmTJgoLC9OkSZPUoUMH59VZAADg8ubVsLNjxw4lnnf9adl5NMOHD9fixYs1efJkFRUVacyYMTp+/LhuvPFGZWZmKjg42PmY2bNny9fXV0OHDlVRUZF69eqlxYsXy8fHp873BwAA1D9eDTs9e/aUYRiV3m+xWJSenq709PRK+/j7+2vu3LmaO3euByoEAAANXb09ZwcAAMAdCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUvPqt5wAAXCgx8eJ9srM9XwfMg5UdAABgaoQdAABgaoQdAABgaoQdAABgaoQdAABgalyNBcDtqnM1DQDUFVZ2AACAqbGyAwBocPgsHtQEKzsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDU6nXYSU9Pl8VicfmJiopy3m8YhtLT0xUTE6OAgAD17NlTe/bs8WLFAACgvqnXYUeS2rdvr9zcXOfPrl27nPfNnDlTs2bN0rx587R9+3ZFRUUpKSlJJ0+e9GLFAACgPqn3YcfX11dRUVHOn6ZNm0r6bVVnzpw5evzxxzVkyBAlJCRoyZIlOnPmjJYvX+7lqgEAQH1R78PODz/8oJiYGMXHx+vuu+/WTz/9JEnKyclRXl6ekpOTnX1tNpt69OihLVu2eKtcAABQz/h6u4Cq3HjjjVq6dKlat26tX375RdOmTVO3bt20Z88e5eXlSZIiIyNdHhMZGakDBw5UOa7dbpfdbnfeLiwslCQ5HA45HA631V82ljvHRHnMs+fVdI79/DxZjTlZrQ6XP3HpKnq58n7heXU5x9XdhsUwDMPDtbjN6dOn1bJlS02ePFldunTRH//4Rx05ckTR0dHOPqNGjdKhQ4e0Zs2aSsdJT0/X1KlTy7UvX75cgYGBHqkdAAC415kzZzRs2DAVFBQoJCSk0n71emXnQkFBQerQoYN++OEHDRo0SJKUl5fnEnby8/PLrfZcaMqUKZowYYLzdmFhoWJjY5WcnFzlZNWUw+FQVlaWkpKSZLVa3TYuXDHPnnf+HA8ezBx7gtXq0KhRWXr99SQ5HMyxO6xaVb6N9wvPq8s5LjsyczENKuzY7XZ99913uvnmmxUfH6+oqChlZWXpuuuukyQVFxdr48aNeu6556ocx2azyWazlWu3Wq0eeWI8NS5cMc+eZ7VaVVzMHHuSw8Ecu0tVbwe8X3heXcxxdcev12Fn0qRJGjBggJo3b678/HxNmzZNhYWFGj58uCwWi9LS0jR9+nS1atVKrVq10vTp0xUYGKhhw4Z5u3QAgJclJpZv8/OTxoyRUlKk4mIpO7vu60Ldq9dh5/Dhw7rnnnv066+/qmnTpurSpYu2bdumuLg4SdLkyZNVVFSkMWPG6Pjx47rxxhuVmZmp4OBgL1cOAADqi3oddjIyMqq832KxKD09Xenp6XVTEAAAaHDqddgBULcqWvaXXJf+AaChqfcfKggAAHApCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUfL1dAAAA3pKYePE+2dmerwOexcoOAAAwNVZ2gMtEdf4HCwBmxMoOAAAwNVZ2AACoAuf1NHys7AAAAFMj7AAAAFPjMBZgApx8DHgXh7rqN1Z2AACAqRF2AACAqRF2AACAqXHODuBFnGsDAJ7Hyg4AADA104Sd+fPnKz4+Xv7+/urUqZM+//xzb5cEAADqAVOEnRUrVigtLU2PP/64vvrqK918883q16+fDh486O3SAACAl5ki7MyaNUupqal68MEH1bZtW82ZM0exsbFasGCBt0sDAABe1uBPUC4uLtbOnTv16KOPurQnJydry5YtXqoKAABX7rogoTofTsiHHLpq8GHn119/VWlpqSIjI13aIyMjlZeXV+Fj7Ha77Ha783ZBQYEk6dixY3I4HG6rzeFw6MyZM0pJOaqSEmul/d59122bvCyVzfPRo0dltVY+z+40dOjF+1Tneb2iwayt/jbH0lFdcUXdzPHlhzmuGw1/no8evXif6ry3VGec6rjw/dDX16ERI1z/7fPUv3MnT56UJBmGUWW/Bh92ylgsFpfbhmGUayszY8YMTZ06tVx7fHy8R2q7mPBwr2wWHma25zUz09sVmB9zXDca+jy7673Fk+9RF86xp98PT548qdDQ0Ervb/BhJzw8XD4+PuVWcfLz88ut9pSZMmWKJkyY4Lx97tw5HTt2TE2aNKk0INVGYWGhYmNjdejQIYWEhLhtXLhinj2POfY85rhuMM+eV5dzbBiGTp48qZiYmCr7Nfiw4+fnp06dOikrK0uDBw92tmdlZWngwIEVPsZms8lms7m0XXnllR6rMSQkhF+qOsA8ex5z7HnMcd1gnj2vrua4qhWdMg0+7EjShAkTdN9996lz587q2rWrXnvtNR08eFAPPfSQt0sDAABeZoqwc9ddd+no0aN6+umnlZubq4SEBH366aeKi4vzdmkAAMDLTBF2JGnMmDEaM2aMt8twYbPZ9NRTT5U7ZAb3Yp49jzn2POa4bjDPnlcf59hiXOx6LQAAgAaswXzKBwAAQG0QdgAAgKkRdgAAgKkRdgAAgKkRdurQbbfdpubNm8vf31/R0dG67777dOTIEW+XZRr79+9Xamqq4uPjFRAQoJYtW+qpp55ScXGxt0szlWeeeUbdunVTYGCgRz+M83Izf/58xcfHy9/fX506ddLnn3/u7ZJMZdOmTRowYIBiYmJksVj04Ycfersk05kxY4b+8Ic/KDg4WBERERo0aJD27t3r7bIkEXbqVGJiot59913t3btX77//vvbt26c77rjD22WZxr///W+dO3dOr776qvbs2aPZs2frlVde0WOPPebt0kyluLhYd955p/785z97uxTTWLFihdLS0vT444/rq6++0s0336x+/frp4MGD3i7NNE6fPq1rrrlG8+bN83YpprVx40aNHTtW27ZtU1ZWlkpKSpScnKzTp097uzQuPfemjz/+WIMGDZLdbq+zb+u+3Dz//PNasGCBfvrpJ2+XYjqLFy9WWlqaTpw44e1SGrwbb7xR119/vRYsWOBsa9u2rQYNGqQZM2Z4sTJzslgsWrlypQYNGuTtUkztP//5jyIiIrRx40Z1797dq7WwsuMlx44d09tvv61u3boRdDyooKBAYWFh3i4DqFRxcbF27typ5ORkl/bk5GRt2bLFS1UBl66goECS6sV7MGGnjv31r39VUFCQmjRpooMHD+qjjz7ydkmmtW/fPs2dO5fvSEO99uuvv6q0tFSRkZEu7ZGRkcrLy/NSVcClMQxDEyZM0E033aSEhARvl0PYuVTp6emyWCxV/uzYscPZ/7/+67/01VdfKTMzUz4+Prr//vvFkcSq1XSOJenIkSPq27ev7rzzTj344INeqrzhqM0cw70sFovLbcMwyrUBDcXDDz+sb7/9Vu+88463S5Fkou/G8paHH35Yd999d5V9WrRo4fx7eHi4wsPD1bp1a7Vt21axsbHatm2bunbt6uFKG66azvGRI0eUmJiorl276rXXXvNwdeZQ0zmG+4SHh8vHx6fcKk5+fn651R6gIRg3bpw+/vhjbdq0Sc2aNfN2OZIIO5esLLzURtmKjt1ud2dJplOTOf7555+VmJioTp06adGiRbriChYvq+NSXse4NH5+furUqZOysrI0ePBgZ3tWVpYGDhzoxcqAmjEMQ+PGjdPKlSu1YcMGxcfHe7skJ8JOHfniiy/0xRdf6KabblLjxo31008/6b//+7/VsmVLVnXc5MiRI+rZs6eaN2+uF154Qf/5z3+c90VFRXmxMnM5ePCgjh07poMHD6q0tFRff/21JOmqq65So0aNvFtcAzVhwgTdd9996ty5s3NF8uDBg5xv5kanTp3Sjz/+6Lydk5Ojr7/+WmFhYWrevLkXKzOPsWPHavny5froo48UHBzsXK0MDQ1VQECAd4szUCe+/fZbIzEx0QgLCzNsNpvRokUL46GHHjIOHz7s7dJMY9GiRYakCn/gPsOHD69wjrOzs71dWoP28ssvG3FxcYafn59x/fXXGxs3bvR2SaaSnZ1d4et2+PDh3i7NNCp7/120aJG3SzP4nB0AAGBqnNAAAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADAABMjbADwKssFos+/PBD5+1///vf6tKli/z9/XXttddW2gYA1cXXRQDwiBEjRmjJkiWSJF9fX4WFhaljx4665557NGLECOf3luXm5qpx48bOxz311FMKCgrS3r17nV8/UVEbAFQXKzsAPKZv377Kzc3V/v37tXr1aiUmJmr8+PFKSUlRSUmJpN++t8xmszkfs2/fPt10002Ki4tTkyZNKm2rqeLi4kvfIQANEmEHgMfYbDZFRUXpd7/7na6//no99thj+uijj7R69WotXrxYkuthLIvFop07d+rpp5+WxWJRenp6hW3Sb99wf9ddd6lx48Zq0qSJBg4cqP379zu3PWLECA0aNEgzZsxQTEyMWrduXaPHvfDCC4qOjlaTJk00duxYORwOZx+73a7JkycrNjZWNptNrVq10sKFC533/+tf/1L//v3VqFEjRUZG6r777tOvv/7qkTkGcHGEHQB16pZbbtE111yjDz74oNx9ubm5at++vSZOnKjc3FxNmjSpwrYzZ84oMTFRjRo10qZNm7R582Y1atRIffv2dVnB+eyzz/Tdd98pKytLq1atqvbjsrOztW/fPmVnZ2vJkiVavHixM5xJ0v3336+MjAz97W9/03fffadXXnnFeXgtNzdXPXr00LXXXqsdO3ZozZo1+uWXXzR06FDPTSqAKnHODoA616ZNG3377bfl2qOiouTr66tGjRopKipKktSoUaNybW+++aauuOIKvfHGG7JYLJKkRYsW6corr9SGDRuUnJwsSQoKCtIbb7whPz+/Gj2ucePGmjdvnnx8fNSmTRvdeuut+uyzzzRq1Ch9//33evfdd5WVlaXevXtLkn7/+98792HBggW6/vrrNX36dGfbm2++qdjYWH3//ffOFSYAdYewA6DOGYbhDBu1sXPnTv34448KDg52aT979qz27dvnvN2hQwdn0KnJ49q3by8fHx/n7ejoaO3atUuS9PXXX8vHx0c9evSotLbs7OwKT6Tet28fYQfwAsIOgDr33XffKT4+vtaPP3funDp16qS333673H1NmzZ1/j0oKKhWj7NarS73WSwWnTt3TpIUEBBw0doGDBig5557rtx90dHRVT4WgGcQdgDUqfXr12vXrl165JFHaj3G9ddfrxUrVigiIkIhISEef9z5OnTooHPnzmnjxo3Ow1gXbuP9999XixYt5OvLWyxQH3CCMgCPsdvtysvL088//6wvv/xS06dP18CBA5WSkqL777+/1uP+6U9/Unh4uAYOHKjPP/9cOTk52rhxo8aPH6/Dhw+7/XHna9GihYYPH64HHnhAH374oXJycrRhwwa9++67kqSxY8fq2LFjuueee/TFF1/op59+UmZmph544AGVlpbWep8B1B5hB4DHrFmzRtHR0WrRooX69u2r7Oxs/e1vf9NHH33kck5MTQUGBmrTpk1q3ry5hgwZorZt2+qBBx5QUVFRlSs2tX3chRYsWKA77rhDY8aMUZs2bTRq1CidPn1akhQTE6N//vOfKi0tVZ8+fZSQkKDx48crNDTU+UGKAOqWxTAMw9tFAAAAeAr/zQAAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKZG2AEAAKb2/wC0xKmf1rfBXgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "differences = hf_model.model.layers[0].block_sparse_moe(test_tensor)[0] - tl_model.blocks[0].mlp(test_tensor)\n",
    "\n",
    "# Flatten the differences to create a one-dimensional tensor\n",
    "flattened_differences = differences.flatten().cpu().detach().numpy()\n",
    "\n",
    "# Plot the histogram of the differences\n",
    "plt.hist(flattened_differences, bins=50, alpha=0.75, color='blue')\n",
    "plt.title('Differences Between MLP Outputs')\n",
    "plt.xlabel('Difference')\n",
    "plt.ylabel('Frequency')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ac306e1c-9972-466a-8f4a-f3eb56042f53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.45158088207244873"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].block_sparse_moe(test_tensor)[0][0, 0, 0].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e9481397-6e87-435a-a0cf-ef409630d17c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2893219590187073"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].mlp(test_tensor)[0, 0, 0].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "89f55163-fa2e-44f2-b112-7b052a6e85af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MixtralAttention(\n",
       "  (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "  (k_proj): Linear(in_features=4096, out_features=1024, bias=False)\n",
       "  (v_proj): Linear(in_features=4096, out_features=1024, bias=False)\n",
       "  (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "  (rotary_emb): MixtralRotaryEmbedding()\n",
       ")"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].self_attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "f0186768-d8f0-4d55-a94c-606c4ba3f7ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[ 0.0782, -0.5772,  0.0400,  ..., -0.9961, -0.3231, -1.5561]]],\n",
       "        grad_fn=<AddBackward0>),)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0](test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "432bb274-b499-44c9-98d1-777d03425daa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.0833, -0.5810,  0.0387,  ..., -1.0015, -0.3103, -1.5631]]],\n",
       "       grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0](test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "4a440811-e7f0-4092-b8e7-f7cac80dc84a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[False, False, False,  ..., False, False, False]]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0](test_tensor)[0] == tl_model.blocks[0](test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "8ed65bb3-6990-48e5-9ef2-1becd9dfaffc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.3231111466884613, -0.31034794449806213)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0](test_tensor)[0][0, 0, -2].item(), tl_model.blocks[0](test_tensor)[0, 0, -2].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "763f6c2e-b71f-4724-b2f7-f79a9ab29caf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sum(hf_model.model.layers[0](test_tensor)[0] == tl_model.blocks[0](test_tensor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f41aa0eb-6386-476d-ae1a-b5e7e06893aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHFCAYAAAAUpjivAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABDsUlEQVR4nO3de1yUdf7//+cEw3BGEDklIrmaGVhpW2alsiqSYaW1WramZX5sNfP46ZPbr49Ypmml7lqpW55bDx3MzioaWqatZlaePqWGpwIxNfEIo7x/f/Rl1hFQwBmBi8f9duOm13ve1zXv6zUDPHlfh7EZY4wAAAAs6oqqHgAAAIA3EXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXZQbc2ePVs2m8315e/vr5iYGKWkpGjcuHHKy8srsU5GRoZsNptbW2FhoR577DHFxsbKx8dH119/vSTp8OHDuv/++xUVFSWbzaZ77rnnMuxVzbV7926318Nmsyk0NFTXXXedJk+erLNnz1Zqu5988okyMjI8O9hqxmaz6fHHH6/qYXjU9u3b1adPHzVo0EB+fn6KjIxU586d9emnn17Sdl977TXNnj3bM4O8iJMnTyojI0OrVq26LM+HquNb1QMALmbWrFlq2rSpnE6n8vLytGbNGo0fP14vvfSSFi1apA4dOrj6Pvroo0pLS3Nbf+rUqZo+fbqmTJmili1bKjg4WJL03HPP6b333tPMmTPVqFEjRUREXNb9qqkGDRqknj17SpJ+++03ffDBBxo6dKj27dunl19+ucLb++STT/Tqq69aPvBYyeLFi9WzZ09dddVVeuaZZ3T11VfrwIEDmjVrljp37qz//u//1oQJEyq17ddee02RkZHq06ePZwddipMnT2r06NGSpHbt2nn9+VB1CDuo9pKSknTjjTe6lu+9914NHTpUt912m7p166YdO3YoOjpaklS/fn3Vr1/fbf0tW7YoICCgxF/WW7ZsUaNGjfTggw96bKynTp1SQECAx7ZXHTVo0ECtWrVyLaelpWnLli1asGBBpcIOqp+TJ08qMDCw1Md27dqlXr16KTk5WatWrVJQUJDrsT//+c/661//qhdffFEtWrTQ/ffff7mGDFwQh7FQIzVo0EAvv/yyjh07punTp7vazz+MZbPZ9MYbb+jUqVOuQy/Fh8dWrFih7du3u9qLp7ILCws1ZswYNW3aVA6HQ/Xq1dPDDz+sgwcPuo2hYcOGSk9P1+LFi3XDDTfI39/f9Vdibm6u+vfvr/r168vPz0+JiYkaPXq0zpw541q/+LDQSy+9pIkTJyoxMVHBwcG65ZZb9NVXX5XY53//+9/q0qWL6tatK39/fzVq1EhDhgxx67Njxw717NlTUVFRcjgcuuaaa/Tqq6+69SkqKtKYMWN09dVXKyAgQHXq1FHz5s3197//vVKvhSSFhYXJbreXaF+0aJFuueUWBQUFKTg4WJ06ddKmTZtcj/fp08c1vnMPj+3evVt//vOfde2117ptr0uXLrLZbHr77bddbd98841sNps+/PBDV1t56i9V/LVeunSpWrRooYCAADVt2lQzZ86sdM1Kq1VqaqpiY2MVEBCga665Rk899ZROnDjh6jNv3jzZbDatW7euxPrPPvus7Ha7fvnlF1fbihUr1L59e4WGhiowMFC33nqrVq5c6bZe8ffMN998o/vuu0/h4eFq1KhRmeOcNGmSTp48qSlTprgFnWIvv/yy6tSpo+eff77Ec5yv+Htx9+7dkn6v89atW7V69WrXe6Fhw4aSpFWrVslms+nNN9/UsGHDFBMTo4CAALVt29btPSX9PktT2kxNnz59XNvbvXu36tWrJ0kaPXq06/mKZ5QOHjyo//qv/1J8fLzrvXHrrbdqxYoVZdYG1ZgBqqlZs2YZSWbDhg2lPn78+HHj4+Nj2rdv72obNWqUOfdtvW7dOtO5c2cTEBBg1q1bZ9atW2dyc3PNunXrzA033GCuuuoqV/vRo0fN2bNnTVpamgkKCjKjR482mZmZ5o033jBXXnmladasmTl58qRr2wkJCSY2NtZcddVVZubMmSYrK8usX7/e5OTkmPj4eJOQkGCmT59uVqxYYZ577jnjcDhMnz59XOtnZ2cbSaZhw4YmLS3NLFmyxCxZssQkJyeb8PBw89tvv7n6Ll261NjtdtO8eXMze/Zs89lnn5mZM2ea+++/39Vn69atJiwszCQnJ5u5c+ea5cuXm+HDh5srrrjCZGRkuPqNGzfO+Pj4mFGjRpmVK1eapUuXmsmTJ7v1KU3xeMePH2+cTqdxOp3m119/NTNmzDC+vr7m6aefduv//PPPG5vNZh555BHz0UcfmcWLF5tbbrnFBAUFma1btxpjjNm5c6e57777jCTX67Bu3Tpz+vRpM23aNCPJ/PLLL8YYY5xOpwkJCTEBAQGmX79+rucZP3688fX1Nfn5+cYYU+76V/S1rl+/vmnWrJmZO3euWbZsmfnzn/9sJJnVq1dfsG7GGCPJDBw48IJ9nnvuOTNp0iTz8ccfm1WrVplp06aZxMREk5KS4upTUFBgYmJizIMPPui2rtPpNHFxcebPf/6zq23evHnGZrOZe+65xyxevNh8+OGHJj093fj4+JgVK1a4+hV/zyQkJJj/+Z//MZmZmWbJkiVljrNJkyYmOjr6gvvSvXt3I8nk5OS4Pcf5ir/Hs7OzjTHGfPPNN+aqq64yN9xwg+u98M033xhjjMnKyjKSTHx8vLn77rvNhx9+aN58803zhz/8wYSGhppdu3a5ttu2bVvTtm3bEs/Xu3dvk5CQYIwx5vTp02bp0qVGkunbt6/r+Xbu3GmMMaZTp06mXr165p///KdZtWqVWbJkifnf//1fs3DhwgvuO6onwg6qrYuFHWOMiY6ONtdcc41rubQfqr179zZBQUEl1m3btq259tpr3doWLFhgJJl3333XrX3Dhg1GknnttddcbQkJCcbHx8f88MMPbn379+9vgoODzZ49e9zaX3rpJSPJ9Yu+ODwkJyebM2fOuPqtX7/eSDILFixwtTVq1Mg0atTInDp1qsxadOrUydSvX98cPXrUrf3xxx83/v7+5vDhw8YYY9LT0831119f5nbKUjze0r769Onjtg979+41vr6+ZtCgQW7bOHbsmImJiTHdu3d3tQ0cOLDUX4Q7d+40kszcuXONMcasWbPGSDJPPvmkSUxMdPXr2LGjad26tWu5vPWv6Gvt7+/vts1Tp06ZiIgI079//4vWrjxh51xFRUXG6XSa1atXG0nmu+++cz02atQo4+fnZw4cOOBqW7RokVvwOnHihImIiDBdunRx2+7Zs2fNddddZ2666Sa37Uky//u//1uusfn7+5tWrVpdsM///M//GEnm3//+t9tznO/8sGOMMddee22pQaU47LRo0cIUFRW52nfv3m3sdrt59NFHXW3lCTvGGHPw4EEjyYwaNapE3+DgYDNkyJAL7idqDg5joUYzxnh0ex999JHq1KmjLl266MyZM66v66+/XjExMSWu2mjevLmaNGlSYhspKSmKi4tz28Ydd9whSVq9erVb/zvvvFM+Pj5u25SkPXv2SJJ+/PFH7dq1S3379pW/v3+p4z59+rRWrlyprl27KjAw0O15O3furNOnT7sOjd1000367rvvNGDAAC1btkz5+fkVqtHgwYO1YcMGbdiwQVlZWRo7dqzeeustPfDAA64+y5Yt05kzZ/TQQw+5jcXf319t27Yt19UvjRo1UsOGDV2HDTIzM5WcnKy//OUvys7O1q5du1RQUKA1a9a4naRe3vpX9LW+/vrr1aBBA9eyv7+/mjRp4nqdLtVPP/2knj17KiYmRj4+PrLb7Wrbtq2k3698KvbXv/5VkvT666+72l555RUlJyerTZs2kqS1a9fq8OHD6t27t9u+FRUVKS0tTRs2bHA7PCb9fi6cpxR/X5Z26OpS9ezZ0227CQkJat26tbKysjz6PDfddJNmz56tMWPG6KuvvpLT6fTo9nF5cYIyaqwTJ07o0KFDSk5O9tg2Dxw4oN9++01+fn6lPv7rr7+6LcfGxpa6jQ8//LDUc1hK20bdunXdlh0Oh6TfT3aW5Dp/5PwTr8916NAhnTlzRlOmTNGUKVMu+LwjR45UUFCQ3nzzTU2bNk0+Pj5q06aNxo8f73YieFnq16/v1q9du3ay2WwaOXKkli1bpk6dOunAgQOSpD/+8Y+lbuOKK8r3d1b79u21dOlSSb+ff9KxY0clJycrOjpaK1asUOPGjXXq1Cm3sFPe+lf0tT7/dZJ+f62KX6dLcfz4cd1+++3y9/fXmDFj1KRJEwUGBmrfvn3q1q2b23NER0erR48emj59up566ilt3bpVX3zxhdu5a8X1v++++8p8zsOHD7udc1Pae7k0DRo0UHZ29gX7FJ+DEx8fX65tVkRMTEypbd99951Hn2fRokUaM2aM3njjDT3zzDMKDg5W165dNWHChFLHgOqNsIMa6+OPP9bZs2c9esloZGSk6tat6/oFe76QkBC35dL+co2MjFTz5s3dTtA8V1xcXIXGVHwS5f79+8vsEx4eLh8fH/Xq1UsDBw4stU9iYqIkydfXV8OGDdOwYcP022+/acWKFfrb3/6mTp06ad++fWVehXMhxbNR3333nTp16qTIyEhJ0jvvvKOEhIQKb69Y+/btNWPGDK1fv17//ve/9f/9f/+fJOlPf/qTMjMztWfPHgUHB7tdHVbe+lf0tfamzz77TL/88otWrVrlms2Rfr+0vzSDBw/WvHnz9P7772vp0qWqU6eO21WFxfWfMmWKW23OVXwFY7HyzsJ07NhRr776qr766qtSt33y5EllZmYqKSnJFQqKZyQLCgpcYV4qGSjLIzc3t9S2c8Oov7+/jh49WqJfRZ4vMjJSkydP1uTJk7V371598MEHeuqpp5SXl1fmewbVF2EHNdLevXs1YsQIhYWFqX///h7bbnp6uhYuXKizZ8/q5ptvrvQ2PvnkEzVq1Ejh4eGXPKYmTZqoUaNGmjlzpoYNG+b2y6JYYGCgUlJStGnTJjVv3rzM2Yrz1alTR/fdd59+/vlnDRkyRLt371azZs0qPMZvv/1WkhQVFSVJ6tSpk3x9fbVr166LHh45dybr/Mv227dvL5vNpmeeeUZXXHGF6zBNhw4d9N///d/as2eP2rRp4zaLU976e+K19pTioHH+a3vubM25WrZsqdatW2v8+PHasmWL/uu//sttlubWW29VnTp1tG3bNo/fzHDo0KGaOXOmBg0aVOLSc0kaMWKEjhw5oqlTp7raiq+A+v77791m+869gq7YxWbLFixYoGHDhrlqtmfPHq1du1YPPfSQ2/O9/fbbbuHq0KFDWrt2rUJDQ92eS9JFZ+caNGigxx9/XCtXrtSXX355wb6ongg7qPa2bNniOucgLy9PX3zxhWbNmiUfHx+99957rpkPT7j//vv1r3/9S507d9bgwYN10003yW63a//+/crKytLdd9+trl27XnAbzz77rDIzM9W6dWs98cQTuvrqq3X69Gnt3r1bn3zyiaZNm3bBQ1KlefXVV9WlSxe1atVKQ4cOVYMGDbR3714tW7ZM//rXvyRJf//733Xbbbfp9ttv11//+lc1bNhQx44d086dO/Xhhx/qs88+k/T75dvF9y6qV6+e9uzZo8mTJyshIUGNGze+6Fj27t3rOv/nxIkTWrduncaNG6eEhAR169ZN0u+/bJ599lk9/fTT+umnn5SWlqbw8HAdOHBA69evV1BQkOsy/eLDkOPHj9cdd9whHx8fV2CLiopSUlKSli9frpSUFNesU4cOHXT48GEdPnxYEydOrFT9PfFaV8SuXbv0zjvvlGhv1qyZWrdurfDwcD322GMaNWqU7Ha7/vWvf13w0MzgwYPVo0cP2Ww2DRgwwO2x4OBgTZkyRb1799bhw4d13333KSoqSgcPHtR3332ngwcPuoWRimjUqJHmzZunBx98UH/84x81bNgw100FZ86cqU8//VQjRoxQjx49XOt07txZERER6tu3r5599ln5+vpq9uzZ2rdvX4ntJycna+HChVq0aJGuuuoq+fv7ux2qzsvLU9euXdWvXz8dPXpUo0aNkr+/v0aOHOnq06tXL02fPl1/+ctf1K9fPx06dEgTJkxwCzrS77N3CQkJev/999W+fXtFREQoMjJS4eHhSklJUc+ePdW0aVOFhIRow4YNWrp0qes9jhqmqs+QBspSfKVG8Zefn5+Jiooybdu2NWPHjjV5eXkl1rnUq7GM+f0y3pdeeslcd911xt/f3wQHB5umTZua/v37mx07drj6JSQkmDvvvLPUsR88eNA88cQTJjEx0djtdhMREWFatmxpnn76aXP8+HFjzH+ubnrxxRdLrK9SrhBZt26dueOOO0xYWJhxOBymUaNGZujQoW59srOzzSOPPGKuvPJKY7fbTb169Uzr1q3NmDFjXH1efvll07p1axMZGWn8/PxMgwYNTN++fc3u3btL3Zdzt63zrsLy9/c3TZo0MUOGDHFdZnyuJUuWmJSUFBMaGmocDodJSEgw9913n9ulzwUFBebRRx819erVMzabrcTVOUOHDjWSzPPPP++27caNGxtJ5vvvvy/xvOWpvzGX/lqXddXP+c6v27lfxa/z2rVrzS233GICAwNNvXr1zKOPPmq++eYbI8nMmjWrxDYLCgqMw+EwaWlpZT7v6tWrzZ133mkiIiKM3W43V155pbnzzjvN22+/7epT/D1z8ODBi+7HubZu3Wp69+5t6tev76pxWlqa+fjjj0vtv379etO6dWsTFBRkrrzySjNq1CjzxhtvlHi9d+/ebVJTU01ISIjrknhj/nM11rx588wTTzxh6tWrZxwOh7n99tvN119/XeL55syZY6655hrj7+9vmjVrZhYtWlTiaixjjFmxYoW54YYbjMPhMJJM7969zenTp81jjz1mmjdvbkJDQ01AQIC5+uqrzahRo8yJEycqVCdUDzZjPHw5CwDA6z788EPddddd+vjjj9W5c+eqHo7XrVq1SikpKXr77bcveOI1UBoOYwFADbJt2zbt2bNHw4cP1/XXX++6pB5A2bjPDgDUIAMGDNBdd92l8PBwLViwwCv3sgGshsNYAADA0pjZAQAAlkbYAQAAlkbYAQAAlsbVWJKKior0yy+/KCQkhJP9AACoIYwxOnbsmOLi4i74mXuEHUm//PKLVz6wDgAAeN++ffsueGd6wo7+84F/+/btK3E7cZTO6XRq+fLlSk1NLfPTpVEx1NQ7qKvnUVPvoK4Vl5+fr/j4+It+cC9hR//5EL7Q0FDCTjk5nU4FBgYqNDSUb0oPoabeQV09j5p6B3WtvIudgsIJygAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNJ8q3oAAFAdpKRcvE9WlvfHAcDzmNkBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWVqVhZ9y4cfrjH/+okJAQRUVF6Z577tEPP/zg1qdPnz6y2WxuX61atXLrU1BQoEGDBikyMlJBQUG66667tH///su5KwAAoJqq0rCzevVqDRw4UF999ZUyMzN15swZpaam6sSJE2790tLSlJOT4/r65JNP3B4fMmSI3nvvPS1cuFBr1qzR8ePHlZ6errNnz17O3QEAANVQld5BeenSpW7Ls2bNUlRUlDZu3Kg2bdq42h0Oh2JiYkrdxtGjRzVjxgzNmzdPHTp0kCS9+eabio+P14oVK9SpUyfv7QAAAKj2qtXHRRw9elSSFBER4da+atUqRUVFqU6dOmrbtq2ef/55RUVFSZI2btwop9Op1NRUV/+4uDglJSVp7dq1pYadgoICFRQUuJbz8/MlSU6nU06n0+P7ZUXFdaJenkNNvaO8dfXzK8+2PDGimo/3qndQ14orb61sxhjj5bGUizFGd999t44cOaIvvvjC1b5o0SIFBwcrISFB2dnZeuaZZ3TmzBlt3LhRDodD8+fP18MPP+wWXiQpNTVViYmJmj59eonnysjI0OjRo0u0z58/X4GBgZ7fOQAA4HEnT55Uz549dfToUYWGhpbZr9rM7Dz++OP6/vvvtWbNGrf2Hj16uP6flJSkG2+8UQkJCfr444/VrVu3MrdnjJHNZiv1sZEjR2rYsGGu5fz8fMXHxys1NfWCxcJ/OJ1OZWZmqmPHjrLb7VU9HEugpt5R3rqmp198Wx995MGB1WC8V72DulZc8ZGZi6kWYWfQoEH64IMP9Pnnn6t+/foX7BsbG6uEhATt2LFDkhQTE6PCwkIdOXJE4eHhrn55eXlq3bp1qdtwOBxyOBwl2u12O2+wCqJmnkdNveNidS0sLM82PDggC+C96h3UtfzKW6cqvRrLGKPHH39cixcv1meffabExMSLrnPo0CHt27dPsbGxkqSWLVvKbrcrMzPT1ScnJ0dbtmwpM+wAAIDao0pndgYOHKj58+fr/fffV0hIiHJzcyVJYWFhCggI0PHjx5WRkaF7771XsbGx2r17t/72t78pMjJSXbt2dfXt27evhg8frrp16yoiIkIjRoxQcnKy6+osAABQe1Vp2Jk6daokqV27dm7ts2bNUp8+feTj46PNmzdr7ty5+u233xQbG6uUlBQtWrRIISEhrv6TJk2Sr6+vunfvrlOnTql9+/aaPXu2fHx8LufuAACAaqhKw87FLgQLCAjQsmXLLrodf39/TZkyRVOmTPHU0AAAgEXw2VgAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSfKt6AABQlpSUi/fJyvL+OADUbMzsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAAS+PScwA12sUuT/fzkwYMuDxjAVA9MbMDAAAsjbADAAAsjcNYAGqF9HSpsLCqRwGgKjCzAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI377ACoEhf7mAcA8BRmdgAAgKURdgAAgKURdgAAgKVxzg6ACinPuTZZWd4fBwCUFzM7AADA0gg7AADA0gg7AADA0gg7AADA0gg7AADA0gg7AADA0gg7AADA0rjPDgCPq82fe8V9iIDqh5kdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgadxnBwDKqTbfPwioyZjZAQAAllalYWfcuHH64x//qJCQEEVFRemee+7RDz/84NbHGKOMjAzFxcUpICBA7dq109atW936FBQUaNCgQYqMjFRQUJDuuusu7d+//3LuCgAAqKaqNOysXr1aAwcO1FdffaXMzEydOXNGqampOnHihKvPhAkTNHHiRL3yyivasGGDYmJi1LFjRx07dszVZ8iQIXrvvfe0cOFCrVmzRsePH1d6errOnj1bFbsFAACqkSo9Z2fp0qVuy7NmzVJUVJQ2btyoNm3ayBijyZMn6+mnn1a3bt0kSXPmzFF0dLTmz5+v/v376+jRo5oxY4bmzZunDh06SJLefPNNxcfHa8WKFerUqdNl3y8AAFB9VKsTlI8ePSpJioiIkCRlZ2crNzdXqamprj4Oh0Nt27bV2rVr1b9/f23cuFFOp9OtT1xcnJKSkrR27dpSw05BQYEKCgpcy/n5+ZIkp9Mpp9PplX2zmuI6US/PqSk19fOr6hFUjN3udPu3OqjmL/FF1ZT3ak1DXSuuvLWqNmHHGKNhw4bptttuU1JSkiQpNzdXkhQdHe3WNzo6Wnv27HH18fPzU3h4eIk+xeufb9y4cRo9enSJ9uXLlyswMPCS96U2yczMrOohWE51r+mAAVU9gsrp16/61PWTT6p6BJ5R3d+rNRV1Lb+TJ0+Wq1+1CTuPP/64vv/+e61Zs6bEYzabzW3ZGFOi7XwX6jNy5EgNGzbMtZyfn6/4+HilpqYqNDS0EqOvfZxOpzIzM9WxY0fZ7faqHo4l1JSapqdX9Qgqxm53ql+/TL3+ekc5ndWjrh99VNUjuDQ15b1a01DXiis+MnMx1SLsDBo0SB988IE+//xz1a9f39UeExMj6ffZm9jYWFd7Xl6ea7YnJiZGhYWFOnLkiNvsTl5enlq3bl3q8zkcDjkcjhLtdrudN1gFUTPPq+41LSys6hFUjtNpV2Fh9ahrNX55K6S6v1drKupafuWtU5VejWWM0eOPP67Fixfrs88+U2JiotvjiYmJiomJcZvSKyws1OrVq11BpmXLlrLb7W59cnJytGXLljLDDgAAqD2qdGZn4MCBmj9/vt5//32FhIS4zrEJCwtTQECAbDabhgwZorFjx6px48Zq3Lixxo4dq8DAQPXs2dPVt2/fvho+fLjq1q2riIgIjRgxQsnJya6rswAAQO1VpWFn6tSpkqR27dq5tc+aNUt9+vSRJD355JM6deqUBgwYoCNHjujmm2/W8uXLFRIS4uo/adIk+fr6qnv37jp16pTat2+v2bNny8fH53LtCgAAqKaqNOwYYy7ax2azKSMjQxkZGWX28ff315QpUzRlyhQPjg4AAFgBn40FAAAsjbADAAAsjbADAAAsjbADAAAsjbADAAAsrVrcQRkAapOUlIv3ycry/jiA2oKZHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGlceg7ApTyXRANATcPMDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDQuPQeAaohPRgc8h5kdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaYQdAABgaZUKO9nZ2Z4eBwAAgFdUKuz84Q9/UEpKit58802dPn3a02MCAADwmEqFne+++0433HCDhg8frpiYGPXv31/r16/39NgAAAAuWaXCTlJSkiZOnKiff/5Zs2bNUm5urm677TZde+21mjhxog4ePOjpcQIAAFSK7yWt7Ourrl27qnPnznrttdc0cuRIjRgxQiNHjlSPHj00fvx4xcbGemqsAC5BSkpVjwAAqsYlXY319ddfa8CAAYqNjdXEiRM1YsQI7dq1S5999pl+/vln3X333Z4aJwAAQKVUamZn4sSJmjVrln744Qd17txZc+fOVefOnXXFFb9np8TERE2fPl1Nmzb16GABAAAqqlJhZ+rUqXrkkUf08MMPKyYmptQ+DRo00IwZMy5pcAAAAJeqUmFnx44dF+3j5+en3r17V2bzAAAAHlOpc3ZmzZqlt99+u0T722+/rTlz5lzyoAAAADylUmHnhRdeUGRkZIn2qKgojR07ttzb+fzzz9WlSxfFxcXJZrNpyZIlbo/36dNHNpvN7atVq1ZufQoKCjRo0CBFRkYqKChId911l/bv31+Z3QIAABZUqcNYe/bsUWJiYon2hIQE7d27t9zbOXHihK677jo9/PDDuvfee0vtk5aWplmzZrmW/fz83B4fMmSIPvzwQy1cuFB169bV8OHDlZ6ero0bN8rHx6fcYwGAmqY8txPIyvL+OIDqrlJhJyoqSt9//70aNmzo1v7dd9+pbt265d7OHXfcoTvuuOOCfRwOR5knQR89elQzZszQvHnz1KFDB0nSm2++qfj4eK1YsUKdOnUq91gAAIA1VSrs3H///XriiScUEhKiNm3aSJJWr16twYMH6/777/foAFetWqWoqCjVqVNHbdu21fPPP6+oqChJ0saNG+V0OpWamurqHxcXp6SkJK1du7bMsFNQUKCCggLXcn5+viTJ6XTK6XR6dPxWVVwn6uU53q7peZOitYbd7nT7t7bxxtuJ73/voK4VV95a2YwxpqIbLywsVK9evfT222/L1/f3vFRUVKSHHnpI06ZNK3GoqVwDsdn03nvv6Z577nG1LVq0SMHBwUpISFB2draeeeYZnTlzRhs3bpTD4dD8+fP18MMPuwUXSUpNTXXd66c0GRkZGj16dIn2+fPnKzAwsMJjBwAAl9/JkyfVs2dPHT16VKGhoWX2q9TMjp+fnxYtWqTnnntO3333nQICApScnKyEhIRKD7g0PXr0cP0/KSlJN954oxISEvTxxx+rW7duZa5njJHNZivz8ZEjR2rYsGGu5fz8fMXHxys1NfWCxcJ/OJ1OZWZmqmPHjrLb7VU9HEvwdk3T0z2+yRrBbneqX79Mvf56Rzmdte+9+tFHnt8m3//eQV0rrvjIzMVc0mdjNWnSRE2aNLmUTVRIbGysEhISXPf5iYmJUWFhoY4cOaLw8HBXv7y8PLVu3brM7TgcDjkcjhLtdrudN1gFUTPP81ZNCws9vskaxem0q7Cw9r1Xvfntyfe/d1DX8itvnSoVds6ePavZs2dr5cqVysvLU1FRkdvjn332WWU2e1GHDh3Svn37XB8u2rJlS9ntdmVmZqp79+6SpJycHG3ZskUTJkzwyhgAAEDNUqmwM3jwYM2ePVt33nmnkpKSLnjI6EKOHz+unTt3upazs7P17bffKiIiQhEREcrIyNC9996r2NhY7d69W3/7298UGRmprl27SpLCwsLUt29fDR8+XHXr1lVERIRGjBih5ORk19VZAACgdqtU2Fm4cKHeeustde7c+ZKe/Ouvv1bKOTeKKD6Ppnfv3po6dao2b96suXPn6rffflNsbKxSUlK0aNEihYSEuNaZNGmSfH191b17d506dUrt27fX7NmzuccOAACQdAknKP/hD3+45Cdv166dLnQx2LJlyy66DX9/f02ZMkVTpky55PEAAADrqdTHRQwfPlx///vfLxhUAAAAqoNKzeysWbNGWVlZ+vTTT3XttdeWOBt68eLFHhkcAADApapU2KlTp47rJGEAAIDqrFJh59wP5gQAAKjOKnXOjiSdOXNGK1as0PTp03Xs2DFJ0i+//KLjx497bHAAAACXqlIzO3v27FFaWpr27t2rgoICdezYUSEhIZowYYJOnz6tadOmeXqcAAAAlVKpmZ3Bgwfrxhtv1JEjRxQQEOBq79q1q1auXOmxwQEAAFyqSl+N9eWXX5b4dPOEhAT9/PPPHhkYAACAJ1RqZqeoqEhnz54t0b5//363uxsDAABUtUqFnY4dO2ry5MmuZZvNpuPHj2vUqFGX/BESAAAAnlSpw1iTJk1SSkqKmjVrptOnT6tnz57asWOHIiMjtWDBAk+PEQAAoNIqFXbi4uL07bffasGCBfrmm29UVFSkvn376sEHH3Q7YRkAAKCqVSrsSFJAQIAeeeQRPfLII54cDwAAgEdVKuzMnTv3go8/9NBDlRoMAACAp1Uq7AwePNht2el06uTJk/Lz81NgYCBhBwAAVBuVuhrryJEjbl/Hjx/XDz/8oNtuu40TlAEAQLVS6c/GOl/jxo31wgsvlJj1AQAAqEoeCzuS5OPjo19++cWTmwQAALgklTpn54MPPnBbNsYoJydHr7zyim699VaPDAwAAMATKhV27rnnHrdlm82mevXq6U9/+pNefvllT4wLAADAIyoVdoqKijw9DgAAAK+o9E0FAVQfKSlVPQIAqL4qFXaGDRtW7r4TJ06szFMAAAB4RKXCzqZNm/TNN9/ozJkzuvrqqyVJP/74o3x8fNSiRQtXP5vN5plRAgAAVFKlwk6XLl0UEhKiOXPmKDw8XNLvNxp8+OGHdfvtt2v48OEeHSQAAEBlVeo+Oy+//LLGjRvnCjqSFB4erjFjxnA1FgAAqFYqFXby8/N14MCBEu15eXk6duzYJQ8KAADAUyoVdrp27aqHH35Y77zzjvbv36/9+/frnXfeUd++fdWtWzdPjxEAAKDSKnXOzrRp0zRixAj95S9/kdPp/H1Dvr7q27evXnzxRY8OEAAA4FJUKuwEBgbqtdde04svvqhdu3bJGKM//OEPCgoK8vT4AAAALskl3VQwJydHOTk5atOmjQICAmSM4XJzAKhhynNTyqws748D8JZKnbNz6NAhtW/fXk2aNFHnzp2Vk5MjSXr00Ue57BwAAFQrlQo7Q4cOld1u1969exUYGOhq79Gjh5YuXeqxwQEAAFyqSh3GWr58uZYtW6b69eu7tTdu3Fh79uzxyMAAAAA8oVIzOydOnHCb0Sn266+/yuFwXPKgAAAAPKVSYadNmzaaO3eua9lms6moqEgvvviiUvj4ZQAAUI1U6jDWiy++qHbt2unrr79WYWGhnnzySW3dulWHDx/Wl19+6ekxAgAAVFqlZnaaNWum77//XjfddJM6duyoEydOqFu3btq0aZMaNWrk6TECAABUWoVndpxOp1JTUzV9+nSNHj3aG2MCAADwmArP7Njtdm3ZsoWbBwIAgBqhUufsPPTQQ5oxY4ZeeOEFT48HAOBBXDMCVDLsFBYW6o033lBmZqZuvPHGEp+JNXHiRI8MDgAA4FJVKOz89NNPatiwobZs2aIWLVpIkn788Ue3PhzeAgAA1UmFwk7jxo2Vk5OjrP/3iXA9evTQP/7xD0VHR3tlcAAAAJeqQicoG2Pclj/99FOdOHHCowMCAADwpErdZ6fY+eEHAACguqlQ2LHZbCXOyeEcHQAAUJ1V6JwdY4z69Onj+rDP06dP67HHHitxNdbixYs9N0IAAIBLUKGw07t3b7flv/zlLx4dDAAAgKdVKOzMmjXLW+MAUAZuCgcAl+aSTlAGAACo7gg7AADA0gg7AADA0gg7AADA0gg7AADA0ir1qecAgNrl3KsC/fykAQOk9HSpsPA/7f/vYxOBaoeZHQAAYGlVGnY+//xzdenSRXFxcbLZbFqyZInb48YYZWRkKC4uTgEBAWrXrp22bt3q1qegoECDBg1SZGSkgoKCdNddd2n//v2XcS8AAEB1VqVh58SJE7ruuuv0yiuvlPr4hAkTNHHiRL3yyivasGGDYmJi1LFjRx07dszVZ8iQIXrvvfe0cOFCrVmzRsePH1d6errOnj17uXYDAABUY1V6zs4dd9yhO+64o9THjDGaPHmynn76aXXr1k2SNGfOHEVHR2v+/Pnq37+/jh49qhkzZmjevHnq0KGDJOnNN99UfHy8VqxYoU6dOl22fQEAANVTtT1BOTs7W7m5uUpNTXW1ORwOtW3bVmvXrlX//v21ceNGOZ1Otz5xcXFKSkrS2rVryww7BQUFKigocC3n5+dLkpxOp5xOp5f2yFqK60S9PKesmvr5VcVorMNud7r9i0tXVk35cXBp+LlaceWtVbUNO7m5uZKk6Ohot/bo6Gjt2bPH1cfPz0/h4eEl+hSvX5px48Zp9OjRJdqXL1+uwMDASx16rZKZmVnVQ7Cc82s6YEAVDcRi+vXjvepp59f0k0+qaCAWw8/V8jt58mS5+lXbsFPMZrO5LRtjSrSd72J9Ro4cqWHDhrmW8/PzFR8fr9TUVIWGhl7agGsJp9OpzMxMdezYUXa7vaqHYwll1TQ9vQoHZQF2u1P9+mXq9dc7yunkveoJZdX0o4+qcFAWwM/Viis+MnMx1TbsxMTESPp99iY2NtbVnpeX55rtiYmJUWFhoY4cOeI2u5OXl6fWrVuXuW2HwyGHw1Gi3W638warIGrmeefX9Nz7mKDynE67Cgt5r3rS+TXlR4Fn8HO1/Mpbp2p7n53ExETFxMS4TecVFhZq9erVriDTsmVL2e12tz45OTnasmXLBcMOAACoPap0Zuf48ePauXOnazk7O1vffvutIiIi1KBBAw0ZMkRjx45V48aN1bhxY40dO1aBgYHq2bOnJCksLEx9+/bV8OHDVbduXUVERGjEiBFKTk52XZ0FAABqtyoNO19//bVSzrkHefF5NL1799bs2bP15JNP6tSpUxowYICOHDmim2++WcuXL1dISIhrnUmTJsnX11fdu3fXqVOn1L59e82ePVs+Pj6XfX8AAED1U6Vhp127djLGlPm4zWZTRkaGMjIyyuzj7++vKVOmaMqUKV4YIQAAqOmq7Tk7AAAAnkDYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAluZb1QMAarOUlP/8389PGjBASk+XCgurbkwAYDXM7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEvzreoBAACsISXl4n2ysrw/DuB8zOwAAABLY2YHAHDZMPuDqsDMDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDROUAa8pDwnYgIAvI+ZHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGnVOuxkZGTIZrO5fcXExLgeN8YoIyNDcXFxCggIULt27bR169YqHDEAAKhuqnXYkaRrr71WOTk5rq/Nmze7HpswYYImTpyoV155RRs2bFBMTIw6duyoY8eOVeGIAQBAdVLtw46vr69iYmJcX/Xq1ZP0+6zO5MmT9fTTT6tbt25KSkrSnDlzdPLkSc2fP7+KRw0AAKoL36oewMXs2LFDcXFxcjgcuvnmmzV27FhdddVVys7OVm5urlJTU119HQ6H2rZtq7Vr16p///5VOGpYXUpKVY8AAFBe1Trs3HzzzZo7d66aNGmiAwcOaMyYMWrdurW2bt2q3NxcSVJ0dLTbOtHR0dqzZ88Ft1tQUKCCggLXcn5+viTJ6XTK6XR6eC+sqbhOtbVefn6e36bd7nT7F55BXT3P2zWtpT9Wav3P1coob61sxhjj5bF4zIkTJ9SoUSM9+eSTatWqlW699Vb98ssvio2NdfXp16+f9u3bp6VLl5a5nYyMDI0ePbpE+/z58xUYGOiVsQMAAM86efKkevbsqaNHjyo0NLTMftV6Zud8QUFBSk5O1o4dO3TPPfdIknJzc93CTl5eXonZnvONHDlSw4YNcy3n5+crPj5eqampFywW/sPpdCozM1MdO3aU3W6v6uFcdunpnt+m3e5Uv36Zev31jnI6a19NvYW6ep63a/rRRx7fZI1Q23+uVkbxkZmLqVFhp6CgQNu3b9ftt9+uxMRExcTEKDMzUzfccIMkqbCwUKtXr9b48eMvuB2HwyGHw1Gi3W638waroNpas8JC723b6bSrsLD21dTbqKvneaumtfBHipva+nO1Mspbp2oddkaMGKEuXbqoQYMGysvL05gxY5Sfn6/evXvLZrNpyJAhGjt2rBo3bqzGjRtr7NixCgwMVM+ePat66AAAoJqo1mFn//79euCBB/Trr7+qXr16atWqlb766islJCRIkp588kmdOnVKAwYM0JEjR3TzzTdr+fLlCgkJqeKRAwCA6qJah52FCxde8HGbzaaMjAxlZGRcngEBAIAap9rfVBAAAOBSEHYAAIClVevDWACA2qc8dyjPyvL+OGAdzOwAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABL49Jz4DzluewVAFBzEHZQqxBkAKD24TAWAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNN+qHgAAABWVknLxPllZ3h8HagZmdgAAgKUxswMAsCRmf1CMsIMaoTw/tAAAKA2HsQAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKVx6TkAABfA/XpqPmZ2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApXHpOQCg1irPZeWo+ZjZAQAAlkbYAQAAlkbYAQAAlsY5O6hyHDMHAHgTYQdeRZABAFQ1DmMBAABLI+wAAABL4zAWAACXqDyH7LOyvD8OlI6ZHQAAYGmEHQAAYGkcxsIlSU+XCgurehQAAJSNmR0AAGBpzOygVBc72c7PTxow4PKMBQCsoLw/Vy82Y86JzhVnmZmd1157TYmJifL391fLli31xRdfVPWQAABANWCJmZ1FixZpyJAheu2113Trrbdq+vTpuuOOO7Rt2zY1aNCgqod3WXH5IwAA7iwRdiZOnKi+ffvq0UcflSRNnjxZy5Yt09SpUzVu3LgqHt3FXe6Awkc4AAA89bunJvyRXePDTmFhoTZu3KinnnrKrT01NVVr166tolH9B8ECAOBJ/F6puBofdn799VedPXtW0dHRbu3R0dHKzc0tdZ2CggIVFBS4lo8ePSpJOnz4sJxOp0fHd4WHzoo6dOjyPt/FOXXy5ElJh3TFFfbL9aQWR029g7p6HjX1jupX1/L87inP753y/g6rqGPHjkmSjDEX7Ffjw04xm83mtmyMKdFWbNy4cRo9enSJ9sTERK+MzRMiI6t6BCUtX17VI7Aeauod1NXzqKl3VLe6eup3j7d/hx07dkxhYWFlPl7jw05kZKR8fHxKzOLk5eWVmO0pNnLkSA0bNsy1XFRUpMOHD6tu3bplBiS4y8/PV3x8vPbt26fQ0NCqHo4lUFPvoK6eR029g7pWnDFGx44dU1xc3AX71fiw4+fnp5YtWyozM1Ndu3Z1tWdmZuruu+8udR2HwyGHw+HWVqdOHW8O07JCQ0P5pvQwauod1NXzqKl3UNeKudCMTrEaH3YkadiwYerVq5duvPFG3XLLLfrnP/+pvXv36rHHHqvqoQEAgCpmibDTo0cPHTp0SM8++6xycnKUlJSkTz75RAkJCVU9NAAAUMUsEXYkacCAARrA5xdcNg6HQ6NGjSpxOBCVR029g7p6HjX1DurqPTZzseu1AAAAajDLfDYWAABAaQg7AADA0gg7AADA0gg7AADA0gg7KNWRI0fUq1cvhYWFKSwsTL169dJvv/12wXWMMcrIyFBcXJwCAgLUrl07bd261fX44cOHNWjQIF199dUKDAxUgwYN9MQTT7g+m8zqvFFTSfrnP/+pdu3aKTQ0VDab7aLbrOlee+01JSYmyt/fXy1bttQXX3xxwf6rV69Wy5Yt5e/vr6uuukrTpk0r0efdd99Vs2bN5HA41KxZM7333nveGn615em6bt26Vffee68aNmwom82myZMne3H01ZOna/r666/r9ttvV3h4uMLDw9WhQwetX7/em7tgHQYoRVpamklKSjJr1641a9euNUlJSSY9Pf2C67zwwgsmJCTEvPvuu2bz5s2mR48eJjY21uTn5xtjjNm8ebPp1q2b+eCDD8zOnTvNypUrTePGjc299957OXapynmjpsYYM2nSJDNu3Dgzbtw4I8kcOXLEy3tSdRYuXGjsdrt5/fXXzbZt28zgwYNNUFCQ2bNnT6n9f/rpJxMYGGgGDx5stm3bZl5//XVjt9vNO++84+qzdu1a4+PjY8aOHWu2b99uxo4da3x9fc1XX311uXarynmjruvXrzcjRowwCxYsMDExMWbSpEmXaW+qB2/UtGfPnubVV181mzZtMtu3bzcPP/ywCQsLM/v3779cu1VjEXZQwrZt24wktx/269atM5LM//3f/5W6TlFRkYmJiTEvvPCCq+306dMmLCzMTJs2rczneuutt4yfn59xOp2e24Fq6HLUNCsry/Jh56abbjKPPfaYW1vTpk3NU089VWr/J5980jRt2tStrX///qZVq1au5e7du5u0tDS3Pp06dTL333+/h0Zd/XmjrudKSEiodWHH2zU1xpgzZ86YkJAQM2fOnEsfsMVxGAslrFu3TmFhYbr55ptdba1atVJYWJjWrl1b6jrZ2dnKzc1Vamqqq83hcKht27ZlriNJR48eVWhoqHx9LXN/y1JdzppaVWFhoTZu3OhWD0lKTU0tsx7r1q0r0b9Tp076+uuv5XQ6L9inttTYW3WtzS5XTU+ePCmn06mIiAjPDNzCCDsoITc3V1FRUSXao6KiSny6/LnrSCrxSfPR0dFlrnPo0CE999xz6t+//yWOuPq7XDW1sl9//VVnz56tUD1yc3NL7X/mzBn9+uuvF+xTW2rsrbrWZperpk899ZSuvPJKdejQwTMDtzDCTi2SkZEhm812wa+vv/5akmSz2Uqsb4wptf1c5z9e1jr5+fm688471axZM40aNeoS9qpqVaea1hYVrUdp/c9vp8beqWtt582aTpgwQQsWLNDixYvl7+/vgdFam7WPHcDN448/rvvvv/+CfRo2bKjvv/9eBw4cKPHYwYMHS/zlUSwmJkbS73+dxMbGutrz8vJKrHPs2DGlpaUpODhY7733nux2e0V3pdqoLjWtDSIjI+Xj41PiL+ML1SMmJqbU/r6+vqpbt+4F+9SWGnurrrWZt2v60ksvaezYsVqxYoWaN2/u2cFbFDM7tUhkZKSaNm16wS9/f3/dcsstOnr0qNsljf/+97919OhRtW7dutRtJyYmKiYmRpmZma62wsJCrV692m2d/Px8paamys/PTx988EGN/4ukOtS0tvDz81PLli3d6iFJmZmZZdbjlltuKdF/+fLluvHGG10hu6w+taXG3qprbebNmr744ot67rnntHTpUt14442eH7xVVdGJ0ajm0tLSTPPmzc26devMunXrTHJyconLpK+++mqzePFi1/ILL7xgwsLCzOLFi83mzZvNAw884HaZdH5+vrn55ptNcnKy2blzp8nJyXF9nTlz5rLuX1XwRk2NMSYnJ8ds2rTJvP7660aS+fzzz82mTZvMoUOHLtu+XS7Fl/POmDHDbNu2zQwZMsQEBQWZ3bt3G2OMeeqpp0yvXr1c/Ysv5x06dKjZtm2bmTFjRonLeb/88kvj4+NjXnjhBbN9+3bzwgsv1NpLzz1Z14KCArNp0yazadMmExsba0aMGGE2bdpkduzYcdn3ryp4o6bjx483fn5+5p133nH7+Xns2LHLvn81DWEHpTp06JB58MEHTUhIiAkJCTEPPvhgiUuaJZlZs2a5louKisyoUaNMTEyMcTgcpk2bNmbz5s2ux4svjS7tKzs7+/LsWBXyRk2NMWbUqFGl1vTc7VjJq6++ahISEoyfn59p0aKFWb16teux3r17m7Zt27r1X7VqlbnhhhuMn5+fadiwoZk6dWqJbb799tvm6quvNna73TRt2tS8++673t6NasfTdc3Ozi71fXn+dqzM0zVNSEgotaajRo26DHtTs9mM+X9nQAEAAFgQ5+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAqFI2m01LlixxLf/f//2fWrVqJX9/f11//fVltgFAefFBoAC8ok+fPpozZ44kydfXVxEREWrevLkeeOAB9enTR1dc8fvfWjk5OQoPD3etN2rUKAUFBemHH35QcHBwmW0AUF7M7ADwmrS0NOXk5Gj37t369NNPlZKSosGDBys9PV1nzpyR9PunPTscDtc6u3bt0m233aaEhATXpz2X1lZRhYWFl75DAGokwg4Ar3E4HIqJidGVV16pFi1a6G9/+5vef/99ffrpp5o9e7Yk98NYNptNGzdu1LPPPiubzaaMjIxS2yTp559/Vo8ePRQeHq66devq7rvv1u7du13P3adPH91zzz0aN26c4uLi1KRJkwqt99JLLyk2NlZ169bVwIED5XQ6XX0KCgr05JNPKj4+Xg6HQ40bN9aMGTNcj2/btk2dO3dWcHCwoqOj1atXL/36669eqTGAiyPsALis/vSnP+m6667T4sWLSzyWk5Oja6+9VsOHD1dOTo5GjBhRatvJkyeVkpKi4OBgff7551qzZo2Cg4OVlpbmNoOzcuVKbd++XZmZmfroo4/KvV5WVpZ27dqlrKwszZkzR7Nnz3aFM0l66KGHtHDhQv3jH//Q9u3bNW3aNNfhtZycHLVt21bXX3+9vv76ay1dulQHDhxQ9+7dvVdUABfEOTsALrumTZvq+++/L9EeExMjX19fBQcHKyYmRpIUHBxcom3mzJm64oor9MYbb8hms0mSZs2apTp16mjVqlVKTU2VJAUFBemNN96Qn59fhdYLDw/XK6+8Ih8fHzVt2lR33nmnVq5cqX79+unHH3/UW2+9pczMTHXo0EGSdNVVV7n2YerUqWrRooXGjh3raps5c6bi4+P1448/umaYAFw+hB0Al50xxhU2KmPjxo3auXOnQkJC3NpPnz6tXbt2uZaTk5NdQaci61177bXy8fFxLcfGxmrz5s2SpG+//VY+Pj5q27ZtmWPLysoq9UTqXbt2EXaAKkDYAXDZbd++XYmJiZVev6ioSC1bttS//vWvEo/Vq1fP9f+goKBKrWe3290es9lsKioqkiQFBARcdGxdunTR+PHjSzwWGxt7wXUBeAdhB8Bl9dlnn2nz5s0aOnRopbfRokULLVq0SFFRUQoNDfX6eudKTk5WUVGRVq9e7TqMdf5zvPvuu2rYsKF8ffkRC1QHnKAMwGsKCgqUm5urn3/+Wd98843Gjh2ru+++W+np6XrooYcqvd0HH3xQkZGRuvvuu/XFF18oOztbq1ev1uDBg7V//36Pr3euhg0bqnfv3nrkkUe0ZMkSZWdna9WqVXrrrbckSQMHDtThw4f1wAMPaP369frpp5+0fPlyPfLIIzp79myl9xlA5RF2AHjN0qVLFRsbq4YNGyotLU1ZWVn6xz/+offff9/tnJiKCgwM1Oeff64GDRqoW7duuuaaa/TII4/o1KlTF5yxqex655s6daruu+8+DRgwQE2bNlW/fv104sQJSVJcXJy+/PJLnT17Vp06dVJSUpIGDx6ssLAw140UAVxeNmOMqepBAAAAeAt/ZgAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEv7/wGSbZzYVr236AAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "differences = hf_model.model.layers[0](test_tensor)[0] - tl_model.blocks[0](test_tensor)\n",
    "\n",
    "# Flatten the differences to create a one-dimensional tensor\n",
    "flattened_differences = differences.flatten().cpu().detach().numpy()\n",
    "\n",
    "# Plot the histogram of the differences\n",
    "plt.hist(flattened_differences, bins=50, alpha=0.75, color='blue')\n",
    "plt.title('Differences Between Layer Outputs')\n",
    "plt.xlabel('Difference')\n",
    "plt.ylabel('Frequency')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "5172efa2-0066-4ae0-a6a2-530d815b053b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.0925,  0.2487, -0.0235,  ..., -0.3745, -0.1924, -0.1008]]],\n",
       "       grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].attn.forward(test_tensor, test_tensor, test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "92781a06-e16d-43f9-be4c-3ef04b3d4b08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.0925,  0.2487, -0.0235,  ..., -0.3745, -0.1924, -0.1008]]],\n",
       "       grad_fn=<UnsafeViewBackward0>)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].self_attn.forward(test_tensor)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "943cd506-2bb8-45bf-afc7-7f6b4f8043f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[False, False, False,  ..., False,  True, False]]])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(tl_model.blocks[0].attn.forward(test_tensor, test_tensor, test_tensor) == \n",
    " hf_model.model.layers[0].self_attn.forward(test_tensor)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "57ffc181-abed-4784-86eb-6e6b4f174bc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(242)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sum(tl_model.blocks[0].attn.forward(test_tensor, test_tensor, test_tensor) == \n",
    " hf_model.model.layers[0].self_attn.forward(test_tensor)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "7427fd15-3029-45c3-9d12-64c80f1048f1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHFCAYAAAAUpjivAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABOQ0lEQVR4nO3deVhUZf8/8PcIw7Aji2yKiOSSgkta7gEqKIp7mVmKS6ZpJi7ZV61HLJfUXAqf1ApBU0Rb1PLJZVDUDH1U0lwqt3AHcWUThxHu3x/+OI8j+zDDwPH9ui4uPffcc5/PZ2aAN2fOzCiEEAJEREREMlXL1AUQERERGRPDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOGU1sbCwUCoX0ZWlpCXd3dwQFBWHBggVIT08vcp3IyEgoFAqdsby8PIwbNw4eHh4wMzNDq1atAAB3797FkCFD4OrqCoVCgf79+1dBVzXXpUuXdO4PhUIBe3t7tGzZEsuXL0d+fr5e6/7yyy+IjIw0bLHV1O3bt6FSqaBQKHDs2LFi58yfPx9bt24tMv7nn38iMjISly5dMm6RZdSxb98+KBQK7Nu3r0rqeJoQAnFxcejatSscHR2hUqnQsGFDTJgwAVevXtV73Rs3biAyMhInTpwwXLGlSEpKQmRkJO7fv18l+6NKEkRGEhMTIwCImJgYcejQIXHgwAHx/fffi4iICOHg4CCcnJyEWq3Wuc7Vq1fFoUOHdMaWL18uAIioqCiRlJQkTp48KYQQIiIiQlhYWIj169eLQ4cOibNnz1ZZbzVRSkqKACAmTpwoDh06JA4dOiR27Ngh3nnnHQFATJkyRa91J0yYIJ6VHyVLly4VAAQAMW7cuGLn2NjYiPDw8CLj3333nQAgEhMTjVtkGXVkZGSIQ4cOiYyMjCqp40n5+fnitddeEwDE66+/LrZu3SoSExPF559/LurVqydq164tDh48qNfaR48elX7eVIXFixcLACIlJaVK9keVY26ijEXPED8/P7Rt21baHjRoECZPnozOnTtj4MCBOH/+PNzc3AAA9erVQ7169XSuf/r0aVhZWeHdd98tMu7r64s33njDYLXm5ubCysrKYOtVR/Xr10f79u2l7Z49e+L06dPYuHEjlixZYsLKqr81a9bA1dUV3t7e2LhxI5YuXVrjHi/29vY6939VWrhwITZt2oRPP/0UH3zwgTQeGBiI1157De3atcOgQYPw999/o3bt2iapkWTK1GmL5KvwyM7Ro0eLvXzz5s0CgJgzZ440Nnv2bJ2jBPj/f0U/+VW47tNfhX8xazQa8cknn4gmTZoICwsL4eLiIkaMGCHS09N19u/t7S169+4tfvjhB9GqVSuhUqnEBx98IIQQIjU1Vbz99tuibt26QqlUigYNGojIyEih1Wql6xceKVm8eLFYsmSJaNCggbCxsRHt27cvcnRKCCEOHz4swsLChJOTk1CpVKJhw4Zi0qRJOnPOnTsnXn/9dVGnTh1hYWEhmjZtKlasWKEzJz8/X3zyySeicePGwtLSUjg4OAh/f3+xfPnyUu+PJ+t9WlhYmKhfv36R8fj4eNG+fXthbW0tbGxsREhIiPj999+ly8PDw4u9L1JSUsQrr7wimjVrVmQ/AMTmzZulseTkZAFA/PTTT9JYeW5/ISp+X+/YsUO0bt1aWFpaiiZNmojo6OhSb7MnHT58WAAQU6dOFV999ZUAIL799ludOcXdFgEBASU+Zp88CqFWq0XXrl2FnZ2dsLKyEh07dhQJCQk66xd+f5w+fVoMGTJE2NvbC1dXVzFy5Ehx//79MusQQojExMRijzBt27ZNtG/fXlhZWQlbW1vRvXt3kZSUpNf+i6PRaISjo6N4/vnnRUFBQbFz4uLiBADx2WefSWPe3t7FHqEKCAgo0tPTX7NnzxZCPH6c2tjYiNOnT4uuXbsKa2tr4eLiIiZMmCBycnKkNQu/R4o7OvTkeoW3Q0k/g/bs2SMCAgKEk5OTsLS0FF5eXmLgwIE6+6KqxbBDRlNW2MnOzhZmZmaiW7du0tjTYefQoUOiV69ewsrKSnrqJS0tTRw6dEi0bt1aNGzYUBrPyMgQ+fn5omfPnsLGxkbMmTNHqNVq8c0334i6deuKZs2aiQcPHkhre3t7Cw8PD9GwYUOxZs0akZiYKI4cOSJSU1OFl5eX8Pb2FqtXrxYJCQnik08+ESqVSowYMUK6fuEPxgYNGoiePXuKrVu3iq1btwp/f3/h6Oio88N/586dQqlUihYtWojY2Fixd+9esWbNGjFkyBBpzpkzZ6Tgsm7dOrF7924xdepUUatWLREZGSnNW7BggTAzMxOzZ88We/bsETt37hTLly/XmVOcwnoXLlwotFqt0Gq14vbt2yI6OlqYm5uLWbNm6cyfN2+eUCgUYtSoUWL79u3ixx9/FB06dBA2NjbizJkzQgghLly4IF555RUBQLofDh06JB4+fChWrVolAIgbN24IIYTQarXSL/IxY8ZI+1m4cKEwNzcXmZmZQghR7tu/ovd1vXr1RLNmzcS6devErl27xKuvvioAiP3795d6uxUaM2aMACDOnDkjMjMzhbW1tQgMDNSZc+jQIWFlZSV69eol3RZnzpwR6enpYv78+QKA+Pe//y1dVhjKvv32W6FQKET//v3Fjz/+KH7++WcRFhYmzMzMdAJP4fdHkyZNxL/+9S+hVqvF0qVLhUqlEiNHjiyzDiGKDzsbNmwQAERISIjYunWr2LRpk2jTpo2wsLAQv/76a4X3X5ykpCQBQPqDojhZWVmiVq1aokePHtJYecJORkaG9PPmww8/lHq+evWqEOJx2LGwsBD169cX8+bNE7t37xaRkZHC3NxchIWFSWuWN+xcvXpVTJw4UQAQP/74o87PoJSUFGFpaSmCg4PF1q1bxb59+8SGDRvEsGHDxL1790q9jch4GHbIaMoKO0II4ebmJp5//nlp++mwI8T//ip7WkBAgGjevLnO2MaNGwUA8cMPP+iMFz6f/+WXX0pj3t7ewszMrMi5PmPHjhW2trbi8uXLOuOfffaZ9MtOiP/9YPT39xePHj2S5h05ckQAEBs3bpTGfH19ha+vr8jNzS3xtujRo4eoV69ekXMp3n33XWFpaSnu3r0rhHh8dKRVq1YlrlOSwnqL+xoxYoROD1euXBHm5uZi4sSJOmtkZWUJd3d3MXjwYGmspHN2Lly4IACIdevWCSGEOHjwoAAgpk+fLnx8fKR5wcHBomPHjtJ2eW//it7XlpaWOmvm5uYKJycnMXbs2DJvu5ycHGFvby/at28vjYWHhwuFQiEuXLigM7ei5+zk5OQIJycn0adPH53x/Px80bJlS/HSSy9JY4XfH4sWLdKZO378eGFpaalzxKSkOp4OO/n5+cLT01P4+/uL/Px8aV5WVpZwdXXVuW8qsv+nxcfHCwBi1apVJc4RoujPhPKEHSFKP2en8Ajk559/rjM+b948AUA6T6i8YUeIks/Z+f777wUAceLEiVL7pKrFV2ORSQkhDLre9u3bUbt2bfTp0wePHj2Svlq1agV3d/cir0Bp0aIFGjduXGSNoKAgeHp66qwRGhoKANi/f7/O/N69e8PMzExnTQC4fPkyAODcuXO4ePEiRo8eDUtLy2LrfvjwIfbs2YMBAwbA2tpaZ7+9evXCw4cPcfjwYQDASy+9hD/++APjx4/Hrl27kJmZWaHbaNKkSTh69CiOHj2KxMREzJ8/H5s3b8brr78uzdm1axcePXqE4cOH69RiaWmJgICAcr2Sx9fXFw0aNEBCQgIAQK1Ww9/fH2+++SZSUlJw8eJFaDQaHDx4EN27d5euV97bv6L3datWrVC/fn1p29LSEo0bN5bup9Js3rwZmZmZGDVqlDQ2atQoCCEQExNT5vVLk5SUhLt37yI8PFynj4KCAvTs2RNHjx5FTk6OznX69u2rs92iRQs8fPiw2Fc4luXs2bO4ceMGhg0bhlq1/vcrwdbWFoMGDcLhw4fx4MEDo+3/aUKIIq/INJSnz+8bOnQoACAxMdFg+2jVqhUsLCzw9ttvY+3atfjnn38Mtjbpjycok8nk5OTgzp078Pf3N9iaN2/exP3792FhYVHs5bdv39bZ9vDwKHaNn3/+GUqlslxrODs762yrVCoAj092BoBbt24BQJETr590584dPHr0CFFRUYiKiip1vzNmzICNjQ3Wr1+PVatWwczMDC+//DIWLlyocyJ4SerVq6czLzAwEAqFAjNmzMCuXbvQo0cP3Lx5EwDw4osvFrvGk78US9OtWzfs3LkTAJCQkIDg4GD4+/vDzc0NCQkJaNSoEXJzc3XCTnlv/4re10/fT8Dj+6rwfipNdHQ0LC0t0bNnT+mlxi1atECDBg0QGxuLOXPm6ATeiii8rV955ZUS59y9exc2NjbSdlmPuYq4c+cOgOK/Fzw9PVFQUIB79+7B2tq6UvsvDJopKSklzsnJycHt27fRunXr8jdQTubm5kXqdnd3B/C/28AQfH19kZCQgEWLFmHChAnIyclBw4YN8d5772HSpEkG2w9VDMMOmcx//vMf5OfnIzAw0GBruri4wNnZWfoF+zQ7Ozud7eL+gnRxcUGLFi0wb968Ytfw9PSsUE116tQBAFy7dq3EOY6OjjAzM8OwYcMwYcKEYuf4+PgAePxDe8qUKZgyZQru37+PhIQEzJw5Ez169MDVq1d1fimVV+HRqD/++AM9evSAi4sLAOD777+Ht7d3hdcr1K1bN0RHR+PIkSP473//iw8//BAA0LVrV6jValy+fBm2trY6rw4q7+1f0ftaX+fOncPBgwcBQOfI0JN27dqFXr166bV+4W0dFRVV4qukCl+taAyFASA1NbXIZTdu3ECtWrXg6OhY6f20adMGjo6O+Omnn7BgwYJiv/d++uknFBQUIDg4WBqztLSERqMpMvf27dvSbVcejx49wp07d3QCT1paGoD/3QaFR16f3l9Fw1CXLl3QpUsX5Ofn49ixY4iKikJERATc3NwwZMiQCq1FhsGwQyZx5coVTJs2DQ4ODhg7dqzB1g0LC0N8fDzy8/PRrl07vdf45Zdf4Ovra5Af8o0bN4avry/WrFmDKVOmSH8FP8na2hpBQUE4fvw4WrRoUeLRiqfVrl0br7zyCq5fv46IiAhcunQJzZo1q3CNhW/E5urqCgDo0aMHzM3NcfHiRQwaNKjU6z75V/3TL8Pu1q0bFAoFPvroI9SqVQsvv/wyAKB79+54//33cfnyZbz88ss6R3HKe/sb4r4uj+joaADA119/jeeee07nstzcXPTr1w9r1qyRwk5JR4tKOvrRqVMn1K5dG3/++WeRt1eojPIetWrSpAnq1q2LuLg4TJs2TQohOTk5+OGHH9ChQwe9AvTTLCws8P7772PmzJlYvHgxpk+frnN5eno6ZsyYATc3N7z11lvSeIMGDXDy5EmduefOncPZs2d1wk55ji5t2LAB7733nrQdFxcHANIfXG5ubrC0tCyyv23bthVZqzz7MzMzQ7t27dC0aVNs2LABv//+O8OOiTDskNGdPn1aOg8hPT0dv/76K2JiYmBmZoYtW7ZIRz4MYciQIdiwYQN69eqFSZMm4aWXXoJSqcS1a9eQmJiIfv36YcCAAaWu8fHHH0OtVqNjx45477330KRJEzx8+BCXLl3CL7/8glWrVpX6lFRx/v3vf6NPnz5o3749Jk+ejPr16+PKlSvYtWsXNmzYAAD4/PPP0blzZ3Tp0gXvvPMOGjRogKysLFy4cAE///wz9u7dCwDo06eP9N5FderUweXLl7F8+XJ4e3ujUaNGZdZy5coV6fyfnJwcHDp0CAsWLIC3tzcGDhwI4PEvmI8//hizZs3CP//8g549e8LR0RE3b97EkSNHYGNjgzlz5gCA9DTkwoULERoaCjMzMymwubq6ws/PD7t370ZQUJD0S7N79+64e/cu7t69i6VLl+p1+xvivi7Lo0ePsG7dOjz//PM6v4Cf1KdPH/z000+4desW6tSpA39/f+zbtw8///wzPDw8YGdnhyZNmsDPzw8A8NVXX8HOzg6Wlpbw8fGBs7MzoqKiEB4ejrt37+KVV16Bq6srbt26hT/++AO3bt3CypUrK1x7SXU8rVatWli0aBHeeOMNhIWFYezYsdBoNFi8eDHu37+PTz/9tML7LskHH3yAP/74Q/r3tddeg4ODA06ePInFixcjKysL27dvh4ODg3SdYcOG4c0338T48eMxaNAgXL58GYsWLSryc8PX1xdWVlbYsGEDnn/+edja2sLT01M6EmhhYYElS5YgOzsbL774IpKSkjB37lyEhoaic+fOAB4f6X3zzTexZs0a+Pr6omXLljhy5IgUip6+fYHH37fh4eFQKpVo0qQJNmzYgL1796J3796oX78+Hj58iDVr1gCAztO1VMVMfII0ydjT7y1iYWEhXF1dRUBAgJg/f36R90IRovKvxhLi8UucP/vsM9GyZUthaWkpbG1tRdOmTcXYsWPF+fPnpXmF771SnFu3bon33ntP+Pj4CKVSKZycnESbNm3ErFmzRHZ2thCi9PetwVOv3BDi8cuBQ0NDhYODg1CpVMLX11dMnjxZZ05KSooYNWqU9P4yderUER07dhRz586V5ixZskR07NhRuLi4SC+nHT16tLh06VKxvTy5Np56FZalpaVo3LixiIiIEKmpqUWus3XrVhEUFCTs7e2FSqUS3t7e4pVXXtF5ObRGoxFvvfWWqFOnjlAoFEVeoTJ58mQBQMybN09n7UaNGgkA0jtiP6k8t78Qlb+vn35FT3H9Ayj1PYx27twpAIglS5YIIYQ4ceKE6NSpk7C2ttZ5fxshHr8buI+PjzAzMyvyqp/9+/eL3r17CycnJ6FUKkXdunVF7969xXfffSfNKfz+uHXrlk4Nhd9rT97uJdVR0vvsbN26VbRr105YWloKGxsb0a1bN/Hbb7/pzKnI/ktSUFAgNmzYIAIDA0Xt2rWFhYWF8PHxEe+8806RV+AVzl+0aJFo2LChsLS0FG3bthV79+4t9r7buHGjaNq0qVAqlcW+z87JkydFYGCgsLKyEk5OTuKdd97ReTwJ8fhl7G+99ZZwc3MTNjY2ok+fPuLSpUvFfk/PmDFDeHp6ilq1akm36aFDh8SAAQOEt7e3UKlUwtnZWQQEBOi8jxRVPYUQBn45DBERUTUyYsQIfP/998jOzjZ1KWQifOk5ERERyRrDDhEREckan8YiIiIiWeORHSIiIpI1hh0iIiKSNYYdIiIikjW+qSCAgoIC3LhxA3Z2dkb7ADoiIiIyLCEEsrKy4OnpWepn9jHs4PHnv3h5eZm6DCIiItLD1atXS31ne4Yd/O8DA69evQp7e/ti52i1WuzevRshISElfhqzHLBP+XlWemWf8sI+5ccYvWZmZsLLy6vMD/5l2MH/Pvna3t6+1LBjbW0Ne3t7WT8g2af8PCu9sk95YZ/yY8xeyzoFhScoExERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrJmbugAiIioqKKj4cQsLYPx4ICwM2LWramsiqql4ZIeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGTNpGFn5cqVaNGiBezt7WFvb48OHTpgx44d0uUjRoyAQqHQ+Wrfvr3OGhqNBhMnToSLiwtsbGzQt29fXLt2rapbISIiomrK3JQ7r1evHj799FM899xzAIC1a9eiX79+OH78OJo3bw4A6NmzJ2JiYqTrWFhY6KwRERGBn3/+GfHx8XB2dsbUqVMRFhaG5ORkmJmZVV0zRETVUFBQ2XMSE41fB5EpmTTs9OnTR2d73rx5WLlyJQ4fPiyFHZVKBXd392Kvn5GRgejoaHz77bfo3r07AGD9+vXw8vJCQkICevToYdwGiIiIqNqrNufs5OfnIz4+Hjk5OejQoYM0vm/fPri6uqJx48YYM2YM0tPTpcuSk5Oh1WoREhIijXl6esLPzw9JSUlVWj8RERFVTyY9sgMAp06dQocOHfDw4UPY2tpiy5YtaNasGQAgNDQUr776Kry9vZGSkoKPPvoIXbt2RXJyMlQqFdLS0mBhYQFHR0edNd3c3JCWllbiPjUaDTQajbSdmZkJANBqtdBqtcVep3C8pMvlgn3Kz7PSq9z6fOoZe4lSqZX+LU+rJa3zpOp4k8nt/izJs9InYJxey7uWQgghDLZXPeTl5eHKlSu4f/8+fvjhB3zzzTfYv3+/FHielJqaCm9vb8THx2PgwIGIi4vDyJEjdYILAAQHB8PX1xerVq0qdp+RkZGYM2dOkfG4uDhYW1sbpjEiIiIyqgcPHmDo0KHIyMiAvb19ifNMfmTHwsJCOkG5bdu2OHr0KD7//HOsXr26yFwPDw94e3vj/PnzAAB3d3fk5eXh3r17Okd30tPT0bFjxxL3OWPGDEyZMkXazszMhJeXF0JCQkq8sbRaLdRqNYKDg6FUKvXqtSZgn/LzrPQqtz7DwoofVyq1GDNGja+/DsaWLWX3WdI6T9q+vYLFVQG53Z8leVb6BIzTa+EzM2Uxedh5mhCiyJGaQnfu3MHVq1fh4eEBAGjTpg2USiXUajUGDx4M4PHRn9OnT2PRokUl7kOlUkGlUhUZVyqVZd4B5ZkjB+xTfp6VXuXSZ15e6ZdrteXrs6x1AKA631xyuT/L8qz0CRi21/KuY9KwM3PmTISGhsLLywtZWVmIj4/Hvn37sHPnTmRnZyMyMhKDBg2Ch4cHLl26hJkzZ8LFxQUDBgwAADg4OGD06NGYOnUqnJ2d4eTkhGnTpsHf3196dRYRERE920wadm7evIlhw4YhNTUVDg4OaNGiBXbu3Ing4GDk5ubi1KlTWLduHe7fvw8PDw8EBQVh06ZNsLOzk9ZYtmwZzM3NMXjwYOTm5qJbt26IjY3le+wQERERABOHnejo6BIvs7Kywq5du8pcw9LSElFRUYiKijJkaURERCQT1eZ9doiIiIiMgWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkzdzUBRARyUlQUNlzEhONXwcR/Q+P7BAREZGsMewQERGRrPFpLCKiKlaep7qIyHB4ZIeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjR8XQURUQ/FjJ4jKh0d2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWTBp2Vq5ciRYtWsDe3h729vbo0KEDduzYIV0uhEBkZCQ8PT1hZWWFwMBAnDlzRmcNjUaDiRMnwsXFBTY2Nujbty+uXbtW1a0QERFRNWXSsFOvXj18+umnOHbsGI4dO4auXbuiX79+UqBZtGgRli5dihUrVuDo0aNwd3dHcHAwsrKypDUiIiKwZcsWxMfH4+DBg8jOzkZYWBjy8/NN1RYRERFVIyYNO3369EGvXr3QuHFjNG7cGPPmzYOtrS0OHz4MIQSWL1+OWbNmYeDAgfDz88PatWvx4MEDxMXFAQAyMjIQHR2NJUuWoHv37mjdujXWr1+PU6dOISEhwZStERERUTVRbd5UMD8/H9999x1ycnLQoUMHpKSkIC0tDSEhIdIclUqFgIAAJCUlYezYsUhOToZWq9WZ4+npCT8/PyQlJaFHjx7F7kuj0UCj0UjbmZmZAACtVgutVlvsdQrHS7pcLtin/DwrvVaXPi0sjLu+UqnV+dcQquNDo7rcn8b2rPQJGKfX8q6lEEIIg+1VD6dOnUKHDh3w8OFD2NraIi4uDr169UJSUhI6deqE69evw9PTU5r/9ttv4/Lly9i1axfi4uIwcuRIneACACEhIfDx8cHq1auL3WdkZCTmzJlTZDwuLg7W1taGbZCIiIiM4sGDBxg6dCgyMjJgb29f4jyTH9lp0qQJTpw4gfv37+OHH35AeHg49u/fL12uUCh05gshiow9raw5M2bMwJQpU6TtzMxMeHl5ISQkpMQbS6vVQq1WIzg4GEqlsjyt1UjsU36elV6rS59hYcZdX6nUYswYNb7+OhharWH63L7dIMsYVHW5P43tWekTME6vhc/MlMXkYcfCwgLPPfccAKBt27Y4evQoPv/8c3zwwQcAgLS0NHh4eEjz09PT4ebmBgBwd3dHXl4e7t27B0dHR505HTt2LHGfKpUKKpWqyLhSqSzzDijPHDlgn/LzrPRq6j7z8qpmP1qtEnl5humzOj8sTH1/VpVnpU/AsL2Wd51q9z47QghoNBr4+PjA3d0darVauiwvLw/79++XgkybNm2gVCp15qSmpuL06dOlhh0iIiJ6dpj0yM7MmTMRGhoKLy8vZGVlIT4+Hvv27cPOnTuhUCgQERGB+fPno1GjRmjUqBHmz58Pa2trDB06FADg4OCA0aNHY+rUqXB2doaTkxOmTZsGf39/dO/e3ZStERERUTVh0rBz8+ZNDBs2DKmpqXBwcECLFi2wc+dOBAcHAwCmT5+O3NxcjB8/Hvfu3UO7du2we/du2NnZSWssW7YM5ubmGDx4MHJzc9GtWzfExsbCzMzMVG0RERFRNWLSsBMdHV3q5QqFApGRkYiMjCxxjqWlJaKiohAVFWXg6oiIiEgOqt05O0RERESGxLBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyZm7qAoiIjC0oqOw5iYnGr4OITINHdoiIiEjWeGSHiKicynOEiIiqHx7ZISIiIllj2CEiIiJZY9ghIiIiWeM5O0REVCa+oo1qMpMe2VmwYAFefPFF2NnZwdXVFf3798fZs2d15owYMQIKhULnq3379jpzNBoNJk6cCBcXF9jY2KBv3764du1aVbZCRERE1ZRJw87+/fsxYcIEHD58GGq1Go8ePUJISAhycnJ05vXs2ROpqanS1y+//KJzeUREBLZs2YL4+HgcPHgQ2dnZCAsLQ35+flW2Q0RERNWQSZ/G2rlzp852TEwMXF1dkZycjJdfflkaV6lUcHd3L3aNjIwMREdH49tvv0X37t0BAOvXr4eXlxcSEhLQo0cP4zVARERE1V61OmcnIyMDAODk5KQzvm/fPri6uqJ27doICAjAvHnz4OrqCgBITk6GVqtFSEiINN/T0xN+fn5ISkpi2CGicuF76BDJV7UJO0IITJkyBZ07d4afn580HhoaildffRXe3t5ISUnBRx99hK5duyI5ORkqlQppaWmwsLCAo6Ojznpubm5IS0srdl8ajQYajUbazszMBABotVpotdpir1M4XtLlcsE+5edZ6bW0Pi0sqroa41EqtTr/GkJ5HhrluQ0N+RDj41Z+jNFreddSCCGEwfZaCRMmTMB//vMfHDx4EPXq1StxXmpqKry9vREfH4+BAwciLi4OI0eO1AkvABAcHAxfX1+sWrWqyBqRkZGYM2dOkfG4uDhYW1tXvhkiIiIyugcPHmDo0KHIyMiAvb19ifOqxZGdiRMn4qeffsKBAwdKDToA4OHhAW9vb5w/fx4A4O7ujry8PNy7d0/n6E56ejo6duxY7BozZszAlClTpO3MzEx4eXkhJCSkxBtLq9VCrVYjODgYSqWyoi3WGOxTfp6VXkvrMyzMREUZgVKpxZgxanz9dTC0WsPcn9u3lz2nPLdhedYpLz5u5ccYvRY+M1MWk4YdIQQmTpyILVu2YN++ffDx8SnzOnfu3MHVq1fh4eEBAGjTpg2USiXUajUGDx4M4PHRn9OnT2PRokXFrqFSqaBSqYqMK5XKMu+A8syRA/YpP89Kr8X1mZdnomKMSKtVIi/PMPdneR4W5bkNjfHwepYft3JlyF7Lu45Jw86ECRMQFxeHbdu2wc7OTjrHxsHBAVZWVsjOzkZkZCQGDRoEDw8PXLp0CTNnzoSLiwsGDBggzR09ejSmTp0KZ2dnODk5Ydq0afD395denUVERETPLpOGnZUrVwIAAgMDdcZjYmIwYsQImJmZ4dSpU1i3bh3u378PDw8PBAUFYdOmTbCzs5PmL1u2DObm5hg8eDByc3PRrVs3xMbGwszMrCrbISIiomrI5E9jlcbKygq7du0qcx1LS0tERUUhKirKUKURERGRTPCDQImIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1vQKOykpKYaug4iIiMgo9Ao7zz33HIKCgrB+/Xo8fPjQ0DURERERGYxeYeePP/5A69atMXXqVLi7u2Ps2LE4cuSIoWsjIiIiqjS9wo6fnx+WLl2K69evIyYmBmlpaejcuTOaN2+OpUuX4tatW4auk4iIiEgvlTpB2dzcHAMGDMDmzZuxcOFCXLx4EdOmTUO9evUwfPhwpKamGqpOIiIiIr1UKuwcO3YM48ePh4eHB5YuXYpp06bh4sWL2Lt3L65fv45+/foZqk4iIiIivZjrc6WlS5ciJiYGZ8+eRa9evbBu3Tr06tULtWo9zk4+Pj5YvXo1mjZtatBiiYjI8IKCTF0BkXHpFXZWrlyJUaNGYeTIkXB3dy92Tv369REdHV2p4oiIiIgqS6+wc/78+TLnWFhYIDw8XJ/liYiIiAxGr3N2YmJi8N133xUZ/+6777B27dpKF0VERERkKHqFnU8//RQuLi5Fxl1dXTF//vxKF0VERERkKHqFncuXL8PHx6fIuLe3N65cuVLpooiIiIgMRa+w4+rqipMnTxYZ/+OPP+Ds7FzpooiIiIgMRa+wM2TIELz33ntITExEfn4+8vPzsXfvXkyaNAlDhgwxdI1EREREetPr1Vhz587F5cuX0a1bN5ibP16ioKAAw4cP5zk7REREVK3oFXYsLCywadMmfPLJJ/jjjz9gZWUFf39/eHt7G7o+IiIiokrRK+wUaty4MRo3bmyoWoiIiIgMTq+wk5+fj9jYWOzZswfp6ekoKCjQuXzv3r0GKY6IiIiosvQKO5MmTUJsbCx69+4NPz8/KBQKQ9dFREREZBB6hZ34+Hhs3rwZvXr1MnQ9RERERAal10vPLSws8Nxzzxm6FiIiIiKD0yvsTJ06FZ9//jmEEIauh4iIiMig9Hoa6+DBg0hMTMSOHTvQvHlzKJVKnct//PFHgxRHREREVFl6hZ3atWtjwIABhq6FiIiIyOD0CjsxMTGGroOIiIjIKPQ6ZwcAHj16hISEBKxevRpZWVkAgBs3biA7O9tgxRERERFVll5Hdi5fvoyePXviypUr0Gg0CA4Ohp2dHRYtWoSHDx9i1apVhq6TiIiISC96HdmZNGkS2rZti3v37sHKykoaHzBgAPbs2WOw4oiIiIgqS6+wc/DgQXz44YewsLDQGff29sb169fLvc6CBQvw4osvws7ODq6urujfvz/Onj2rM0cIgcjISHh6esLKygqBgYE4c+aMzhyNRoOJEyfCxcUFNjY26Nu3L65du6ZPa0RERCQzeoWdgoIC5OfnFxm/du0a7Ozsyr3O/v37MWHCBBw+fBhqtRqPHj1CSEgIcnJypDmLFi3C0qVLsWLFChw9ehTu7u4IDg6WzhMCgIiICGzZsgXx8fE4ePAgsrOzERYWVmyNRERE9GzRK+wEBwdj+fLl0rZCoUB2djZmz55doY+Q2LlzJ0aMGIHmzZujZcuWiImJwZUrV5CcnAzg8VGd5cuXY9asWRg4cCD8/Pywdu1aPHjwAHFxcQCAjIwMREdHY8mSJejevTtat26N9evX49SpU0hISNCnPSIiIpIRvU5QXrZsGYKCgtCsWTM8fPgQQ4cOxfnz5+Hi4oKNGzfqXUxGRgYAwMnJCQCQkpKCtLQ0hISESHNUKhUCAgKQlJSEsWPHIjk5GVqtVmeOp6cn/Pz8kJSUhB49ehTZj0ajgUajkbYzMzMBAFqtFlqtttjaCsdLulwu2Kf8PCu9ltbnU8+412hKpVbn3+rEkA8xPm7lxxi9lncthdDzMx9yc3OxceNG/P777ygoKMALL7yAN954Q+eE5YoQQqBfv364d+8efv31VwBAUlISOnXqhOvXr8PT01Oa+/bbb+Py5cvYtWsX4uLiMHLkSJ3wAgAhISHw8fHB6tWri+wrMjISc+bMKTIeFxcHa2trveonIiKiqvXgwQMMHToUGRkZsLe3L3GeXkd2AMDKygqjRo3CqFGj9F1Cx7vvvouTJ0/i4MGDRS5TKBQ620KIImNPK23OjBkzMGXKFGk7MzMTXl5eCAkJKfHG0mq1UKvVCA4OLvLxGHLCPuXnWem1tD7DwkxUlBEolVqMGaPG118HQ6utXvfn9u2GW4uPW/kxRq+Fz8yURa+ws27dulIvHz58eIXWmzhxIn766SccOHAA9erVk8bd3d0BAGlpafDw8JDG09PT4ebmJs3Jy8vDvXv34OjoqDOnY8eOxe5PpVJBpVIVGVcqlWXeAeWZIwfsU36elV6L6zMvz0TFGJFWq0ReXvW6P43x8HqWH7dyZchey7uOXmFn0qRJOttarRYPHjyAhYUFrK2tyx12hBCYOHEitmzZgn379sHHx0fnch8fH7i7u0OtVqN169YAgLy8POzfvx8LFy4EALRp0wZKpRJqtRqDBw8GAKSmpuL06dNYtGiRPu0RERGRjOgVdu7du1dk7Pz583jnnXfw/vvvl3udCRMmIC4uDtu2bYOdnR3S0tIAAA4ODrCysoJCoUBERATmz5+PRo0aoVGjRpg/fz6sra0xdOhQae7o0aMxdepUODs7w8nJCdOmTYO/vz+6d++uT3tEREQkI3qfs/O0Ro0a4dNPP8Wbb76Jv//+u1zXWblyJQAgMDBQZzwmJgYjRowAAEyfPh25ubkYP3487t27h3bt2mH37t067+ezbNkymJubY/DgwcjNzUW3bt0QGxsLMzMzg/RGRERlCwoqe05iovHrIHqawcIOAJiZmeHGjRvlnl+eF4IpFApERkYiMjKyxDmWlpaIiopCVFRUufdNREREzwa9ws5PP/2ksy2EQGpqKlasWIFOnToZpDAiIiIiQ9Ar7PTv319nW6FQoE6dOujatSuWLFliiLqIiIiIDEKvsFNQUGDoOoiIiIiMQq/PxiIiIiKqKfQ6svPkuw+XZenSpfrsgoiIiMgg9Ao7x48fx++//45Hjx6hSZMmAIBz587BzMwML7zwgjSvrI90ICIiIjI2vcJOnz59YGdnh7Vr10of0XDv3j2MHDkSXbp0wdSpUw1aJBERyQPfi4dMQa9zdpYsWYIFCxbofBaVo6Mj5s6dy1djERERUbWiV9jJzMzEzZs3i4ynp6cjKyur0kURERERGYpeYWfAgAEYOXIkvv/+e1y7dg3Xrl3D999/j9GjR2PgwIGGrpGIiIhIb3qds7Nq1SpMmzYNb775JrRa7eOFzM0xevRoLF682KAFEhEREVWGXmHH2toaX375JRYvXoyLFy9CCIHnnnsONjY2hq6PiIiIqFIq9aaCqampSE1NRePGjWFjY1OuD/YkIiIiqkp6hZ07d+6gW7duaNy4MXr16oXU1FQAwFtvvcWXnRMREVG1olfYmTx5MpRKJa5cuQJra2tp/LXXXsPOnTsNVhwRERFRZel1zs7u3buxa9cu1KtXT2e8UaNGuHz5skEKIyIiIjIEvY7s5OTk6BzRKXT79m2oVKpKF0VERERkKHqFnZdffhnr1q2TthUKBQoKCrB48WIElee9wImIiIiqiF5PYy1evBiBgYE4duwY8vLyMH36dJw5cwZ3797Fb7/9ZugaiYiIiPSm15GdZs2a4eTJk3jppZcQHByMnJwcDBw4EMePH4evr6+hayQiIiLSW4WP7Gi1WoSEhGD16tWYM2eOMWoiIiIiMpgKH9lRKpU4ffo0FAqFMeohIiIiMii9nsYaPnw4oqOjDV0LERERkcHpdYJyXl4evvnmG6jVarRt27bIZ2ItXbrUIMURERERVVaFws4///yDBg0a4PTp03jhhRcAAOfOndOZw6e3iIiIqDqpUNhp1KgRUlNTkZiYCODxx0N88cUXcHNzM0pxRERERJVVoXN2nv5U8x07diAnJ8egBREREREZkl4nKBd6OvwQERERVTcVCjsKhaLIOTk8R4eIiIiqswqdsyOEwIgRI6QP+3z48CHGjRtX5NVYP/74o+EqJCIiIqqECoWd8PBwne0333zToMUQERERGVqFwk5MTIyx6iAiIiIyikqdoExERERU3THsEBERkawx7BAREZGsmTTsHDhwAH369IGnpycUCgW2bt2qc/mIESOkl7sXfrVv315njkajwcSJE+Hi4gIbGxv07dsX165dq8IuiIiIqDozadjJyclBy5YtsWLFihLn9OzZE6mpqdLXL7/8onN5REQEtmzZgvj4eBw8eBDZ2dkICwtDfn6+scsnIiKiGkCvTz03lNDQUISGhpY6R6VSwd3dvdjLMjIyEB0djW+//Rbdu3cHAKxfvx5eXl5ISEhAjx49DF4zERER1SwmDTvlsW/fPri6uqJ27doICAjAvHnz4OrqCgBITk6GVqtFSEiINN/T0xN+fn5ISkoqMexoNBpoNBppOzMzEwCg1Wqh1WqLvU7heEmXywX7lJ9npdfS+rSwqOpqjEep1Or8K0daLR+3cmSMXsu7lkJUkw+4UigU2LJlC/r37y+Nbdq0Cba2tvD29kZKSgo++ugjPHr0CMnJyVCpVIiLi8PIkSN1ggsAhISEwMfHB6tXry52X5GRkZgzZ06R8bi4OFhbWxu0LyIiIjKOBw8eYOjQocjIyIC9vX2J86r1kZ3XXntN+r+fnx/atm0Lb29v/Oc//8HAgQNLvJ4QotTP7JoxYwamTJkibWdmZsLLywshISEl3lharRZqtRrBwcFQKpV6dFMzsE/5eVZ6La3PsDATFWUESqUWY8ao8fXXwdBq5Xl/bt/Ox60cGaPXwmdmylKtw87TPDw84O3tjfPnzwMA3N3dkZeXh3v37sHR0VGal56ejo4dO5a4jkqlkj7f60lKpbLMO6A8c+SAfcrPs9JrcX3m5ZmoGCPSapXIy5Pn/fnk3fcsP27lypC9lnedGvU+O3fu3MHVq1fh4eEBAGjTpg2USiXUarU0JzU1FadPny417BAREdGzw6RHdrKzs3HhwgVpOyUlBSdOnICTkxOcnJwQGRmJQYMGwcPDA5cuXcLMmTPh4uKCAQMGAAAcHBwwevRoTJ06Fc7OznBycsK0adPg7+8vvTqLiIiInm0mDTvHjh1DUFCQtF14Hk14eDhWrlyJU6dOYd26dbh//z48PDwQFBSETZs2wc7OTrrOsmXLYG5ujsGDByM3NxfdunVDbGwszMzMqrwfIiIiqn5MGnYCAwNR2ovBdu3aVeYalpaWiIqKQlRUlCFLIyIiIpmoUScoExGR/AUFPX5vpPHjH7+SrrgTzBMTq74uqrlq1AnKRERERBXFsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyZm7qAoiIKiMo6PG/FhbA+PFAWBiQl2famoioeuGRHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNZOGnQMHDqBPnz7w9PSEQqHA1q1bdS4XQiAyMhKenp6wsrJCYGAgzpw5ozNHo9Fg4sSJcHFxgY2NDfr27Ytr165VYRdERFTVgoLK/iIqZNKwk5OTg5YtW2LFihXFXr5o0SIsXboUK1aswNGjR+Hu7o7g4GBkZWVJcyIiIrBlyxbEx8fj4MGDyM7ORlhYGPLz86uqDSIiIqrGTPo+O6GhoQgNDS32MiEEli9fjlmzZmHgwIEAgLVr18LNzQ1xcXEYO3YsMjIyEB0djW+//Rbdu3cHAKxfvx5eXl5ISEhAjx49qqwXIiIiqp6q7ZsKpqSkIC0tDSEhIdKYSqVCQEAAkpKSMHbsWCQnJ0Or1erM8fT0hJ+fH5KSkkoMOxqNBhqNRtrOzMwEAGi1Wmi12mKvUzhe0uVywT7lR+69Wlg8/lep1Or8K1fss/xqwkNe7t+fTzJGr+Vdq9qGnbS0NACAm5ubzribmxsuX74szbGwsICjo2OROYXXL86CBQswZ86cIuO7d++GtbV1qXWp1epy1V/TsU/5kWuv48frbo8ZI88+n8Y+y/bLLwYsxMjk+v1ZHEP2+uDBg3LNq7Zhp5BCodDZFkIUGXtaWXNmzJiBKVOmSNuZmZnw8vJCSEgI7O3ti72OVquFWq1GcHAwlEplBTqoWdin/Mi917Cwx/8qlVqMGaPG118HQ6uVX5+F2Gf5bd9u4KKMQO7fn08yRq+Fz8yUpdqGHXd3dwCPj954eHhI4+np6dLRHnd3d+Tl5eHevXs6R3fS09PRsWPHEtdWqVRQqVRFxpVKZZl3QHnmyAH7lJ+a2Ks+r6jRapXIy6tZfeqDfZatJj3ca+L3p74M2Wt516m277Pj4+MDd3d3ncNdeXl52L9/vxRk2rRpA6VSqTMnNTUVp0+fLjXsEBER0bPDpEd2srOzceHCBWk7JSUFJ06cgJOTE+rXr4+IiAjMnz8fjRo1QqNGjTB//nxYW1tj6NChAAAHBweMHj0aU6dOhbOzM5ycnDBt2jT4+/tLr84iIiKiZ5tJw86xY8cQ9MRx6sLzaMLDwxEbG4vp06cjNzcX48ePx71799CuXTvs3r0bdnZ20nWWLVsGc3NzDB48GLm5uejWrRtiY2NhZmZW5f0QERFR9WPSsBMYGAghRImXKxQKREZGIjIyssQ5lpaWiIqKQlRUlBEqJCIiopqu2p6zQ0RERGQIDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQka9U67ERGRkKhUOh8ubu7S5cLIRAZGQlPT09YWVkhMDAQZ86cMWHFREREVN2Ym7qAsjRv3hwJCQnStpmZmfT/RYsWYenSpYiNjUXjxo0xd+5cBAcH4+zZs7CzszNFuUREVE0EBZU9JzHR+HWQ6VXrIzsAYG5uDnd3d+mrTp06AB4f1Vm+fDlmzZqFgQMHws/PD2vXrsWDBw8QFxdn4qqJiIiouqj2R3bOnz8PT09PqFQqtGvXDvPnz0fDhg2RkpKCtLQ0hISESHNVKhUCAgKQlJSEsWPHlrimRqOBRqORtjMzMwEAWq0WWq222OsUjpd0uVywT/mpyb1aWJR/rlKp1flXrtinYZn626Imf39WlDF6Le9aCiGEMNheDWzHjh148OABGjdujJs3b2Lu3Ln4+++/cebMGZw9exadOnXC9evX4enpKV3n7bffxuXLl7Fr164S142MjMScOXOKjMfFxcHa2toovRAREZFhPXjwAEOHDkVGRgbs7e1LnFetw87TcnJy4Ovri+nTp6N9+/bo1KkTbty4AQ8PD2nOmDFjcPXqVezcubPEdYo7suPl5YXbt2+XeGNptVqo1WoEBwdDqVQarqlqhn3KT3XtNSzMsOsplVqMGaPG118HQ6utPn0aGvs0rO3bjbZ0uVTX709jMEavmZmZcHFxKTPsVPunsZ5kY2MDf39/nD9/Hv379wcApKWl6YSd9PR0uLm5lbqOSqWCSqUqMq5UKsu8A8ozRw7Yp/xUt17z8oyzrlarRF5e9enTWNinYVSXb4nq9v1pTIbstbzr1Kiwo9Fo8Ndff6FLly7w8fGBu7s71Go1WrduDQDIy8vD/v37sXDhQhNXSvRsK8+rYIiIqkq1DjvTpk1Dnz59UL9+faSnp2Pu3LnIzMxEeHg4FAoFIiIiMH/+fDRq1AiNGjXC/PnzYW1tjaFDh5q6dCIiIqomqnXYuXbtGl5//XXcvn0bderUQfv27XH48GF4e3sDAKZPn47c3FyMHz8e9+7dQ7t27bB7926+xw4RERFJqnXYiY+PL/VyhUKByMhIREZGVk1BREREVONU+zcVJCIiIqoMhh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikrVq/dJzIiIiYyrPu30nJhq/DjIuHtkhIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWePHRRAREVUSP3aieuORHSIiIpI1Htkhogopz1+wRETVCY/sEBERkazxyA4RSXjUhojkiEd2iIiISNZ4ZIeIiKgUPOJZ8/HIDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGt9UkOgZERYG5OWZugoioqrHIztEREQkaww7REREJGuyeRrryy+/xOLFi5GamormzZtj+fLl6NKli6nLIiIiAlD8Z2xZWADjx1fsaebERMPW9SyQRdjZtGkTIiIi8OWXX6JTp05YvXo1QkND8eeff6J+/fqmLo+oUsrzIYT84UdEVDJZhJ2lS5di9OjReOuttwAAy5cvx65du7By5UosWLDAxNURlcxQn6Zc2jqFfzkSkTzwD6CKq/FhJy8vD8nJyfi///s/nfGQkBAkJSWZqCqqrgp/SJR26Jg/JIiI5KXGh53bt28jPz8fbm5uOuNubm5IS0sr9joajQYajUbazsjIAADcvXsXWq222OtotVo8ePAAd+7cgVKpLHd9gweXPWfz5nIvZ3T69llT1JJOyX/cJ3AHtWrp9nnnjmH2VZ77vlaVvESg5F7lhX3KC/usjG7dyp5jqN895f09Z4zfL1lZWQAAIUSp82p82CmkUCh0toUQRcYKLViwAHPmzCky7uPjY5TayuLiYpLdPvN27y5+XI73R0m9yg37lBf2aVxV+bPO2PvKysqCg4NDiZfX+LDj4uICMzOzIkdx0tPTixztKTRjxgxMmTJF2i4oKMDdu3fh7OxcYkDKzMyEl5cXrl69Cnt7e8M1UM2wT/l5Vnpln/LCPuXHGL0KIZCVlQVPT89S59X4sGNhYYE2bdpArVZjwIAB0rharUa/fv2KvY5KpYJKpdIZq127drn2Z29vL/sHJMA+5ehZ6ZV9ygv7lB9D91raEZ1CNT7sAMCUKVMwbNgwtG3bFh06dMBXX32FK1euYNy4caYujYiIiExMFmHntddew507d/Dxxx8jNTUVfn5++OWXX+Dt7W3q0oiIiMjEZBF2AGD8+PEYb8Q3E1GpVJg9e3aRp7/khn3Kz7PSK/uUF/YpP6bsVSHKer0WERERUQ3GDwIlIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWWPY0UODBg2gUCh0vp7+IFI50Wg0aNWqFRQKBU6cOGHqcoyib9++qF+/PiwtLeHh4YFhw4bhxo0bpi7LoC5duoTRo0fDx8cHVlZW8PX1xezZs5H39CehysC8efPQsWNHWFtbl/sNQ2uCL7/8Ej4+PrC0tESbNm3w66+/mrokgztw4AD69OkDT09PKBQKbN261dQlGcWCBQvw4osvws7ODq6urujfvz/Onj1r6rIMbuXKlWjRooX0RoIdOnTAjh07qrwOhh09Fb6nT+HXhx9+aOqSjGb69OllvhV3TRcUFITNmzfj7Nmz+OGHH3Dx4kW88sorpi7LoP7++28UFBRg9erVOHPmDJYtW4ZVq1Zh5syZpi7N4PLy8vDqq6/inXfeMXUpBrNp0yZERERg1qxZOH78OLp06YLQ0FBcuXLF1KUZVE5ODlq2bIkVK1aYuhSj2r9/PyZMmIDDhw9DrVbj0aNHCAkJQU5OjqlLM6h69erh008/xbFjx3Ds2DF07doV/fr1w5kzZ6q2EEEV5u3tLZYtW2bqMqrEL7/8Ipo2bSrOnDkjAIjjx4+buqQqsW3bNqFQKEReXp6pSzGqRYsWCR8fH1OXYTQxMTHCwcHB1GUYxEsvvSTGjRunM9a0aVPxf//3fyaqyPgAiC1btpi6jCqRnp4uAIj9+/ebuhSjc3R0FN98802V7pNHdvS0cOFCODs7o1WrVpg3b54snwq4efMmxowZg2+//RbW1tamLqfK3L17Fxs2bEDHjh2hVCpNXY5RZWRkwMnJydRlUBny8vKQnJyMkJAQnfGQkBAkJSWZqCoypIyMDACQ9fdjfn4+4uPjkZOTgw4dOlTpvhl29DBp0iTEx8cjMTER7777LpYvX27Ud282BSEERowYgXHjxqFt27amLqdKfPDBB7CxsYGzszOuXLmCbdu2mboko7p48SKioqL4GXI1wO3bt5Gfnw83NzedcTc3N6SlpZmoKjIUIQSmTJmCzp07w8/Pz9TlGNypU6dga2sLlUqFcePGYcuWLWjWrFmV1sCw8/9FRkYWOen46a9jx44BACZPnoyAgAC0aNECb731FlatWoXo6GjcuXPHxF2Urbx9RkVFITMzEzNmzDB1yXqryH0KAO+//z6OHz+O3bt3w8zMDMOHD4eoAW8wXtE+AeDGjRvo2bMnXn31Vbz11lsmqrxi9OlTbhQKhc62EKLIGNU87777Lk6ePImNGzeauhSjaNKkCU6cOIHDhw/jnXfeQXh4OP78888qrYEfF/H/3b59G7dv3y51ToMGDWBpaVlk/Pr166hXrx4OHz6Mdu3aGatEgyhvn0OGDMHPP/+s84M0Pz8fZmZmeOONN7B27Vpjl1pplblPr127Bi8vLyQlJVX54daKqmifN27cQFBQENq1a4fY2FjUqlUz/ubR5/6MjY1FREQE7t+/b+TqjCsvLw/W1tb47rvvMGDAAGl80qRJOHHiBPbv32/C6oxHoVBgy5Yt6N+/v6lLMZqJEydi69atOHDgAHx8fExdTpXo3r07fH19sXr16irbp2w+CLSyXFxc4OLiotd1jx8/DgDw8PAwZElGUd4+v/jiC8ydO1favnHjBnr06IFNmzZV+0BXqDL3aeHfABqNxpAlGUVF+rx+/TqCgoLQpk0bxMTE1JigA1Tu/qzpLCws0KZNG6jVap2wo1ar0a9fPxNWRvoSQmDixInYsmUL9u3b98wEHeBx71X9s5Vhp4IOHTqEw4cPIygoCA4ODjh69CgmT54svU+LXDzdi62tLQDA19cX9erVM0VJRnPkyBEcOXIEnTt3hqOjI/755x/861//gq+vb7U/qlMRN27cQGBgIOrXr4/PPvsMt27dki5zd3c3YWWGd+XKFdy9exdXrlxBfn6+9P5Qzz33nPRYrmmmTJmCYcOGoW3btujQoQO++uorXLlyRXbnXGVnZ+PChQvSdkpKCk6cOAEnJydZ/YydMGEC4uLisG3bNtjZ2UnnXjk4OMDKysrE1RnOzJkzERoaCi8vL2RlZSE+Ph779u3Dzp07q7aQKn3tlwwkJyeLdu3aCQcHB2FpaSmaNGkiZs+eLXJyckxdmlGlpKTI9qXnJ0+eFEFBQcLJyUmoVCrRoEEDMW7cOHHt2jVTl2ZQMTExAkCxX3ITHh5ebJ+JiYmmLq1S/v3vfwtvb29hYWEhXnjhBVm+TDkxMbHY+y48PNzUpRlUSd+LMTExpi7NoEaNGiU9ZuvUqSO6desmdu/eXeV18JwdIiIikrWa84Q9ERERkR4YdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdojIpBQKBbZu3Spt//3332jfvj0sLS3RqlWrEseIyDQOHDiAPn36wNPTs8j3rzE0aNCg2A/+nTBhQrnXYNghIqMYMWKE9ENJqVTCzc0NwcHBWLNmDQoKCqR5qampCA0NlbZnz54NGxsbnD17Fnv27ClxjIhMIycnBy1btsSKFSuqZH9Hjx5Famqq9KVWqwEAr776arnXYNghIqPp2bMnUlNTcenSJezYsQNBQUGYNGkSwsLC8OjRIwCPP5dLpVJJ17l48SI6d+4Mb29vODs7lzhWUXl5eZVviIgQGhqKuXPnYuDAgcVenpeXh+nTp6Nu3bqwsbFBu3btsG/fPr33V6dOHbi7u0tf27dvh6+vLwICAsq9BsMOERmNSqWCu7s76tatixdeeAEzZ87Etm3bsGPHDsTGxgLQfRpLoVAgOTkZH3/8MRQKBSIjI4sdAx5/gvtrr70GR0dHODs7o1+/frh06ZK07xEjRqB///5YsGABPD090bhx4wpd77PPPoOHhwecnZ0xYcIEaLVaaY5Go8H06dPh5eUFlUqFRo0aITo6Wrr8zz//RK9evWBraws3NzcMGzYMt2/fNsptTFTdjBw5Er/99hvi4+Nx8uRJvPrqq+jZsyfOnz9f6bXz8vKwfv16jBo1CgqFotzXY9ghoirVtWtXtGzZEj/++GORy1JTU9G8eXNMnToVqampmDZtWrFjDx48QFBQEGxtbXHgwAEcPHgQtra26Nmzp84RnD179uCvv/6CWq3G9u3by329xMREXLx4EYmJiVi7di1iY2OlcAYAw4cPR3x8PL744gv89ddfWLVqlfRp6qmpqQgICECrVq1w7Ngx7Ny5Ezdv3sTgwYONd6MSVRMXL17Exo0b8d1336FLly7w9fXFtGnT0LlzZ8TExFR6/a1bt+L+/fsYMWJEha5nXuk9ExFVUNOmTXHy5Mki4+7u7jA3N4etrS3c3d0BALa2tkXG1qxZg1q1auGbb76R/rqLiYlB7dq1sW/fPoSEhAAAbGxs8M0338DCwqJC13N0dMSKFStgZmaGpk2bonfv3tizZw/GjBmDc+fOYfPmzVCr1ejevTsAoGHDhlIPK1euxAsvvID58+dLY2vWrIGXlxfOnTsnHWEikqPff/8dQogij3ONRiM9BX3p0iX4+PiUus6ECROKPScoOjoaoaGh8PT0rFBdDDtEVOWEEBU6BP205ORkXLhwAXZ2djrjDx8+xMWLF6Vtf39/KehU5HrNmzeHmZmZtO3h4YFTp04BAE6cOAEzM7MSzxdITk5GYmKidKTnSRcvXmTYIVkrKCiAmZkZkpOTdb6HAEjfE3Xr1sVff/1V6jqOjo5Fxi5fvoyEhIRijwqXhWGHiKrcX3/9VeZfdqUpKChAmzZtsGHDhiKX1alTR/q/jY2NXtdTKpU6lykUCukVZFZWVmXW1qdPHyxcuLDIZR4eHqVel6ima926NfLz85Geno4uXboUO0epVKJp06YVXjsmJgaurq7o3bt3ha/LsENEVWrv3r04deoUJk+erPcaL7zwAjZt2gRXV1fY29sb/XpP8vf3R0FBAfbv3y89jfX0Pn744Qc0aNAA5ub8EUvyk52djQsXLkjbKSkpOHHiBJycnNC4cWO88cYbGD58OJYsWYLWrVvj9u3b2Lt3L/z9/dGrVy+99llQUICYmBiEh4fr9X3FE5SJyGg0Gg3S0tJw/fp1/P7775g/fz769euHsLAwDB8+XO9133jjDbi4uKBfv3749ddfkZKSgv3792PSpEm4du2awa/3pAYNGiA8PByjRo3C1q1bkZKSgn379mHz5s0AHp9rcPfuXbz++us4cuQI/vnnH+zevRujRo1Cfn6+3j0TVRfHjh1D69at0bp1awDAlClT0Lp1a/zrX/8C8PgIzPDhwzF16lQ0adIEffv2xX//+194eXnpvc+EhARcuXIFo0aN0uv6/LODiIxm586d8PDwgLm5ORwdHdGyZUt88cUXCA8PR61a+v+tZW1tjQMHDuCDDz7AwIEDkZWVhbp166Jbt26lHrHR93pPW7lyJWbOnInx48fjzp07qF+/PmbOnAkA8PT0xG+//YYPPvgAPXr0gEajgbe3N3r27Fmpnomqi8DAQAghSrxcqVRizpw5mDNnjsH2GRISUuo+y6IQlbk2ERERUTXHPzOIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjW/h/F2I8B3XCYNQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "differences = tl_model.blocks[0].attn.forward(test_tensor, test_tensor, test_tensor) - hf_model.model.layers[0].self_attn.forward(test_tensor)[0]\n",
    "\n",
    "# Flatten the differences to create a one-dimensional tensor\n",
    "flattened_differences = differences.flatten().cpu().detach().numpy()\n",
    "\n",
    "# Plot the histogram of the differences\n",
    "plt.hist(flattened_differences, bins=50, alpha=0.75, color='blue')\n",
    "plt.title('Differences Between Attention Outputs')\n",
    "plt.xlabel('Difference')\n",
    "plt.ylabel('Frequency')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}

## mixtral-playing.ipynb

      
Display the source blob

    
Display the rendered blob

    
    Raw
  

              mixtral-playing.ipynb
            
          
        Loading

      Sorry, something went wrong. Reload?
      Sorry, we cannot display this file.
      Sorry, this file is invalid so it cannot be displayed.
      
          Viewer requires iframe.