Skip to content

Instantly share code, notes, and snippets.

@Solaxun

Solaxun/makemore_part1.ipynb

Last active August 25, 2023 03:01
Show Gist options
  • Save Solaxun/7f70d0a763f0c8897d9b951c84c25ca5 to your computer and use it in GitHub Desktop.
Save Solaxun/7f70d0a763f0c8897d9b951c84c25ca5 to your computer and use it in GitHub Desktop.
Download ZIP
Makemore Part 1
Display the source blob
Display the rendered blob
Raw
makemore_part1.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 896,
"id": "a1307a48",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[(('n', '.'), 6763), (('a', '.'), 6640), (('a', 'n'), 5438)]\n"
]
}
],
"source": [
"import torch\n",
"import torch.nn.functional as F\n",
"from matplotlib import pyplot as plt\n",
"from collections import defaultdict, Counter\n",
"\n",
"names = ['.' + name + '.' for name in open('names.txt').read().splitlines()]\n",
"# just to see an example of the most common bigrams (not used anywhere)\n",
"bigram_counts = Counter((b,e) for name in names for b,e in zip(name,name[1:]))\n",
"print(bigram_counts.most_common(3))\n",
"\n",
"## use dict[letter:dict[letter:cnt]] instead of 2d matrix\n",
"bigram_map = defaultdict(lambda: defaultdict(int)) #{start_letter: {next_letter:count}}\n",
"for name in names:\n",
" for b,e in zip(name,name[1:]):\n",
" bigram_map[b][e] += 1"
]
},
{
"cell_type": "code",
"execution_count": 897,
"id": "7495d7e0",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"before smoothing, missing g,q,x:\n",
"[('.', 516), ('a', 2590), ('b', 112), ('c', 51), ('d', 24), ('e', 818), ('f', 1), ('h', 5), ('i', 1256), ('j', 7), ('k', 1), ('l', 5), ('m', 168), ('n', 20), ('o', 452), ('p', 38), ('r', 97), ('s', 35), ('t', 4), ('u', 139), ('v', 3), ('w', 2), ('y', 287), ('z', 11)]\n",
"\n",
"after smoothing, g,q,x included:\n",
"[('.', 517), ('a', 2591), ('b', 113), ('c', 52), ('d', 25), ('e', 819), ('f', 2), ('g', 1), ('h', 6), ('i', 1257), ('j', 8), ('k', 2), ('l', 6), ('m', 169), ('n', 21), ('o', 453), ('p', 39), ('q', 1), ('r', 98), ('s', 36), ('t', 5), ('u', 140), ('v', 4), ('w', 3), ('x', 1), ('y', 288), ('z', 12)]\n"
]
}
],
"source": [
"### smoothing ###\n",
"\n",
"# some bigrams won't appear in the data, so the sparse table above will \n",
"# not have them. For example the below start row for char '.' (first row)\n",
"# is missing 'g,q,and x' as ending chars. We 'smooth' this out by adding 1\n",
"# to every bigram. We do this smoothing so that later when we view each\n",
"# character as a one-hot encoded value, the shapes are consistent. Otherwise\n",
"# in this particular example. '.' would be missing 3 characters so instead of \n",
"# 27 possible values (26 a-z plus '.' start char) there would be 24. This\n",
"# mismatch becomes an issue when trying to calcualte each row as a softmax\n",
"# probability between index 0 and 27, because the dimensions don't match.\n",
"# everything needs to be 27 dimensions so we know when comparing '.' to \n",
"# some other character (maybe 'a') that the '3rd' index is 'c' in every case\n",
"# e.g. '.abc' c is index 3\n",
"print('before smoothing, missing g,q,x:')\n",
"print(sorted(bigram_map['m'].items(),key=lambda x:x[0]))\n",
"print()\n",
"# do smoothing by defaulting all counts to 1\n",
"alpha = '.abcdefghijklmnopqrstuvwxyz'\n",
"ones = [a1+a2 for a1 in alpha for a2 in alpha]\n",
"for b,e in ones:\n",
" bigram_map[b][e] += 1\n",
" \n",
"# now smoothed -\n",
"print('after smoothing, g,q,x included:')\n",
"print(sorted(bigram_map['m'].items(),key=lambda x:x[0])) "
]
},
{
"cell_type": "code",
"execution_count": 886,
"id": "11ca03c3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"cexze\n",
"momasurailezitynn\n",
"konimittain\n",
"llayn\n",
"ka\n",
"da\n",
"staiyaubrtthrigotai\n",
"moliellavo\n",
"ke\n",
"teda\n",
"ka\n",
"emimmsade\n",
"enkaviyny\n",
"ftlspihinivenvorhlasu\n",
"dsor\n",
"br\n",
"jol\n",
"pen\n",
"aisan\n",
"ja\n"
]
}
],
"source": [
"g = torch.Generator().manual_seed(2147483647)\n",
"\n",
"for i in range(20):\n",
" res = ['.']\n",
" while True:\n",
" char = res[-1]\n",
" opts = bigram_map[char] # char -> nextchar:count\n",
" # highest_prob = max(opts,key=lambda x:opts[x])\n",
" sorted_opts = sorted(opts.items())\n",
" cnts = [x[1] for x in sorted_opts] # frequency of next char\n",
" probs = torch.tensor([c/sum(cnts) for c in cnts]) # convert freq to probs\n",
" pick_ix = torch.multinomial(probs,num_samples=1,replacement=True,generator=g).item()\n",
" char = sorted_opts[pick_ix][0]\n",
" res.append(char)\n",
" if char == '.':\n",
" break\n",
" print(\"\".join(res)[1:-1])\n"
]
},
{
"cell_type": "code",
"execution_count": 887,
"id": "ac035892",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"tensor(-559951.5625) 228146 tensor(2.4544)\n"
]
}
],
"source": [
"## AK is not looking at the loss of every mapped (single instance) bigram\n",
"## he is generating all the bigrams again, and adding the loss.. so bigrams that\n",
"## appear more than once add to that loss and overall count\n",
"n = 0\n",
"logloss = 0\n",
"for name in names:\n",
" for b,e in zip(name,name[1:]):\n",
" cntmap = bigram_map[b] #{letter:cnt}\n",
" total = sum(cntmap.values())\n",
" prob = cntmap[e] / total\n",
" n += 1\n",
" logloss += torch.log(torch.tensor(prob))\n",
"\n",
"loss = -logloss/n\n",
"print(logloss,n,loss)"
]
},
{
"cell_type": "code",
"execution_count": 888,
"id": "c9254c80",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[('.', 1), ('a', 4411), ('b', 1307), ('c', 1543), ('d', 1691), ('e', 1532), ('f', 418), ('g', 670), ('h', 875), ('i', 592), ('j', 2423), ('k', 2964), ('l', 1573), ('m', 2539), ('n', 1147), ('o', 395), ('p', 516), ('q', 93), ('r', 1640), ('s', 2056), ('t', 1309), ('u', 79), ('v', 377), ('w', 308), ('x', 135), ('y', 536), ('z', 930)]\n",
"tensor([3.1192e-05, 1.3759e-01, 4.0767e-02, 4.8129e-02, 5.2745e-02, 4.7785e-02,\n",
" 1.3038e-02, 2.0898e-02, 2.7293e-02, 1.8465e-02, 7.5577e-02, 9.2452e-02,\n",
" 4.9064e-02, 7.9195e-02, 3.5777e-02, 1.2321e-02, 1.6095e-02, 2.9008e-03,\n",
" 5.1154e-02, 6.4130e-02, 4.0830e-02, 2.4641e-03, 1.1759e-02, 9.6070e-03,\n",
" 4.2109e-03, 1.6719e-02, 2.9008e-02])\n",
"\n",
"c\n"
]
}
],
"source": [
"### why i can't match AK lecture results for sampled names ###\n",
"\n",
"## we get a different index (3, or 'c' than AK - 13, or 'm')\n",
"## random seed not guaranteed to be consistent on diff machines\n",
"g = torch.Generator().manual_seed(2147483647)\n",
"start = '.' \n",
"opts = sorted(bigram_map[start].items())\n",
"print(opts)\n",
"cnts = [x[1] for x in opts]\n",
"probs = torch.tensor([c/sum(cnts) for c in cnts])\n",
"print(probs)\n",
"\n",
"ix = torch.multinomial(probs,num_samples=1,replacement=True,generator=g).item()\n",
"print()\n",
"print(opts[ix][0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "947ea2f1",
"metadata": {},
"outputs": [],
"source": [
"## questions -\n",
"# 1.) is the loss on the actual prob from bigrams the \"floor\" since it's actually what the data shows?\n",
"# 2.) for the nn approach, the goal is to get as close as possible to the prob shown in the bigram matrix?\n",
"# for example ... if 'eb' = 10%, that's the best we can hope the nn to learn, because that's what the data shows.\n",
"\n",
"## for trigram: \n",
"# change inputs to 27 * 2 = 54 dimension each, since input is 2 chars\n",
"# each neuron now has 54 weights\n",
"# the \"second half\" of the neuron weights will be irrelevant, it's just padding to make\n",
"# dimension match for matmul. When we compare to label, it will be 0-27, so we can\n",
"# guide the weights by focusing on that. \n",
"\n",
"# general question.. when to flatten dims vs leave as higher dim? Instead of `ngram x 54` we could\n",
"# do `2 x ngram x 27` but now this gets hard to wrap my head around with weight dims / broadcasting matmul"
]
},
{
"cell_type": "markdown",
"id": "8b18ed32",
"metadata": {},
"source": [
"### Neural Net Version"
]
},
{
"cell_type": "code",
"execution_count": 1009,
"id": "99de38ba",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0.])"
]
},
"execution_count": 1009,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# first collect inputs/labels\n",
"bigrams = [(b,e) for name in names for b,e in zip(name,name[1:])]\n",
"\n",
"## characters\n",
"xs = [b[0] for b in bigrams]\n",
"ys = [b[1] for b in bigrams]\n",
"\n",
"## index in string: 0-27\n",
"ixs_x = torch.tensor([alpha.index(x) for x in xs])\n",
"ixs_y = torch.tensor([alpha.index(y) for y in ys])\n",
"\n",
"## one-hot encode\n",
"x_enc = F.one_hot(ixs_x,num_classes=27).float()\n",
"x_enc[0]"
]
},
{
"cell_type": "code",
"execution_count": 1010,
"id": "da95f5a0",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"tensor(3.8164, grad_fn=<NegBackward0>)\n",
"tensor(3.3178, grad_fn=<NegBackward0>)\n",
"tensor(3.0799, grad_fn=<NegBackward0>)\n",
"tensor(2.9335, grad_fn=<NegBackward0>)\n",
"tensor(2.8402, grad_fn=<NegBackward0>)\n",
"tensor(2.7773, grad_fn=<NegBackward0>)\n",
"tensor(2.7315, grad_fn=<NegBackward0>)\n",
"tensor(2.6965, grad_fn=<NegBackward0>)\n",
"tensor(2.6684, grad_fn=<NegBackward0>)\n",
"tensor(2.6455, grad_fn=<NegBackward0>)\n",
"tensor(2.6265, grad_fn=<NegBackward0>)\n",
"tensor(2.6108, grad_fn=<NegBackward0>)\n",
"tensor(2.5976, grad_fn=<NegBackward0>)\n",
"tensor(2.5865, grad_fn=<NegBackward0>)\n",
"tensor(2.5770, grad_fn=<NegBackward0>)\n",
"tensor(2.5688, grad_fn=<NegBackward0>)\n",
"tensor(2.5616, grad_fn=<NegBackward0>)\n",
"tensor(2.5553, grad_fn=<NegBackward0>)\n",
"tensor(2.5496, grad_fn=<NegBackward0>)\n",
"tensor(2.5445, grad_fn=<NegBackward0>)\n",
"tensor(2.5399, grad_fn=<NegBackward0>)\n",
"tensor(2.5357, grad_fn=<NegBackward0>)\n",
"tensor(2.5318, grad_fn=<NegBackward0>)\n",
"tensor(2.5283, grad_fn=<NegBackward0>)\n",
"tensor(2.5250, grad_fn=<NegBackward0>)\n",
"tensor(2.5220, grad_fn=<NegBackward0>)\n",
"tensor(2.5192, grad_fn=<NegBackward0>)\n",
"tensor(2.5165, grad_fn=<NegBackward0>)\n",
"tensor(2.5141, grad_fn=<NegBackward0>)\n",
"tensor(2.5118, grad_fn=<NegBackward0>)\n",
"tensor(2.5097, grad_fn=<NegBackward0>)\n",
"tensor(2.5077, grad_fn=<NegBackward0>)\n",
"tensor(2.5058, grad_fn=<NegBackward0>)\n",
"tensor(2.5040, grad_fn=<NegBackward0>)\n",
"tensor(2.5023, grad_fn=<NegBackward0>)\n",
"tensor(2.5007, grad_fn=<NegBackward0>)\n",
"tensor(2.4992, grad_fn=<NegBackward0>)\n",
"tensor(2.4978, grad_fn=<NegBackward0>)\n",
"tensor(2.4964, grad_fn=<NegBackward0>)\n",
"tensor(2.4952, grad_fn=<NegBackward0>)\n",
"tensor(2.4940, grad_fn=<NegBackward0>)\n",
"tensor(2.4928, grad_fn=<NegBackward0>)\n",
"tensor(2.4917, grad_fn=<NegBackward0>)\n",
"tensor(2.4907, grad_fn=<NegBackward0>)\n",
"tensor(2.4897, grad_fn=<NegBackward0>)\n",
"tensor(2.4887, grad_fn=<NegBackward0>)\n",
"tensor(2.4878, grad_fn=<NegBackward0>)\n",
"tensor(2.4870, grad_fn=<NegBackward0>)\n",
"tensor(2.4861, grad_fn=<NegBackward0>)\n",
"tensor(2.4853, grad_fn=<NegBackward0>)\n",
"tensor(2.4846, grad_fn=<NegBackward0>)\n",
"tensor(2.4838, grad_fn=<NegBackward0>)\n",
"tensor(2.4832, grad_fn=<NegBackward0>)\n",
"tensor(2.4825, grad_fn=<NegBackward0>)\n",
"tensor(2.4818, grad_fn=<NegBackward0>)\n",
"tensor(2.4812, grad_fn=<NegBackward0>)\n",
"tensor(2.4806, grad_fn=<NegBackward0>)\n",
"tensor(2.4801, grad_fn=<NegBackward0>)\n",
"tensor(2.4795, grad_fn=<NegBackward0>)\n",
"tensor(2.4790, grad_fn=<NegBackward0>)\n",
"tensor(2.4785, grad_fn=<NegBackward0>)\n",
"tensor(2.4780, grad_fn=<NegBackward0>)\n",
"tensor(2.4775, grad_fn=<NegBackward0>)\n",
"tensor(2.4771, grad_fn=<NegBackward0>)\n",
"tensor(2.4766, grad_fn=<NegBackward0>)\n",
"tensor(2.4762, grad_fn=<NegBackward0>)\n",
"tensor(2.4758, grad_fn=<NegBackward0>)\n",
"tensor(2.4754, grad_fn=<NegBackward0>)\n",
"tensor(2.4750, grad_fn=<NegBackward0>)\n",
"tensor(2.4746, grad_fn=<NegBackward0>)\n",
"tensor(2.4743, grad_fn=<NegBackward0>)\n",
"tensor(2.4739, grad_fn=<NegBackward0>)\n",
"tensor(2.4736, grad_fn=<NegBackward0>)\n",
"tensor(2.4733, grad_fn=<NegBackward0>)\n",
"tensor(2.4730, grad_fn=<NegBackward0>)\n",
"tensor(2.4726, grad_fn=<NegBackward0>)\n",
"tensor(2.4723, grad_fn=<NegBackward0>)\n",
"tensor(2.4721, grad_fn=<NegBackward0>)\n",
"tensor(2.4718, grad_fn=<NegBackward0>)\n",
"tensor(2.4715, grad_fn=<NegBackward0>)\n",
"tensor(2.4713, grad_fn=<NegBackward0>)\n",
"tensor(2.4710, grad_fn=<NegBackward0>)\n",
"tensor(2.4708, grad_fn=<NegBackward0>)\n",
"tensor(2.4705, grad_fn=<NegBackward0>)\n",
"tensor(2.4703, grad_fn=<NegBackward0>)\n",
"tensor(2.4700, grad_fn=<NegBackward0>)\n",
"tensor(2.4698, grad_fn=<NegBackward0>)\n",
"tensor(2.4696, grad_fn=<NegBackward0>)\n",
"tensor(2.4694, grad_fn=<NegBackward0>)\n",
"tensor(2.4692, grad_fn=<NegBackward0>)\n",
"tensor(2.4690, grad_fn=<NegBackward0>)\n",
"tensor(2.4688, grad_fn=<NegBackward0>)\n",
"tensor(2.4686, grad_fn=<NegBackward0>)\n",
"tensor(2.4684, grad_fn=<NegBackward0>)\n",
"tensor(2.4683, grad_fn=<NegBackward0>)\n",
"tensor(2.4681, grad_fn=<NegBackward0>)\n",
"tensor(2.4679, grad_fn=<NegBackward0>)\n",
"tensor(2.4677, grad_fn=<NegBackward0>)\n",
"tensor(2.4676, grad_fn=<NegBackward0>)\n",
"tensor(2.4674, grad_fn=<NegBackward0>)\n"
]
}
],
"source": [
"W = torch.randn((27,27),requires_grad=True)\n",
"epochs = 100\n",
"for i in range(epochs): \n",
" ### forward pass\n",
" logits = x_enc @ W\n",
" counts = torch.exp(logits) # \"fake\" counts of bigrams\n",
" probs = counts/torch.sum(counts,1,keepdim=True) \n",
" \n",
" ### backward pass\n",
" \n",
" # want near zero to be hugely negative, near 1 to approach zero.. \n",
" # then since loss is positive negate result. this way near zero #'s\n",
" # which are 'uncertain' penalize loss. Near 1 or 'certain' do not.\n",
" neg_log_loss = -torch.log(probs[torch.arange(len(ixs_x)),ixs_y]).mean()\n",
" print(neg_log_loss)\n",
" # now backprop\n",
" neg_log_loss.backward()\n",
" W.data -= 70 * W.grad\n",
" W.grad.zero_()"
]
},
{
"cell_type": "code",
"execution_count": 1012,
"id": "91914c71",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"cexze\n",
"momasurailezityha\n",
"konimittain\n",
"llayn\n",
"ka\n",
"da\n",
"staiyaubrtthrigotai\n",
"moliellavo\n",
"ke\n",
"teda\n",
"ka\n",
"emimmsade\n",
"enkaviyny\n",
"ftlspehinivenvtahlasu\n",
"dsor\n",
"br\n",
"jol\n",
"pen\n",
"aisan\n",
"ja\n"
]
}
],
"source": [
"### inference - take One-Hot encoded input and return prediction index of output\n",
"g = torch.Generator().manual_seed(2147483647)\n",
"\n",
"for i in range(20):\n",
" res = ['.'] # beg at start char\n",
" while True:\n",
" char = res[-1]\n",
" char_ix = torch.tensor(alpha.index(char)) # char->ix e.g. ('.' -> 0), ('a' -> 1)\n",
" input_enc = F.one_hot(char_ix,num_classes=27).float() # one-hot\n",
" logits = input_enc @ W\n",
" counts = torch.exp(logits) # \"fake\" counts of bigrams\n",
" probs = counts/counts.sum()\n",
" pred_ix = torch.multinomial(probs,num_samples=1,replacement=True,generator=g).item() # sample from probs\n",
" char = alpha[pred_ix] # get predicted char for input\n",
" res.append(char)\n",
" if char == '.':\n",
" break \n",
" print(\"\".join(res)[1:-1])\n",
" "
]
},
{
"cell_type": "markdown",
"id": "773b6414",
"metadata": {},
"source": [
"## Trigrams Neural Net"
]
},
{
"cell_type": "code",
"execution_count": 1169,
"id": "fbc8657f",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[0., 0., 0., 1., 0., 0., 0., 0., 0.],\n",
" [0., 0., 0., 0., 0., 0., 0., 0., 1.]])"
]
},
"execution_count": 1169,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"F.one_hot(torch.tensor([3,8])).float()"
]
},
{
"cell_type": "code",
"execution_count": 1013,
"id": "bf76f089",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([196113, 54])"
]
},
"execution_count": 1013,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"trigrams = [(a,b,c) for name in names for a,b,c in zip(name,name[1:],name[2:])]\n",
"# input was one-hot 0-27 of one char, now needs to be two chars... \n",
"# concat two one-hot vectors?\n",
"## characters\n",
"xs_0 = [b[0] for b in trigrams]\n",
"xs_1 = [b[1] for b in trigrams]\n",
"ys = [b[2] for b in trigrams]\n",
"\n",
"## index in string: 0-54\n",
"ixs_x0 = torch.tensor([alpha.index(x) for x in xs_0])\n",
"ixs_x1 = torch.tensor([alpha.index(x) for x in xs_1])\n",
"ixs_y = torch.tensor([alpha.index(y) for y in ys])\n",
"\n",
"# one-hot encode\n",
"x0_enc = F.one_hot(ixs_x0,num_classes=27).float()\n",
"x1_enc = F.one_hot(ixs_x1,num_classes=27).float()\n",
"\n",
"# each input should now be a concat of two one-hot vectors (first two chars of trigram)\n",
"x_enc = torch.cat((x0_enc,x1_enc),1)\n",
"x_enc.shape"
]
},
{
"cell_type": "code",
"execution_count": 1160,
"id": "c44fd4fc",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"tensor(4.4450, grad_fn=<NegBackward0>)\n",
"tensor(3.5424, grad_fn=<NegBackward0>)\n",
"tensor(3.1695, grad_fn=<NegBackward0>)\n",
"tensor(2.9636, grad_fn=<NegBackward0>)\n",
"tensor(2.8278, grad_fn=<NegBackward0>)\n",
"tensor(2.7327, grad_fn=<NegBackward0>)\n",
"tensor(2.6638, grad_fn=<NegBackward0>)\n",
"tensor(2.6117, grad_fn=<NegBackward0>)\n",
"tensor(2.5712, grad_fn=<NegBackward0>)\n",
"tensor(2.5387, grad_fn=<NegBackward0>)\n",
"tensor(2.5121, grad_fn=<NegBackward0>)\n",
"tensor(2.4900, grad_fn=<NegBackward0>)\n",
"tensor(2.4714, grad_fn=<NegBackward0>)\n",
"tensor(2.4554, grad_fn=<NegBackward0>)\n",
"tensor(2.4415, grad_fn=<NegBackward0>)\n",
"tensor(2.4293, grad_fn=<NegBackward0>)\n",
"tensor(2.4184, grad_fn=<NegBackward0>)\n",
"tensor(2.4087, grad_fn=<NegBackward0>)\n",
"tensor(2.3999, grad_fn=<NegBackward0>)\n",
"tensor(2.3920, grad_fn=<NegBackward0>)\n",
"tensor(2.3847, grad_fn=<NegBackward0>)\n",
"tensor(2.3781, grad_fn=<NegBackward0>)\n",
"tensor(2.3720, grad_fn=<NegBackward0>)\n",
"tensor(2.3664, grad_fn=<NegBackward0>)\n",
"tensor(2.3611, grad_fn=<NegBackward0>)\n",
"tensor(2.3563, grad_fn=<NegBackward0>)\n",
"tensor(2.3518, grad_fn=<NegBackward0>)\n",
"tensor(2.3476, grad_fn=<NegBackward0>)\n",
"tensor(2.3437, grad_fn=<NegBackward0>)\n",
"tensor(2.3401, grad_fn=<NegBackward0>)\n",
"tensor(2.3366, grad_fn=<NegBackward0>)\n",
"tensor(2.3334, grad_fn=<NegBackward0>)\n",
"tensor(2.3303, grad_fn=<NegBackward0>)\n",
"tensor(2.3275, grad_fn=<NegBackward0>)\n",
"tensor(2.3247, grad_fn=<NegBackward0>)\n",
"tensor(2.3222, grad_fn=<NegBackward0>)\n",
"tensor(2.3197, grad_fn=<NegBackward0>)\n",
"tensor(2.3174, grad_fn=<NegBackward0>)\n",
"tensor(2.3152, grad_fn=<NegBackward0>)\n",
"tensor(2.3131, grad_fn=<NegBackward0>)\n",
"tensor(2.3111, grad_fn=<NegBackward0>)\n",
"tensor(2.3092, grad_fn=<NegBackward0>)\n",
"tensor(2.3074, grad_fn=<NegBackward0>)\n",
"tensor(2.3057, grad_fn=<NegBackward0>)\n",
"tensor(2.3040, grad_fn=<NegBackward0>)\n",
"tensor(2.3024, grad_fn=<NegBackward0>)\n",
"tensor(2.3009, grad_fn=<NegBackward0>)\n",
"tensor(2.2995, grad_fn=<NegBackward0>)\n",
"tensor(2.2981, grad_fn=<NegBackward0>)\n",
"tensor(2.2967, grad_fn=<NegBackward0>)\n",
"tensor(2.2954, grad_fn=<NegBackward0>)\n",
"tensor(2.2942, grad_fn=<NegBackward0>)\n",
"tensor(2.2930, grad_fn=<NegBackward0>)\n",
"tensor(2.2918, grad_fn=<NegBackward0>)\n",
"tensor(2.2907, grad_fn=<NegBackward0>)\n",
"tensor(2.2896, grad_fn=<NegBackward0>)\n",
"tensor(2.2886, grad_fn=<NegBackward0>)\n",
"tensor(2.2876, grad_fn=<NegBackward0>)\n",
"tensor(2.2867, grad_fn=<NegBackward0>)\n",
"tensor(2.2857, grad_fn=<NegBackward0>)\n",
"tensor(2.2848, grad_fn=<NegBackward0>)\n",
"tensor(2.2839, grad_fn=<NegBackward0>)\n",
"tensor(2.2831, grad_fn=<NegBackward0>)\n",
"tensor(2.2823, grad_fn=<NegBackward0>)\n",
"tensor(2.2815, grad_fn=<NegBackward0>)\n",
"tensor(2.2807, grad_fn=<NegBackward0>)\n",
"tensor(2.2800, grad_fn=<NegBackward0>)\n",
"tensor(2.2793, grad_fn=<NegBackward0>)\n",
"tensor(2.2786, grad_fn=<NegBackward0>)\n",
"tensor(2.2779, grad_fn=<NegBackward0>)\n",
"tensor(2.2772, grad_fn=<NegBackward0>)\n",
"tensor(2.2766, grad_fn=<NegBackward0>)\n",
"tensor(2.2759, grad_fn=<NegBackward0>)\n",
"tensor(2.2753, grad_fn=<NegBackward0>)\n",
"tensor(2.2747, grad_fn=<NegBackward0>)\n",
"tensor(2.2742, grad_fn=<NegBackward0>)\n",
"tensor(2.2736, grad_fn=<NegBackward0>)\n",
"tensor(2.2731, grad_fn=<NegBackward0>)\n",
"tensor(2.2725, grad_fn=<NegBackward0>)\n",
"tensor(2.2720, grad_fn=<NegBackward0>)\n",
"tensor(2.2715, grad_fn=<NegBackward0>)\n",
"tensor(2.2710, grad_fn=<NegBackward0>)\n",
"tensor(2.2705, grad_fn=<NegBackward0>)\n",
"tensor(2.2701, grad_fn=<NegBackward0>)\n",
"tensor(2.2696, grad_fn=<NegBackward0>)\n",
"tensor(2.2692, grad_fn=<NegBackward0>)\n",
"tensor(2.2687, grad_fn=<NegBackward0>)\n",
"tensor(2.2683, grad_fn=<NegBackward0>)\n",
"tensor(2.2679, grad_fn=<NegBackward0>)\n",
"tensor(2.2675, grad_fn=<NegBackward0>)\n",
"tensor(2.2671, grad_fn=<NegBackward0>)\n",
"tensor(2.2667, grad_fn=<NegBackward0>)\n",
"tensor(2.2663, grad_fn=<NegBackward0>)\n",
"tensor(2.2659, grad_fn=<NegBackward0>)\n",
"tensor(2.2656, grad_fn=<NegBackward0>)\n",
"tensor(2.2652, grad_fn=<NegBackward0>)\n",
"tensor(2.2649, grad_fn=<NegBackward0>)\n",
"tensor(2.2645, grad_fn=<NegBackward0>)\n",
"tensor(2.2642, grad_fn=<NegBackward0>)\n",
"tensor(2.2639, grad_fn=<NegBackward0>)\n",
"tensor(2.2636, grad_fn=<NegBackward0>)\n",
"tensor(2.2633, grad_fn=<NegBackward0>)\n",
"tensor(2.2629, grad_fn=<NegBackward0>)\n",
"tensor(2.2627, grad_fn=<NegBackward0>)\n",
"tensor(2.2624, grad_fn=<NegBackward0>)\n",
"tensor(2.2621, grad_fn=<NegBackward0>)\n",
"tensor(2.2618, grad_fn=<NegBackward0>)\n",
"tensor(2.2615, grad_fn=<NegBackward0>)\n",
"tensor(2.2612, grad_fn=<NegBackward0>)\n",
"tensor(2.2610, grad_fn=<NegBackward0>)\n",
"tensor(2.2607, grad_fn=<NegBackward0>)\n",
"tensor(2.2605, grad_fn=<NegBackward0>)\n",
"tensor(2.2602, grad_fn=<NegBackward0>)\n",
"tensor(2.2600, grad_fn=<NegBackward0>)\n",
"tensor(2.2597, grad_fn=<NegBackward0>)\n",
"tensor(2.2595, grad_fn=<NegBackward0>)\n",
"tensor(2.2593, grad_fn=<NegBackward0>)\n",
"tensor(2.2590, grad_fn=<NegBackward0>)\n",
"tensor(2.2588, grad_fn=<NegBackward0>)\n",
"tensor(2.2586, grad_fn=<NegBackward0>)\n",
"tensor(2.2584, grad_fn=<NegBackward0>)\n",
"tensor(2.2582, grad_fn=<NegBackward0>)\n",
"tensor(2.2580, grad_fn=<NegBackward0>)\n",
"tensor(2.2577, grad_fn=<NegBackward0>)\n",
"tensor(2.2575, grad_fn=<NegBackward0>)\n",
"tensor(2.2573, grad_fn=<NegBackward0>)\n",
"tensor(2.2572, grad_fn=<NegBackward0>)\n",
"tensor(2.2570, grad_fn=<NegBackward0>)\n",
"tensor(2.2568, grad_fn=<NegBackward0>)\n",
"tensor(2.2566, grad_fn=<NegBackward0>)\n",
"tensor(2.2564, grad_fn=<NegBackward0>)\n",
"tensor(2.2562, grad_fn=<NegBackward0>)\n",
"tensor(2.2561, grad_fn=<NegBackward0>)\n",
"tensor(2.2559, grad_fn=<NegBackward0>)\n",
"tensor(2.2557, grad_fn=<NegBackward0>)\n",
"tensor(2.2555, grad_fn=<NegBackward0>)\n",
"tensor(2.2554, grad_fn=<NegBackward0>)\n",
"tensor(2.2552, grad_fn=<NegBackward0>)\n",
"tensor(2.2550, grad_fn=<NegBackward0>)\n",
"tensor(2.2549, grad_fn=<NegBackward0>)\n",
"tensor(2.2547, grad_fn=<NegBackward0>)\n",
"tensor(2.2546, grad_fn=<NegBackward0>)\n",
"tensor(2.2544, grad_fn=<NegBackward0>)\n",
"tensor(2.2543, grad_fn=<NegBackward0>)\n",
"tensor(2.2541, grad_fn=<NegBackward0>)\n",
"tensor(2.2540, grad_fn=<NegBackward0>)\n",
"tensor(2.2538, grad_fn=<NegBackward0>)\n",
"tensor(2.2537, grad_fn=<NegBackward0>)\n",
"tensor(2.2536, grad_fn=<NegBackward0>)\n",
"tensor(2.2534, grad_fn=<NegBackward0>)\n",
"tensor(2.2533, grad_fn=<NegBackward0>)\n",
"tensor(2.2532, grad_fn=<NegBackward0>)\n",
"tensor(2.2530, grad_fn=<NegBackward0>)\n",
"tensor(2.2529, grad_fn=<NegBackward0>)\n",
"tensor(2.2528, grad_fn=<NegBackward0>)\n",
"tensor(2.2527, grad_fn=<NegBackward0>)\n",
"tensor(2.2525, grad_fn=<NegBackward0>)\n",
"tensor(2.2524, grad_fn=<NegBackward0>)\n",
"tensor(2.2523, grad_fn=<NegBackward0>)\n",
"tensor(2.2522, grad_fn=<NegBackward0>)\n",
"tensor(2.2521, grad_fn=<NegBackward0>)\n",
"tensor(2.2519, grad_fn=<NegBackward0>)\n",
"tensor(2.2518, grad_fn=<NegBackward0>)\n",
"tensor(2.2517, grad_fn=<NegBackward0>)\n",
"tensor(2.2516, grad_fn=<NegBackward0>)\n",
"tensor(2.2515, grad_fn=<NegBackward0>)\n",
"tensor(2.2514, grad_fn=<NegBackward0>)\n",
"tensor(2.2513, grad_fn=<NegBackward0>)\n",
"tensor(2.2512, grad_fn=<NegBackward0>)\n",
"tensor(2.2511, grad_fn=<NegBackward0>)\n",
"tensor(2.2510, grad_fn=<NegBackward0>)\n",
"tensor(2.2509, grad_fn=<NegBackward0>)\n",
"tensor(2.2508, grad_fn=<NegBackward0>)\n",
"tensor(2.2507, grad_fn=<NegBackward0>)\n",
"tensor(2.2506, grad_fn=<NegBackward0>)\n",
"tensor(2.2505, grad_fn=<NegBackward0>)\n",
"tensor(2.2504, grad_fn=<NegBackward0>)\n",
"tensor(2.2503, grad_fn=<NegBackward0>)\n",
"tensor(2.2502, grad_fn=<NegBackward0>)\n",
"tensor(2.2501, grad_fn=<NegBackward0>)\n",
"tensor(2.2500, grad_fn=<NegBackward0>)\n",
"tensor(2.2499, grad_fn=<NegBackward0>)\n",
"tensor(2.2498, grad_fn=<NegBackward0>)\n",
"tensor(2.2497, grad_fn=<NegBackward0>)\n",
"tensor(2.2497, grad_fn=<NegBackward0>)\n",
"tensor(2.2496, grad_fn=<NegBackward0>)\n",
"tensor(2.2495, grad_fn=<NegBackward0>)\n",
"tensor(2.2494, grad_fn=<NegBackward0>)\n",
"tensor(2.2493, grad_fn=<NegBackward0>)\n",
"tensor(2.2492, grad_fn=<NegBackward0>)\n",
"tensor(2.2492, grad_fn=<NegBackward0>)\n",
"tensor(2.2491, grad_fn=<NegBackward0>)\n",
"tensor(2.2490, grad_fn=<NegBackward0>)\n",
"tensor(2.2489, grad_fn=<NegBackward0>)\n",
"tensor(2.2488, grad_fn=<NegBackward0>)\n",
"tensor(2.2488, grad_fn=<NegBackward0>)\n",
"tensor(2.2487, grad_fn=<NegBackward0>)\n",
"tensor(2.2486, grad_fn=<NegBackward0>)\n",
"tensor(2.2485, grad_fn=<NegBackward0>)\n",
"tensor(2.2485, grad_fn=<NegBackward0>)\n"
]
}
],
"source": [
"# now 54 neurons, still 27 weights each. Need 54 neurons so each neuron has it's 'vote' for an input\n",
"W = torch.randn((54,27),requires_grad=True)\n",
"epochs = 200\n",
"losses = []\n",
"for i in range(epochs): \n",
" ### forward pass\n",
" logits = x_enc @ W\n",
" counts = torch.exp(logits) # \"fake\" counts of bigrams\n",
" probs = counts/torch.sum(counts,1,keepdim=True) \n",
" \n",
" ### backward pass\n",
" \n",
" # want near zero to be hugely negative, near 1 to approach zero.. \n",
" # then since loss is positive negate result. this way near zero #'s\n",
" # which are 'uncertain' penalize loss. Near 1 or 'certain' do not.\n",
" neg_log_loss = -torch.log(probs[torch.arange(len(ixs_x1)),ixs_y]).mean()\n",
" print(neg_log_loss)\n",
" losses.append([i, neg_log_loss.data])\n",
" # now backprop\n",
" neg_log_loss.backward()\n",
" W.data -= 50 * W.grad\n",
" W.grad.zero_()"
]
},
{
"cell_type": "code",
"execution_count": 1161,
"id": "ffa9833a",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[<matplotlib.lines.Line2D at 0x2e2784e50>]"
]
},
"execution_count": 1161,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvD0lEQVR4nO3de3SU9b3v8c8zl0xCSIZrSGKCgDcUBBFsxe1ti9J6Qd1aq7uuVtvq2fTgbVvWstjVZXtaF562x1KXLYhFK2W3uM8JdmtRKi5J1KXscomKqEhrJFwSIkIu5DIzmfmdP+aSmdwnl3kymfdrrVnMc5mZ35MncT7+ft/n91jGGCMAAACbOOxuAAAAyGyEEQAAYCvCCAAAsBVhBAAA2IowAgAAbEUYAQAAtiKMAAAAWxFGAACArVx2N6A/QqGQjhw5ory8PFmWZXdzAABAPxhj1NTUpOLiYjkcPfd/pEUYOXLkiEpLS+1uBgAAGICDBw+qpKSkx+1pEUby8vIkhQ8mPz/f5tYAAID+aGxsVGlpaex7vCdpEUaiQzP5+fmEEQAA0kxfJRYUsAIAAFsNKoysXLlSlmXpgQce6HGf8vJyWZbV5fHxxx8P5qMBAMAoMeBhmh07dmjt2rWaM2dOv/bft29fwhDL5MmTB/rRAABgFBlQz8jJkyd1++236+mnn9b48eP79ZqCggIVFhbGHk6ncyAfDQAARpkBhZFly5bp2muv1ZVXXtnv18ybN09FRUVatGiRtm3b1uu+Pp9PjY2NCQ8AADA6JT1Ms3HjRu3evVs7duzo1/5FRUVau3at5s+fL5/Ppz/84Q9atGiRysvLdemll3b7mpUrV+onP/lJsk0DAABpyDLGmP7ufPDgQS1YsECvvvqq5s6dK0m6/PLLdd5552nVqlX9/tAlS5bIsiy9+OKL3W73+Xzy+Xyx5eh1yg0NDVzaCwBAmmhsbJTX6+3z+zupYZpdu3aprq5O8+fPl8vlksvlUkVFhZ544gm5XC4Fg8F+vc+FF16o/fv397jd4/HE5hRhbhEAAEa3pIZpFi1apD179iSs+/a3v62ZM2fqoYce6ndRamVlpYqKipL5aAAAMEolFUby8vI0e/bshHW5ubmaOHFibP2KFSt0+PBhrV+/XpK0atUqTZs2TbNmzZLf79eGDRtUVlamsrKyIToEAACQzoZ8OviamhpVV1fHlv1+v5YvX67Dhw8rJydHs2bN0ubNm3XNNdcM9UcDAIA0lFQBq136WwADAABGjv5+f6fFjfKGS9muQ9pzuEFfnV2oC2dMtLs5AABkpIy+UV7FJ5/r929/pg+PMKkaAAB2yegw4nKEb2kcDI34kSoAAEatjA4jzkgYCYRCNrcEAIDMldFhxOWM9IwE6RkBAMAumR1GHOHDb2eYBgAA22R0GHFSMwIAgO0yOoxEC1jpGQEAwD4ZHUackZqR9iAFrAAA2CWjwwg9IwAA2C/Dw0j48KkZAQDAPhkeRugZAQDAbhkdRqI1I0EmPQMAwDYZHUZiPSNMegYAgG0yOow4mfQMAADbZXQYcTuZ9AwAALtldBhxxgpYqRkBAMAuGR1GXEwHDwCA7TI6jERrRgIUsAIAYJuMDiP0jAAAYL/MDiNOakYAALBbZocRekYAALBdRocR5hkBAMB+GR1GmIEVAAD7ZXQYcXKjPAAAbJfRYcTFjfIAALBdZocRakYAALBdRocRJ1fTAABgu4wOIxSwAgBgv4wOI9woDwAA+2V0GHE7w4fPMA0AAPbJ6DDCpb0AANgvo8NIbDp4akYAALBNRocRekYAALBfRocR7toLAID9MjuMMOkZAAC2y/AwEu4ZMUYKEUgAALBFRocRZ2SYRqJ3BAAAu2R0GIn2jEjMNQIAgF0yOow448JIgCJWAABskdFhxO3oOHzmGgEAwB4ZHUYcDktWpHOEmhEAAOyR0WFEipuFlTACAIAtMj6McOdeAADslfFhJDbxGTUjAADYIuPDCPenAQDAXhkfRtxOakYAALBTxocRakYAALBXxoeRaM0IPSMAANgj48NItGckQAErAAC2yPgwwjwjAADYizDipGYEAAA7ZXwYcVIzAgCArTI+jLiYZwQAAFtlfBiJXdpLASsAALbI+DDSUcBKzQgAAHYgjDgZpgEAwE6EEQpYAQCwVcaHEWpGAACwV8aHERf3pgEAwFYZH0acXNoLAICtMj6MuJ3UjAAAYKeMDyPUjAAAYK+MDyPcKA8AAHtlfBiJ9owEKGAFAMAWgwojK1eulGVZeuCBB3rdr6KiQvPnz1d2drZmzJihNWvWDOZjh1R00rMgwzQAANhiwGFkx44dWrt2rebMmdPrflVVVbrmmmt0ySWXqLKyUg8//LDuu+8+lZWVDfSjh1R00jOupgEAwB4DCiMnT57U7bffrqefflrjx4/vdd81a9Zo6tSpWrVqlc4++2zddddd+s53vqNf/vKXA2rwUHNSMwIAgK0GFEaWLVuma6+9VldeeWWf+77zzjtavHhxwrqvfOUr2rlzpwKBwEA+fki5mGcEAABbuZJ9wcaNG7V7927t2LGjX/vX1tZqypQpCeumTJmi9vZ2HTt2TEVFRV1e4/P55PP5YsuNjY3JNrPfnNEb5QUpYAUAwA5J9YwcPHhQ999/vzZs2KDs7Ox+v86yrIRlY0y366NWrlwpr9cbe5SWlibTzKTQMwIAgL2SCiO7du1SXV2d5s+fL5fLJZfLpYqKCj3xxBNyuVwKBoNdXlNYWKja2tqEdXV1dXK5XJo4cWK3n7NixQo1NDTEHgcPHkymmUnhrr0AANgrqWGaRYsWac+ePQnrvv3tb2vmzJl66KGH5HQ6u7xm4cKFeumllxLWvfrqq1qwYIHcbne3n+PxeOTxeJJp2oDRMwIAgL2SCiN5eXmaPXt2wrrc3FxNnDgxtn7FihU6fPiw1q9fL0launSpnnzyST344IO6++679c4772jdunX605/+NESHMDjRmpEgk54BAGCLIZ+BtaamRtXV1bHl6dOn6+WXX1Z5ebnOO+88/fSnP9UTTzyhm2++eag/ekBc3JsGAABbJX01TWfl5eUJy7///e+77HPZZZdp9+7dg/2oYeFk0jMAAGyV8femcTuZ9AwAADtlfBhxxgpYqRkBAMAOGR9GXEwHDwCArTI+jERrRgIUsAIAYIuMDyP0jAAAYC/CiJOaEQAA7EQYoWcEAABbZXwYYZ4RAADslfFhhBlYAQCwV8aHESc3ygMAwFYZH0Zc3CgPAABbEUaoGQEAwFYZH0acXE0DAICtMj6MUMAKAIC9Mj6McKM8AADslfFhxO0M/wgYpgEAwB4ZH0a4tBcAAHtlfBiJTQdPzQgAALbI+DAS7RkJUDMCAIAtMj6MdEx6Rs8IAAB2IIww6RkAALYijESGaYyRQgQSAABSLuPDiDMyTCPROwIAgB0yPoxEe0YkJj4DAMAOGR9GnA56RgAAsFPGhxG3o+NHwFwjAACkXsaHEYfDkhXpHKFnBACA1Mv4MCLFzcJKGAEAIOUII4qbhTVIASsAAKlGGFHHxGf0jAAAkHqEEXVMCU/NCAAAqUcYETUjAADYiTCijpoRJj0DACD1CCOKu1ke84wAAJByhBHF94wQRgAASDXCiDoKWKkZAQAg9Qgj6ihgpWYEAIDUI4xIcjLPCAAAtiGMKK5nhAJWAABSjjAiClgBALATYUSSO1bASs0IAACpRhgRPSMAANiJMCJulAcAgJ0II+roGQlQwAoAQMoRRhR/ozxqRgAASDXCiDpmYKVmBACA1COMiJoRAADsRBhR3NU01IwAAJByhBFxbxoAAOxEGBHzjAAAYCfCiCSXM1IzwjANAAApRxhR/DANYQQAgFQjjKhjmIaraQAASD3CiDp6RgIUsAIAkHKEEUnO6F17qRkBACDlCCOiZgQAADsRRsQMrAAA2IkwInpGAACwE2FEHTUj7UEKWAEASDXCiDp6RhimAQAg9QgjkpyRmhGGaQAASD3CiCS3k54RAADsQhhR/I3yqBkBACDVCCOKm4GVSc8AAEg5woikbLdTktQWCNrcEgAAMg9hRFJOJIy0EkYAAEi5pMLI6tWrNWfOHOXn5ys/P18LFy7UK6+80uP+5eXlsiyry+Pjjz8edMOHUk5WJIz4CSMAAKSaK5mdS0pK9Nhjj+n000+XJD333HO64YYbVFlZqVmzZvX4un379ik/Pz+2PHny5AE2d3jkMEwDAIBtkgojS5YsSVh+9NFHtXr1am3fvr3XMFJQUKBx48YNqIGpkM0wDQAAthlwzUgwGNTGjRvV3NyshQsX9rrvvHnzVFRUpEWLFmnbtm19vrfP51NjY2PCYziNYZgGAADbJB1G9uzZo7Fjx8rj8Wjp0qV64YUXdM4553S7b1FRkdauXauysjJt2rRJZ511lhYtWqQ33nij189YuXKlvF5v7FFaWppsM5MSqxmhZwQAgJSzjDFJTa7h9/tVXV2t+vp6lZWV6Xe/+50qKip6DCSdLVmyRJZl6cUXX+xxH5/PJ5/PF1tubGxUaWmpGhoaEmpPhkp9i1/n/a+tkqT9j14tt5OLjAAAGKzGxkZ5vd4+v7+TqhmRpKysrFgB64IFC7Rjxw79+te/1lNPPdWv11944YXasGFDr/t4PB55PJ5kmzZg0ZoRKVzEShgBACB1Bv2ta4xJ6MXoS2VlpYqKigb7sUPK43LICk/CylANAAApllTPyMMPP6yrr75apaWlampq0saNG1VeXq4tW7ZIklasWKHDhw9r/fr1kqRVq1Zp2rRpmjVrlvx+vzZs2KCysjKVlZUN/ZEMgmVZynE71eIPqs3P/WkAAEilpMLI0aNH9c1vflM1NTXyer2aM2eOtmzZoquuukqSVFNTo+rq6tj+fr9fy5cv1+HDh5WTk6NZs2Zp8+bNuuaaa4b2KIbAmKxwGKFnBACA1Eq6gNUO/S2AGYyL//frOnSiVS/8z4s0b+r4YfkMAAAySX+/v6nUjOD+NAAA2IMwEhGda4Qp4QEASC3CSERsSngKWAEASCnCSATDNAAA2IMwEtFxf5p2m1sCAEBmIYxE0DMCAIA9CCMR2VnUjAAAYAfCSAQ9IwAA2IMwEhENI1zaCwBAahFGInJiwzSEEQAAUokwEhHtGWmhZwQAgJQijETQMwIAgD0IIxHUjAAAYA/CSEQ2V9MAAGALwkgEwzQAANiDMBLBMA0AAPYgjERE703TQs8IAAApRRiJoGYEAAB7EEYiYjUjhBEAAFKKMBIRrRnxt4cUDBmbWwMAQOYgjEREw4hEESsAAKlEGInwuDp+FAzVAACQOoSRCIfDUrY7/ONgrhEAAFKHMBJnTJZLEj0jAACkEmEkTrRuhJ4RAABShzASJzZMQ88IAAApQxiJw1wjAACkHmEkTuz+NAzTAACQMoSRONEp4bk/DQAAqUMYiTOGYRoAAFKOMBInNkxDGAEAIGUII3FiBawM0wAAkDKEkTjRmhGGaQAASB3CSJwcClgBAEg5wkgcakYAAEg9wkgcJj0DACD1CCNxKGAFACD1CCNxcihgBQAg5QgjcagZAQAg9QgjcbKzuJoGAIBUI4zEYZgGAIDUI4zEid6bhrv2AgCQOoSROPSMAACQeoSRONHp4JvpGQEAIGUII3G8Y9ySJH97iCtqAABIEcJInDyPS06HJUmqbwnY3BoAADIDYSSOZVkalxPuHalv9dvcGgAAMgNhpJPoUM2JZnpGAABIBcJIJ+PHZEmSGugZAQAgJQgjnUSHaU5QMwIAQEoQRjoZF+kZoYAVAIDUIIx0Mm4MBawAAKQSYaST2NU0FLACAJAShJFOxuVGhmnoGQEAICUII51QwAoAQGoRRjqJXdpLGAEAICUII51QwAoAQGoRRjqJhpETLQEZY2xuDQAAox9hpJPoPCPhO/eGbG4NAACjH2Gkk9wsp1yRO/eeaGGoBgCA4UYY6cSyLGZhBQAghQgj3aCIFQCA1CGMdGN8NIzQMwIAwLAjjHTDm8MwDQAAqUIY6cb42OW9DNMAADDckgojq1ev1pw5c5Sfn6/8/HwtXLhQr7zySq+vqaio0Pz585Wdna0ZM2ZozZo1g2pwKkRrRhpa6RkBAGC4JRVGSkpK9Nhjj2nnzp3auXOnrrjiCt1www3au3dvt/tXVVXpmmuu0SWXXKLKyko9/PDDuu+++1RWVjYkjR8u0atpTjTTMwIAwHBzJbPzkiVLEpYfffRRrV69Wtu3b9esWbO67L9mzRpNnTpVq1atkiSdffbZ2rlzp375y1/q5ptvHnirh1nH1TT0jAAAMNwGXDMSDAa1ceNGNTc3a+HChd3u884772jx4sUJ677yla9o586dCgRG7hf9uBxulgcAQKok1TMiSXv27NHChQvV1tamsWPH6oUXXtA555zT7b61tbWaMmVKwropU6aovb1dx44dU1FRUbev8/l88vl8seXGxsZkmzkoFLACAJA6SfeMnHXWWXr33Xe1fft2fe9739Mdd9yhDz/8sMf9LctKWI7efK7z+ngrV66U1+uNPUpLS5Nt5qB4GaYBACBlkg4jWVlZOv3007VgwQKtXLlSc+fO1a9//etu9y0sLFRtbW3Curq6OrlcLk2cOLHHz1ixYoUaGhpij4MHDybbzEEZH5sO3s+dewEAGGZJD9N0ZoxJGFKJt3DhQr300ksJ61599VUtWLBAbre7x/f0eDzyeDyDbdqARQtYA0GjFn9QuZ5B/5gAAEAPkuoZefjhh/Xmm2/qs88+0549e/TDH/5Q5eXluv322yWFezS+9a1vxfZfunSpDhw4oAcffFAfffSRnnnmGa1bt07Lly8f2qMYYjlup7Kc4R8NQzUAAAyvpP6X/+jRo/rmN7+pmpoaeb1ezZkzR1u2bNFVV10lSaqpqVF1dXVs/+nTp+vll1/Wv//7v+s3v/mNiouL9cQTT4zoy3ql6J173apr8ulEs1+njMuxu0kAAIxaSYWRdevW9br997//fZd1l112mXbv3p1Uo0aCiWM9qmvy6djJ7oegAADA0ODeND0ozA/XrNQ2tNncEgAARjfCSA8KveGhmRrCCAAAw4ow0oMib7YkekYAABhuhJEeFEbCSE0jYQQAgOFEGOlBR89Iq80tAQBgdCOM9IBhGgAAUoMw0oNoAWtjW7uafe02twYAgNGLMNKDsR6X8iLTwNdSNwIAwLAhjPSikKEaAACGHWGkF7EraggjAAAMG8JILwrzuaIGAIDhRhjpRRE9IwAADDvCSC+iV9RQMwIAwPAhjPSCnhEAAIYfYaQXsatpuLQXAIBhQxjpRbRn5HizX22BoM2tAQBgdCKM9MKb41a2O/wjqmv02dwaAABGJ8JILyzLUlGkiLWGy3sBABgWhJE+xOYaoW4EAIBhQRjpQ7Ru5Eg9YQQAgOFAGOlDyYQxkqQDXzTb3BIAAEYnwkgfTpucK0n6x+cnbW4JAACjE2GkD6dNHitJ+sfn9IwAADAcCCN9mBHpGTne7NfxZr/NrQEAYPQhjPRhTJZLp4wLX977KUM1AAAMOcJIP8ygbgQAgGFDGOkH6kYAABg+hJF+OK0gEkbq6BkBAGCoEUb6gct7AQAYPoSRfjg9Mkxz8ESrfO3cvRcAgKFEGOmHyXke5XlcCoaMqr9osbs5AACMKoSRfrAsSzOidSMM1QAAMKQII/3UUTfCFTUAAAwlwkg/xS7v5YoaAACGFGGkn06PDNN8XNtkc0sAABhdCCP9NKfEK0nad7RJrX6uqAEAYKgQRvqpyJujKfkeBUNGew432N0cAABGDcJIEs4rHSdJevfgCXsbAgDAKEIYScJ5peMlSe8erLe3IQAAjCKEkSTEekaq621tBwAAowlhJAlzSrxyWNKRhjbVNbbZ3RwAAEYFwkgScj0unTklT5JUyVANAABDgjCSpOhQTSVDNQAADAnCSJK4ogYAgKFFGEnSeVPHSZLeP9Sg9mDI3sYAADAKEEaSdEZBnsaNcavFH+QSXwAAhgBhJElOh6VLzpgsSSrf97nNrQEAIP0RRgbgsjMjYeSTOptbAgBA+iOMDEA0jHxwuFF1Tcw3AgDAYBBGBmBynkezT8mXJL35yTGbWwMAQHojjAzQ5WcWSJLKP6FuBACAwSCMDNBlZ4WHat7c/7mCIWNzawAASF+EkQGaVzpO+dku1bcEVFnNBGgAAAwUYWSAXE6HrpgZHqp58b0jNrcGAID0RRgZhH85v0RSOIz425mNFQCAgSCMDMI/nTZRBXke1bcEVL6POUcAABgIwsgguJwO3XBesSRp0+7DNrcGAID0RBgZpJsiQzWvf1yn+ha/za0BACD9EEYG6eyifM0szJM/GNJL79fY3RwAANIOYWQIfG1+uHfkubc/U4g5RwAASAphZAjcekGp8jwu/b3upLZRyAoAQFIII0MgL9utb3x5qiRp7Ruf2twaAADSC2FkiNz5T9Pkclj676rjeu9gvd3NAQAgbRBGhkiRN0fXRy7zXVPxD5tbAwBA+iCMDKF/u/Q0WZb0yge1epfeEQAA+oUwMoTOKszTzZF5R372lw9lDFfWAADQl6TCyMqVK3XBBRcoLy9PBQUFuvHGG7Vv375eX1NeXi7Lsro8Pv7440E1fKRavvgs5bid2nnghLZ8UGt3cwAAGPGSCiMVFRVatmyZtm/frq1bt6q9vV2LFy9Wc3Nzn6/dt2+fampqYo8zzjhjwI0eyQq92br70hmSpJWvfKy2QNDmFgEAMLK5ktl5y5YtCcvPPvusCgoKtGvXLl166aW9vragoEDjxo1LuoHp6N8unaHnd1Sr+niLHt/6iR6+5my7mwQAwIg1qJqRhoYGSdKECRP63HfevHkqKirSokWLtG3btl739fl8amxsTHikk1yPS4/eeK4k6ek3P9WuA8dtbhEAACPXgMOIMUYPPvigLr74Ys2ePbvH/YqKirR27VqVlZVp06ZNOuuss7Ro0SK98cYbPb5m5cqV8nq9sUdpaelAm2mbK8+ZopvPL5Ex0vL/+75a/QzXAADQHcsM8JKPZcuWafPmzXrrrbdUUlKS1GuXLFkiy7L04osvdrvd5/PJ5/PFlhsbG1VaWqqGhgbl5+cPpLm2aGgNaPGvKnS00aebzj9F/+eWubIsy+5mAQCQEo2NjfJ6vX1+fw+oZ+Tee+/Viy++qG3btiUdRCTpwgsv1P79+3vc7vF4lJ+fn/BIR94ct35163lyWNKm3Ye1/p0DdjcJAIARJ6kwYozRPffco02bNun111/X9OnTB/ShlZWVKioqGtBr081Fp02KFbD+9C8favunX9jcIgAARpakrqZZtmyZ/vjHP+q//uu/lJeXp9ra8DwaXq9XOTk5kqQVK1bo8OHDWr9+vSRp1apVmjZtmmbNmiW/368NGzaorKxMZWVlQ3woI9d3L56u9w416KX3juju9Tv1/P9YqHOK07O3BwCAoZZUz8jq1avV0NCgyy+/XEVFRbHH888/H9unpqZG1dXVsWW/36/ly5drzpw5uuSSS/TWW29p8+bNuummm4buKEY4y7L085vn6IJp49XU1q5vPfM3fXas77lZAADIBAMuYE2l/hbAjHQNrQHdtna7PqppVLE3Wxvu+rJmTB5rd7MAABgWw1rAioHx5ri1/jtf0ozJuTrS0KavP/WO9h5psLtZAADYijCSYpPzPPrPf1uoc4rydeykX7et3a43Pvnc7mYBAGAbwogNJo316E//40J9adoENbW1685n/6Z1b1Vxl18AQEYijNjEm+PWH+76km6ZX6KQCV/2e9/Gd9XYFrC7aQAApBRhxEYel1M//9oc/ei6c+RyWHrpvSO69ok3tevACbubBgBAyhBGbGZZlr578XT959KFKhmfo4PHW/W1NW/rZ3/5kPvZAAAyAmFkhDh/6nhtvu8S3TTvFBkj/e6tKl31qwptfr+GWhIAwKhGGBlBvDluPX7reXr2zgtU7M3WoROtWvbH3framnf07sF6u5sHAMCwYNKzEarF366n36jSmop/qDUQHq658bxi3X/lmZo+Kdfm1gEA0Lf+fn8TRka42oY2/eKv+1S2+5AkybKka2YXaellp+ncEq/NrQMAoGeEkVFmz6EG/eq1T/T6x3WxdZecMUl3XzJDF58+SQ6HZWPrAADoijAySn1U06inKv6hl96vUTAUPnWlE3J02wVTdcv8EhXkZ9vcQgAAwggjo9zB4y1a91aVynYdUpOvXZLkdFhaNLNA/zLvFP3zzAJlu502txIAkMkIIxmixd+uze/XaOOOgwmTpeVmOXXlOVN07blFuuysyfK4CCYAgNQijGSgT4426f/tOqTN79focH1rbH1ullP/dPokXTGzQP88s0BTGMoBAKQAYSSDGWNUebBef3mvRi/vqVFtY1vC9lnF+br49Em6cMZELZg2XnnZbptaCgAYzQgjkCSFQkZ7jzTq9Y/r9Pq+Or1/qF7xZ9xhSbNP8erL0ydEwskEeXMIJwCAwSOMoFvHTvr05v7Ptf0fx/XfVV/osy9aErZbljRjUq7mlozTuSVezSkZp1nF+RTDAgCSRhhBv9Q0tOq/Pw0Hk+2fHlfVseYu+7gcls6ckqdzT/HqrMI8zSzM05mFeZo01mNDiwEA6YIwggE5dtKnPYca9N6her1/qEHvH6rXsZP+bvedNDZLZ07J05lT8nTGlLGaPjFX0yblqjA/m0nYAACEEQwNY4xqGtr0/qF6fXikUR/XNumTo006cLxFPf3mZLsdmjYxN/yYlKvpk8Zo2sRcnToxVwV5HoIKAGQIwgiGVYu/XX+vO6mPa5u0r7ZJn35+Up990aKDx1vUHur5V8rttFTkzVHxuGydMm6MThmfo1Pinhd5s6lPAYBRor/f364UtgmjyJgsl+aUjNOcknEJ6wPBkA6faFXVF8367Fj4UfVFi6qOndSR+jYFgkbVx1tUfbxF0vFu3zs/26WC/GxNyfeoIC9bBXkeTc7zaEp++HlB5N9cD7++ADAa8F9zDCm306Fpk8LDMzorcVt7MKS6Jp8O17fq8InW8L/xz0+0qjUQVGNbuxrbTurvdSd7/azcLKcmjM3ShFyPJoxxa3xulibmZml8bpYmjMnShNzwI7o+P9vNEBEAjECEEaSMy+lQ8bgcFY/L0QXTum43xqixrV2fN7XpaKNPdU1tqmv0dTxv8unzJp+ONrapxR9Usz+o5uOtOni8teubdcNhSePHZGncGLfyc9zKz3bLm+NWfo5L+dkd66LL4W1u5We7lJftVpbLMbQ/EACAJMIIRhDLsuTNCYeA0wvyet33pK9dnzf5dLzZp+PNAZ1o9uuLZr9OtPh1vLnjcaLFr+Mn/WrytStkpC8i+w1Ejtup/ByXxnrCj9zoI8up3Lh1Y7KcsedjI8u5Ca9xKsftlGXRSwMAEmEEaSoaCKZPyu3X/v72kOpbwkGkviWgxraAGlsD4SGh1oAaWqPr2ju2RbafjNwVuTUQVGsgqKPyDbr9Ditcd5Ptdiony6EcdzighJedHctxz3OyItvjXpMdty3H7ZTH5VSWyyGPyyGP2yGPyyknQ1MARjjCCDJClssRLnwdwE0C24MhnfS1q7G1XQ2tATX5AmrxBdXsDweVZl+7mn3B8L/+jucnI8stvmDHfv6gJClkwr070aAznJwOKxxOXI5uw0qWM/q803aXUx63I257eFuW05Lb6Yg84p87lOUKL7scHc+739eiZwhADGEE6IPL6dC4MVkaNyZr0O8VChm1BsJhpcUfjPW2tMU9b/UH1RZ7Hgpvj6yP7d95OfLc3x6Srz2UcHl1MGTU4g+qxR+UFBj0MQwVt9OSyxEOJlmujtDiclrK6iHAuCL/Oh0OuRyWnA4rshx+L5fDktNpRbY55O68HNs3sn+n5ei+3W2LLrvj9otfdlqWHA7JaYXXE7aA/iOMACnkcFixWpPh1B4MyR8MxcKJLxCSrz0Yft7e8bxje6flhO1B+QLh9/MFQgoEQwqEjALtkefBkPxBo0AwpPZgSIGgkT+yPrxPeLmzQNAoEAyqdeTkoyFlWYoEFCsWUBxWuKcq/NyKPXc6rIR9HQ5LToe6rou+j8OSM/JejoR14ZDU2/s4O2+3wvvEb7escFsdkX/Dy+H9os+j263YfpF1jt63W3E/i8TPids30uZutzsS1yVsd6jLZzvjXktAHLkII8Ao5HI65HI6NASdOUPCGKNgyCQGlWBI7QnBpeN5eyTcxO8baDcKhEIKhozag5H3C4UUDBq1h7pfbg+F4vY1Ciax3B55ffT92kNG7cFQ3Lbwfj0fs9RuTHhMDiOC1SVghZedcWElfpsUXZYsdWyzrI73shT+V52Wo+9nSXI4Ol6v2D5xn6dO7xn5V0pcDr88utzxuq6fH9fOuGPo0q5On/W1+SWafYo39SdGhBEAKWBZllxOSy6nlKPRM8NuNGRFg0nQGIUSnqubdeF/g6GOR8gYBUOKe574XrHtPbxXxzopGAopGFLH+3T7nur2c4LGKJyfjEKRf03ccZi49SGjyHJ4eyj6vJftxkSOIeFzwtsT3zvueai7NiW+T//PV/h14cotQmJn808dTxgBgHQTH7JgD9NtgIqELWNkOgWlLqEnFLevkaTofonvJSUum4TP7nhfo/A6E3ke3RZbDoVjUPS9TOS9ev6sTq+Pb1un4+/x9er4rN5ef3rB2NSfwAjCCAAgbcWGLUQ9SDpjSkkAAGArwggAALAVYQQAANiKMAIAAGxFGAEAALYijAAAAFsRRgAAgK0IIwAAwFaEEQAAYCvCCAAAsBVhBAAA2IowAgAAbEUYAQAAtkqLu/aayC2RGxsbbW4JAADor+j3dvR7vCdpEUaampokSaWlpTa3BAAAJKupqUler7fH7ZbpK66MAKFQSEeOHFFeXp4syxqy921sbFRpaakOHjyo/Pz8IXvfkYRjTH+j/fgkjnE0GO3HJ3GMA2GMUVNTk4qLi+Vw9FwZkhY9Iw6HQyUlJcP2/vn5+aP2FyuKY0x/o/34JI5xNBjtxydxjMnqrUckigJWAABgK8IIAACwVUaHEY/Ho0ceeUQej8fupgwbjjH9jfbjkzjG0WC0H5/EMQ6ntChgBQAAo1dG94wAAAD7EUYAAICtCCMAAMBWhBEAAGCrjA4jv/3tbzV9+nRlZ2dr/vz5evPNN+1u0oCsXLlSF1xwgfLy8lRQUKAbb7xR+/btS9jnzjvvlGVZCY8LL7zQphYn78c//nGX9hcWFsa2G2P04x//WMXFxcrJydHll1+uvXv32tji5E2bNq3LMVqWpWXLlklKv3P4xhtvaMmSJSouLpZlWfrzn/+csL0/58zn8+nee+/VpEmTlJubq+uvv16HDh1K4VH0rrdjDAQCeuihh3TuuecqNzdXxcXF+ta3vqUjR44kvMfll1/e5bzedtttKT6SnvV1HvvzezmSz2Nfx9fd36RlWfrFL34R22ckn8P+fD+MhL/FjA0jzz//vB544AH98Ic/VGVlpS655BJdffXVqq6utrtpSauoqNCyZcu0fft2bd26Ve3t7Vq8eLGam5sT9vvqV7+qmpqa2OPll1+2qcUDM2vWrIT279mzJ7bt5z//uR5//HE9+eST2rFjhwoLC3XVVVfF7muUDnbs2JFwfFu3bpUk3XLLLbF90ukcNjc3a+7cuXryySe73d6fc/bAAw/ohRde0MaNG/XWW2/p5MmTuu666xQMBlN1GL3q7RhbWlq0e/du/ehHP9Lu3bu1adMmffLJJ7r++uu77Hv33XcnnNennnoqFc3vl77Oo9T37+VIPo99HV/8cdXU1OiZZ56RZVm6+eabE/YbqeewP98PI+Jv0WSoL33pS2bp0qUJ62bOnGl+8IMf2NSioVNXV2ckmYqKiti6O+64w9xwww32NWqQHnnkETN37txut4VCIVNYWGgee+yx2Lq2tjbj9XrNmjVrUtTCoXf//feb0047zYRCIWNMep9DSeaFF16ILffnnNXX1xu32202btwY2+fw4cPG4XCYLVu2pKzt/dX5GLvzt7/9zUgyBw4ciK277LLLzP333z+8jRsi3R1jX7+X6XQe+3MOb7jhBnPFFVckrEunc9j5+2Gk/C1mZM+I3+/Xrl27tHjx4oT1ixcv1ttvv21Tq4ZOQ0ODJGnChAkJ68vLy1VQUKAzzzxTd999t+rq6uxo3oDt379fxcXFmj59um677TZ9+umnkqSqqirV1tYmnE+Px6PLLrssbc+n3+/Xhg0b9J3vfCfh5pDpfg6j+nPOdu3apUAgkLBPcXGxZs+enbbntaGhQZZlady4cQnr/+M//kOTJk3SrFmztHz58rTq0ZN6/70cTefx6NGj2rx5s7773e922ZYu57Dz98NI+VtMixvlDbVjx44pGAxqypQpCeunTJmi2tpam1o1NIwxevDBB3XxxRdr9uzZsfVXX321brnlFp166qmqqqrSj370I11xxRXatWtXWswm+OUvf1nr16/XmWeeqaNHj+pnP/uZLrroIu3duzd2zro7nwcOHLCjuYP25z//WfX19brzzjtj69L9HMbrzzmrra1VVlaWxo8f32WfdPw7bWtr0w9+8AN94xvfSLgB2e23367p06ersLBQH3zwgVasWKH33nsvNkw30vX1ezmazuNzzz2nvLw83XTTTQnr0+Ucdvf9MFL+FjMyjETF/x+nFD5Rndelm3vuuUfvv/++3nrrrYT1t956a+z57NmztWDBAp166qnavHlzlz+skejqq6+OPT/33HO1cOFCnXbaaXruuedixXKj6XyuW7dOV199tYqLi2Pr0v0cdmcg5ywdz2sgENBtt92mUCik3/72twnb7r777tjz2bNn64wzztCCBQu0e/dunX/++aluatIG+nuZjufxmWee0e23367s7OyE9elyDnv6fpDs/1vMyGGaSZMmyel0dkl0dXV1XdJhOrn33nv14osvatu2bSopKel136KiIp166qnav39/ilo3tHJzc3Xuuedq//79satqRsv5PHDggF577TXdddddve6XzuewP+essLBQfr9fJ06c6HGfdBAIBPT1r39dVVVV2rp1a5+3ZT///PPldrvT8rxKXX8vR8t5fPPNN7Vv374+/y6lkXkOe/p+GCl/ixkZRrKysjR//vwuXWhbt27VRRddZFOrBs4Yo3vuuUebNm3S66+/runTp/f5mi+++EIHDx5UUVFRClo49Hw+nz766CMVFRXFukfjz6ff71dFRUVans9nn31WBQUFuvbaa3vdL53PYX/O2fz58+V2uxP2qamp0QcffJA25zUaRPbv36/XXntNEydO7PM1e/fuVSAQSMvzKnX9vRwN51EK91bOnz9fc+fO7XPfkXQO+/p+GDF/i0NSBpuGNm7caNxut1m3bp358MMPzQMPPGByc3PNZ599ZnfTkva9733PeL1eU15ebmpqamKPlpYWY4wxTU1N5vvf/755++23TVVVldm2bZtZuHChOeWUU0xjY6PNre+f73//+6a8vNx8+umnZvv27ea6664zeXl5sfP12GOPGa/XazZt2mT27Nlj/vVf/9UUFRWlzfFFBYNBM3XqVPPQQw8lrE/Hc9jU1GQqKytNZWWlkWQef/xxU1lZGbuSpD/nbOnSpaakpMS89tprZvfu3eaKK64wc+fONe3t7XYdVoLejjEQCJjrr7/elJSUmHfffTfhb9Pn8xljjPn73/9ufvKTn5gdO3aYqqoqs3nzZjNz5kwzb968tDjG/v5ejuTz2NfvqTHGNDQ0mDFjxpjVq1d3ef1IP4d9fT8YMzL+FjM2jBhjzG9+8xtz6qmnmqysLHP++ecnXAqbTiR1+3j22WeNMca0tLSYxYsXm8mTJxu3222mTp1q7rjjDlNdXW1vw5Nw6623mqKiIuN2u01xcbG56aabzN69e2PbQ6GQeeSRR0xhYaHxeDzm0ksvNXv27LGxxQPz17/+1Ugy+/btS1ifjudw27Zt3f5e3nHHHcaY/p2z1tZWc88995gJEyaYnJwcc911142oY+7tGKuqqnr829y2bZsxxpjq6mpz6aWXmgkTJpisrCxz2mmnmfvuu8988cUX9h5YnN6Osb+/lyP5PPb1e2qMMU899ZTJyckx9fX1XV4/0s9hX98PxoyMv0Ur0lgAAABbZGTNCAAAGDkIIwAAwFaEEQAAYCvCCAAAsBVhBAAA2IowAgAAbEUYAQAAtiKMAAAAWxFGAACArQgjAADAVoQRAABgK8IIAACw1f8HP6vja6eC8UkAAAAASUVORK5CYII=",
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"foo = [l[0] for l in losses]\n",
"bar = [l[1]for l in losses]\n",
"plt.plot(foo,bar)"
]
},
{
"cell_type": "code",
"execution_count": 1139,
"id": "21a294f2",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"adexze\n",
"n\n",
"eegllurailazitynnellin\n",
"hittain\n",
"yllayn\n",
"ema\n",
"far\n",
"vre\n",
"niyaubrtthrigotai\n",
"aid\n",
"ia\n",
"a\n",
"kanio\n",
"in\n",
"stteda\n",
"mek\n",
"ie\n",
"y\n",
"ia\n",
"nsede\n"
]
}
],
"source": [
"### inference - take One-Hot encoded input and return prediction index of output\n",
"import random\n",
"g = torch.Generator().manual_seed(2147483647)\n",
"\n",
"for i in range(20):\n",
" # instead of starting with '.e' pick a random start of trigram. Otherwise since\n",
" # trigram will always start with '.e' instead of just '.' like in the bigram example\n",
" # we will always start with a letter beginning with 'e'\n",
" res = list(random.choice([prefix[0:2] for prefix in trigrams]))\n",
"# res = ['.','e']\n",
" while True:\n",
" chars = res[-2:]\n",
" char_ixs = torch.tensor([torch.tensor(alpha.index(c)) for c in chars]) # alpha index of each prefix\n",
" input_enc = F.one_hot(char_ixs,num_classes=27).float() # one-hot each\n",
" input_enc = torch.cat(tuple(input_enc)) # merge the 2,27 one-hot into 1x54\n",
" \n",
" ## same as before from here forward\n",
" logits = input_enc @ W\n",
" counts = torch.exp(logits) # \"fake\" counts of bigrams\n",
" probs = counts/counts.sum()\n",
" pred_ix = torch.multinomial(probs,num_samples=1,replacement=True,generator=g).item() # sample from probs\n",
" char = alpha[pred_ix] # get predicted char for input\n",
" res.append(char)\n",
" if char == '.':\n",
" break \n",
" print(\"\".join(res)[1:-1])\n",
" "
]
},
{
"cell_type": "markdown",
"id": "d85c3b3c",
"metadata": {},
"source": [
"## Trigrams Table Lookup"
]
},
{
"cell_type": "code",
"execution_count": 1113,
"id": "e1db79c9",
"metadata": {},
"outputs": [],
"source": [
"trigram_map = defaultdict(lambda: defaultdict(lambda: defaultdict(int))) # oh boy\n",
"for name in names:\n",
" for a,b,c in zip(name,name[1:],name[2:]):\n",
" trigram_map[a][b][c] += 1\n",
"\n",
"# do smoothing by defaulting all counts to 1\n",
"alpha = '.abcdefghijklmnopqrstuvwxyz'\n",
"ones = [a1+a2+a3 for a1 in alpha for a2 in alpha for a3 in alpha]\n",
"for a,b,c in ones:\n",
" bigram_map[a][b][c] += 1\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 1137,
"id": "2ea0dc36",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"idrendun\n",
"e\n",
"saldece\n",
"mundle\n",
"rshiyonischan\n",
"la\n",
"omer\n",
"mi\n",
"heo\n",
"waryn\n",
"ngjorisleil\n",
"ann\n",
"ee\n",
"lei\n",
"nslilyonolleya\n",
"coper\n",
"sh\n",
"l\n",
"thaid\n",
"ny\n"
]
}
],
"source": [
"g = torch.Generator().manual_seed(2147483647)\n",
"\n",
"for i in range(20):\n",
"# res = ['.','e']\n",
" res = list(random.choice([prefix[0:2] for prefix in trigrams]))\n",
" while True:\n",
" a,b = res[-2:]\n",
" opts = trigram_map[a][b] # char -> nextchar:count\n",
" sorted_opts = sorted(opts.items())\n",
" cnts = [x[1] for x in sorted_opts] # frequency of next char\n",
" probs = torch.tensor([c/sum(cnts) for c in cnts]) # convert freq to probs\n",
" pick_ix = torch.multinomial(probs,num_samples=1,replacement=True,generator=g).item()\n",
" char = sorted_opts[pick_ix][0]\n",
" res.append(char)\n",
" if char == '.':\n",
" break\n",
" print(\"\".join(res)[1:-1])\n"
]
},
{
"cell_type": "code",
"execution_count": 1116,
"id": "e2cb2932",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"tensor(-404377.3750) 196113 tensor(2.0620)\n"
]
}
],
"source": [
"n = 0\n",
"logloss = 0\n",
"for name in names:\n",
" for a,b,c in zip(name,name[1:],name[2:]):\n",
" cntmap = trigram_map[a][b] #{letter:cnt}\n",
" total = sum(cntmap.values())\n",
" prob = cntmap[c] / total\n",
" n += 1\n",
" logloss += torch.log(torch.tensor(prob))\n",
"\n",
"loss = -logloss/n\n",
"print(logloss,n,loss)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment