cjams

for s in data_trn[:nr_trn]:
    cur_tokens += list(s)

init_token_count = len(cur_tokens)
nr_actual_tokens = len(stoi)
nr_desired_tokens = 5000

print(f"Initial token count: {init_token_count}")

while nr_actual_tokens < nr_desired_tokens:

cjams

class LayerNorm():
    def __init__(self, device, num_features):
        self.out = None
        self.gamma = torch.ones(num_features, device=device)
        self.bias = torch.zeros(num_features, device=device)

    def __call__(self, x: torch.Tensor):
        assert x.ndim == 2

        H = x.shape[1]

cjams

%matplotlib inline

def plot_loss(trn_loss, val_loss=None, title="Loss Curves"):
    plt.figure(figsize=(10, 6))
    plt.xticks(fontsize=12)
    plt.yticks(fontsize=12)
    plt.title(title)

    legends = []

cjams

model = [
    Embedding(device=device, num_embeddings=vocab_size, embedding_dim=embed_dim),
    Flatten(input_dim1=ctx_window, input_dim2=embed_dim),
    Linear(device=device, in_features=ctx_window*embed_dim, out_features=hidden_size, bias=True),
    Tanh(),
    Linear(device=device, in_features=hidden_size, out_features=hidden_size, bias=True),
    Tanh(),
    Linear(device=device, in_features=hidden_size, out_features=hidden_size, bias=True),
    Tanh(),
    Linear(device=device, in_features=hidden_size, out_features=hidden_size, bias=True),

cjams

story = ‘’
ctx = [0] * ctx_window # start with context full of “special” characters

while True:
    x = torch.tensor([ctx], device=device)

    for layer in model:
        x = layer(x)

    counts = x.exp()

cjams

# Computing perplexity
with torch.no_grad():
    test_strs = data_val[21800:]
    total_nll = 0.0
    total_tokens = 0

    for test_str in test_strs:
        seq_nll = 0.0
        ctx = [0] * ctx_window

cjams

def plot_loss(trn_loss, val_loss=None, title=”Loss Curves”):
    plt.figure(figsize=(10, 6)) 
    plt.xticks(fontsize=12)
    plt.yticks(fontsize=12)
    plt.title(title)

    legends = []

    assert len(trn_loss) % 1000 == 0
    plt.plot(torch.tensor(trn_loss).view(-1, 1000).mean(dim=1))

cjams

model = [
    Embedding(device=device, num_embeddings=vocab_size, embedding_dim=embed_dim),
    Flatten(device=device, input_dim1=ctx_window, input_dim2=embed_dim),
    Linear(device=device, in_features=ctx_window*embed_dim, out_features=hidden_size, bias=True),
    Tanh(device=device),
    Linear(device=device, in_features=hidden_size, out_features=vocab_size, bias=False)
]

params = [p for layer in model for p in layer.params()]

cjams

ctx_window = 8
max_step = 100000
batch_size = 64
embed_dim = 32
hidden_size = 256
lr = 1e-3
vocab_size = len(stoi)
device = torch.device(”cuda” if torch.cuda.is_available() else “cpu”)

cjams

class Embedding():
    def __init__(self, device, num_embeddings, embedding_dim):
        self.out = None
        self.weight = torch.randn(num_embeddings, embedding_dim, device=device)

    def __call__(self, x):
        self.out = F.embedding(x, self.weight)
        return self.out

    def params(self):