ChunML

ChunML

import * as tf from '@tensorflow/tfjs';
import React from 'react';
import ReactDOM from 'react-dom';

class CharacterTable {
  constructor(chars) {
    // chars must be a list of unique characters
    this.chars = chars;
    this.charIndices = {};
    this.indicesChar = {};

ChunML

What a ridiculous concept !
[[8, 5, 21, 22, 23]]
quel concept ridicule ! <end>
Your idea is not entirely crazy .
[[24, 25, 6, 26, 27, 28, 1]]
votre idee n est pas completement folle . <end>
A man s worth lies in what he is .
[[5, 29, 30, 31, 32, 9, 8, 7, 6, 1]]
la valeur d un homme reside dans ce qu il est . <end>
What he did is very wrong .

ChunML

encoder = Encoder(en_vocab_size, MODEL_SIZE, NUM_LAYERS, H)
decoder = Decoder(fr_vocab_size, MODEL_SIZE, NUM_LAYERS, H)

NUM_EPOCHS = 100

start_time = time.time()
for e in range(NUM_EPOCHS):
    for batch, (source_seq, target_seq_in, target_seq_out) in enumerate(dataset.take(-1)):
        loss = train_step(source_seq, target_seq_in,
                          target_seq_out)

ChunML

@tf.function
def train_step(source_seq, target_seq_in, target_seq_out):
    with tf.GradientTape() as tape:
        padding_mask = 1 - tf.cast(tf.equal(source_seq, 0), dtype=tf.float32)
        
        # Manually add two more dimentions
        # so that the mask's shape becomes (batch_size, 1, 1, seq_len)
        padding_mask = tf.expand_dims(padding_mask, axis=1)
        padding_mask = tf.expand_dims(padding_mask, axis=1)

ChunML

class MultiHeadAttention(tf.keras.Model):
    def __init__(self, model_size, h):
        super(MultiHeadAttention, self).__init__()
        self.key_size = model_size // h
        self.h = h
        self.wq = tf.keras.layers.Dense(model_size) #[tf.keras.layers.Dense(key_size) for _ in range(h)]
        self.wk = tf.keras.layers.Dense(model_size) #[tf.keras.layers.Dense(key_size) for _ in range(h)]
        self.wv = tf.keras.layers.Dense(model_size) #[tf.keras.layers.Dense(value_size) for _ in range(h)]
        self.wo = tf.keras.layers.Dense(model_size)
    

ChunML

Epoch 90 Loss 0.0023
Average elapsed time: 3.77s
A man s worth lies in what he is .
[[5, 29, 30, 31, 32, 9, 8, 7, 6, 1]]
la valeur d un homme reside dans ce qu il est . <end>

Epoch 100 Loss 0.0018
Average elapsed time: 3.76s
What a ridiculous concept !
[[8, 5, 21, 22, 23]]

ChunML

encoder = Encoder(en_vocab_size, MODEL_SIZE, NUM_LAYERS, H)
decoder = Decoder(fr_vocab_size, MODEL_SIZE, NUM_LAYERS, H)

NUM_EPOCHS = 100

start_time = time.time()
for e in range(NUM_EPOCHS):
    for batch, (source_seq, target_seq_in, target_seq_out) in enumerate(dataset.take(-1)):
        loss = train_step(source_seq, target_seq_in,
                          target_seq_out)

ChunML

@tf.function
def train_step(source_seq, target_seq_in, target_seq_out):
    with tf.GradientTape() as tape:
        # padding_mask of the source sequence
        # to be used in the Encoder
        # and the middle Multi-Head Attention of the Decoder
        padding_mask = 1 - tf.cast(tf.equal(source_seq, 0), dtype=tf.float32)

        encoder_output = encoder(source_seq, padding_mask)
        

ChunML

    def call(self, sequence, encoder_output, padding_mask):
        # EMBEDDING AND POSITIONAL EMBEDDING
        embed_out = embedding(sequence)
        embed_out += pes[:sequence.shape[1], :]
        
        bot_sub_in = embed_out
        
        for i in range(self.num_layers):
            # BOTTOM MULTIHEAD SUB LAYER