Skip to content

Instantly share code, notes, and snippets.

@owahltinez

owahltinez/goemotions-dataset-generation.ipynb Secret

Created October 18, 2022 22:52
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save owahltinez/ebd6587dea4c56127367d8967f23be8b to your computer and use it in GitHub Desktop.
Save owahltinez/ebd6587dea4c56127367d8967f23be8b to your computer and use it in GitHub Desktop.
Download ZIP
GoEmotions Dataset: Generation
Display the source blob
Display the rendered blob
Raw
goemotions-dataset-generation.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/owahltinez/ebd6587dea4c56127367d8967f23be8b/goemotions-dataset-generation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"id": "c-6AaDi8w6n9",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "e249c693-b2f4-4d50-ab21-8dfe6c129059"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\u001b[K |████████████████████████████████| 511.8 MB 8.6 kB/s \n",
"\u001b[K |████████████████████████████████| 438 kB 71.7 MB/s \n",
"\u001b[K |████████████████████████████████| 5.8 MB 52.3 MB/s \n",
"\u001b[K |████████████████████████████████| 1.6 MB 58.8 MB/s \n",
"\u001b[K |████████████████████████████████| 4.6 MB 5.2 MB/s \n",
"\u001b[?25h"
]
}
],
"source": [
"#@title Install Dependencies\n",
"!pip install -q \"tensorflow==2.9.*\"\n",
"!pip install -q \"tensorflow-text==2.9.*\""
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"id": "_tOll1k9jIPV",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "311a85f2-1ec1-4e69-c09b-3b955f39f435"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Downloading data from https://huggingface.co/owahltinez/goemotions_bert/resolve/main/sentiment140_goemotions.csv\n",
"136831517/136831517 [==============================] - 2s 0us/step\n"
]
}
],
"source": [
"#@title Download the Re-Classified Dataset\n",
"import tensorflow as tf\n",
"\n",
"batch_size = 128\n",
"url_root = 'https://huggingface.co/owahltinez/goemotions_bert/resolve/main'\n",
"url_data = f'{url_root}/sentiment140_goemotions.csv'\n",
"data_path = tf.keras.utils.get_file(origin=url_data)\n",
"ds = tf.data.experimental.make_csv_dataset(data_path, batch_size)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"id": "XEwLHtGKj-dF"
},
"outputs": [],
"source": [
"#@title Extracting text samples for a single emotion\n",
"\n",
"def filter_dataset_by_emotion(ds, emotion, batch_size):\n",
" ds = ds.unbatch()\n",
" ds = ds.filter(lambda x: tf.equal(x['label'], emotion))\n",
" return ds.batch(batch_size).prefetch(tf.data.AUTOTUNE)\n",
"\n",
"ds_emo = filter_dataset_by_emotion(ds, 'disappointment', batch_size)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"id": "UvQeFT9dMRTS"
},
"outputs": [],
"source": [
"#@title Preprocessing text for the miniature GPT model\n",
"\n",
"def prepare_lm_inputs_labels(text, vectorized_layer):\n",
" text = tf.expand_dims(text, -1)\n",
" tokenized_sentences = vectorize_layer(text)\n",
" x = tokenized_sentences[:, :-1]\n",
" y = tokenized_sentences[:, 1:]\n",
" return x, y\n",
"\n",
"maxlen = 80 # Max sequence size\n",
"vocab_size = 20000 # Only consider the top 20k words\n",
"\n",
"# Create a vectorization layer to tokenize text.\n",
"vectorize_layer = tf.keras.layers.TextVectorization(\n",
" max_tokens=vocab_size - 1,\n",
" output_mode=\"int\",\n",
" output_sequence_length=maxlen + 1,\n",
")\n",
"\n",
"# Adapt the vectorization layer to our text samples.\n",
"vectorize_layer.adapt(ds.map(lambda x: x['sentence']).take(10_000))\n",
"vocab = vectorize_layer.get_vocabulary()\n",
"\n",
"# Build a tokenization index for input prompts.\n",
"word_to_index = {}\n",
"for index, word in enumerate(vocab):\n",
" word_to_index[word] = index\n",
"\n",
"text_ds = ds_emo.map(lambda x: x['sentence'])\n",
"text_ds = text_ds.map(lambda x: prepare_lm_inputs_labels(x, vectorize_layer))\n",
"text_ds = text_ds.prefetch(tf.data.AUTOTUNE)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"id": "avq_UvtAw1-H"
},
"outputs": [],
"source": [
"#@title Implementing the miniature GPT architecture\n",
"import numpy as np\n",
"\n",
"\n",
"embed_dim = 256 # Embedding size for each token\n",
"num_heads = 2 # Number of attention heads\n",
"feed_forward_dim = 256 # Hidden layer size in feed forward network inside transformer\n",
"\n",
"\n",
"def causal_attention_mask(batch_size, n_dest, n_src, dtype):\n",
" i = tf.range(n_dest)[:, None]\n",
" j = tf.range(n_src)\n",
" m = i >= j - n_src + n_dest\n",
" mask = tf.cast(m, dtype)\n",
" mask = tf.reshape(mask, [1, n_dest, n_src])\n",
" mult = tf.concat(\n",
" [tf.expand_dims(batch_size, -1), tf.constant([1, 1], dtype=tf.int32)], 0)\n",
" return tf.tile(mask, mult)\n",
"\n",
"\n",
"class TransformerBlock(tf.keras.layers.Layer):\n",
" def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):\n",
" super(TransformerBlock, self).__init__()\n",
" self.att = tf.keras.layers.MultiHeadAttention(num_heads, embed_dim)\n",
" self.ffn = tf.keras.Sequential([\n",
" tf.keras.layers.Dense(ff_dim, activation=\"relu\"),\n",
" tf.keras.layers.Dense(embed_dim),\n",
" ])\n",
" self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
" self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
" self.dropout1 = tf.keras.layers.Dropout(rate)\n",
" self.dropout2 = tf.keras.layers.Dropout(rate)\n",
"\n",
" def call(self, inputs):\n",
" input_shape = tf.shape(inputs)\n",
" batch_size = input_shape[0]\n",
" seq_len = input_shape[1]\n",
" causal_mask = causal_attention_mask(batch_size, seq_len, seq_len, tf.bool)\n",
" attention_output = self.att(inputs, inputs, attention_mask=causal_mask)\n",
" attention_output = self.dropout1(attention_output)\n",
" out1 = self.layernorm1(inputs + attention_output)\n",
" ffn_output = self.ffn(out1)\n",
" ffn_output = self.dropout2(ffn_output)\n",
" return self.layernorm2(out1 + ffn_output)\n",
"\n",
"class TokenAndPositionEmbedding(tf.keras.layers.Layer):\n",
" def __init__(self, maxlen, vocab_size, embed_dim):\n",
" super(TokenAndPositionEmbedding, self).__init__()\n",
" self.token_emb = tf.keras.layers.Embedding(\n",
" input_dim=vocab_size, output_dim=embed_dim)\n",
" self.pos_emb = tf.keras.layers.Embedding(\n",
" input_dim=maxlen, output_dim=embed_dim)\n",
"\n",
" def call(self, x):\n",
" maxlen = tf.shape(x)[-1]\n",
" positions = tf.range(start=0, limit=maxlen, delta=1)\n",
" positions = self.pos_emb(positions)\n",
" x = self.token_emb(x)\n",
" return x + positions\n",
"\n",
"\n",
"class TextGenerator(tf.keras.callbacks.Callback):\n",
" def __init__(self, max_tokens, start_tokens, index_to_word, top_k=10):\n",
" self.max_tokens = max_tokens\n",
" self.start_tokens = start_tokens\n",
" self.index_to_word = index_to_word\n",
" self.k = top_k\n",
"\n",
" def sample_from(self, logits):\n",
" logits, indices = tf.math.top_k(logits, k=self.k, sorted=True)\n",
" indices = np.asarray(indices).astype(\"int32\")\n",
" preds = tf.keras.activations.softmax(tf.expand_dims(logits, 0))[0]\n",
" preds = np.asarray(preds).astype(\"float32\")\n",
" return np.random.choice(indices, p=preds)\n",
"\n",
" def detokenize(self, number):\n",
" return self.index_to_word[number]\n",
"\n",
" def on_epoch_end(self, epoch, logs=None):\n",
" start_tokens = [_ for _ in self.start_tokens]\n",
" num_tokens_generated = 0\n",
" tokens_generated = []\n",
" while num_tokens_generated <= self.max_tokens:\n",
" pad_len = maxlen - len(start_tokens)\n",
" sample_index = len(start_tokens) - 1\n",
" if pad_len < 0:\n",
" x = start_tokens[:maxlen]\n",
" sample_index = maxlen - 1\n",
" elif pad_len > 0:\n",
" x = start_tokens + [0] * pad_len\n",
" else:\n",
" x = start_tokens\n",
" x = np.array([x])\n",
" y, _ = self.model.predict(x, verbose=0)\n",
" sample_token = self.sample_from(y[0][sample_index])\n",
" tokens_generated.append(sample_token)\n",
" start_tokens.append(sample_token)\n",
" num_tokens_generated = len(tokens_generated)\n",
" txt = \" \".join(\n",
" [self.detokenize(_) for _ in self.start_tokens + tokens_generated])\n",
" print(f\"\\ngenerated text:\\n{txt}\\n\")\n",
"\n",
"\n",
"def create_model():\n",
" inputs = tf.keras.layers.Input(shape=(maxlen,), dtype=tf.int32)\n",
" embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\n",
" x = embedding_layer(inputs)\n",
" transformer_block = TransformerBlock(embed_dim, num_heads, feed_forward_dim)\n",
" x = transformer_block(x)\n",
" outputs = tf.keras.layers.Dense(vocab_size)(x)\n",
" return tf.keras.Model(inputs=inputs, outputs=[outputs, x])"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "tZf_No8nNcro",
"outputId": "1a72deb6-23a6-4952-d691-ab1fa50cf44a"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Epoch 1/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 1.7985 - dense_2_loss: 1.7985\n",
"generated text:\n",
"i feel like you didnt have a new the [UNK] and i have to get my new [UNK] \n",
"\n",
"1000/1000 [==============================] - 189s 178ms/step - loss: 1.7985 - dense_2_loss: 1.7985\n",
"Epoch 2/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.9474 - dense_2_loss: 0.9474\n",
"generated text:\n",
"i feel like a bit of [UNK] \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.9474 - dense_2_loss: 0.9474\n",
"Epoch 3/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.9000 - dense_2_loss: 0.9000\n",
"generated text:\n",
"i feel like this time with the [UNK] [UNK] \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.9000 - dense_2_loss: 0.9000\n",
"Epoch 4/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.8709 - dense_2_loss: 0.8709\n",
"generated text:\n",
"i feel like this sucks so im not so so i cant do \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.8709 - dense_2_loss: 0.8709\n",
"Epoch 5/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.8466 - dense_2_loss: 0.8466\n",
"generated text:\n",
"i feel like that i am lost in the [UNK] [UNK] [UNK] \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.8466 - dense_2_loss: 0.8466\n",
"Epoch 6/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.8239 - dense_2_loss: 0.8239\n",
"generated text:\n",
"i feel like i missed [UNK] today \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.8239 - dense_2_loss: 0.8239\n",
"Epoch 7/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.8025 - dense_2_loss: 0.8025\n",
"generated text:\n",
"i feel like [UNK] and i think the only one of the only 1 [UNK] was a little bit disappointed \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.8025 - dense_2_loss: 0.8025\n",
"Epoch 8/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.7834 - dense_2_loss: 0.7834\n",
"generated text:\n",
"i feel like i missed the last night of my life \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.7834 - dense_2_loss: 0.7834\n",
"Epoch 9/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.7638 - dense_2_loss: 0.7638\n",
"generated text:\n",
"i feel like i missed my [UNK] \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.7638 - dense_2_loss: 0.7638\n",
"Epoch 10/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.7456 - dense_2_loss: 0.7456\n",
"generated text:\n",
"i feel like this is going to rain i miss that [UNK] \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.7456 - dense_2_loss: 0.7456\n",
"Epoch 11/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.7282 - dense_2_loss: 0.7282\n",
"generated text:\n",
"i feel like i missed the game \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.7282 - dense_2_loss: 0.7282\n",
"Epoch 12/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.7119 - dense_2_loss: 0.7119\n",
"generated text:\n",
"i feel like im going to work today \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.7119 - dense_2_loss: 0.7119\n",
"Epoch 13/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.6966 - dense_2_loss: 0.6966\n",
"generated text:\n",
"i feel like a little girl that was a total [UNK] \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.6966 - dense_2_loss: 0.6966\n",
"Epoch 14/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.6814 - dense_2_loss: 0.6814\n",
"generated text:\n",
"i feel like eating [UNK] i dont even know im upset i guess i will not get ready 4 more to get on the new [UNK] \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.6814 - dense_2_loss: 0.6814\n",
"Epoch 15/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.6667 - dense_2_loss: 0.6667\n",
"generated text:\n",
"i feel like i missed it \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.6667 - dense_2_loss: 0.6667\n",
"Epoch 16/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.6539 - dense_2_loss: 0.6539\n",
"generated text:\n",
"i feel like a little depressed right now \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.6539 - dense_2_loss: 0.6539\n",
"Epoch 17/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.6404 - dense_2_loss: 0.6404\n",
"generated text:\n",
"i feel like shit im not happy to do i have a headache and i have to leave this is boring \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.6404 - dense_2_loss: 0.6404\n",
"Epoch 18/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.6281 - dense_2_loss: 0.6281\n",
"generated text:\n",
"i feel like a [UNK] i am missing the [UNK] on that note to self all the time \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.6281 - dense_2_loss: 0.6281\n",
"Epoch 19/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.6164 - dense_2_loss: 0.6164\n",
"generated text:\n",
"i feel like i had a lot of shit lately i just dont really want to go to work tomorrow \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.6164 - dense_2_loss: 0.6164\n",
"Epoch 20/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.6052 - dense_2_loss: 0.6052\n",
"generated text:\n",
"i feel like i missed out on the beach at all day [UNK] \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.6052 - dense_2_loss: 0.6052\n",
"Epoch 21/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.5946 - dense_2_loss: 0.5946\n",
"generated text:\n",
"i feel like shit i cant wait to see [UNK] on facebook chat [UNK] \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.5946 - dense_2_loss: 0.5946\n",
"Epoch 22/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.5840 - dense_2_loss: 0.5840\n",
"generated text:\n",
"i feel like im gonna have a bad headache \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.5840 - dense_2_loss: 0.5840\n",
"Epoch 23/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.5737 - dense_2_loss: 0.5737\n",
"generated text:\n",
"i feel like im gonna go home tonight because im bored \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.5737 - dense_2_loss: 0.5737\n",
"Epoch 24/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.5647 - dense_2_loss: 0.5647\n",
"generated text:\n",
"i feel like a [UNK] right [UNK] but i cant \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.5647 - dense_2_loss: 0.5647\n",
"Epoch 25/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.5552 - dense_2_loss: 0.5552\n",
"generated text:\n",
"i feel like i didnt want to sleep yet again i have to go to sleep but didnt do any sence i suck \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.5552 - dense_2_loss: 0.5552\n",
"Epoch 26/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.5466 - dense_2_loss: 0.5466\n",
"generated text:\n",
"i feel like a piece of [UNK] or something wrong but i know it doesnt help me find out on that \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.5466 - dense_2_loss: 0.5466\n",
"Epoch 27/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.5386 - dense_2_loss: 0.5386\n",
"generated text:\n",
"i feel like a failure cuz i will miss [UNK] \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.5386 - dense_2_loss: 0.5386\n",
"Epoch 28/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.5306 - dense_2_loss: 0.5306\n",
"generated text:\n",
"i feel like a failure i am not a happy camper isnt working \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.5306 - dense_2_loss: 0.5306\n",
"Epoch 29/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.5232 - dense_2_loss: 0.5232\n",
"generated text:\n",
"i feel like im getting ready for work today but this is really boring day \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.5232 - dense_2_loss: 0.5232\n",
"Epoch 30/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.5160 - dense_2_loss: 0.5160\n",
"generated text:\n",
"i feel like i dont wanna be here but im going to get a [UNK] \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.5160 - dense_2_loss: 0.5160\n",
"Epoch 31/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.5089 - dense_2_loss: 0.5089\n",
"generated text:\n",
"i feel like this is being really bad its going to be a long time for the week \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.5089 - dense_2_loss: 0.5089\n",
"Epoch 32/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.5026 - dense_2_loss: 0.5026\n",
"generated text:\n",
"i feel like shit is going to have an exam on wednesday now so i have to study chemistry work amp i have to wait til 4pm over \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.5026 - dense_2_loss: 0.5026\n",
"Epoch 33/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4962 - dense_2_loss: 0.4962\n",
"generated text:\n",
"i feel like not a single life right now \n",
"\n",
"1000/1000 [==============================] - 176s 176ms/step - loss: 0.4962 - dense_2_loss: 0.4962\n",
"Epoch 34/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4903 - dense_2_loss: 0.4903\n",
"generated text:\n",
"i feel like i dont feel good \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4903 - dense_2_loss: 0.4903\n",
"Epoch 35/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4847 - dense_2_loss: 0.4847\n",
"generated text:\n",
"i feel like i am so bored of studying for fricken assignments these day the new least i dont have any more time to start \n",
"\n",
"1000/1000 [==============================] - 180s 180ms/step - loss: 0.4847 - dense_2_loss: 0.4847\n",
"Epoch 36/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4790 - dense_2_loss: 0.4790\n",
"generated text:\n",
"i feel like i missed a [UNK] \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4790 - dense_2_loss: 0.4790\n",
"Epoch 37/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4740 - dense_2_loss: 0.4740\n",
"generated text:\n",
"i feel like i cant work today \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4740 - dense_2_loss: 0.4740\n",
"Epoch 38/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4688 - dense_2_loss: 0.4688\n",
"generated text:\n",
"i feel like a [UNK] out of my life too but its not as much as [UNK] as i expected cant go \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4688 - dense_2_loss: 0.4688\n",
"Epoch 39/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4641 - dense_2_loss: 0.4641\n",
"generated text:\n",
"i feel like [UNK] i was going to sleep in [UNK] but it was too hot for the [UNK] but im still homesick \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.4641 - dense_2_loss: 0.4641\n",
"Epoch 40/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4596 - dense_2_loss: 0.4596\n",
"generated text:\n",
"i feel like shit [UNK] \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.4596 - dense_2_loss: 0.4596\n",
"Epoch 41/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4550 - dense_2_loss: 0.4550\n",
"generated text:\n",
"i feel like i cant watch it anymore but im still at the moment maybe we have to \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.4550 - dense_2_loss: 0.4550\n",
"Epoch 42/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4513 - dense_2_loss: 0.4513\n",
"generated text:\n",
"i feel like a failure at [UNK] \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.4513 - dense_2_loss: 0.4513\n",
"Epoch 43/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4470 - dense_2_loss: 0.4470\n",
"generated text:\n",
"i feel like ive been up so upset for a couple hours to go but it was too hot as well as i work on my phone charger \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4470 - dense_2_loss: 0.4470\n",
"Epoch 44/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4431 - dense_2_loss: 0.4431\n",
"generated text:\n",
"i feel like i am going to miss the sun \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4431 - dense_2_loss: 0.4431\n",
"Epoch 45/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4396 - dense_2_loss: 0.4396\n",
"generated text:\n",
"i feel like shit today i got a little hurt from my knee the lake [UNK] \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.4396 - dense_2_loss: 0.4396\n",
"Epoch 46/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4360 - dense_2_loss: 0.4360\n",
"generated text:\n",
"i feel like a [UNK] pills helped him alot i think its reputation anyway \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4360 - dense_2_loss: 0.4360\n",
"Epoch 47/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4325 - dense_2_loss: 0.4325\n",
"generated text:\n",
"i feel like i am not going to sleep well \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.4325 - dense_2_loss: 0.4325\n",
"Epoch 48/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4291 - dense_2_loss: 0.4291\n",
"generated text:\n",
"i feel like i cant sleep for this stupid thunderstorm and not even [UNK] [UNK] [UNK] \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4291 - dense_2_loss: 0.4291\n",
"Epoch 49/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4259 - dense_2_loss: 0.4259\n",
"generated text:\n",
"i feel like going to rain go out and go away but its friday for me \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.4259 - dense_2_loss: 0.4259\n",
"Epoch 50/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4230 - dense_2_loss: 0.4230\n",
"generated text:\n",
"i feel like a regular emails thing but im [UNK] just trying to be [UNK] boring \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4230 - dense_2_loss: 0.4230\n",
"Epoch 51/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4200 - dense_2_loss: 0.4200\n",
"generated text:\n",
"i feel like i cant believe the metal cable in any of the world tour missed check this time to sat to pay attention \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4200 - dense_2_loss: 0.4200\n",
"Epoch 52/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4169 - dense_2_loss: 0.4169\n",
"generated text:\n",
"i feel like [UNK] in [UNK] i missed out on a saturday morning at home \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4169 - dense_2_loss: 0.4169\n",
"Epoch 53/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4144 - dense_2_loss: 0.4144\n",
"generated text:\n",
"i feel like a little bit [UNK] a little [UNK] hehe np will be mine \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.4144 - dense_2_loss: 0.4144\n",
"Epoch 54/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4118 - dense_2_loss: 0.4118\n",
"generated text:\n",
"i feel like a wasted one day of my whole life is wasted \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.4118 - dense_2_loss: 0.4118\n",
"Epoch 55/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4090 - dense_2_loss: 0.4090\n",
"generated text:\n",
"i feel like [UNK] [UNK] i was too upset but the best i dont wanna \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4090 - dense_2_loss: 0.4090\n",
"Epoch 56/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4066 - dense_2_loss: 0.4066\n",
"generated text:\n",
"i feel like im getting sick \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.4066 - dense_2_loss: 0.4066\n",
"Epoch 57/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4038 - dense_2_loss: 0.4038\n",
"generated text:\n",
"i feel like a failure im missing my good tv right now \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4038 - dense_2_loss: 0.4038\n",
"Epoch 58/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.4019 - dense_2_loss: 0.4019\n",
"generated text:\n",
"i feel like i am in the office today but it has to be done [UNK] \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.4019 - dense_2_loss: 0.4019\n",
"Epoch 59/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3995 - dense_2_loss: 0.3995\n",
"generated text:\n",
"i feel like a failure thats screwed for the extra day \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3995 - dense_2_loss: 0.3995\n",
"Epoch 60/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3969 - dense_2_loss: 0.3969\n",
"generated text:\n",
"i feel like i havent talked to tay bear in forever and i would do a picture though \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3969 - dense_2_loss: 0.3969\n",
"Epoch 61/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3951 - dense_2_loss: 0.3951\n",
"generated text:\n",
"i feel like i am going to go to the movies that is not fun \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3951 - dense_2_loss: 0.3951\n",
"Epoch 62/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3930 - dense_2_loss: 0.3930\n",
"generated text:\n",
"i feel like im losing my [UNK] \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.3930 - dense_2_loss: 0.3930\n",
"Epoch 63/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3906 - dense_2_loss: 0.3906\n",
"generated text:\n",
"i feel like this is really upset that i cant even find my camera that i could really do a little bit \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.3906 - dense_2_loss: 0.3906\n",
"Epoch 64/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3889 - dense_2_loss: 0.3889\n",
"generated text:\n",
"i feel like going to work at a [UNK] lazy day \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3889 - dense_2_loss: 0.3889\n",
"Epoch 65/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3867 - dense_2_loss: 0.3867\n",
"generated text:\n",
"i feel like shit cause i lost six hours and its a memory stick is still stuck on a good sign \n",
"\n",
"1000/1000 [==============================] - 178s 177ms/step - loss: 0.3867 - dense_2_loss: 0.3867\n",
"Epoch 66/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3851 - dense_2_loss: 0.3851\n",
"generated text:\n",
"i feel like shit at work today i miss [UNK] \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3851 - dense_2_loss: 0.3851\n",
"Epoch 67/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3832 - dense_2_loss: 0.3832\n",
"generated text:\n",
"i feel like a bit upset but i didnt get a good song for a couple of months \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.3832 - dense_2_loss: 0.3832\n",
"Epoch 68/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3811 - dense_2_loss: 0.3811\n",
"generated text:\n",
"i feel like i am unemployed as level now \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.3811 - dense_2_loss: 0.3811\n",
"Epoch 69/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3798 - dense_2_loss: 0.3798\n",
"generated text:\n",
"i feel like im going to miss this \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3798 - dense_2_loss: 0.3798\n",
"Epoch 70/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3777 - dense_2_loss: 0.3777\n",
"generated text:\n",
"i feel like a failure enjoyment of mack the knife \n",
"\n",
"1000/1000 [==============================] - 178s 177ms/step - loss: 0.3777 - dense_2_loss: 0.3777\n",
"Epoch 71/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3761 - dense_2_loss: 0.3761\n",
"generated text:\n",
"i feel like im still bored \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3761 - dense_2_loss: 0.3761\n",
"Epoch 72/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3746 - dense_2_loss: 0.3746\n",
"generated text:\n",
"i feel like a regular [UNK] i just dont know its best but im [UNK] bored of the point of cops tie soccer [UNK] \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3746 - dense_2_loss: 0.3746\n",
"Epoch 73/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3728 - dense_2_loss: 0.3728\n",
"generated text:\n",
"i feel like i havent talked to the greatest while [UNK] as you [UNK] \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3728 - dense_2_loss: 0.3728\n",
"Epoch 74/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3715 - dense_2_loss: 0.3715\n",
"generated text:\n",
"i feel like going through work i need a new one more followers so far its depressing even [UNK] of work [UNK] \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3715 - dense_2_loss: 0.3715\n",
"Epoch 75/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3699 - dense_2_loss: 0.3699\n",
"generated text:\n",
"i feel like i dont have time to go to work tomorrow and my work is going to happen off the phone is depressing me \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3699 - dense_2_loss: 0.3699\n",
"Epoch 76/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3683 - dense_2_loss: 0.3683\n",
"generated text:\n",
"i feel like a loser i never got to see my friends stupid free [UNK] \n",
"\n",
"1000/1000 [==============================] - 178s 177ms/step - loss: 0.3683 - dense_2_loss: 0.3683\n",
"Epoch 77/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3669 - dense_2_loss: 0.3669\n",
"generated text:\n",
"i feel like i am stuck on a train [UNK] and its snowing april accident on fire day so fuzzy and doesnt have enough time \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.3669 - dense_2_loss: 0.3669\n",
"Epoch 78/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3654 - dense_2_loss: 0.3654\n",
"generated text:\n",
"i feel like a [UNK] at my mind right bout that healthy society xx \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3654 - dense_2_loss: 0.3654\n",
"Epoch 79/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3640 - dense_2_loss: 0.3640\n",
"generated text:\n",
"i feel like im not good at [UNK] anymore i miss my old friends [UNK] \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3640 - dense_2_loss: 0.3640\n",
"Epoch 80/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3627 - dense_2_loss: 0.3627\n",
"generated text:\n",
"i feel like a little dizzy and im not going my 2nd today \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3627 - dense_2_loss: 0.3627\n",
"Epoch 81/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3611 - dense_2_loss: 0.3611\n",
"generated text:\n",
"i feel like a failure i still havent even started studying reading ever [UNK] i just cant upgrade yet \n",
"\n",
"1000/1000 [==============================] - 178s 177ms/step - loss: 0.3611 - dense_2_loss: 0.3611\n",
"Epoch 82/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3598 - dense_2_loss: 0.3598\n",
"generated text:\n",
"i feel like a piece of the [UNK] [UNK] is missing cali too \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3598 - dense_2_loss: 0.3598\n",
"Epoch 83/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3586 - dense_2_loss: 0.3586\n",
"generated text:\n",
"i feel like this has been a pile of wood had an [UNK] variety of music going through my mind and got a nasty sunburn bad idea \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3586 - dense_2_loss: 0.3586\n",
"Epoch 84/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3573 - dense_2_loss: 0.3573\n",
"generated text:\n",
"i feel like i am thinking about [UNK] my [UNK] in the rain sucks i cant go out \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3573 - dense_2_loss: 0.3573\n",
"Epoch 85/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3560 - dense_2_loss: 0.3560\n",
"generated text:\n",
"i feel like going to work i feel shitty but im tired \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3560 - dense_2_loss: 0.3560\n",
"Epoch 86/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3550 - dense_2_loss: 0.3550\n",
"generated text:\n",
"i feel like im on my phone as well [UNK] \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3550 - dense_2_loss: 0.3550\n",
"Epoch 87/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3535 - dense_2_loss: 0.3535\n",
"generated text:\n",
"i feel like i dont want to work on the computer but im bored of work on it \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3535 - dense_2_loss: 0.3535\n",
"Epoch 88/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3524 - dense_2_loss: 0.3524\n",
"generated text:\n",
"i feel like the worst baker in the history of the world \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.3524 - dense_2_loss: 0.3524\n",
"Epoch 89/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3513 - dense_2_loss: 0.3513\n",
"generated text:\n",
"i feel like [UNK] i never said it would probably be like a loser \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.3513 - dense_2_loss: 0.3513\n",
"Epoch 90/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3500 - dense_2_loss: 0.3500\n",
"generated text:\n",
"i feel like going to work but not so happy about that \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3500 - dense_2_loss: 0.3500\n",
"Epoch 91/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3491 - dense_2_loss: 0.3491\n",
"generated text:\n",
"i feel like a bit of a sudden \n",
"\n",
"1000/1000 [==============================] - 177s 177ms/step - loss: 0.3491 - dense_2_loss: 0.3491\n",
"Epoch 92/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3480 - dense_2_loss: 0.3480\n",
"generated text:\n",
"i feel like shit i was at the last week or the movie was just so bored \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3480 - dense_2_loss: 0.3480\n",
"Epoch 93/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3470 - dense_2_loss: 0.3470\n",
"generated text:\n",
"i feel like this is not fun \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3470 - dense_2_loss: 0.3470\n",
"Epoch 94/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3460 - dense_2_loss: 0.3460\n",
"generated text:\n",
"i feel like going to the movies but i really wanted to be in the car and im back was a little cry \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3460 - dense_2_loss: 0.3460\n",
"Epoch 95/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3447 - dense_2_loss: 0.3447\n",
"generated text:\n",
"i feel like this for my life i guess imma just dont like that [UNK] hasnt let me yet \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3447 - dense_2_loss: 0.3447\n",
"Epoch 96/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3439 - dense_2_loss: 0.3439\n",
"generated text:\n",
"i feel like the worst baker in the history of the world \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3439 - dense_2_loss: 0.3439\n",
"Epoch 97/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3428 - dense_2_loss: 0.3428\n",
"generated text:\n",
"i feel like the worst baker in the history of the world \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3428 - dense_2_loss: 0.3428\n",
"Epoch 98/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3419 - dense_2_loss: 0.3419\n",
"generated text:\n",
"i feel like i should be working with my phone i can never get my tax amp checking out asylum [UNK] sat night \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3419 - dense_2_loss: 0.3419\n",
"Epoch 99/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3411 - dense_2_loss: 0.3411\n",
"generated text:\n",
"i feel like im getting sick again its a little bit of a sudden or something \n",
"\n",
"1000/1000 [==============================] - 178s 177ms/step - loss: 0.3411 - dense_2_loss: 0.3411\n",
"Epoch 100/100\n",
"1000/1000 [==============================] - ETA: 0s - loss: 0.3399 - dense_2_loss: 0.3399\n",
"generated text:\n",
"i feel like i am going to be a little shaky \n",
"\n",
"1000/1000 [==============================] - 178s 178ms/step - loss: 0.3399 - dense_2_loss: 0.3399\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<keras.callbacks.History at 0x7fa131154b90>"
]
},
"metadata": {},
"execution_count": 11
}
],
"source": [
"#@title Training the model and monitoring results\n",
"\n",
"start_prompt = \"i feel like\"\n",
"start_tokens = [word_to_index.get(_, 1) for _ in start_prompt.split()]\n",
"num_tokens_generated = 40\n",
"text_gen_callback = TextGenerator(num_tokens_generated, start_tokens, vocab)\n",
"\n",
"model = create_model()\n",
"optimizer = tf.keras.optimizers.Adam(learning_rate=1E-4)\n",
"loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
"model.compile(optimizer, loss=[loss_fn, None], jit_compile=True)\n",
"model.fit(text_ds, epochs=100, steps_per_epoch=1_000, callbacks=[text_gen_callback])"
]
},
{
"cell_type": "code",
"source": [
"#@title Generate text for arbitrary prompts\n",
"\n",
"def sample_from(logits, top_k=10):\n",
" logits, indices = tf.math.top_k(logits, k=top_k, sorted=True)\n",
" indices = np.asarray(indices).astype(\"int32\")\n",
" preds = tf.keras.activations.softmax(tf.expand_dims(logits, 0))[0]\n",
" preds = np.asarray(preds).astype(\"float32\")\n",
" return np.random.choice(indices, p=preds)\n",
"\n",
"def generate(model, start_prompt, max_tokens=80):\n",
" start_tokens_ = [word_to_index.get(_, 1) for _ in start_prompt.split()]\n",
" start_tokens = [_ for _ in start_tokens_]\n",
" num_tokens_generated = 0\n",
" tokens_generated = []\n",
" while num_tokens_generated <= max_tokens:\n",
" pad_len = maxlen - len(start_tokens)\n",
" sample_index = len(start_tokens) - 1\n",
" if pad_len < 0:\n",
" x = start_tokens[:maxlen]\n",
" sample_index = maxlen - 1\n",
" elif pad_len > 0:\n",
" x = start_tokens + [0] * pad_len\n",
" else:\n",
" x = start_tokens\n",
" x = np.array([x])\n",
" y, _ = model.predict(x, verbose=0)\n",
" sample_token = sample_from(y[0][sample_index])\n",
" tokens_generated.append(sample_token)\n",
" start_tokens.append(sample_token)\n",
" num_tokens_generated = len(tokens_generated)\n",
" return (' '.join(iter(map(lambda x: vocab[x], start_tokens_ + tokens_generated)))).strip()"
],
"metadata": {
"id": "64hQTwftY0th"
},
"execution_count": 19,
"outputs": []
},
{
"cell_type": "code",
"source": [
"for _ in range(10): print(generate(model, 'my day is'))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Bbn8kuXIaBs2",
"outputId": "b14d20c7-a1ad-4a99-fae7-8e0c00c20cc8"
},
"execution_count": 23,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"my day is [UNK]\n",
"my day is gone and all week gotta go away i dont have it\n",
"my day is gone and its not going out to go to school\n",
"my day is gone still havent done any classes in the time\n",
"my day is ruined cant even stand food bye bye\n",
"my day is so bad today i need to get my hair done tonight\n",
"my day is [UNK] not so much of [UNK] or something i can do bec i cant work amp that is still icky\n",
"my day is [UNK]\n",
"my day is pretty much not going on to work at 530 and i got a headache but dont feel good\n",
"my day is gone nothing of css or html or anything but i got 2 websites now which look very professional\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"for _ in range(10): print(generate(model, 'last night'))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "M5unFu-5bGOt",
"outputId": "bec5509b-e411-4f3c-be5b-93c81f27667d"
},
"execution_count": 29,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"last night was so bad\n",
"last night at the museum 2 last [UNK] dreamt that was last night though she was on this pretty bad place\n",
"last night of bible school im gonna have after babysitting tonight\n",
"last night of bible school i looked cause my parents cut cable\n",
"last night was [UNK] i was really disappointed when i got told you i had to cancel it\n",
"last night at the museum 2 last [UNK] felt good and i feel pretty good last week either\n",
"last night of bible school im babysitting because im gonna miss 1st amp not dancing\n",
"last night was [UNK] i was supposed to go see but the weekend after a long weekend but i got no plans for physio\n",
"last night of a bad start of [UNK] i am not going to have to work with the day\n",
"last night of bible school i shouldnt have from my 2nd grade 1st amp not [UNK]\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"for _ in range(10): print(generate(model, 'every time i'))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "db9xuKo6bS1r",
"outputId": "a5757f6b-1433-47f2-9adf-101ac513dc50"
},
"execution_count": 31,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"every time i see everything on [UNK] i used to see up on this beautiful day\n",
"every time i missed it i have to be at the end\n",
"every time i think im going to sleep in 2 months this week and i missed it\n",
"every time i see the outside today unfortunately this is the last summer i have\n",
"every time i get to the new pad who are going to throw up the shining over 15 blast\n",
"every time i just forget about the suffering from my [UNK] [UNK] post\n",
"every time i think ive missed the i get together last hands on the diagnosis\n",
"every time i get so out of my bday\n",
"every time i watched the mtv movie awards not australian films that yet\n",
"every time i think the hills are over i gotta hit ur show up cuz i dont think it is ever happening again\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"for _ in range(10): print(generate(model, 'i feel'))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "7eoIOBcdbvrI",
"outputId": "6155dd90-0370-49f9-99f9-7be966a01504"
},
"execution_count": 33,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"i feel so bad today [UNK]\n",
"i feel bad for the [UNK] in need of followers i thought this is it the matter that hes leaving please\n",
"i feel like this for a couple of days i cant seem to get rich asap once i can wake up\n",
"i feel like im losing my voice my throat hurts\n",
"i feel so bored and i am not even at all\n",
"i feel like im in the sun insted of stuck in the office\n",
"i feel like i am going to go [UNK] just [UNK]\n",
"i feel like a loser stayed home from the rest till i hope they didnt make it better\n",
"i feel so bad for the summer amp it was a long day but im really tired of it\n",
"i feel good today is bad im in a bad mood\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"for _ in range(10): print(generate(model, 'my phone'))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "qw2nWay0b-7A",
"outputId": "8deb2c8a-6aa8-448d-ee42-dda70c621578"
},
"execution_count": 36,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"my phone is broken i cant come\n",
"my phone is dead and im not feeling well [UNK]\n",
"my phone was acting sucks\n",
"my phone died before my oldest son didnt get to take a headache for you\n",
"my phone died again\n",
"my phone just left shoe box just made these pairs i was only 13 and they were wrong with only thing left\n",
"my phone is acting career no idea going its not going anywhere\n",
"my phone does not work on my phone\n",
"my phone does not work for the first day back i feel like i havent cried when my eyes work on my back time\n",
"my phone doesnt work im so bored\n"
]
}
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"collapsed_sections": [],
"provenance": [],
"authorship_tag": "ABX9TyMjCAhhkcDC5N6nCrBHPJZz",
"include_colab_link": true
},
"gpuClass": "standard",
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
@Cash-paid
Copy link

Good

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment