Last active
August 2, 2020 04:55
-
-
Save uni-3/559aeb67e86fb022b8480263b7608b1c to your computer and use it in GitHub Desktop.
matrix factorization with tensorflow
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"nbformat": 4, | |
"nbformat_minor": 0, | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 3", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.7.6" | |
}, | |
"colab": { | |
"name": "mf_keras.ipynb", | |
"provenance": [], | |
"collapsed_sections": [], | |
"include_colab_link": true | |
} | |
}, | |
"cells": [ | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"id": "view-in-github", | |
"colab_type": "text" | |
}, | |
"source": [ | |
"<a href=\"https://colab.research.google.com/gist/uni-3/559aeb67e86fb022b8480263b7608b1c/mf_keras.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "5wLON6eufKPP", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"import datetime\n", | |
"import numpy as np\n", | |
"import pandas as pd\n", | |
"import sklearn.model_selection\n", | |
"import tensorflow as tf\n", | |
"import matplotlib.pyplot as plt\n", | |
"import os\n", | |
"import sys" | |
], | |
"execution_count": 2, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"id": "G3HSls6VfKPu", | |
"colab_type": "text" | |
}, | |
"source": [ | |
"#### setup dataset" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "v79IwzJXfKPw", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 526 | |
}, | |
"outputId": "ec80c737-9fc5-4a12-a199-2b6a0d31c663" | |
}, | |
"source": [ | |
"# download dataset\n", | |
"!mkdir -p data\n", | |
"!curl 'http://files.grouplens.org/datasets/movielens/ml-100k.zip' -o ./data/ml-100k.zip\n", | |
"!unzip -o ./data/ml-100k.zip -d data" | |
], | |
"execution_count": 18, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"text": [ | |
" % Total % Received % Xferd Average Speed Time Time Time Current\n", | |
" Dload Upload Total Spent Left Speed\n", | |
"\r 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\r100 4808k 100 4808k 0 0 12.0M 0 --:--:-- --:--:-- --:--:-- 12.0M\n", | |
"Archive: ./data/ml-100k.zip\n", | |
" inflating: data/ml-100k/allbut.pl \n", | |
" inflating: data/ml-100k/mku.sh \n", | |
" inflating: data/ml-100k/README \n", | |
" inflating: data/ml-100k/u.data \n", | |
" inflating: data/ml-100k/u.genre \n", | |
" inflating: data/ml-100k/u.info \n", | |
" inflating: data/ml-100k/u.item \n", | |
" inflating: data/ml-100k/u.occupation \n", | |
" inflating: data/ml-100k/u.user \n", | |
" inflating: data/ml-100k/u1.base \n", | |
" inflating: data/ml-100k/u1.test \n", | |
" inflating: data/ml-100k/u2.base \n", | |
" inflating: data/ml-100k/u2.test \n", | |
" inflating: data/ml-100k/u3.base \n", | |
" inflating: data/ml-100k/u3.test \n", | |
" inflating: data/ml-100k/u4.base \n", | |
" inflating: data/ml-100k/u4.test \n", | |
" inflating: data/ml-100k/u5.base \n", | |
" inflating: data/ml-100k/u5.test \n", | |
" inflating: data/ml-100k/ua.base \n", | |
" inflating: data/ml-100k/ua.test \n", | |
" inflating: data/ml-100k/ub.base \n", | |
" inflating: data/ml-100k/ub.test \n" | |
], | |
"name": "stdout" | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "9R6dj_ctfP71", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 36 | |
}, | |
"outputId": "d01ef12c-bb98-459c-b6f9-a1e544f723a6" | |
}, | |
"source": [ | |
"!ls data" | |
], | |
"execution_count": 19, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"text": [ | |
"ml-100k ml-100k.zip\n" | |
], | |
"name": "stdout" | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "dA0a3uwXfKP7", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"def id_to_index(df):\n", | |
" # id to index\n", | |
" user_to_index = {uid: idx for idx, uid in enumerate(set(df['user_id']))}\n", | |
" item_to_index = {iid: idx for idx, iid in enumerate(set(df['item_id']))}\n", | |
" index_to_user = {idx: uid for idx, uid in enumerate(set(df['user_id']))}\n", | |
" index_to_item = {idx: iid for idx, iid in enumerate(set(df['item_id']))}\n", | |
" \n", | |
" return user_to_index, item_to_index, \\\n", | |
" index_to_user, index_to_item\n", | |
"\n", | |
"def load_movielens(input_file=\"./data/ml-100k/u.data\") -> (pd.DataFrame, dict, dict, dict, dict):\n", | |
" headers = ['user_id', 'item_id', 'rating', 'timestamp']\n", | |
"\n", | |
"\n", | |
" df = pd.read_csv(input_file,\n", | |
" sep='\\t',\n", | |
" names=headers, \n", | |
" header=None,\n", | |
" dtype={\n", | |
" 'user_id': np.int32,\n", | |
" 'item_id': np.int32,\n", | |
" 'rating': np.float32,\n", | |
" 'timestamp': np.int32,\n", | |
" })\n", | |
"\n", | |
" # id to index\n", | |
" user_to_index, item_to_index, \\\n", | |
" index_to_user, index_to_item = id_to_index(df)\n", | |
"\n", | |
" # add index col\n", | |
" df['user_id_index'] = df['user_id'].map(user_to_index)\n", | |
" df['item_id_index'] = df['item_id'].map(item_to_index)\n", | |
" return df, user_to_index, item_to_index,\\\n", | |
" index_to_user, index_to_item\n" | |
], | |
"execution_count": 20, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "0SFz5z2bfKQD", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"def prep_data(df, col_names=['user_id', 'item_id'], target='rating', norm=True, n_nega_sample: int=None) -> (pd.DataFrame, int, int):\n", | |
" \"\"\"\n", | |
"\n", | |
" Parameters\n", | |
" ----------\n", | |
" df: pd.DataFrame\n", | |
" [user_id, item_id, score]\n", | |
" col_names: list[string]\n", | |
" name of user_id and item_id\n", | |
" target: string\n", | |
" score\n", | |
" norm: bool\n", | |
" whether normalize rating or not\n", | |
" n_nega_sample: int\n", | |
" number of negative sampling from dataset\n", | |
" if None do not sampling\n", | |
" Returns\n", | |
" -------\n", | |
"\n", | |
" \"\"\"\n", | |
"\n", | |
" if norm:\n", | |
" df[target] = 1\n", | |
" n_user = df[col_names[0]].nunique()\n", | |
" n_item = df[col_names[1]].nunique()\n", | |
"\n", | |
" df_mat = pd.pivot_table(df, index=col_names[0],\n", | |
" columns=col_names[1], values=target, fill_value=0\n", | |
" ).stack().reset_index().rename(columns={0:'rating'})\n", | |
"\n", | |
" # positive samples\n", | |
" df_posi = df_mat[df_mat['rating']>0].reset_index(drop=True)\n", | |
" df_com = df_posi\n", | |
"\n", | |
" if n_nega_sample is not None:\n", | |
" df_nega = df_mat[df_mat['rating']==0].reset_index(drop=True)\n", | |
"\n", | |
" df_nega_samples = df_nega.groupby(col_names[0]) \\\n", | |
" .apply(lambda x: x.sample(n=min(n_nega_sample, len(x)), random_state=14)) \\\n", | |
" .reset_index(drop=True)\n", | |
"\n", | |
" df_com = pd.concat([df_posi, df_nega_samples]).reset_index(drop=True)\n", | |
"\n", | |
" return df_com, n_user, n_item\n" | |
], | |
"execution_count": 21, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "EUvuur09fKQM", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"df_raw, user_to_index, item_to_index,\\\n", | |
" index_to_user, index_to_item = load_movielens()\n", | |
"df_raw.shape\n", | |
"# prep data\n", | |
"col_names = ['user_id_index', 'item_id_index']\n", | |
"target = 'rating'\n", | |
"df, n_user, n_item = prep_data(df_raw,\n", | |
" #col_names=col_names,\n", | |
" n_nega_sample=5, norm=None\n", | |
" )\n", | |
"#df = df.astype({'user_id_index': 'int', 'user_id_index': 'int'})" | |
], | |
"execution_count": 22, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "8GHGqf3lfKQY", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 206 | |
}, | |
"outputId": "4699eab1-19bd-4a56-d26e-e9688fcfc6ff" | |
}, | |
"source": [ | |
"df_raw.head()" | |
], | |
"execution_count": 23, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/html": [ | |
"<div>\n", | |
"<style scoped>\n", | |
" .dataframe tbody tr th:only-of-type {\n", | |
" vertical-align: middle;\n", | |
" }\n", | |
"\n", | |
" .dataframe tbody tr th {\n", | |
" vertical-align: top;\n", | |
" }\n", | |
"\n", | |
" .dataframe thead th {\n", | |
" text-align: right;\n", | |
" }\n", | |
"</style>\n", | |
"<table border=\"1\" class=\"dataframe\">\n", | |
" <thead>\n", | |
" <tr style=\"text-align: right;\">\n", | |
" <th></th>\n", | |
" <th>user_id</th>\n", | |
" <th>item_id</th>\n", | |
" <th>rating</th>\n", | |
" <th>timestamp</th>\n", | |
" <th>user_id_index</th>\n", | |
" <th>item_id_index</th>\n", | |
" </tr>\n", | |
" </thead>\n", | |
" <tbody>\n", | |
" <tr>\n", | |
" <th>0</th>\n", | |
" <td>196</td>\n", | |
" <td>242</td>\n", | |
" <td>3.0</td>\n", | |
" <td>881250949</td>\n", | |
" <td>195</td>\n", | |
" <td>241</td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th>1</th>\n", | |
" <td>186</td>\n", | |
" <td>302</td>\n", | |
" <td>3.0</td>\n", | |
" <td>891717742</td>\n", | |
" <td>185</td>\n", | |
" <td>301</td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th>2</th>\n", | |
" <td>22</td>\n", | |
" <td>377</td>\n", | |
" <td>1.0</td>\n", | |
" <td>878887116</td>\n", | |
" <td>21</td>\n", | |
" <td>376</td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th>3</th>\n", | |
" <td>244</td>\n", | |
" <td>51</td>\n", | |
" <td>2.0</td>\n", | |
" <td>880606923</td>\n", | |
" <td>243</td>\n", | |
" <td>50</td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th>4</th>\n", | |
" <td>166</td>\n", | |
" <td>346</td>\n", | |
" <td>1.0</td>\n", | |
" <td>886397596</td>\n", | |
" <td>165</td>\n", | |
" <td>345</td>\n", | |
" </tr>\n", | |
" </tbody>\n", | |
"</table>\n", | |
"</div>" | |
], | |
"text/plain": [ | |
" user_id item_id rating timestamp user_id_index item_id_index\n", | |
"0 196 242 3.0 881250949 195 241\n", | |
"1 186 302 3.0 891717742 185 301\n", | |
"2 22 377 1.0 878887116 21 376\n", | |
"3 244 51 2.0 880606923 243 50\n", | |
"4 166 346 1.0 886397596 165 345" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 23 | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "u-G9piITfKQh", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 206 | |
}, | |
"outputId": "e2c4c6b0-b9de-4be0-9f05-153715552961" | |
}, | |
"source": [ | |
"df.head()" | |
], | |
"execution_count": 24, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/html": [ | |
"<div>\n", | |
"<style scoped>\n", | |
" .dataframe tbody tr th:only-of-type {\n", | |
" vertical-align: middle;\n", | |
" }\n", | |
"\n", | |
" .dataframe tbody tr th {\n", | |
" vertical-align: top;\n", | |
" }\n", | |
"\n", | |
" .dataframe thead th {\n", | |
" text-align: right;\n", | |
" }\n", | |
"</style>\n", | |
"<table border=\"1\" class=\"dataframe\">\n", | |
" <thead>\n", | |
" <tr style=\"text-align: right;\">\n", | |
" <th></th>\n", | |
" <th>user_id</th>\n", | |
" <th>item_id</th>\n", | |
" <th>rating</th>\n", | |
" </tr>\n", | |
" </thead>\n", | |
" <tbody>\n", | |
" <tr>\n", | |
" <th>0</th>\n", | |
" <td>1</td>\n", | |
" <td>1</td>\n", | |
" <td>5</td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th>1</th>\n", | |
" <td>1</td>\n", | |
" <td>2</td>\n", | |
" <td>3</td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th>2</th>\n", | |
" <td>1</td>\n", | |
" <td>3</td>\n", | |
" <td>4</td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th>3</th>\n", | |
" <td>1</td>\n", | |
" <td>4</td>\n", | |
" <td>3</td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th>4</th>\n", | |
" <td>1</td>\n", | |
" <td>5</td>\n", | |
" <td>3</td>\n", | |
" </tr>\n", | |
" </tbody>\n", | |
"</table>\n", | |
"</div>" | |
], | |
"text/plain": [ | |
" user_id item_id rating\n", | |
"0 1 1 5\n", | |
"1 1 2 3\n", | |
"2 1 3 4\n", | |
"3 1 4 3\n", | |
"4 1 5 3" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 24 | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "fsINgd_RfKQo", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"X_train, X_test, y_train, y_test \\\n", | |
" = sklearn.model_selection.train_test_split(df[['user_id', 'item_id']], df[target]\n", | |
" , test_size=0.05, random_state=1)\n", | |
" " | |
], | |
"execution_count": 25, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "ARWEtqnYfKQw", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 36 | |
}, | |
"outputId": "10c48f0b-c529-4f9e-8f45-87b8a3c34c84" | |
}, | |
"source": [ | |
"X_train.shape" | |
], | |
"execution_count": 26, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/plain": [ | |
"(99479, 2)" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 26 | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "2le9n0mSfKQ7", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 36 | |
}, | |
"outputId": "651ec478-4b2e-431d-bca5-66023f2f69ba" | |
}, | |
"source": [ | |
"X_test.shape" | |
], | |
"execution_count": 27, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/plain": [ | |
"(5236, 2)" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 27 | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "fXJYnroCfKRG", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 36 | |
}, | |
"outputId": "77ee3de1-3f36-4f05-f30b-04e74f7a3591" | |
}, | |
"source": [ | |
"X_train['user_id'].nunique()" | |
], | |
"execution_count": 28, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/plain": [ | |
"943" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 28 | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "LKeZLlLVfKRN", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 36 | |
}, | |
"outputId": "7811ae25-e96e-46f3-9ad9-4a6ab01a0f3e" | |
}, | |
"source": [ | |
"X_test['user_id'].nunique()" | |
], | |
"execution_count": 29, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/plain": [ | |
"877" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 29 | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "G7C06ajwfKRZ", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"user_id_input = tf.keras.layers.Input(shape=[1], name='user')\n", | |
"item_id_input = tf.keras.layers.Input(shape=[1], name='item')\n", | |
"\n", | |
"\n", | |
"embedding_size = 8\n", | |
"user_embedding = tf.keras.layers.Embedding(output_dim=embedding_size, input_dim=n_user+1,\n", | |
" #embeddings_initializer='glorot_uniform',\n", | |
" #embeddings_regularizer=tf.keras.regularizers.l2(l=0.01),\n", | |
" input_length=1,\n", | |
" name='user_embedding')(user_id_input)\n", | |
"item_embedding = tf.keras.layers.Embedding(output_dim=embedding_size, input_dim=n_item+1,\n", | |
" #embeddings_initializer='glorot_uniform',\n", | |
" #embeddings_regularizer=tf.keras.regularizers.l2(l=0.01),\n", | |
" input_length=1,\n", | |
" name='item_embedding')(item_id_input)\n", | |
"\n", | |
"user_vecs = tf.keras.layers.Reshape([embedding_size])(user_embedding)\n", | |
"item_vecs = tf.keras.layers.Reshape([embedding_size])(item_embedding)\n", | |
"\n", | |
"# Final prediction layer\n", | |
"# inner product(score)\n", | |
"y = tf.keras.layers.Dot(1, normalize=False, name=\"concat\")([user_vecs, item_vecs])\n", | |
"#y = tf.linalg.matmul([user_vecs, item_vecs])\n", | |
"\n", | |
"model = tf.keras.Model(inputs=[user_id_input, item_id_input], outputs=y)\n", | |
"\n", | |
"model.compile(loss='mse',\n", | |
" optimizer=\"adam\"\n", | |
" )" | |
], | |
"execution_count": 30, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "h6tqn0vWfKRj", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 74 | |
}, | |
"outputId": "a4a60d7b-490b-4bf3-cc86-69bec60b2995" | |
}, | |
"source": [ | |
"!pip install pydot-ng graphviz " | |
], | |
"execution_count": 31, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"text": [ | |
"Requirement already satisfied: pydot-ng in /usr/local/lib/python3.6/dist-packages (2.0.0)\n", | |
"Requirement already satisfied: graphviz in /usr/local/lib/python3.6/dist-packages (0.10.1)\n", | |
"Requirement already satisfied: pyparsing>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from pydot-ng) (2.4.7)\n" | |
], | |
"name": "stdout" | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "pDfJmM3_fKRr", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 369 | |
}, | |
"outputId": "537daf61-f6be-4bd1-9f81-0088e8be180e" | |
}, | |
"source": [ | |
"# show model\n", | |
"tf.keras.utils.plot_model(\n", | |
" model, #to_file='model.png',\n", | |
" show_shapes=False, show_layer_names=True,\n", | |
" rankdir='TB', expand_nested=True, dpi=96\n", | |
")\n" | |
], | |
"execution_count": 32, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAhcAAAFgCAIAAACHdMz7AAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3deUBU5d4H8OfMAjMDDKCxKKvgloqaISFiKeZNM3EBhNxyK9AULSu66jUyKc3esFTqmmalXVnUa4prahIiIKXmgqBiQIgIssnOLOf947x3Xi7CCIeBZ2b4fv7ynDPznN858zx+OcucYViWJQAAALwIaBcAAAAGDCkCAAD8IUUAAIA/pAgAAPAnol1Am3z++eepqam0qwCeRo0a9fbbb9OuontJTU39/PPPaVcBHWUQY8cwjkVSU1PT0tJoVwF8pKWl4S+ArvfXX3/t37+fdhXQIYYydgzjWIQQ4u3tnZCQQLsKaLegoCDaJXRfGDIGzVDGjmEciwAAgH5CigAAAH9IEQAA4A8pAgAA/CFFAACAP6QIAADwhxQBAAD+kCIAAMAfUgQAAPhDigAAAH9IEQAA4A8pAgAA/CFFAACAP6QIAADwhxTRgbS0tKefflogEDAMY2dnt2HDhi5b9YEDB9zc3BiGYRjG3t5+zpw5XbZqMALHjh2ztLQ8cuQIlbVj4BgHg/l9EX3m7e198+bNiRMnnjx5Mjs728rKqstWHRAQEBAQ0Ldv34cPHxYVFXXZesE4sCxLce0YOMYBxyKGp66uzsfHh3YVYAwmT55cWVk5ZcoUbtK4u5Zxbx1FSBHDs2vXruLiYtpVgBEy7q5l3FtHkfGkSHh4uImJib29PTf55ptvmpmZMQzz8OFDbk5SUpKXl5dMJpPL5R4eHo8ePSKEqFSqdevWOTs7S6XSoUOHxsXFEUI+/fRTmUxmYWFRXFy8atUqBweH7OzsEydOyOXyqKiothQTExNjZmYmk8l++umnSZMmyeVyR0fHffv2cUu//PJLiURia2sbFhbWq1cviUTi4+OTnp7elg1ZuXLlqlWrcnJyGIbp27dvG3dOcnLyoEGDLC0tJRKJh4fHyZMnCSGLFy/mzgu7u7tfvnyZELJgwQKZTGZpaXn48OF27Zw2lgF65fz5887OzgzDbNu2jbTUtVrsAFu2bDEzMxMIBM8++6ydnZ1YLDYzMxsxYsSYMWOcnJwkEomVldV7772nWQsGjvEPHNYQBAYGBgYGPvFls2fPtrOz00xu3ryZEFJSUsKybHV1tVwu37RpU11dXVFR0YwZM7j577zzjqmp6f79+8vLy1evXi0QCDIyMliWXbNmDSFkxYoVW7dunTFjxs2bNxMTEy0sLNavX9/a2l966SVCSHl5OTfJtXDmzJnKysri4uIxY8aYmZk1NjZyS0NDQ83MzDIzM+vr62/cuDFy5EgLC4v8/PwnbgjLsgEBAe7u7k1X7e7ubmlpqWXPJCQkREZGlpWVlZaWent79+zZU9OUUCi8d++e5pWzZs06fPgw9++27xwtq27jZwe6xf3P9cSX/fXXX4SQrVu3cpPNulZrHeCDDz4ghKSnp9fU1Dx8+HDixImEkKNHj5aUlNTU1ISHhxNCrly5wjWCgcNv4LCGM3a6S4pcv36dEJKYmNj09XV1dTKZLCQkhJusra01NTVdunQp+5/Pu66uru1FtjgYNC1s376dEHLnzh1uMjQ0tGn3zcjIIIR8+OGHT9wQltdgaOrjjz8mhBQXF7Mse/r0aULIhg0buEWVlZX9+vVTKpWs7naOoYwEI9PxFNHSAbgUqaqq4hZ9//33hJBr165xkxcvXiSExMbGtrFUDJzWGMrYMZ4zWtq5ubnZ2trOmTMnMjIyNzeXm5mdnV1bWztkyBBuUiqV2tvbZ2VldUYBJiYmhBCFQtHiUk9PT5lM1kmrbkYsFhNCVCoVIcTPz69///7ffvsty7KEkNjY2JCQEKFQSLp254AeansH4Pq2UqnkJrkO1lpXby8MHP3XXVJEKpWePXvW19c3KirKzc0tJCSkrq6upqaGELJ27VrmP/Ly8mpra6lUaGpqWlJS0kmNHz16dOzYsTY2Nqampk3PWTMMExYWdvfu3TNnzhBCfvjhh0WLFnGL9GrnQNczlA6AgUNdd0kRQsjgwYOPHDlSWFgYERERFxf32Wef2djYEEKio6ObHp2lpqZ2fW0KhaKiosLR0VGHbf7666/R0dGEkPz8/OnTp9vb26enp1dWVm7atKnpy+bPny+RSHbu3JmdnS2Xy11cXLj5+rNzgAqD6AAYOPrAqL51KBKJWjvyLSwsrKioGDRokI2NzSeffHLq1KnMzEzulpIrV650cZ2PO3fuHMuy3t7e3KSWDWm733//3czMjBBy7do1hUKxdOlSNzc3QgjDME1fZm1tHRwcHBsba2Fh8frrr2vm68/OASoMogNg4OgDozoW6du3b1lZ2aFDhxQKRUlJSV5enmZRYWFhWFhYVlZWY2Pj5cuX8/LyvL29JRLJggUL9u3bFxMT8+jRI5VKVVBQcP/+/RYbP378eNtvWGwLtVpdXl6uVCqvXr26cuVKZ2fn+fPnP3FDCCE9evQoLCzMzc2tqqpqccwoFIoHDx6cO3eOGwzOzs6EkNOnT9fX19++fVtzZ6TGkiVLGhoaEhMTNd8+I4S0a+eAcWjatYRCoU46AAaO8Q+czrlor2NtvFehtLR03LhxEomkT58+y5cvf/fddwkhffv2zc/Pz83N9fHxsba2FgqFvXv3XrNmDXdDRUNDQ0REhLOzs0gksrGxCQgIuHHjxqZNm6RSKSHEyclpz549XOPHjh2zsLDQ3JXRVFpa2uDBgwUCASHE3t4+Kipq+/btMpmMENKvX7+cnJwdO3bI5XJCiIuLy61bt1iWDQ0NFYvFDg4OIpFILpdPmzYtJyenLRvCsuylS5dcXFykUqmvr+9XX33l7u7e2od78OBBrsGIiIgePXpYWVkFBQVxXw5wd3fX3B/Jsuwzzzzz97//vdl2tX3ndPyzA91qyz1aW7du5b5dIZPJ/P392f/uWkVFRS12gC1btnB929XVNTk5eePGjZaWloQQOzu7H3/8MTY21s7OjhBibW29b98+FgOH78BhDWfsMCzVB+m0UVBQECEkISGBdiE6ExYWlpCQUFpaSruQ/zN58uRt27b16dNH5y0b32dnEOLj44ODgw1idLdL9xk4xHDGjlGd0TIs3C2DFGkO6q9evcr9+Ua3HoC2wMDRN0Z1dR3aJSIiYsmSJSzLLliwYM+ePbTLATAMGDjN4FiEgtWrV+/evbuysrJPnz779++nVYZMJhs4cOCLL74YGRk5aNAgWmUAtBEGjn7CdRHoXPjsqDDW6yLdiqGMHRyLAAAAf0gRAADgDykCAAD8IUUAAIA/pAgAAPCHFAEAAP6QIgAAwB9SBAAA+EOKAAAAf0gRAADgDykCAAD8IUUAAIA/pAgAAPBnML8vkpaWxj3hEgxLWlqat7c37Sq6KQwZg2YoY8cwUmTUqFG0S+hEv/32GyHE09OTdiGdwtvb27g/Pv3k5OQUGBhIu4rOdfjwYU9Pz969e9MupLMYytgxjN8XMW4zZ84khMTHx9MuBMCQMAwTFxfHDR+gCNdFAACAP6QIAADwhxQBAAD+kCIAAMAfUgQAAPhDigAAAH9IEQAA4A8pAgAA/CFFAACAP6QIAADwhxQBAAD+kCIAAMAfUgQAAPhDigAAAH9IEQAA4A8pAgAA/CFFAACAP6QIAADwhxQBAAD+kCIAAMAfUgQAAPhDigAAAH9IEQAA4A8pAgAA/CFFAACAP6QIAADwhxQBAAD+kCIAAMAfUgQAAPhDigAAAH9IEQAA4A8pAgAA/CFFAACAP4ZlWdo1dDvffffdli1bVCoVN1lSUkIIsbGx4SaFQuHKlSvnz59PqzwA/TR37twrV65oJnNzc21sbMzMzLhJsVh85MgRBwcHStV1XyLaBXRHo0aNWrBgQbOZDx480Pzb29u7aysCMAADBgzYu3dv0znV1dWafw8cOBARQgXOaFEwYMAADw8PhmEeX8QwjIeHx8CBA7u+KgA99+qrr7Y4agghYrEYh++0IEXomDdvnlAofHy+SCR67bXXur4eAP3n7u7+zDPPCAQt/K+lVCqDg4O7viQgSBFaZs2apbku0hQGA4AW8+bNezxFGIbx8vJydXWlUREgRSjp3bu3j49Ps/EgEAh8fHwcHR1pVQWg54KDg9VqdbOZAoFg3rx5VOoBghShaO7cuc1O8jIMg8EAoIW9vf2YMWMePxscEBBApR4gSBGKgoKCHr9UiMEAoN3cuXObTgoEgnHjxtnZ2dGqB5Ai1PTo0WPChAki0f/dbC0UCidMmNCzZ0+6VQHouaCgoGangpvlCnQxpAhNc+bM0ZzkZVkWgwHgieRy+cSJE5v++TV16lS6JXVzSBGapk6damJiwv1bLBb7+/vTrQfAIMyZM4e7xVEkEvn7+1taWtKuqFtDitBkZmbm7+8vFotFItG0adPMzc1pVwRgAPz9/aVSKSFEpVLNnj2bdjndHVKEstmzZyuVSpVKNWvWLNq1ABgGiUQyY8YMQohMJps0aRLtcrq7/3qOVkFBwYULF2iV0j2pVCqJRMKybHV1dXx8PO1yuhedfDsHo4YKJycnQsjIkSMPHz5Mu5Zux8nJadSoUf8/zTYRFxdHrzCArhYXF8d2GEYNdDeBgYFNh0ALz/TFs+K72C+//MIwzNixY2kX0r209lw/fjBqul5kZOTatWs1N2tB1wgKCmo2Bx8AfS+88ALtEgAMDyJET+AzoK/FZ5QCgHaIED2B/78AAIA/pAgAAPCHFAEAAP6QIgAAwB9SBAAA+EOKAAAAf0gRAADgDykCAAD8IUUAAIA/pAgAAPCHFAEAAP6QIgAAwF+3S5GRI0cKhcLhw4d3pJHFixdbWFgwDHPlypW2LD127JilpeWRI0c6stInOnDggJubG9MSV1dXHg0a8b7qekawXUbcHzB2OqLbpUhGRsa4ceM62MjOnTu/+eabti/tmh+fCAgIuHv3rru7u6WlJffrMUqlsra29sGDBzKZjEeDRryvup4RbJcR9weMnY7opo9W1u0vFD3R5MmTKysru3KNHKFQKJVKpVJp//79eTfSTfZVZ2u2XXV1dePHjzfEn9rtJv0BY6ftut2xCEcsFnewBe39Q4e9h2XZhISEHTt2dKSRQ4cO8X5vd9tXXWPXrl3FxcW0q+Cju/UHjJ02vbnZL0hr/5Xp5cuXi8ViOzs7bnLp0qXcEV9JSQk359y5cyNHjpRKpRYWFkOGDKmsrOQOD//xj384OTlJJBIPD4/Y2FiWZTdt2iSVSs3NzR88ePD222/37t07KytL+9pbbCc6OlomkzEMM2LECFtbW5FIJJPJnnnmGV9fX0dHR1NTU0tLy3fffVfTyPjx462trQcMGCCTySQSia+vb3JysvZVsCyrVqs//fTT/v37m5iYyOVyJycnQsjly5efuDQ5OZmb3Lp1K8uy27dvl8lkUqn00KFDEydOtLCwcHBw+Ne//tW0gKioqP79+0skkp49e7q4uAwfPry8vJxbevz4cQsLiw0bNrS2i5oelT+uW+0r7YhOf3dd+2uabdeKFStMTEy4Aeju7s7qqGNrh7GDsaOTsRMYGNjsd9fbnSIsy86ePVuTIizLbt68WZMi1dXVcrl806ZNdXV1RUVFM2bM4Oa/8847pqam+/fvLy8vX716tUAgyMjIYFl2zZo1hJAVK1Zs3bp1xowZN2/e1L7q1tr54IMPCCHp6ek1NTUPHz6cOHEiIeTo0aMlJSU1NTXh4eGEkCtXrnCNjB8/3s3N7c8//1QoFNevX3/uueckEsmtW7e0r2LNmjUMw/zP//xPeXl5bW3t9u3bm3662pf+9ddfmk9Xs9VnzpyprKwsLi4eM2aMmZlZY2MjtzQqKkooFP7000+1tbW///67nZ3d2LFjNXsgMTHRwsJi/fr1re2iZiNhxYoV165da/qC7rOvtOvKFGEf266AgAAuPzg66djaYexg7Ohk7HR6ily/fp0QkpiY2PT1dXV1MpksJCSEm6ytrTU1NV26dKlmO+vq6tpSvZZ2uE+3qqqKW/T9998TQjQ94OLFi4QQTdqPHz9+2LBhmmavXr1KCHnnnXe0rKK2tlYmk02YMEHzrn379mk+P+1L2VY+Xc1Wc13hzp073OTIkSO9vLw0Tb3xxhsCgaChoaEtu4hlWXd392aHmy2OBOwr/UkRXXVsLTB22gJjpy376vEU0fF1ETc3N1tb2zlz5kRGRubm5nIzs7Oza2trhwwZwk1KpVJ7e/usrKz2Nt72drjTBUqlkpvkzk4qFIoWm/Xw8LC0tOQ+49ZWcefOndra2vHjx7fYgvalT8RVqymvvr6ebXKvhUqlEovFQqGw7Q02+3uqLWvvtvtKH3RSx+6CVRhff8DY4dGyjlNEKpWePXvW19c3KirKzc0tJCSkrq6upqaGELJ27VrNLdh5eXm1tbXtbVxX7TxOLBZzO7e1VRQUFBBCbGxsWny79qXt9fLLL//+++8//fRTXV3db7/9dujQoVdeeYX3/4xbtmzRdFadMOJ9RUvndewuWIUR9weMnTbS/T1agwcPPnLkSGFhYURERFxc3GeffcZtdnR0dNODoNTU1Pa2rKt2mlEqlWVlZc7OzlpWIZFICCENDQ0ttqB9aXtFRkb6+fnNnz9fLpfPmDFj5syZWu4Z72LYV52hkzp2F6wC/aHtjHhf8UkRkUjU2lFbYWFhZmYmIcTGxuaTTz4ZMWJEZmYmdx9Ci1+/bBddtdPML7/8olarR4wYoWUVQ4YMEQgESUlJLbagfWl73bhxIycnp6SkRKFQ5Ofnx8TEWFtbd7DN+/fvL1iwoOO1dYd91fU6qWN3wSq6Q3/A2HkiPinSt2/fsrKyQ4cOKRSKkpKSvLw8zaLCwsKwsLCsrKzGxsbLly/n5eV5e3tLJJIFCxbs27cvJibm0aNHKpWqoKDg/v377V2vrtohhDQ2NlZWViqVykuXLoWHh7u4uMyfP1/LKmxsbAICAvbv379r165Hjx5dvXq16Y3V2pe217Jly5ydnaurq1tcevz4cblcHhUV1cbWWJatq6s7cOCAXC7nV4/h7it91qNHj8LCwtzc3KqqKqFQqKuO3RqMHYKx8990OXaaHlK18W6T0tLScePGSSSSPn36LF++/N133yWE9O3bNz8/Pzc318fHx9raWigU9u7de82aNUqlkmXZhoaGiIgIZ2dnkUjE7YsbN25w3xchhDg5Oe3Zs+eJ622tnS1btnDfWXF1dU1OTt64caOlpSUhxM7O7scff4yNjbWzsyOEWFtb79u3j2XZ3bt3jxs3jrvju2fPnq+++mpeXp72VbAsW1VVtXjx4p49e5qbm/v6+q5bt44Q4ujo+Mcff2hfunXrVnt7e0KITCbz9/fn7uMmhPTr1y8nJ2fHjh1cH3VxceHuAjx79mzPnj01H5BYLH766acPHDjAlXfs2LHW7nk/ePDg4zeZaKxdu5Zl2W61r7QjXXiPVrPtYln20qVLLi4uUqnU19e3qKhIJx1bO4wdjB2djB3d3OkLnWr79u0rV67UTDY0NLz11lumpqa1tbUUq9JPHdlXXZki0DUwdtqO9756PEW66XO09FZRUVF4eHjTE6YmJibOzs4KhUKhUHCHbsDBvoKm0B/aTrf7So+eo5WVldXik5k5ISEhtAvsClKpVCwW79q168GDBwqForCwcOfOnevWrQsJCeF9ctZYYV9pYOwQ9If20O2+0qNjkYEDB7KG//TsDrK0tDx16tT69ev79+9fU1Njbm4+ePDgjRs3vvHGG7RL0zvYVxoYOwT9oT10u6/0KEWAM2bMmJ9//pl2FYYB+wqaQn9oOx3uKz06owUAAAYHKQIAAPwhRQAAgD+kCAAA8IcUAQAA/pAiAADAH1IEAAD4Q4oAAAB/SBEAAOAPKQIAAPwhRQAAgD+kCAAA8IcUAQAA/lp4pm98fHzX1wFg0DBqoJsoKChwdHRsOqeFFAkODu6qegCMBEYNdB+BgYFNJxn8uI3+mzp1akVFRVJSEu1CAKhxcXF5/fXX165dS7sQaA7XRQzAihUrfv3114yMDNqFANBx7969/Px8X19f2oVAC5AiBsDPz2/YsGFbt26lXQgAHb/++qtYLPby8qJdCLQAKWIYVqxYERsbe+/ePdqFAFCQkpLyzDPPyGQy2oVAC5AihmHWrFk9e/b85z//SbsQAArOnz+P01l6CyliGExNTUNDQ7/++uu6ujratQB0qUePHl2/fn306NG0C4GWIUUMxtKlS6uqqv71r3/RLgSgS6WmpqpUKh8fH9qFQMuQIgbD1tY2ODg4OjoaN2dDt5KSktKvXz97e3vahUDLkCKGZNWqVZmZmWfOnKFdCEDXOX/+PE5n6TOkiCHx8PB44YUXvvjiC9qFAHQRpVKZkZGBFNFnSBEDs2LFiqNHj2ZlZdEuBKArXLp0qbq6Gjdo6TOkiIHx9/d3c3Pbtm0b7UIAukJKSspTTz01YMAA2oVAq5AiBkYgECxbtmz37t1lZWW0awHodCkpKaNHj2YYhnYh0CqkiOFZtGiRWCzetWsX7UIAOt2FCxdwUUTPIUUMj4WFxcKFC7dt26ZUKmnXAtCJ7ty5c//+fVwU0XNIEYMUHh5+7969f//737QLAehE58+fNzU1HTFiBO1CQBukiEFydXX19/ffsmUL7UIAOlFKSoqXl5epqSntQkAbpIihWrly5YULF9LT02kXAtBZ8BBGg4AUMVTPP//8yJEjv/zyS9qFAHSKhw8fZmdn49K6/kOKGLBly5YlJCQUFBTQLgRA9y5cuEAIGTVqFO1C4AmQIgYsJCTkqaeeiomJoV0IgO6lpKQMHjy4R48etAuBJ0CKGDATE5OwsLCvv/66pqaGdi0AOoaHMBoKpIhhW7JkSX19/d69e2kXAqBLDQ0Nv//+O1LEICBFDJuNjc2sWbPwoyNgZC5evNjQ0IAbtAwCUsTgrVy58tatW6dOnaJdCIDOpKSk9O7du0+fPrQLgSdj8DesEXjxxRfFYvHx48dpFwKgG1OmTJHJZHFxcbQLgSfDsYgxWLFixcmTJ2/evEm7EAAdYFkWD2E0IEgRY/DKK68MGDAA30AE45CZmVlWVoYUMRRIEWPAMMybb775ww8/lJaW0q4FoKPOnz9vbm4+bNgw2oVAmyBFjMTChQslEsk333xDuxCAjkpJSfH29haJRLQLgTZBihgJmUy2cOHCrVu3KhQK2rUAdAj3+4a0q4C2QooYj2XLlhUXF+/fv592IQD8FRUV3b17F98UMSBIEePh4uIyffr0zz//nHYhAPwlJycLhcLnnnuOdiHQVkgRo7JixYrffvstNTVVM6e0tDQpKYliSQDaXb16taioSDOZkpIyfPhwCwsLiiVBuyBFjMro0aO9vLy430DMzMx84403HBwccHQC+mznzp29evVycXF57bXXdu7cefr0aVwUMSy4C8LYLF++/LXXXvP19b1w4YJYLG5sbGz6hx6AvnFwcBAKhfn5+T/++OOPP/6oUqlyc3NzcnJeeOEFHx8fT09P/GKunkOKGI+Ghoa4uLgNGzao1er09HSWZRsbGwkhDx8+pF0aQKscHR255zCpVCpuTk1NzfHjx0+dOqVQKGxsbG7fvm1paUm1RtAGKWIMCgoKtm3b9tVXX9XU1HADUqlUapaWlZXRKw3gCRwcHNRqdbOZarVarVYzDBMZGYkI0XN4GqMxSEtL8/Pzq6+vb/HTFAgECoVCIMA1MNBHd+7c6dev3+PzRSLR0KFDMzIy0HX1HD4eY+Dt7Z2YmCgSiRiGeXypWq2uqKjo+qoA2sLR0bHFfksI+eGHHxAh+g+fkJHw8/NLSEhobTTi+VqgtyQSyeP39YpEor///e+DBw+mUhK0C1LEeEydOvW7775rMUhwgR30Wa9evZpOCoVCJyen1atX06oH2gUpYlTmzp37xRdfPD4fxyKgz1xdXZtOqtXqnTt3SiQSSuVA+yBFjM3y5cvXrFnT9IhEKBTiWAT0mYuLi1gs5v4tFosXLlzo5+dHtyRoO6SIEdqwYUN4eLjmsqRQKMSxCOgzBwcHrrsyDCOXyzdv3ky7ImgHpIhxio6Onjt3rlAoJIQwDIMUAX3m4ODAfcOJZdmYmBhra2vaFUE7IEWME8Mw33zzzYQJE8RisUKhwBkt0GeOjo4qlUokEk2cOHHmzJm0y4H2QYoYLbFYfPDgQS8vL7VaXVxcTLscgFY5OjoSQkQiUUxMDO1aoP1YaF1gYCDtzwd0iXaHYlmWjYuLo70bQF/ExcXR7o86gOdoPYG3t/dbb71Fu4oOqaqq+uabb95++23ahdCUmprKPTBfTyBLmvnoo4/WrFnTrb6pHhwcTLsE3UCKPIGjo6MRnKidMWMGrljqVYoYQafSrTFjxjT77qHRM5oU6UbJ350hQkDPdbcIMSZIEQAA4A8pAgAA/CFFAACAP6QIAADwhxQBAAD+kCIAAMAfUgQAAPhDigAAAH9IEQAA4A8pAgAA/CFFAACAP6QIAADwhxQBAAD+kCKULViwQCKRMAxTX19Pt5IDBw64ubkxTUgkkj59+ixcuPDPP//k16b+bF23ooe7Xa1WR0dH+/j4tP0t6JAGg/bPZOm1wMDAwMDAzl7LmjVrCCF1dXWdvaK2cHd3t7S0ZFlWpVI9ePDghx9+kMlktra2Dx8+5Negnmwd96tQdGvgdE0lerLbObdu3Ro9ejQhZNiwYe19r7F2SJZlibH81iGORaBlAoHA1tZ27ty5y5YtKy4uPn36NO2KwCD98ccf77///pIlS4YPH96RdtAh9RZSRF8wDEO7hJb17duXEFJUVNSRRvR264ybPuz2YcOGHThwYPbs2aampjppEB1S3yBFOurTTz+VyWQWFhbFxcWrVq1ycHDIzs5WqVTr1q1zdnaWSqVDhw7V/Mh2UlKSl5eXTCaTy+UeHh6PHj3i5gsEgqNHj06aNMnS0rJXr17ffvutpv3k5ORBgwZZWlpKJBIPD4+TJ08SQgzQ0HIAABUDSURBVL788kuJRGJraxsWFtarVy+JROLj45Oenq55V2sFnDhxQi6XR0VFtX0Db9++TQgZNmzYExunvnVGw7A6VUegQxoD2qfU9Fobr4twZ1pXrFixdevWGTNm3Lx585133jE1Nd2/f395efnq1asFAkFGRkZ1dbVcLt+0aVNdXV1RUdGMGTNKSko0bz9z5kxFRUVZWdnLL79sampaU1PDNZ6QkBAZGVlWVlZaWurt7d2zZ09ufmhoqJmZWWZmZn19/Y0bN0aOHGlhYZGfn88tbbEAlmUTExMtLCzWr1/f2rZoTkOzLFteXv7dd9/JZLLJkyc3fY3ebp0WBnddxIA6VRs999xzj18X6bYdkjWi6yJ6Ma70VrtSRHO9rq6uTiaThYSEcJO1tbWmpqZLly69fv06ISQxMVH723/44QdCyPXr1x9f0ccff0wIKS4uZlk2NDRUM7pYls3IyCCEfPjhh1oKaMsmu7u7N/0jg2GYDRs2NDY2al5goFtnoCliHJ2K02KKPJGxdkjWiFIEZ7R0Lzs7u7a2dsiQIdykVCq1t7fPyspyc3OztbWdM2dOZGRkbm5ua28Xi8WEEIVC0doilUr1+CJPT0+ZTJaVlaWlgDbWrxkt7777LsuylpaW3HqNY+sMVHfe7eiQeg4pons1NTWEkLVr12ruc8/Ly6utrZVKpWfPnvX19Y2KinJzcwsJCamrq3tia0ePHh07dqyNjY2pqel7772n5ZWmpqYlJSVaCmjvhvzjH/+wt7dfvXr1X3/9ZXxbZ1iw2wk6pL5CiuiejY0NISQ6OrrpQV9qaiohZPDgwUeOHCksLIyIiIiLi/vss8+0N5Wfnz99+nR7e/v09PTKyspNmza19kqFQlFRUeHo6Ki9gHaxsLDYuHFjVVXV0qVLjW/rDAt2O0GH1FdIEd1zcnKSSCRXrlxpNr+wsDAzM5MQYmNj88knn4wYMYKb1OLatWsKhWLp0qVubm7cd25be+W5c+dYlvX29tZSAA/z5s177rnnEhMT4+PjuTnGtHUGBLudgw6ph5AiuieRSBYsWLBv376YmJhHjx6pVKqCgoL79+8XFhaGhYVlZWU1NjZevnw5Ly+P64VaODs7E0JOnz5dX19/+/btpjcXEkLUanV5eblSqbx69erKlSudnZ3nz5+vpQBCyPHjx9t1YyXDMF9++SXDMOHh4eXl5Xq+dUbMWHc7OqQxaMsl+G6rLfdobdq0SSqVEkKcnJz27NnDzWxoaIiIiHB2dhaJRDY2NgEBATdu3MjNzfXx8bG2thYKhb17916zZo1SqdS8vV+/fjk5OXv37rW2tiaEODo6cveNRERE9OjRw8rKKigoaNu2bYQQd3f3/Pz80NBQsVjs4OAgEonkcvm0adNycnI0VbVYAMuyx44ds7Cw2LBhw+MbkpKS0r9/f65X9O7dOywsTLOIGy1WVlaffPKJPm+dFoZ1j5ZhdSrtUlNTR48e3atXL65r2dvb+/j4JCUlcUu7bYdkjegeLb0YV3qra56jxU9oaGiPHj1oV9FZdL51hpUitBh3p+qIztgzRpMiOKNlwFq8B9FoGPfW6S3s9tZgz7QGKQIAPGVlZTGtCwkJoV0gdAWkiEFavXr17t27Kysr+/Tps3//ftrl6Jhxb53e4rHbBw4cqOVER2xsbGfX3DXQIbVjWJalXYP+CgoKIoQkJCTQLgQ6Kj4+Pjg4WB96u/5UAnQxDBMXFzdz5kzahXQUjkUAAIA/pAgAAPCHFAEAAP6QIgAAwB9SBAAA+EOKAAAAf0gRAADgDykCAAD8IUUAAIA/pAgAAPCHFAEAAP6QIgAAwB9SBAAA+BPRLkDf7d+/n2EY2lWAsUGnAqOBJ8Nrk5qa+tdff9GuwjD8/PPPu3btWrx48Ysvvki7llbpw1O4CwoKLly4QLuKJ1MqlTt37kxKSlq8ePH48eNpl2OcfHx8HB0daVfRUUgR0JkNGzasW7dux44dixcvpl0LdEhZWVlgYGBGRsbevXunTp1KuxzQazijBTqzdu3aurq6sLAwc3Nz/Fqq4bp9+/aUKVOqq6uTkpJGjBhBuxzQd0gR0KWoqKjGxsZ58+aZmZlNmTKFdjnQbj///PPMmTMHDhx47tw5e3t72uWAAcA9WqBjn3766aJFiwIDA48fP067FmifHTt2TJ48+W9/+9vZs2cRIdBGSBHQMYZhYmJi5s6dGxgYmJSURLscaBOVSvX++++HhYWtXr06NjZWKpXSrggMBq6uQ6dQqVSzZ88+fvz4zz//7OXlRbsc0KaqqurVV189c+bMrl27Zs2aRbscMDBIEegsCoVixowZ58+fP3PmDC7S6q27d+9OmTKlvLz8p59+GjlyJO1ywPDgjBZ0FrFYfODAAW9v74kTJ2ZmZtIuB1qQkpIyatQosViclpaGCAF+kCLQiUxMTA4cOPD000/7+fllZ2fTLgf+y7fffuvn5zdmzJgLFy44OzvTLgcMFVIEOpdMJjty5Iizs/OECRNyc3NplwOEEMKybGRk5KJFi8LCwuLj42UyGe2KwIDhugh0hYqKCj8/v6qqqqSkpN69e9Mup1urrq6eM2fOiRMnduzYMW/ePNrlgMFDikAXKSkpGTt2rEqlSkpKsrOzo11ON3Xv3j1/f/+8vLyDBw8+//zztMsBY4AzWtBFbGxsTp06pVAoXnrppbKyMtrldEdpaWmenp6NjY2//fYbIgR0BSkCXcfBweGXX36pqKh4+eWXq6qqaJfTvcTHx/v5+Q0bNuz8+fOurq60ywHjgRSBLuXs7Pzzzz/n5+dPmjSppqaGdjndAsuymzZtCgkJmTt3bmJioqWlJe2KwKjgughQcO3atXHjxo0YMeLw4cMSiYR2Ocasvr5+8eLFcXFxW7ZsefPNN2mXA0YIKQJ0XLlyxc/P7/nnn09ISBCLxbTLMU7379+fOnVqTk5OQkKCn58f7XLAOOGMFtAxfPjwY8eOnT179tVXX1UqlbTLMUJ//PGHt7d3eXl5SkoKIgQ6D1IEqPH29j527NiJEycWL16sVqtpl2NUDh48OHr06P79+1+8eHHgwIG0ywFjhhQBmnx9fQ8ePBgbGxseHk67FuPxxRdfBAUFzZo169ixY9bW1rTLASOH3zoEyv72t7/FxsYGBQWJxeLo6Gja5Ri2hoaG0NDQvXv3fvzxxxEREbTLgW4BKQL0TZs2bd++fSEhIVZWVh988AHtcgxVaWlpQEDApUuXDh069Morr9AuB7oLpAjohcDAwJ07dy5atEgikeCPaB6uX7/u7+/PMExqaurgwYNplwPdCFIE9MX8+fOrq6uXL18uFovffvtt2uUYkpMnTwYHBw8ePPjf//63ra0t7XKge0GKgB5ZtmyZQqFYtWqVhYXF66+/Trscw7Bjx44333wzKCjo22+/xVc4oeshRUC/vPXWW+Xl5UuWLDEzM8NvgGunVCpXrlwZExOzbt26yMhI2uVAN4UUAb2zfv36hoaGefPmicXioKAg2uXoqfLy8sDAwPT09IMHD06bNo12OdB9IUVAH23cuJH7MSWZTDZ58mTa5eidO3fuTJkyhfvVr2effZZ2OdCt4VuHoI8Yhtm2bdu8efOCgoLOnTtHuxz9kpycPGrUKKlUmpaWhggB6pAioKcYhvn666+nTZv2yiuvJCcn0y5HX3zzzTfjx48fN27c+fPnHR0daZcDgBQBPSYUCr///vvx48dPmTLl999/p10OZSqV6v333w8NDX377bdjY2NlMhntigAIwZPhQf81NjZOnz794sWL586d67bfp6uqqpo9e/apU6d27tw5Z84c2uUA/D+kCBiAurq6l19+OTMzMykpqRs+ofbPP/+cMmVKWVnZoUOHvLy8aJcD8F9wRgsMgFQqTUxM7N+//4QJE/7888/HX2DED5a/cOGCt7e3UChMTU1FhIAeQoqAYTAzM0tMTLSzs5swYUJhYWHTRTt27NiwYQOtwnSiqKiooaHh8fmxsbHjx4/39PRMTk52cXHp+sIAnghntMCQlJSUjB07VqlUJiUl2dvbE0I+//zzd955x9LS8t69e4Z7wXn69Onm5uZ79uzRzGFZ9sMPP/zwww/Dw8Ojo6MFAvzBB/qKBTAoRUVFAwYMGDp0aGlp6UcffcR1Y6FQGBMTQ7s0nk6dOkUIYRjm448/5uZUV1dPnz7dxMRk9+7dVEsDeDIci4DhycvLe/755x0dHS9cuMDNYRjG0dHxzz//FAqFdGtrr8bGxkGDBuXm5qpUKoZh9u3bN2bMGH9//9zc3P37948dO5Z2gQBPgMNkMDzOzs4TJkxITU3VzGFZtqCg4NChQxSr4mfLli1chHCT8+bN8/T0rK+vz8jIQISAQcCxCBgYtVr9+uuvf/fdd83uyxIIBEOHDr18+TKtwngoKipyd3evra3VzBEKhebm5mlpad3whmYwUDgWAUOiVCpnz579/fffP35rr1qtvnLlyvnz56kUxs9bb72lUCiazlGpVLW1tTNnzqypqaFVFUC7IEXAkHzwwQexsbGtfTtELBZv3Lixi0vi7fz583Fxcc1ShBCiUCiysrLmzp1rxF+CAWMixI/bgAEZN26cp6fnlStXSktLGYZptlStVt+5cycoKMjGxoZKeW2nVCpffvnlsrKyFk8pq9XqrKwslmXHjRvX9bUBtAuORcCQCASCKVOmXL9+PS4urk+fPgKBoFmWiESi6OhoWuW13VdffZWdna25qN6UWCwmhIwcOdLZ2RmXLUH/4eo6GCq1Wn3gwIH33nsvPz+fu2+dmy8Wi/Py8nr16kW3PC2Ki4vd3d2rq6ubzhSJREql8qmnnpo1a9aiRYuGDh1KqzyAdsGxCBgqgUAQFBR069at3bt3Ozk5NT0u2b59O93atIuIiNA874RhGKFQKBKJJkyYEB8ff//+/S+++AIRAgYExyJgDBobG3fv3h0ZGVlSUqJSqeRyeWFhoZmZGe26WpCenj5q1CiWZcVisUKhGDp06BtvvDFr1ixra2vapQHwgRQBCoKCgjqjWbVafffu3Zs3bzY0NAwfPrxv376dsZaOYFn2zJkzFRUVJiYmrq6urq6ucrm8480mJCR0vBEAfpAiQAHDMN7e3p30g68qleru3bsFBQVjx459/D4uunJzcwsLC11dXe3t7XXygMWCgoK0tDSMYqAIKQIUMAwTFxc3c+bMzltFbW0ty7L6dlKrvr5eIpHosMH4+Pjg4GCMYqBIRLsAgE6hn0+J122EAOgD3KMFAAD8IUUAAIA/pAgAAPCHFAEAAP6QIgAAwB9SBAAA+EOKAAAAf0gRAADgDykCAAD8IUUAAIA/pAgAAPCHFAEAAP6QIgAAwB9SBEAHDhw44ObmxjRhYmJia2s7duzYzZs3l5eX0y4QoLMgRQB0ICAg4O7du+7u7paWlizLqtXq4uLi+Pj4Pn36REREDB48+LfffqNdI0CnQIoAtKqurs7Hx4fHGxmGsbKyGjt27O7du+Pj4x88eDB58uTKyspOXSkAFUgRgFbt2rWruLi4g40EBgbOnz+/uLj466+/7rKVAnQZpAjorz179nh6ekokEjMzM1dX148++ogQwrLs559//vTTT5uamlpbW0+bNi0rK4t7fUxMjJmZmUwm++mnnyZNmiSXyx0dHfft2/fENpOTkwcNGmRpaSmRSDw8PE6ePEkIWbly5apVq3JychiG6du3LyHkxIkTcrk8KiqqvRsyf/58Qsjx48e5SS2b8PhKAfQdC9DlCCFxcXHaXxMdHU0I+eSTT0pLS8vKyv75z3/Onj2bZdl169aZmJjs2bOnoqLi6tWrI0aMeOqpp4qKirh3rVmzhhBy5syZysrK4uLiMWPGmJmZNTY2am8zISEhMjKyrKystLTU29u7Z8+e3OsDAgLc3d01JSUmJlpYWKxfv761mjXXRZp59OgRIcTJyYmb1L4JzVaqXVxcHEYx0IX+BxQ8MUUaGxutrKzGjRunmaNUKrds2VJbW2tubh4SEqKZf/HiRUKI5n92LkXq6uq4ye3btxNC7ty5o6XNZqv++OOPCSHFxcVsO/9DZ1tPEZZluSslLMs+cROQImBYcEYL9NHVq1crKipeeuklzRyhULhixYobN25UV1d7enpq5o8cOdLExCQ9Pb3FdkxMTAghCoVCS5vN3iIWiwkhKpVKd1tDampqWJaVy+WEkPZuAoCeE9EuAKAF3CkgKyurZvMrKioIIebm5k1nWllZVVVV8W6TEHL06NHNmzffuHHj0aNHXOTo1q1btwghAwcOJB3bBAA9hGMR0Ee9e/cmhDx8+LDZfC4Dmv2HW1FR4ejoyLvN/Pz86dOn29vbp6enV1ZWbtq0qSOVt+jEiROEkEmTJpGObQKAHkKKgD5ydXXt0aPHqVOnms0fMmSIubl502/wpaenNzY2Pvvss7zbvHbtmkKhWLp0qZubm0QiYRim4/U3VVRUFB0d7ejouHDhwg5uAoAeQoqAPjI1NV29evWvv/4aHh5+7949tVpdVVWVmZkpkUhWrVp18ODBvXv3Pnr06Nq1a0uWLOnVq1doaCjvNp2dnQkhp0+frq+vv337dtPrEz169CgsLMzNza2qqlIoFMePH3/inb4sy1ZXV6vVapZlS0pK4uLiRo8eLRQKDx06xF0XeeImNFtph/YjQBegfHUfuiXShjt9WZbdtm2bh4eHRCKRSCTPPPPM9u3bWZZVq9WbN2/u16+fWCy2traePn16dnY29/rt27fLZDJCSL9+/XJycnbs2MH9x+3i4nLr1i0tbUZERPTo0cPKyiooKGjbtm2EEHd39/z8/EuXLrm4uEilUl9f36KiomPHjllYWGzYsOHxUg8fPjx06FCZTGZiYiIQCMh/vr7u5eW1fv360tLSpi/WsgksyzZbqfZdhHu0gDqGZVmaIQbdEsMwcXFxM2fOpF2IwYuPjw8ODsYoBopwRgsAAPhDigAAAH9IEQAA4A8pAgAA/CFFAACAP6QIAADwhxQBAAD+kCIAAMAfUgQAAPhDigAAAH9IEQAA4A8pAgAA/CFFAACAP6QIAADwhxQBAAD+kCIAAMAfUgQAAPjDbx0CBQzDeHt7Ozo60i7E4BUUFKSlpWEUA0VIEaAgKCiIdglGJSEhgXYJ0H0hRQAAgD9cFwEAAP6QIgAAwB9SBAAA+EOKAAAAf/8L0WSbJDWxUFsAAAAASUVORK5CYII=\n", | |
"text/plain": [ | |
"<IPython.core.display.Image object>" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 32 | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "sviUZ9_UfKRx", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 451 | |
}, | |
"outputId": "398f354f-85c0-4334-889e-da48b23265fa" | |
}, | |
"source": [ | |
"model.summary()" | |
], | |
"execution_count": 33, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"text": [ | |
"Model: \"model\"\n", | |
"__________________________________________________________________________________________________\n", | |
"Layer (type) Output Shape Param # Connected to \n", | |
"==================================================================================================\n", | |
"user (InputLayer) [(None, 1)] 0 \n", | |
"__________________________________________________________________________________________________\n", | |
"item (InputLayer) [(None, 1)] 0 \n", | |
"__________________________________________________________________________________________________\n", | |
"user_embedding (Embedding) (None, 1, 8) 7552 user[0][0] \n", | |
"__________________________________________________________________________________________________\n", | |
"item_embedding (Embedding) (None, 1, 8) 13464 item[0][0] \n", | |
"__________________________________________________________________________________________________\n", | |
"reshape (Reshape) (None, 8) 0 user_embedding[0][0] \n", | |
"__________________________________________________________________________________________________\n", | |
"reshape_1 (Reshape) (None, 8) 0 item_embedding[0][0] \n", | |
"__________________________________________________________________________________________________\n", | |
"concat (Dot) (None, 1) 0 reshape[0][0] \n", | |
" reshape_1[0][0] \n", | |
"==================================================================================================\n", | |
"Total params: 21,016\n", | |
"Trainable params: 21,016\n", | |
"Non-trainable params: 0\n", | |
"__________________________________________________________________________________________________\n" | |
], | |
"name": "stdout" | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "2kQ3kZPtfKR5", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 1000 | |
}, | |
"outputId": "3dce9058-1bac-4195-ee2a-0f11f9055af8" | |
}, | |
"source": [ | |
"import time\n", | |
"\n", | |
"checkpoint_callback = tf.keras.callbacks.ModelCheckpoint('./models_fm', \n", | |
" monitor='val_loss',\n", | |
" save_best_only=True,\n", | |
" freq='epoch'\n", | |
" )\n", | |
"log_dir = os.path.join(\"./models_fm/logs\" , datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n", | |
"tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n", | |
"\n", | |
"history = model.fit(x=[X_train[\"user_id\"], X_train[\"item_id\"]]\n", | |
" ,y=y_train\n", | |
" ,batch_size=128, epochs=20\n", | |
" ,validation_split=0.1\n", | |
" ,callbacks=[checkpoint_callback, tensorboard_callback]\n", | |
" ,shuffle=True\n", | |
" #,verbose=2\n", | |
")\n", | |
"\n", | |
"import pickle\n", | |
"with open(os.path.join(\"models_fm\", \"history.pkl\"), \"wb\") as f:\n", | |
" pickle.dump(history.history, f)" | |
], | |
"execution_count": 34, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"text": [ | |
"Epoch 1/20\n", | |
"699/700 [============================>.] - ETA: 0s - loss: 12.4237WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:1817: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n", | |
"Instructions for updating:\n", | |
"If using Keras pass *_constraint arguments to layers.\n", | |
"INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 3ms/step - loss: 12.4219 - val_loss: 9.6668\n", | |
"Epoch 2/20\n", | |
"681/700 [============================>.] - ETA: 0s - loss: 5.5264INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 3ms/step - loss: 5.4549 - val_loss: 2.7801\n", | |
"Epoch 3/20\n", | |
"682/700 [============================>.] - ETA: 0s - loss: 2.0878INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 3ms/step - loss: 2.0768 - val_loss: 1.6733\n", | |
"Epoch 4/20\n", | |
"697/700 [============================>.] - ETA: 0s - loss: 1.4977INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 2ms/step - loss: 1.4976 - val_loss: 1.3814\n", | |
"Epoch 5/20\n", | |
"699/700 [============================>.] - ETA: 0s - loss: 1.3131INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 3ms/step - loss: 1.3134 - val_loss: 1.2714\n", | |
"Epoch 6/20\n", | |
"698/700 [============================>.] - ETA: 0s - loss: 1.2366INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 2ms/step - loss: 1.2371 - val_loss: 1.2249\n", | |
"Epoch 7/20\n", | |
"692/700 [============================>.] - ETA: 0s - loss: 1.2012INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 2ms/step - loss: 1.2008 - val_loss: 1.2033\n", | |
"Epoch 8/20\n", | |
"675/700 [===========================>..] - ETA: 0s - loss: 1.1818INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 2ms/step - loss: 1.1818 - val_loss: 1.1919\n", | |
"Epoch 9/20\n", | |
"675/700 [===========================>..] - ETA: 0s - loss: 1.1696INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 3ms/step - loss: 1.1706 - val_loss: 1.1882\n", | |
"Epoch 10/20\n", | |
"678/700 [============================>.] - ETA: 0s - loss: 1.1647INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 3ms/step - loss: 1.1639 - val_loss: 1.1855\n", | |
"Epoch 11/20\n", | |
"700/700 [==============================] - 1s 2ms/step - loss: 1.1593 - val_loss: 1.1859\n", | |
"Epoch 12/20\n", | |
"695/700 [============================>.] - ETA: 0s - loss: 1.1563INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 3ms/step - loss: 1.1562 - val_loss: 1.1836\n", | |
"Epoch 13/20\n", | |
"693/700 [============================>.] - ETA: 0s - loss: 1.1534INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 3ms/step - loss: 1.1537 - val_loss: 1.1825\n", | |
"Epoch 14/20\n", | |
"700/700 [==============================] - 1s 2ms/step - loss: 1.1516 - val_loss: 1.1844\n", | |
"Epoch 15/20\n", | |
"675/700 [===========================>..] - ETA: 0s - loss: 1.1484INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 3ms/step - loss: 1.1494 - val_loss: 1.1810\n", | |
"Epoch 16/20\n", | |
"700/700 [==============================] - 1s 2ms/step - loss: 1.1475 - val_loss: 1.1822\n", | |
"Epoch 17/20\n", | |
"700/700 [==============================] - 1s 2ms/step - loss: 1.1457 - val_loss: 1.1812\n", | |
"Epoch 18/20\n", | |
"700/700 [==============================] - 1s 2ms/step - loss: 1.1438 - val_loss: 1.1815\n", | |
"Epoch 19/20\n", | |
"700/700 [==============================] - 1s 2ms/step - loss: 1.1416 - val_loss: 1.1813\n", | |
"Epoch 20/20\n", | |
"681/700 [============================>.] - ETA: 0s - loss: 1.1383INFO:tensorflow:Assets written to: ./models_fm/assets\n", | |
"700/700 [==============================] - 2s 2ms/step - loss: 1.1389 - val_loss: 1.1804\n" | |
], | |
"name": "stdout" | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "TKC1qsN-fKSA", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"import pickle\n", | |
"with open(os.path.join(\"models_fm\", \"history.pkl\"), \"wb\") as f:\n", | |
" pickle.dump(history.history, f)" | |
], | |
"execution_count": 35, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "1ZsBmNlOfKSH", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 36 | |
}, | |
"outputId": "f563fad7-bb19-4554-e39e-4a4abd29236f" | |
}, | |
"source": [ | |
"model.layers[3].get_weights()[0].shape" | |
], | |
"execution_count": 36, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/plain": [ | |
"(1683, 8)" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 36 | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "DWVtifB6fKSM", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 36 | |
}, | |
"outputId": "ddadb22e-c25b-48c1-cc26-18c9fa3b381d" | |
}, | |
"source": [ | |
"model.layers[2].get_weights()[0].shape" | |
], | |
"execution_count": 37, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/plain": [ | |
"(944, 8)" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 37 | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "w2IDZ5FEfKSW", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"# output embeddings\n", | |
"pd.DataFrame(model.layers[2].get_weights()[0]).to_csv('./user_emb_8.csv')\n", | |
"pd.DataFrame(model.layers[3].get_weights()[0]).to_csv('./item_emb_8.csv')\n" | |
], | |
"execution_count": 38, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "eoL-CMGWfKSg", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 206 | |
}, | |
"outputId": "40f783ea-09c6-4289-e803-827f48b68168" | |
}, | |
"source": [ | |
"df_scores = pd.DataFrame(model.layers[2].get_weights()[0] @ model.layers[3].get_weights()[0].T).stack().reset_index()\n", | |
"df_scores.columns = ['user_id', 'item_id', 'rating']\n", | |
"df_scores.to_csv('./scores.csv', index=False)\n", | |
"df_scores.head()\n" | |
], | |
"execution_count": 39, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/html": [ | |
"<div>\n", | |
"<style scoped>\n", | |
" .dataframe tbody tr th:only-of-type {\n", | |
" vertical-align: middle;\n", | |
" }\n", | |
"\n", | |
" .dataframe tbody tr th {\n", | |
" vertical-align: top;\n", | |
" }\n", | |
"\n", | |
" .dataframe thead th {\n", | |
" text-align: right;\n", | |
" }\n", | |
"</style>\n", | |
"<table border=\"1\" class=\"dataframe\">\n", | |
" <thead>\n", | |
" <tr style=\"text-align: right;\">\n", | |
" <th></th>\n", | |
" <th>user_id</th>\n", | |
" <th>item_id</th>\n", | |
" <th>rating</th>\n", | |
" </tr>\n", | |
" </thead>\n", | |
" <tbody>\n", | |
" <tr>\n", | |
" <th>0</th>\n", | |
" <td>0</td>\n", | |
" <td>0</td>\n", | |
" <td>0.001375</td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th>1</th>\n", | |
" <td>0</td>\n", | |
" <td>1</td>\n", | |
" <td>-0.060442</td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th>2</th>\n", | |
" <td>0</td>\n", | |
" <td>2</td>\n", | |
" <td>-0.048946</td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th>3</th>\n", | |
" <td>0</td>\n", | |
" <td>3</td>\n", | |
" <td>-0.048313</td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th>4</th>\n", | |
" <td>0</td>\n", | |
" <td>4</td>\n", | |
" <td>-0.050197</td>\n", | |
" </tr>\n", | |
" </tbody>\n", | |
"</table>\n", | |
"</div>" | |
], | |
"text/plain": [ | |
" user_id item_id rating\n", | |
"0 0 0 0.001375\n", | |
"1 0 1 -0.060442\n", | |
"2 0 2 -0.048946\n", | |
"3 0 3 -0.048313\n", | |
"4 0 4 -0.050197" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 39 | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "TSybDjVyfKSm", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"# umap\n", | |
"import pandas as pd\n", | |
"import umap\n", | |
"import time\n", | |
"import matplotlib.pyplot as plt\n", | |
"\n", | |
"#df_user_emb = pd.read_csv('./user_emb_8.csv')\n", | |
"#df_item_emb = pd.read_csv('./item_emb_8.csv')\n", | |
"#embedding = umap.UMAP().fit_transform(df_item_emb[[\"0\",\"1\",\"2\",\"3\",\"4\",\"5\",\"6\",\"7\"]].values)\n", | |
"#pd.DataFrame(embedding).to_csv('item_emb_umap.csv')" | |
], | |
"execution_count": 40, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "fCTc45hqfKSs", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 36 | |
}, | |
"outputId": "fec48ccf-bb58-454e-80a5-cdbf9161354d" | |
}, | |
"source": [ | |
"pd.DataFrame(model.layers[2].get_weights()[0] @ model.layers[3].get_weights()[0].T).shape" | |
], | |
"execution_count": 41, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/plain": [ | |
"(944, 1683)" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 41 | |
} | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"id": "Q3bJS9QofKSx", | |
"colab_type": "text" | |
}, | |
"source": [ | |
"#### https://www.tensorflow.org/tensorboard/tensorboard_in_notebooks" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "vABcwTIifKSy", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"" | |
], | |
"execution_count": 41, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "KXE5j0eFfKS4", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"!kill 437" | |
], | |
"execution_count": null, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "X_vFtJ3BfKS8", | |
"colab_type": "code", | |
"colab": {}, | |
"outputId": "352afdaf-e71b-4bc5-8653-a528205e8142" | |
}, | |
"source": [ | |
"from tensorboard import notebook\n", | |
"notebook.list() # View open TensorBoard instances" | |
], | |
"execution_count": null, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"text": [ | |
"No known TensorBoard instances running.\n" | |
], | |
"name": "stdout" | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "A7gnzyCCfKTB", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"" | |
], | |
"execution_count": null, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "shMtu5fefKTG", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"%load_ext tensorboard" | |
], | |
"execution_count": null, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "Wrn5VyNEfKTL", | |
"colab_type": "code", | |
"colab": {}, | |
"outputId": "aa89e246-ccf7-4318-d239-44f6280bb1d6" | |
}, | |
"source": [ | |
"%reload_ext tensorboard\n", | |
"%tensorboard --logdir ./models_fm/logs --host 0.0.0.0" | |
], | |
"execution_count": null, | |
"outputs": [ | |
{ | |
"output_type": "display_data", | |
"data": { | |
"text/html": [ | |
"\n", | |
" <iframe id=\"tensorboard-frame-8722f0df074f4d28\" width=\"100%\" height=\"800\" frameborder=\"0\">\n", | |
" </iframe>\n", | |
" <script>\n", | |
" (function() {\n", | |
" const frame = document.getElementById(\"tensorboard-frame-8722f0df074f4d28\");\n", | |
" const url = new URL(\"/\", window.location);\n", | |
" url.port = 6006;\n", | |
" frame.src = url;\n", | |
" })();\n", | |
" </script>\n", | |
" " | |
], | |
"text/plain": [ | |
"<IPython.core.display.HTML object>" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
} | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "anW_lEhufKTO", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"y_pred = model.predict(x=[X_test[\"user_id\"], X_test[\"item_id\"]])\n" | |
], | |
"execution_count": null, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "8eJWWrGSfKTU", | |
"colab_type": "code", | |
"colab": {}, | |
"outputId": "8f3ed7a7-9686-4b9b-a29f-cb8803b09f97" | |
}, | |
"source": [ | |
"y_pred.shape" | |
], | |
"execution_count": null, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/plain": [ | |
"(20000, 1)" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 27 | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "qNN3BO0IfKTa", | |
"colab_type": "code", | |
"colab": {}, | |
"outputId": "76b03505-0c3b-4a16-95a0-369ad3257b46" | |
}, | |
"source": [ | |
"print(f'''user {X_test[\"user_id\"].values[0]}のitem {X_test[\"item_id\"].values[0]}に対する\\n\n", | |
"予測値: {y_pred[0][0]}\\n\n", | |
"真の値: {y_test.values[0]}\n", | |
"''')" | |
], | |
"execution_count": null, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"text": [ | |
"user 265のitem 284に対する\n", | |
"\n", | |
"予測値: 3.259321689605713\n", | |
"\n", | |
"真の値: 4\n", | |
"\n" | |
], | |
"name": "stdout" | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "EIDZr1JtfKTf", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"from sklearn.metrics import mean_squared_error\n", | |
"# calc RMSE\n", | |
"np.sqrt(mean_squared_error(y_test, y_pred))" | |
], | |
"execution_count": null, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "sSehlno-fKTm", | |
"colab_type": "code", | |
"colab": {}, | |
"outputId": "c0e12881-2498-41cb-e625-2881b2c3d1f3" | |
}, | |
"source": [ | |
"# 重みとオプティマイザを含む全く同じモデルを再作成\n", | |
"model_s = './models_fm'\n", | |
"model_l = tf.keras.models.load_model(model_s)\n", | |
"\n", | |
"model_l.summary()\n" | |
], | |
"execution_count": null, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"text": [ | |
"Model: \"model\"\n", | |
"__________________________________________________________________________________________________\n", | |
"Layer (type) Output Shape Param # Connected to \n", | |
"==================================================================================================\n", | |
"user (InputLayer) [(None, 1)] 0 \n", | |
"__________________________________________________________________________________________________\n", | |
"item (InputLayer) [(None, 1)] 0 \n", | |
"__________________________________________________________________________________________________\n", | |
"user_embedding (Embedding) (None, 1, 30) 2400000 user[0][0] \n", | |
"__________________________________________________________________________________________________\n", | |
"item_embedding (Embedding) (None, 1, 30) 2400000 item[0][0] \n", | |
"__________________________________________________________________________________________________\n", | |
"reshape (Reshape) (None, 30) 0 user_embedding[0][0] \n", | |
"__________________________________________________________________________________________________\n", | |
"reshape_1 (Reshape) (None, 30) 0 item_embedding[0][0] \n", | |
"__________________________________________________________________________________________________\n", | |
"dot (Dot) (None, 1) 0 reshape[0][0] \n", | |
" reshape_1[0][0] \n", | |
"==================================================================================================\n", | |
"Total params: 4,800,000\n", | |
"Trainable params: 4,800,000\n", | |
"Non-trainable params: 0\n", | |
"__________________________________________________________________________________________________\n" | |
], | |
"name": "stdout" | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "dZ_qj4ogfKTq", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"y_pred = model_l.predict(x=[X_test[\"user_id\"], X_test[\"item_id\"]])\n" | |
], | |
"execution_count": null, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "JwUAGj8SfKTt", | |
"colab_type": "code", | |
"colab": {}, | |
"outputId": "43a2fd06-3a40-4373-cf2c-d8d41761e86e" | |
}, | |
"source": [ | |
"from sklearn.metrics import mean_squared_error\n", | |
"# calc RMSE\n", | |
"np.sqrt(mean_squared_error(y_test, y_pred))" | |
], | |
"execution_count": null, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/plain": [ | |
"0.93278754" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 28 | |
} | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"id": "fbltKPwrfKTy", | |
"colab_type": "text" | |
}, | |
"source": [ | |
"### predict topn" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "97LJTYJ3fKTz", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"import numpy as np\n", | |
"import pandas as pd\n", | |
"import tensorflow as tf\n", | |
"\n", | |
"import itertools\n", | |
"\n", | |
"def all(model: tf.keras.Model, users: list, items: list, n: int=None):\n", | |
" \"\"\"\n", | |
" 全組み合わせについてスコアを取得\n", | |
"\n", | |
" Parameters\n", | |
" ----------\n", | |
" model: tf.keras.Model\n", | |
" user: list\n", | |
" items: list\n", | |
" n: int, default None\n", | |
" if is not None, return topn each user\n", | |
"\n", | |
" Returns\n", | |
" -------\n", | |
" df\n", | |
"\n", | |
" \"\"\"\n", | |
" # 全組み合わせ\n", | |
" ul = np.array([u for u, _ in itertools.product(list(set(users)),\n", | |
" list(set(items)))])\n", | |
" il = np.array([i for _, i in itertools.product(list(set(users)),\n", | |
" list(set(items)))])\n", | |
" y_pred = model.predict(x=[il, ul]).flatten()\n", | |
"\n", | |
" df = pd.DataFrame(\n", | |
" {\n", | |
" \"user_id_index\": ul,\n", | |
" \"item_id_index\": il,\n", | |
" \"score\": np.array(y_pred)\n", | |
" }\n", | |
" )\n", | |
"\n", | |
" # return topn each user\n", | |
" if n is not None:\n", | |
" return df.groupby([\"user_id_index\"]).apply(lambda row: row.nlargest(n, 'score')).reset_index(drop=True)\n", | |
"\n", | |
"\n", | |
" return df\n", | |
"\n", | |
"\n", | |
"def topn(model: tf.keras.Model, user_id: int, items: list, n=10):\n", | |
" \"\"\"\n", | |
"\n", | |
" Parameters\n", | |
" ----------\n", | |
" model\n", | |
" user_id: int, str\n", | |
" user_id\n", | |
" items: list\n", | |
" item_ids\n", | |
" n: int, default 10\n", | |
" topn\n", | |
"\n", | |
" Returns\n", | |
" -------\n", | |
" df\n", | |
"\n", | |
" \"\"\"\n", | |
" y_pred = model.predict(x=[np.array(items),\n", | |
" np.array([user_id]*len(items))])\n", | |
"\n", | |
" # to rating\n", | |
" #y_rat = [int(abs(round(i[0]))) for i in y]\n", | |
" y = [i[0] for i in y_pred]\n", | |
" # index of topn\n", | |
" idx = np.argsort(y)[::-1][:n]\n", | |
"\n", | |
" df = pd.DataFrame(\n", | |
" {\n", | |
" \"user_id_index\": [user_id] * n,\n", | |
" \"item_id_index\": np.array(items)[idx],\n", | |
" \"score\": np.array(y)[idx]\n", | |
" }\n", | |
" )\n", | |
"\n", | |
" return df" | |
], | |
"execution_count": null, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "z5p5C-j8fKT7", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"# id\n", | |
"n = 5\n", | |
"pred_user = user_to_index[user_id]\n", | |
"pred_items = list(set(X_test['item_id_index']))\n", | |
"df_pred = topn(model_l, pred_user, pred_items, n=n)\n", | |
"\n", | |
"# index to id\n", | |
"df_pred['user_id'] = df_pred['user_id_index'].map(index_to_user)\n", | |
"df_pred['item_id'] = df_pred['item_id_index'].map(index_to_item)\n", | |
"df_pred" | |
], | |
"execution_count": null, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "zdvtSt1TfKUA", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"# 全組み合わせのtopn Noneの時は全の組み合わせを取得\n", | |
"df_preds_topn = all(model_l,\n", | |
" X_train['user_id_index'],\n", | |
" X_train['item_id_index'], n=n)\n", | |
"df_preds_topn['user_id'] = df_preds_topn['user_id_index'].map(index_to_user)\n", | |
"df_preds_topn['item_id'] = df_preds_topn['item_id_index'].map(index_to_item)" | |
], | |
"execution_count": null, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "6MVhIQZ1fKUD", | |
"colab_type": "code", | |
"colab": {}, | |
"outputId": "ceb25a5b-e3f3-4d49-bcaf-73dd0f7c4979" | |
}, | |
"source": [ | |
"history.history['loss']" | |
], | |
"execution_count": null, | |
"outputs": [ | |
{ | |
"output_type": "execute_result", | |
"data": { | |
"text/plain": [ | |
"[12.634256362915039,\n", | |
" 5.796047687530518,\n", | |
" 2.128021240234375,\n", | |
" 1.5114867687225342,\n", | |
" 1.3187335729599,\n", | |
" 1.2392791509628296,\n", | |
" 1.2014925479888916,\n", | |
" 1.1819627285003662,\n", | |
" 1.170408010482788,\n", | |
" 1.163235068321228,\n", | |
" 1.158171534538269,\n", | |
" 1.154823660850525,\n", | |
" 1.151853084564209,\n", | |
" 1.1492549180984497,\n", | |
" 1.1466904878616333,\n", | |
" 1.144012451171875,\n", | |
" 1.141165018081665,\n", | |
" 1.1381170749664307,\n", | |
" 1.134507179260254,\n", | |
" 1.1303595304489136]" | |
] | |
}, | |
"metadata": { | |
"tags": [] | |
}, | |
"execution_count": 195 | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "CIsdTxlKfKUI", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"" | |
], | |
"execution_count": null, | |
"outputs": [] | |
} | |
] | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment