There are difference between GMF, MLP AND NeuMF
MODEL_CHECKPOINT = "model.ckpt"
def _create_gmf(n_users, n_items, regs=None, embedding_size=10, implicit=False):
"""Create a GMF model
Args:
n_users (int): number of user embeddings
n_items (int): number of item embeddings
regs[Optional[List[int]]]: regularization parameters for embedding layer
embedding_size(int): size of each user and item embedding
Returns:
"keras.models.Model": GMF model
"""
# To control where and how TensorFlow sessions are created, we
# do imports here make keras work well with multiprocessing.
from keras import Input, Model
from keras.layers import Embedding, Flatten, multiply, Dense
from keras.regularizers import l2
if regs:
assert len(regs) == 2
regs = regs or [0, 0]
user_input = Input(shape=(1,), dtype="int32", name="user_input")
item_input = Input(shape=(1,), dtype="int32", name="item_input")
mf_embedding_user = Embedding(
input_dim=n_users,
output_dim=embedding_size,
name="user_embedding",
embeddings_initializer="normal",
embeddings_regularizer=l2(regs[0]),
input_length=1,
)
mf_embedding_item = Embedding(
input_dim=n_items,
output_dim=embedding_size,
name="item_embedding",
embeddings_initializer="normal",
embeddings_regularizer=l2(regs[1]),
input_length=1,
)
# Crucial to flatten an embedding vector!
user_latent = Flatten()(mf_embedding_user(user_input))
item_latent = Flatten()(mf_embedding_item(item_input))
# Element-wise product of user and item embeddings
predict_vector = multiply([user_latent, item_latent])
# TODO apply Dropout after the mutliply operation?
# Final prediction layer
if implicit:
LOG.info("GMF model to Predict binary implicit preference")
# prediction = Lambda(lambda x: K.sigmoid(K.sum(x)), output_shape=(1,))(predict_vector)
prediction = Dense(
1,
activation="sigmoid",
kernel_initializer="lecun_uniform",
name="prediction",
)(predict_vector)
else:
LOG.info("Prediction layer configured for explicit feedback")
prediction = Dense(
1,
activation="linear",
kernel_initializer="lecun_uniform",
name="prediction",
)(predict_vector)
model = Model(inputs=[user_input, item_input], outputs=prediction)
return model
def _create_gmf_constrained(
n_users, n_items, regs=None, embedding_size=10, dropout=None, implicit=False
):
"""Create a GMF model
Args:
n_users (int): number of user embeddings
n_items (int): number of item embeddings
regs[Optional[List[int]]]: regularization parameters for embedding layer
embedding_size(int): size of each user and item embedding
Returns:
"keras.models.Model": GMF model
"""
# To control where and how TensorFlow sessions are created, we
# do imports here make keras work well with multiprocessing.
from keras import Input, Model
from keras.layers import Embedding, Flatten, multiply, Dense, Lambda, Dropout
from keras.regularizers import l2
from keras.layers.merge import Maximum, Minimum
if regs:
assert len(regs) == 2
regs = regs or [0, 0]
embedding_size = embedding_size
user_input = Input(shape=(1,), dtype="int32", name="user_input")
item_input = Input(shape=(1,), dtype="int32", name="item_input")
mf_embedding_user = Embedding(
input_dim=n_users,
output_dim=embedding_size,
name="user_embedding",
embeddings_initializer="normal",
embeddings_regularizer=l2(regs[0]),
input_length=1,
)
mf_embedding_item = Embedding(
input_dim=n_items,
output_dim=embedding_size,
name="item_embedding",
embeddings_initializer="normal",
embeddings_regularizer=l2(regs[1]),
input_length=1,
)
# Crucial to flatten an embedding vector!
user_latent = Flatten()(mf_embedding_user(user_input))
item_latent = Flatten()(mf_embedding_item(item_input))
# Element-wise product of user and item embeddings
predict_vector = multiply([user_latent, item_latent])
# TODO apply Dropout after the mutliply operation?
if dropout:
assert isinstance(dropout, float)
predict_vector = Dropout(dropout)(predict_vector)
# Final prediction layer
if implicit:
LOG.info("GMF model to Predict binary implicit preference")
# prediction = Lambda(lambda x: K.sigmoid(K.sum(x)), output_shape=(1,))(predict_vector)
prediction = Dense(
1,
activation="sigmoid",
kernel_initializer="lecun_uniform",
name="prediction",
)(predict_vector)
else:
LOG.info("Prediction layer configured for explicit feedback")
from keras import backend as K
def custom_relu(x):
return K.minimum(5.0, K.relu(x=x))
prediction = Dense(
1,
# activation="linear",
activation=custom_relu,
kernel_initializer="lecun_uniform",
name="prediction",
)(predict_vector)
model = Model(inputs=[user_input, item_input], outputs=prediction)
return model
def _user_embeddings_from_weights(
n_users: int, embed_size: int, weights: List[np.ndarray]
) -> np.ndarray:
"""Returns user embedding from a list of weights
Args:
n_users (int): total number of users
embed_size (int): size of the user embedding
weights (List[np.ndarray]): weights from underlying neural network
Returns
np.ndarray : weights corresponding to user embedding
"""
# user_embed_idx: Optional[
# int
# ] = USER_EMBEDDING_LAYER_IDX # Important: only relevant for get_mf_model()
found = False
user_embed_idx = None
for idx, weight in enumerate(weights):
if weight.shape == (n_users, embed_size):
user_embed_idx = idx
if found:
raise ValueError(
f"Found two layers matching user embedding size: {embed_size}"
)
else:
found = True
if user_embed_idx is None:
raise ValueError("Failed to find user embeddings in input weights")
user_embedding = weights[user_embed_idx]
if np.isnan(user_embedding).any():
warnings.warn("Converting NaNs to numbers in user embeddings")
user_embedding = np.nan_to_num(user_embedding)
return user_embedding, user_embed_idx
user_embeddings_from_weights = _user_embeddings_from_weights
GMF = _create_gmf
GMFConstrained = _create_gmf_constrained
from typing import List
from keras import Input, Model
from keras.layers import Embedding, Flatten, concatenate, Dense
from keras.regularizers import l2
from .._base import getLogger
import logging
LOG = logging.getLogger("fedfast")
def get_model(n_users, n_items, layers: List[int], reg_layers: List[int]):
assert len(layers) == len(reg_layers)
num_layer = len(layers) # Number of layers in the MLP
user_input = Input(shape=(1,), dtype="int32", name="user_input")
item_input = Input(shape=(1,), dtype="int32", name="item_input")
mlp_embedding_user = Embedding(
input_dim=n_users,
output_dim=int(layers[0] / 2),
name="user_embedding",
embeddings_initializer="normal",
embeddings_regularizer=l2(reg_layers[0]),
input_length=1,
)
mlp_embedding_item = Embedding(
input_dim=n_items,
output_dim=int(layers[0] / 2),
name="item_embedding",
embeddings_initializer="normal",
embeddings_regularizer=l2(reg_layers[0]),
input_length=1,
)
# Crucial to flatten an embedding vector!
user_latent = Flatten()(mlp_embedding_user(user_input))
item_latent = Flatten()(mlp_embedding_item(item_input))
# The 0-th layer is the concatenation of embedding layers
vector = concatenate([user_latent, item_latent])
# MLP layers
for idx in range(1, num_layer):
layer = Dense(
layers[idx],
kernel_regularizer=l2(reg_layers[idx]),
activation="relu",
name="layer%d" % idx,
)
vector = layer(vector)
# Final prediction layer
prediction = Dense(
1, activation="sigmoid", kernel_initializer="lecun_uniform", name="prediction"
)(vector)
model = Model(inputs=[user_input, item_input], outputs=prediction)
return model
from typing import List
from keras import Input, Model
from keras.layers import Embedding, Flatten, multiply, concatenate, Dense
from keras.regularizers import l2
from .._base import getLogger
import logging
LOG = logging.getLogger("fedfast")
__all__ = ["NCF"]
def get_model(
n_users, n_items, layers: List[int], reg_layers: List[int], mf_dim=10, reg_mf=0
):
assert len(layers) == len(reg_layers)
num_layer = len(layers) # Number of layers in the MLP
# Input variables
user_input = Input(shape=(1,), dtype="int32", name="user_input")
item_input = Input(shape=(1,), dtype="int32", name="item_input")
# Embedding layer
mf_embedding_user = Embedding(
input_dim=n_users,
output_dim=mf_dim,
name="mf_embedding_user",
embeddings_initializer="normal",
embeddings_regularizer=l2(reg_mf),
input_length=1,
)
mf_embedding_item = Embedding(
input_dim=n_items,
output_dim=mf_dim,
name="mf_embedding_item",
embeddings_initializer="normal",
embeddings_regularizer=l2(reg_mf),
input_length=1,
)
mlp_embedding_user = Embedding(
input_dim=n_users,
output_dim=int(layers[0] / 2),
name="mlp_embedding_user",
embeddings_initializer="normal",
embeddings_regularizer=l2(reg_layers[0]),
input_length=1,
)
mlp_embedding_item = Embedding(
input_dim=n_items,
output_dim=int(layers[0] / 2),
name="mlp_embedding_item",
embeddings_initializer="normal",
embeddings_regularizer=l2(reg_layers[0]),
input_length=1,
)
# MF part
mf_user_latent = Flatten()(mf_embedding_user(user_input))
mf_item_latent = Flatten()(mf_embedding_item(item_input))
mf_vector = multiply([mf_user_latent, mf_item_latent]) # element-wise multiply
# MLP part
mlp_user_latent = Flatten()(mlp_embedding_user(user_input))
mlp_item_latent = Flatten()(mlp_embedding_item(item_input))
mlp_vector = concatenate([mlp_user_latent, mlp_item_latent])
for idx in range(1, num_layer):
layer = Dense(
layers[idx],
kernel_initializer=l2(reg_layers[idx]),
activation="relu",
name="layer%d" % idx,
)
mlp_vector = layer(mlp_vector)
# Concatenate MF and MLP parts
# mf_vector = Lambda(lambda x: x * alpha)(mf_vector)
# mlp_vector = Lambda(lambda x : x * (1-alpha))(mlp_vector)
predict_vector = concatenate([mf_vector, mlp_vector])
# Final prediction layer
prediction = Dense(
1, activation="sigmoid", kernel_initializer="lecun_uniform", name="prediction"
)(predict_vector)
model = Model(inputs=[user_input, item_input], outputs=prediction)
return model
NCF = get_model
python - How to get word vectors from Keras Embedding Layer - Stack Overflow
class NCF:
"""Neural Collaborative Filtering (NCF) implementation
:Citation:
He, Xiangnan, Lizi Liao, Hanwang Zhang, Liqiang Nie, Xia Hu, and Tat-Seng Chua. "Neural collaborative filtering."
In Proceedings of the 26th International Conference on World Wide Web, pp. 173-182. International World Wide Web
Conferences Steering Committee, 2017. Link: https://www.comp.nus.edu.sg/~xiangnan/papers/ncf.pdf
"""
def __init__(
self,
n_users,
n_items,
model_type="NeuMF",
n_factors=8,
layer_sizes=[16, 8, 4],
n_epochs=50,
batch_size=64,
learning_rate=5e-3,
verbose=1,
seed=None,
):
"""Constructor
Args:
n_users (int): Number of users in the dataset.
n_items (int): Number of items in the dataset.
model_type (str): Model type.
n_factors (int): Dimension of latent space.
layer_sizes (list): Number of layers for MLP.
n_epochs (int): Number of epochs for training.
batch_size (int): Batch size.
learning_rate (float): Learning rate.
verbose (int): Whether to show the training output or not.
seed (int): Seed.
"""
# seed
tf.compat.v1.set_random_seed(seed)
np.random.seed(seed)
self.seed = seed
self.n_users = n_users
self.n_items = n_items
self.model_type = model_type.lower()
self.n_factors = n_factors
self.layer_sizes = layer_sizes
self.n_epochs = n_epochs
self.verbose = verbose
self.batch_size = batch_size
self.learning_rate = learning_rate
# check model type
model_options = ["gmf", "mlp", "neumf"]
if self.model_type not in model_options:
raise ValueError(
"Wrong model type, please select one of this list: {}".format(
model_options
)
)
# ncf layer input size
self.ncf_layer_size = n_factors + layer_sizes[-1]
# create ncf model
self._create_model()
# set GPU use with demand growth
gpu_options = tf.compat.v1.GPUOptions(allow_growth=True)
# set TF Session
self.sess = tf.compat.v1.Session(
config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)
)
# parameters initialization
self.sess.run(tf.compat.v1.global_variables_initializer())
def _create_model(
self,
):
# reset graph
tf.compat.v1.reset_default_graph()
with tf.compat.v1.variable_scope("input_data", reuse=tf.compat.v1.AUTO_REUSE):
# input: index of users, items and ground truth
self.user_input = tf.compat.v1.placeholder(tf.int32, shape=[None, 1])
self.item_input = tf.compat.v1.placeholder(tf.int32, shape=[None, 1])
self.labels = tf.compat.v1.placeholder(tf.float32, shape=[None, 1])
with tf.compat.v1.variable_scope("embedding", reuse=tf.compat.v1.AUTO_REUSE):
# set embedding table
self.embedding_gmf_P = tf.Variable(
tf.random.truncated_normal(
shape=[self.n_users, self.n_factors],
mean=0.0,
stddev=0.01,
seed=self.seed,
),
name="embedding_gmf_P",
dtype=tf.float32,
)
self.embedding_gmf_Q = tf.Variable(
tf.random.truncated_normal(
shape=[self.n_items, self.n_factors],
mean=0.0,
stddev=0.01,
seed=self.seed,
),
name="embedding_gmf_Q",
dtype=tf.float32,
)
# set embedding table
self.embedding_mlp_P = tf.Variable(
tf.random.truncated_normal(
shape=[self.n_users, int(self.layer_sizes[0] / 2)],
mean=0.0,
stddev=0.01,
seed=self.seed,
),
name="embedding_mlp_P",
dtype=tf.float32,
)
self.embedding_mlp_Q = tf.Variable(
tf.random.truncated_normal(
shape=[self.n_items, int(self.layer_sizes[0] / 2)],
mean=0.0,
stddev=0.01,
seed=self.seed,
),
name="embedding_mlp_Q",
dtype=tf.float32,
)
with tf.compat.v1.variable_scope("gmf", reuse=tf.compat.v1.AUTO_REUSE):
# get user embedding p and item embedding q
self.gmf_p = tf.reduce_sum(
input_tensor=tf.nn.embedding_lookup(
params=self.embedding_gmf_P, ids=self.user_input
),
axis=1,
)
self.gmf_q = tf.reduce_sum(
input_tensor=tf.nn.embedding_lookup(
params=self.embedding_gmf_Q, ids=self.item_input
),
axis=1,
)
# get gmf vector
self.gmf_vector = self.gmf_p * self.gmf_q
with tf.compat.v1.variable_scope("mlp", reuse=tf.compat.v1.AUTO_REUSE):
# get user embedding p and item embedding q
self.mlp_p = tf.reduce_sum(
input_tensor=tf.nn.embedding_lookup(
params=self.embedding_mlp_P, ids=self.user_input
),
axis=1,
)
self.mlp_q = tf.reduce_sum(
input_tensor=tf.nn.embedding_lookup(
params=self.embedding_mlp_Q, ids=self.item_input
),
axis=1,
)
# concatenate user and item vector
output = tf.concat([self.mlp_p, self.mlp_q], 1)
# MLP Layers
for layer_size in self.layer_sizes[1:]:
output = slim.layers.fully_connected(
output,
num_outputs=layer_size,
activation_fn=tf.nn.relu,
weights_initializer=tf.compat.v1.keras.initializers.VarianceScaling(
scale=1.0,
mode="fan_avg",
distribution="uniform",
seed=self.seed,
),
)
self.mlp_vector = output
# self.output = tf.sigmoid(tf.reduce_sum(self.mlp_vector, axis=1, keepdims=True))
with tf.compat.v1.variable_scope("ncf", reuse=tf.compat.v1.AUTO_REUSE):
if self.model_type == "gmf":
# GMF only
output = slim.layers.fully_connected(
self.gmf_vector,
num_outputs=1,
activation_fn=None,
biases_initializer=None,
weights_initializer=tf.compat.v1.keras.initializers.VarianceScaling(
scale=1.0,
mode="fan_avg",
distribution="uniform",
seed=self.seed,
),
)
self.output = tf.sigmoid(output)
elif self.model_type == "mlp":
# MLP only
output = slim.layers.fully_connected(
self.mlp_vector,
num_outputs=1,
activation_fn=None,
biases_initializer=None,
weights_initializer=tf.compat.v1.keras.initializers.VarianceScaling(
scale=1.0,
mode="fan_avg",
distribution="uniform",
seed=self.seed,
),
)
self.output = tf.sigmoid(output)
elif self.model_type == "neumf":
# concatenate GMF and MLP vector
self.ncf_vector = tf.concat([self.gmf_vector, self.mlp_vector], 1)
# get predicted rating score
output = slim.layers.fully_connected(
self.ncf_vector,
num_outputs=1,
activation_fn=None,
biases_initializer=None,
weights_initializer=tf.compat.v1.keras.initializers.VarianceScaling(
scale=1.0,
mode="fan_avg",
distribution="uniform",
seed=self.seed,
),
)
self.output = tf.sigmoid(output)
with tf.compat.v1.variable_scope("loss", reuse=tf.compat.v1.AUTO_REUSE):
# set loss function
self.loss = tf.compat.v1.losses.log_loss(self.labels, self.output)
with tf.compat.v1.variable_scope("optimizer", reuse=tf.compat.v1.AUTO_REUSE):
# set optimizer
self.optimizer = tf.compat.v1.train.AdamOptimizer(
learning_rate=self.learning_rate
).minimize(self.loss)
def save(self, dir_name):
"""Save model parameters in `dir_name`
Args:
dir_name (str): directory name, which should be a folder name instead of file name
we will create a new directory if not existing.
"""
# save trained model
if not os.path.exists(dir_name):
os.makedirs(dir_name)
saver = tf.compat.v1.train.Saver()
saver.save(self.sess, os.path.join(dir_name, MODEL_CHECKPOINT))
def load(self, gmf_dir=None, mlp_dir=None, neumf_dir=None, alpha=0.5):
"""Load model parameters for further use.
GMF model --> load parameters in `gmf_dir`
MLP model --> load parameters in `mlp_dir`
NeuMF model --> load parameters in `neumf_dir` or in `gmf_dir` and `mlp_dir`
Args:
gmf_dir (str): Directory name for GMF model.
mlp_dir (str): Directory name for MLP model.
neumf_dir (str): Directory name for neumf model.
alpha (float): the concatenation hyper-parameter for gmf and mlp output layer.
Returns:
object: Load parameters in this model.
"""
# load pre-trained model
if self.model_type == "gmf" and gmf_dir is not None:
saver = tf.compat.v1.train.Saver()
saver.restore(self.sess, os.path.join(gmf_dir, MODEL_CHECKPOINT))
elif self.model_type == "mlp" and mlp_dir is not None:
saver = tf.compat.v1.train.Saver()
saver.restore(self.sess, os.path.join(mlp_dir, MODEL_CHECKPOINT))
elif self.model_type == "neumf" and neumf_dir is not None:
saver = tf.compat.v1.train.Saver()
saver.restore(self.sess, os.path.join(neumf_dir, MODEL_CHECKPOINT))
elif self.model_type == "neumf" and gmf_dir is not None and mlp_dir is not None:
# load neumf using gmf and mlp
self._load_neumf(gmf_dir, mlp_dir, alpha)
else:
raise NotImplementedError
def _load_neumf(self, gmf_dir, mlp_dir, alpha):
"""Load gmf and mlp model parameters for further use in NeuMF.
NeuMF model --> load parameters in `gmf_dir` and `mlp_dir`
"""
# load gmf part
variables = tf.compat.v1.global_variables()
# get variables with 'gmf'
var_flow_restore = [
val for val in variables if "gmf" in val.name and "ncf" not in val.name
]
# load 'gmf' variable
saver = tf.compat.v1.train.Saver(var_flow_restore)
# restore
saver.restore(self.sess, os.path.join(gmf_dir, MODEL_CHECKPOINT))
# load mlp part
variables = tf.compat.v1.global_variables()
# get variables with 'gmf'
var_flow_restore = [
val for val in variables if "mlp" in val.name and "ncf" not in val.name
]
# load 'gmf' variable
saver = tf.compat.v1.train.Saver(var_flow_restore)
# restore
saver.restore(self.sess, os.path.join(mlp_dir, MODEL_CHECKPOINT))
# concat pretrain h_from_gmf and h_from_mlp
vars_list = tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope="ncf"
)
assert len(vars_list) == 1
ncf_fc = vars_list[0]
# get weight from gmf and mlp
gmf_fc = tf.train.load_variable(gmf_dir, ncf_fc.name)
mlp_fc = tf.train.load_variable(mlp_dir, ncf_fc.name)
# load fc layer by tf.concat
assign_op = tf.compat.v1.assign(
ncf_fc, tf.concat([alpha * gmf_fc, (1 - alpha) * mlp_fc], axis=0)
)
self.sess.run(assign_op)
def fit(self, data):
"""Fit model with training data
Args:
data (NCFDataset): initilized Dataset in ./dataset.py
"""
# get user and item mapping dict
self.user2id = data.user2id
self.item2id = data.item2id
self.id2user = data.id2user
self.id2item = data.id2item
# loop for n_epochs
for epoch_count in range(1, self.n_epochs + 1):
# negative sampling for training
train_begin = time()
# initialize
train_loss = []
# calculate loss and update NCF parameters
for user_input, item_input, labels in data.train_loader(self.batch_size):
user_input = np.array([self.user2id[x] for x in user_input])
item_input = np.array([self.item2id[x] for x in item_input])
labels = np.array(labels)
feed_dict = {
self.user_input: user_input[..., None],
self.item_input: item_input[..., None],
self.labels: labels[..., None],
}
# get loss and execute optimization
loss, _ = self.sess.run([self.loss, self.optimizer], feed_dict)
train_loss.append(loss)
train_time = time() - train_begin
# output every self.verbose
if self.verbose and epoch_count % self.verbose == 0:
logger.info(
"Epoch %d [%.2fs]: train_loss = %.6f "
% (epoch_count, train_time, sum(train_loss) / len(train_loss))
)
def predict(self, user_input, item_input, is_list=False):
"""Predict function of this trained model
Args:
user_input (list or element of list): userID or userID list
item_input (list or element of list): itemID or itemID list
is_list (bool): if true, the input is list type
noting that list-wise type prediction is faster than element-wise's.
Returns:
list or float: A list of predicted rating or predicted rating score.
"""
if is_list:
output = self._predict(user_input, item_input)
return list(output.reshape(-1))
else:
output = self._predict(np.array([user_input]), np.array([item_input]))
return float(output.reshape(-1)[0])
def _predict(self, user_input, item_input):
# index converting
user_input = np.array([self.user2id[x] for x in user_input])
item_input = np.array([self.item2id[x] for x in item_input])
# get feed dict
feed_dict = {
self.user_input: user_input[..., None],
self.item_input: item_input[..., None],
}
# calculate predicted score
return self.sess.run(self.output, feed_dict)- [ ]
- Should I implement NCF, which embeddings should I choose?
- Check out Altair for visualization
Using LightFM with BPR loss: https://github.dev/amitkaps/recommendation
- GitHub - amitkaps/recommendation: Recommendation System using ML and DL
- recoder/embedding.py at master · amoussawi/recoder · GitHub
- good docstrings
- Annoy
- Negative sampling
- GitHub - datadynamo/strata_ny_2017_recommender_tutorial: Jupyter Notebooks for Strata Data Conference NY 2017 Deep Learning for Recommender Systems Tutorial
Tutorial — Recoder 0.4.0 documentation
Use tf.truncated_normal_initializer(mean=0, stddev=.05)
Set dtype to tf.float32
def _create_gmf(n_users, n_items, regs=None, embedding_size=10, implicit=False):
"""Create a GMF model
Args:
n_users (int): number of user embeddings
n_items (int): number of item embeddings
regs[Optional[List[int]]]: regularization parameters for embedding layer
embedding_size(int): size of each user and item embedding
Returns:
"keras.models.Model": GMF model
"""
# To control where and how TensorFlow sessions are created, we
# do imports here make keras work well with multiprocessing.
from tensorflow.keras import Input, Model
from tensorflow.keras.layers import Dense, Embedding, Flatten, multiply
from tensorflow.keras.regularizers import l2
if regs:
assert len(regs) == 2
regs = regs or [0, 0]
user_input = Input(shape=(1,), dtype="int32", name="user_input")
item_input = Input(shape=(1,), dtype="int32", name="item_input")
mf_embedding_user = Embedding(
input_dim=n_users,
output_dim=embedding_size,
name="user_embedding",
embeddings_initializer="normal",
embeddings_regularizer=l2(regs[0]),
input_length=1,
)
mf_embedding_item = Embedding(
input_dim=n_items,
output_dim=embedding_size,
name="item_embedding",
embeddings_initializer="normal",
embeddings_regularizer=l2(regs[1]),
input_length=1,
)
# Crucial to flatten an embedding vector!
user_latent = Flatten()(mf_embedding_user(user_input))
item_latent = Flatten()(mf_embedding_item(item_input))
# Element-wise product of user and item embeddings
predict_vector = multiply([user_latent, item_latent])
# TODO apply Dropout after the mutliply operation?
# Final prediction layer
if implicit:
LOG.debug("GMF model to predict binary implicit preference")
# prediction = Lambda(lambda x: K.sigmoid(K.sum(x)), output_shape=(1,))(predict_vector)
prediction = Dense(
1,
activation="sigmoid",
kernel_initializer="lecun_uniform",
name="prediction",
)(predict_vector)
else:
LOG.debug("Prediction layer configured for explicit feedback")
prediction = Dense(
1,
activation="linear",
kernel_initializer="lecun_uniform",
name="prediction",
)(predict_vector)
model = Model(inputs=[user_input, item_input], outputs=prediction)
return model
GMF = _create_gmf
model = GMF(
n_users=train_df["user_id"].nunique(),
n_items=train_df["item_id"].nunique(),
embedding_size=10,
regs=[0, 0],
)
model.compile(optimizer="adam", loss="binary_crossentropy")@contextmanager
def timed_op(name: str = None, logger=None):
t0 = time.time()
try:
yield
finally:
duration = time.time() - t0
duration_str = str(timedelta(seconds=duration))
if logger is not None:
logger.info("Finished %s in %s", name, duration_str)
https://github.dev/amitkaps/recommendation/blob/master/MovieLens/14-Image-Features.ipynb
from sklearn.neighbors import NearestNeighbours
def get_similar(embedding, k):
model_similar_items = NearestNeighbors(n_neighbors=k, algorithm="ball_tree").fit(embedding)
distances, indices = model_similar_items.kneighbors(embedding)
return distances, indices
Source: GitHub - amitkaps/recommendation: Recommendation System using ML and DL Implementation of Implicit MF: recommendation/13-Implicit-MF.ipynb at master · amitkaps/recommendation · GitHub
from keras.models import Model
from keras.layers import Input, Embedding, Flatten, Dot, Add, Lambda, Activation, Reshape
from keras.regularizers import l2
from keras.constraints import non_neg
def ImplicitMF (n_users, n_items, n_factors):
# Item Layer
item_input = Input(shape=[1], name='Item')
item_embedding = Embedding(n_items, n_factors,
embeddings_regularizer=l2(1e-6),
name='ItemEmbedding')(item_input)
item_vec = Flatten(name='FlattenItemsE')(item_embedding)
# User Layer
user_input = Input(shape=[1], name='User')
user_embedding = Embedding(n_users, n_factors,
embeddings_regularizer=l2(1e-6),
name='UserEmbedding')(user_input)
user_vec = Flatten(name='FlattenUsersE')(user_embedding)
# Dot Product of Item and User
rating = Dot(axes=1, name='DotProduct')([item_vec, user_vec])
# Model Creation
model = Model([user_input, item_input], rating)
# Compile Model
model.compile(loss='binary_crossentropy', optimizer="sgd")
return modeldef Deep_MF(n_users, n_items, n_factors):
# Item Layer
item_input = Input(shape=[1], name='Item')
item_embedding = Embedding(n_items, n_factors, embeddings_regularizer=l2(1e-6),
embeddings_initializer='glorot_normal',
name='ItemEmbedding')(item_input)
item_vec = Flatten(name='FlattenItemE')(item_embedding)
# Item Bias
item_bias = Embedding(n_items, 1, embeddings_regularizer=l2(1e-6),
embeddings_initializer='glorot_normal',
name='ItemBias')(item_input)
item_bias_vec = Flatten(name='FlattenItemBiasE')(item_bias)
# User Layer
user_input = Input(shape=[1], name='User')
user_embedding = Embedding(n_users, n_factors, embeddings_regularizer=l2(1e-6),
embeddings_initializer='glorot_normal',
name='UserEmbedding')(user_input)
user_vec = Flatten(name='FlattenUserE')(user_embedding)
# User Bias
user_bias = Embedding(n_users, 1, embeddings_regularizer=l2(1e-6),
embeddings_initializer='glorot_normal',
name='UserBias')(user_input)
user_bias_vec = Flatten(name='FlattenUserBiasE')(user_bias)
# Dot Product of Item and User & then Add Bias
Concat = Concatenate(name='Concat')([item_vec, user_vec])
ConcatDrop = Dropout(0.5)(Concat)
kernel_initializer='he_normal'
# Use Dense to learn non-linear dense representation
Dense_1 = Dense(10, kernel_initializer='glorot_normal', name="Dense1")(ConcatDrop)
Dense_1_Drop = Dropout(0.5)(Dense_1)
Dense_2 = Dense(1, kernel_initializer='glorot_normal', name="Dense2")(Dense_1_Drop)
AddBias = Add(name="AddBias")([Dense_2, item_bias_vec, user_bias_vec])
# Scaling for each user
y = Activation('sigmoid')(AddBias)
rating_output = Lambda(lambda x: x * (max_rating - min_rating) + min_rating)(y)
# Model Creation
model = Model([user_input, item_input], rating_output)
# Compile Model
model.compile(loss='mean_squared_error', optimizer=Adam(lr=0.001))
return modeldef Neural_CF(n_users, n_items, n_factors):
# Item Layer
item_input = Input(shape=[1], name='Item')
# Item Embedding MF
item_embedding_mf = Embedding(n_items, n_factors, embeddings_regularizer=l2(1e-6),
embeddings_initializer='he_normal',
name='ItemEmbeddingMF')(item_input)
item_vec_mf = Flatten(name='FlattenItemMF')(item_embedding_mf)
# Item embedding MLP
item_embedding_mlp = Embedding(n_items, n_factors, embeddings_regularizer=l2(1e-6),
embeddings_initializer='he_normal',
name='ItemEmbeddingMLP')(item_input)
item_vec_mlp = Flatten(name='FlattenItemMLP')(item_embedding_mlp)
# User Layer
user_input = Input(shape=[1], name='User')
# User Embedding MF
user_embedding_mf = Embedding(n_users, n_factors, embeddings_regularizer=l2(1e-6),
embeddings_initializer='he_normal',
name='UserEmbeddingMF')(user_input)
user_vec_mf = Flatten(name='FlattenUserMF')(user_embedding_mf)
# User Embedding MF
user_embedding_mlp = Embedding(n_users, n_factors, embeddings_regularizer=l2(1e-6),
embeddings_initializer='he_normal',
name='UserEmbeddingMLP')(user_input)
user_vec_mlp = Flatten(name='FlattenUserMLP')(user_embedding_mlp)
# Multiply MF paths
DotProductMF = Dot(axes=1, name='DotProductMF')([item_vec_mf, user_vec_mf])
# Concat MLP paths
ConcatMLP = Concatenate(name='ConcatMLP')([item_vec_mlp, user_vec_mlp])
# Use Dense to learn non-linear dense representation
Dense_1 = Dense(50, name="Dense1")(ConcatMLP)
Dense_2 = Dense(20, name="Dense2")(Dense_1)
# Concatenate MF and MLP paths
Concat = Concatenate(name="ConcatAll")([DotProductMF, Dense_2])
# Use Dense to learn non-linear dense representation
Pred = Dense(1, name="Pred")(Concat)
# Item Bias
item_bias = Embedding(n_items, 1, embeddings_regularizer=l2(1e-5), name='ItemBias')(item_input)
item_bias_vec = Flatten(name='FlattenItemBiasE')(item_bias)
# User Bias
user_bias = Embedding(n_users, 1, embeddings_regularizer=l2(1e-5), name='UserBias')(user_input)
user_bias_vec = Flatten(name='FlattenUserBiasE')(user_bias)
# Pred with bias added
PredAddBias = Add(name="AddBias")([Pred, item_bias_vec, user_bias_vec])
# Scaling for each user
y = Activation('sigmoid')(PredAddBias)
rating_output = Lambda(lambda x: x * (max_rating - min_rating) + min_rating)(y)
# Model Creation
model = Model([user_input, item_input], rating_output)
# Compile Model
model.compile(loss='mean_squared_error', optimizer="adam")
return modelThe GitHub repo for this project is GitHub - PreferredAI/cornac: A Comparative Framework for Multimodal Recommender Systems
with tf.variable_scope(scope):
user_emb = tf.get_variable(
"user_emb",
shape=[num_users, emb_size],
dtype=tf.float32,
initializer=tf.random_normal_initializer(stddev=0.01, seed=seed),
regularizer=tf.contrib.layers.l2_regularizer(scale=reg_user),
)
item_emb = tf.get_variable(
"item_emb",
shape=[num_items, emb_size],
dtype=tf.float32,
initializer=tf.random_normal_initializer(stddev=0.01, seed=seed),
regularizer=tf.contrib.layers.l2_regularizer(scale=reg_item),
)
return tf.nn.embedding_lookup(user_emb, uid), tf.nn.embedding_lookup(item_emb, iid)When merging user and item embedding, use tf.multiply
Cornac does it well. For GMF, user and item embeddings are multiplied. But in MLP, they’re concatenated. See https://github.com/PreferredAI/cornac/blob/master/cornac/models/ncf/ops.py
Batch size is 256. num_neg = 4, learning rate = 0.001
This is Cornac’s implementation of GMF: https://github.com/PreferredAI/cornac/blob/master/cornac/models/ncf/recom_gmf.py
Their MLP implementation is : https://github.com/PreferredAI/cornac/blob/master/cornac/models/ncf/recom_mlp.py
# Import necessary libraries
from keras.layers import Input, Embedding, Flatten, Dot, Add, Concatenate
from keras.models import Model
# Define the input and embedding layers
input_user = Input(shape=(1,))
input_item = Input(shape=(1,))
embedding_user = Embedding(input_dim=num_users, output_dim=latent_dim, input_length=1)(input_user)
embedding_item = Embedding(input_dim=num_items, output_dim=latent_dim, input_length=1)(input_item)
# Flatten the embedding layers
flat_user = Flatten()(embedding_user)
flat_item = Flatten()(embedding_item)
# Use the Dot layer to compute the dot product of the user and item embeddings
dot_product = Dot(axes=1)([flat_user, flat_item])
# Add the user and item bias terms
user_bias = Embedding(input_dim=num_users, output_dim=1, input_length=1)(input_user)
item_bias = Embedding(input_dim=num_items, output_dim=1, input_length=1)(input_item)
user_bias = Flatten()(user_bias)
item_bias = Flatten()(item_bias)
bias_term = Add()([user_bias, item_bias])
# Concatenate the dot product and bias term to create the final prediction
output = Concatenate()([dot_product, bias_term])
# Create and compile the model
model = Model(inputs=[input_user, input_item], outputs=output)
model.compile(optimizer="adam", loss="mean_squared_error")
# Train the model on the input data
model.fit([users, items], ratings)