Skip to content

Instantly share code, notes, and snippets.

Hannes Hapke hanneshapke

Block or report user

Report or block hanneshapke

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
View model_inference_tf_example.py
import base64
import googleapiclient.discovery
from example_pb2 import Example
from feature_pb2 import BytesList, Feature, Features
def _convert_to_pb(value):
""" Serialize a given sentence to the ProtoBuf Structure required to model the tf.Example data structure.
Feel free to add more features and different data types if your models reqiures different inputs. An overview of
View .bashrc
export CURRENT_DEV=kreuzberg
alias latest_dev='cd ~/development/$CURRENT_DEV'
# ssh tunnel
alias ssd='~/bin/ssh_host_color.sh ubuntu@remote -p 823 -L 6006:gpu:6006'
# add additional paths to the PYTHONPATH
export PYTHONPATH=$PYTHONPATH:~/development/additional_package
# git shortcuts
@hanneshapke
hanneshapke / redis.py
Created May 24, 2018
Load word vectors into a redis db
View redis.py
import bz2
import pickle
from django.conf import settings
from djang_redis import get_redis_connection
from tqdm import tqdm
from .constants import GOOGLE_WORD2VEC_MODEL_NAME
@hanneshapke
hanneshapke / redis_vectors.py
Last active May 24, 2018
Mimicking Gensim's KeyedVectors class
View redis_vectors.py
import bz2
import numpy as np
import pickle
from django.conf import settings
from django_redis import get_redis_connection
from gensim.models.keyedvectors import KeyedVectors
from .constants import GOOGLE_WORD2VEC_MODEL_NAME
from .redis import load_word2vec_model_into_redis, query_redis
View get_highlighted_tokens.py
def get_highlighted_tokens(tokens, matrix, model, layer_name, threshold, y_labels):
indices = get_token_indices(model, layer_name, threshold, matrix, y_labels)
ctokens = []
for i, t in enumerate(tokens):
if i in indices.keys():
_color = color(indices[i], threshold=threshold)
ctokens.append(cstr(t, color=_color))
else:
ctokens.append(t)
View get_token_indicies.py
def get_token_indices(model, layer_name, threshold, matrix, y_labels):
heatmap = get_heatmap(model=model, layer_name=layer_name, matrix=matrix, y_labels=y_labels)
_, output_dim = get_conv_layer(model, layer_name)
# depending on the ration between the input and layer output shape, we need to calculate
# how many original tokens have contributed to the layer output
dim_ratio = matrix.shape[1] / output_dim
if dim_ratio < 1.5:
window_size = 1
else:
@hanneshapke
hanneshapke / generate_color_for_heatmap_value.py
Last active Apr 16, 2018
Generate text color depending on the heat-map value
View generate_color_for_heatmap_value.py
def color(hvalue, threshold, max=1, cdefault='black', colors=['red', 'yellow', 'green', 'cyan', 'blue']):
num_colors = len(colors)
if hvalue < threshold:
return cdefault
for i, color in enumerate(colors):
if hvalue > (max - (max - threshold) / num_colors * (i + 1)):
return color
@hanneshapke
hanneshapke / generate_text_tag.py
Created Apr 11, 2018
Generate the text tag with the color attribute
View generate_text_tag.py
def cstr(s, color='black'):
return "<text style=\"color:{}\">{}</text>".format(color, s)
@hanneshapke
hanneshapke / plot_heatmap.py
Last active Apr 16, 2018
Plot the layer heatmap
View plot_heatmap.py
def plot_heatmap(heatmap, height_ratio=0.05):
# calculating how often the vector should be repeated to display a height relative to the vector length
repeat_vector_n_times = int(heatmap.shape[0] * height_ratio)
plt.matshow([heatmap] * repeat_vector_n_times)
@hanneshapke
hanneshapke / norm_heatmap.py
Last active Apr 16, 2018
Normalize the visualization heat map
View norm_heatmap.py
def norm_heatmap(heatmap):
# element-wise maximum calculation, basically setting all negative values to zero
heatmap = np.maximum(heatmap, 0)
# normalizing the heatmap to values between 0 and 1
norm_heatmap = heatmap / np.max(heatmap)
return norm_heatmap
You can’t perform that action at this time.