Skip to content

Instantly share code, notes, and snippets.

Avatar
👐
We are hiring! https://jobs.jina.ai

Han Xiao hanxiao

👐
We are hiring! https://jobs.jina.ai
View GitHub Profile
@hanxiao
hanxiao / test.py
Created Jul 13, 2018
get mean rank of positive labels
View test.py
def get_pos_mean_rank(self, score, label):
_, d = tf.nn.top_k(score, k=self.size_support)
f = tf.gather(label, d)
g = tf.where(tf.not_equal(f, 0))
return tf.reduce_mean(g)
View gist:c210f9e232db50167228bc67e2700e34
datasets = [tf.data.Dataset...]
def concat_datasets(datasets):
ds0 = tf.data.Dataset.from_tensors(datasets[0])
for ds1 in datasets[1:]:
ds0 = ds0.concatenate(tf.data.Dataset.from_tensors(ds1))
return ds0
ds = tf.data.Dataset.zip(tuple(datasets)).flat_map(
lambda *args: concat_datasets(args)
@hanxiao
hanxiao / test.py
Created Jul 4, 2018
reject sampling with tf dataset api
View test.py
(tf.data.Dataset.apply(tf.contrib.data.rejection_resample(class_func=lambda x: tf.py_func(self.get_class_label, [x],
[tf.int64],
name='class_label_fn'),
target_dist=tf.constant(self.uniform_dist, dtype=tf.float32),
initial_dist=tf.constant(self.init_dist, dtype=tf.float32)))
.map(lambda _, x: x)
.shuffle(buffer_size=1000)
.batch(self._args.batch_size)
.map(lambda x: tf.py_func(self._make_batch, [x], tf.string, name='train_mkbatch_fn'))
.prefetch(self._args.batch_size * self._args.prefetch_factor))
@hanxiao
hanxiao / mlstm.py
Last active May 2, 2018
MLSTM to match passage encodes with question encodes
View mlstm.py
# Python 3.6 + TF 1.6
# Han Xiao (artex.xh@gmail.com)
import tensorflow as tf
import tensorflow.contrib as tc
#### Usage: matching passage encodes with question encodes
ml = MatchLSTMLayer(hidden_size=16,
control_gate=False,
@hanxiao
hanxiao / p2ready.sh
Last active Feb 11, 2018
Prepare a p2 instance for Tensorflow
View p2ready.sh
#!/bin/bash
# bash <(curl -fsSL https://gist.githubusercontent.com/hanxiao/46237d60f640ab0e70e2a8365868b6db/raw/673cff6bfab2e0ae9fb94f3f1c121a507616d131/p2ready.sh)
# install python 3.6 using pyenv
sudo apt-get -y update
sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev
curl -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash
echo 'export PATH="/home/ubuntu/.pyenv/bin:$PATH"' >> ~/.bash_profile
echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile
@hanxiao
hanxiao / gist:a5c08238a5600b48f1e0f08ae68cab83
Created May 4, 2017
python in class decorator with parameter
View gist:a5c08238a5600b48f1e0f08ae68cab83
class Baaar:
def __init__(self):
self.z = 10
def wrapper(v:int = 0):
def f2(f):
def f2_v(self, *args, **kwargs):
print('z=%d' % self.z)
return f(self, *args, **kwargs) + v
@hanxiao
hanxiao / test.py
Created May 4, 2017
python decorator wrapper function
View test.py
def wrapper(v:int = 0):
def f2(f):
def f2_v(*args, **kwargs):
return f(*args, **kwargs) + v
return f2_v
return f2
@wrapper(v=4)
def foo(x:int):