Skip to content

Instantly share code, notes, and snippets.

@hanxiao
hanxiao / test.py
Created May 4, 2017 21:17
python decorator wrapper function
def wrapper(v:int = 0):
def f2(f):
def f2_v(*args, **kwargs):
return f(*args, **kwargs) + v
return f2_v
return f2
@wrapper(v=4)
def foo(x:int):
@hanxiao
hanxiao / gist:a5c08238a5600b48f1e0f08ae68cab83
Created May 4, 2017 21:28
python in class decorator with parameter
class Baaar:
def __init__(self):
self.z = 10
def wrapper(v:int = 0):
def f2(f):
def f2_v(self, *args, **kwargs):
print('z=%d' % self.z)
return f(self, *args, **kwargs) + v
@hanxiao
hanxiao / p2ready.sh
Last active February 11, 2018 21:31
Prepare a p2 instance for Tensorflow
#!/bin/bash
# bash <(curl -fsSL https://gist.githubusercontent.com/hanxiao/46237d60f640ab0e70e2a8365868b6db/raw/673cff6bfab2e0ae9fb94f3f1c121a507616d131/p2ready.sh)
# install python 3.6 using pyenv
sudo apt-get -y update
sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev
curl -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash
echo 'export PATH="/home/ubuntu/.pyenv/bin:$PATH"' >> ~/.bash_profile
echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile
@hanxiao
hanxiao / mlstm.py
Last active May 2, 2018 04:12
MLSTM to match passage encodes with question encodes
# Python 3.6 + TF 1.6
# Han Xiao (artex.xh@gmail.com)
import tensorflow as tf
import tensorflow.contrib as tc
#### Usage: matching passage encodes with question encodes
ml = MatchLSTMLayer(hidden_size=16,
control_gate=False,
@hanxiao
hanxiao / test.py
Created July 4, 2018 11:05
reject sampling with tf dataset api
(tf.data.Dataset.apply(tf.contrib.data.rejection_resample(class_func=lambda x: tf.py_func(self.get_class_label, [x],
[tf.int64],
name='class_label_fn'),
target_dist=tf.constant(self.uniform_dist, dtype=tf.float32),
initial_dist=tf.constant(self.init_dist, dtype=tf.float32)))
.map(lambda _, x: x)
.shuffle(buffer_size=1000)
.batch(self._args.batch_size)
.map(lambda x: tf.py_func(self._make_batch, [x], tf.string, name='train_mkbatch_fn'))
.prefetch(self._args.batch_size * self._args.prefetch_factor))
@hanxiao
hanxiao / gist:c210f9e232db50167228bc67e2700e34
Created July 5, 2018 01:55
Interleaving multiple datasets
datasets = [tf.data.Dataset...]
def concat_datasets(datasets):
ds0 = tf.data.Dataset.from_tensors(datasets[0])
for ds1 in datasets[1:]:
ds0 = ds0.concatenate(tf.data.Dataset.from_tensors(ds1))
return ds0
ds = tf.data.Dataset.zip(tuple(datasets)).flat_map(
lambda *args: concat_datasets(args)
@hanxiao
hanxiao / test.py
Created July 13, 2018 02:43
get mean rank of positive labels
def get_pos_mean_rank(self, score, label):
_, d = tf.nn.top_k(score, k=self.size_support)
f = tf.gather(label, d)
g = tf.where(tf.not_equal(f, 0))
return tf.reduce_mean(g)
@hanxiao
hanxiao / metadata.tsv
Created February 18, 2024 07:23
Projector Data Files
We can make this file beautiful and searchable if this error is corrected: No tabs found in this TSV file in line 0.
hello
world
@hanxiao
hanxiao / project_config.json
Created February 18, 2024 07:23
Projector Meta Files
{"embeddings":[{"tensorName":"jina-embeddings-v2-base-en","tensorShape":[2,768],"tensorPath":"https://gist.githubusercontent.com/hanxiao/ba5fcb4d59dd6ceecbe1c5e6f01bea71/raw/e1037c1789246de1941dcd0a8910d2e03a96c935/tensor.tsv","metadataPath":"https://gist.githubusercontent.com/hanxiao/ba5fcb4d59dd6ceecbe1c5e6f01bea71/raw/94954abda49de8615a048f8d2e64b5de848e27a1/metadata.tsv"}]}
@hanxiao
hanxiao / metadata.tsv
Created February 18, 2024 07:24
Projector Data Files
We can make this file beautiful and searchable if this error is corrected: No tabs found in this TSV file in line 0.
hello
world