Skip to content

Instantly share code, notes, and snippets.

@SuoXC
SuoXC / safe_backup.py
Created August 20, 2020 11:49
rsync sometimes delete files with no backup, so this script is a solution to safely sync things
import filecmp
import os
import pathlib
import shutil
def no_change(src_file, dest_file):
equal = filecmp.cmp(src_file.resolve(), dest_file.resolve(), False)
return equal
@SuoXC
SuoXC / recursive_unzip.py
Created December 6, 2019 04:16
use zipfile to unzip a zipfile recursively to target directory
import os
import zipfile
def recursive_unzip(zip_file, target_dir):
print("export: ", zip_file)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
@SuoXC
SuoXC / bench_concurrency_models.py
Last active May 11, 2019 08:42
python concurrency models compare
import random
import time
from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED
from memory_profiler import profile
random.seed(10)
total_iterations = 1000
@SuoXC
SuoXC / __init__.py
Last active March 11, 2019 09:13
auto register sub modules in current dir
import os
import importlib
# 自动注册当前文件夹下所有模块
module_dir = os.path.dirname(__file__)
for i in os.listdir(module_dir):
module_name = os.path.basename(i).split('.')[0]
importlib.import_module('.' + module_name, __package__)
@SuoXC
SuoXC / filter_json_logs.sh
Created January 16, 2019 10:41
use *jq* to filter json logs and convert some data into csv mode
pattern="*.log" #change to your file pattern
# need to install jq
function filter_function(){
file=$1
cat $file |jq -r 'select(.userid==100) |[.userid, .name, .age, .type] |@csv'
}
files=$(ls $pattern)
for file in $files ; do
echo "current_file:$file"
@SuoXC
SuoXC / .htaccess
Created January 1, 2019 12:45
apache php single entrance config file, place it in root dir of project
<IfModule mod_rewrite.c>
<IfModule mod_negotiation.c>
Options -MultiViews
</IfModule>
RewriteEngine On
# Redirect Trailing Slashes...
RewriteRule ^(.*)/$ /$1 [L,R=301]
@SuoXC
SuoXC / wgan_gp_loss.py
Created December 19, 2018 06:23
wgan gp loss definition
import tensorflow as tf
def gradient_panalty(real, fake, discriminator, alpha, gp_lambda=10):
# alpha = tf.placeholder(shape=[None, 1, 1, 1], dtype=tf.float32)
# alpha = tf.random_uniform(shape=[batch_size, 1, 1, 1], minval=0., maxval=1.)
interpolated = real + alpha * (fake - real)
logit = discriminator(interpolated)
grad = tf.gradients(logit, interpolated)[0] # gradient of D(interpolated)
grad_norm = tf.norm(tf.layers.flatten(grad), axis=1) # l2 norm
gp = gp_lambda * tf.reduce_mean(tf.square(grad_norm - 1.))
return gp
@SuoXC
SuoXC / export_model.py
Last active December 14, 2018 03:09
convert keras or tensorflow or tensorflow-keras model to a predictable saved_model for tensorflow-serving and python predictor
import tensorflow as tf
def export_submodel(sess, inputs, outputs, output_dir):
output_nodes = list(inputs.values()) + list(outputs.values())
output_node_names = [node.name.replace(":0", "") for node in output_nodes]
graph = tf.get_default_graph()
graph_def = graph.as_graph_def()
# sess = tf.get_default_session()
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess,
@SuoXC
SuoXC / async_queue.py
Last active November 22, 2018 15:52
gpu training util, to accelerate dataloader's speed
from queue import Queue
from threading import Thread
class AsyncQueue:
def __init__(self, generator, cache_size=10):
self._q = Queue(maxsize=cache_size)
self._generator = generator
self._input_thread = Thread(target=self.append_to_queue)
self._input_thread.setDaemon(True)
@SuoXC
SuoXC / aria2.conf
Created September 14, 2018 15:03
aria2 downloader on manjaro linux
enable-rpc=true
rpc-allow-origin-all=true
rpc-listen-all=true
#rpc-secret=xiaocong
dir=/home/suo/下载/aria2
#是否启用https加密,启用之后要设置公钥,私钥的文件路径
#rpc-secure=true
#启用加密设置公钥