Skip to content

Instantly share code, notes, and snippets.

View MInner's full-sized avatar

Ben Usman MInner

View GitHub Profile
@MInner
MInner / gist:d31604fb0b10c25a99dfe414ba3406eb
Last active November 25, 2021 23:40
prepare_images_for_arxiv.sh
find . -name '*.png' | while read line; do echo ${line}; convert -resize 70% ${line} ${line}.small.png; pngquant --speed 1 ${line}.small.png -o ${line}.quant.png; done
find . -name '*.small.png' | while read line; do rm $line; done
read -p "Press [Enter] if you are happy with *.quant.png images, otherwise press Ctrl-C."
find . -name '*.quant.png' | while read line; do mv ${line} ${line:0:-10}; done
for EXT in png pdf; do find -name "*.${EXT}" | while read line; do echo $line; done; done | while read line; do if ! grep -q -r --include \*.tex ${line#*/} .; then echo ${line}; rm ${line}; fi; done
use std::error::Error;
enum TaskError<'a, C> {
CustomError(C),
StdError(Box<dyn Error + 'a>)
}
type TaskResult<'a, R, E> = std::result::Result<R, TaskError<'a, E>>;
impl<'a, E: Error + 'a, C> From<E> for TaskError<'a, C> {
@MInner
MInner / gist:cc1704f5a22271073d694b45f60ba25d
Last active August 5, 2021 14:50
loop over two zipped arrays in bash
a=(x y z); b=(q w e); for i in ${!a[@]}; do echo ${a[i]}-${b[i]}; done
import sys
import glob
import tqdm
import imageio
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import tensorflow as tf
import tensorflow_datasets as tfds
spec = tfds.features.FeaturesDict({
'image': tfds.features.Tensor(shape=(28, 28, 1), dtype=tf.float32),
'label': tfds.features.ClassLabel(names=['no', 'yes']),
'metadata': {
'id': tf.int64,
'language': tf.string,
},
@MInner
MInner / run_release_gpu.py
Created July 27, 2018 22:49
Release GPU memory after tensorflow session is closed
def run_release_gpu(func):
def parallel_wrapper(output_dict, *argv, **kwargs):
ret = func(*argv, **kwargs)
if ret is not None:
output_dict['ret'] = ret
def outer_wrapper(*argv, **kwargs):
same_process = kwargs.pop('same_process', False)
if same_process:
return func(*argv, **kwargs)
@MInner
MInner / watch-nvidia-smi-user
Created May 5, 2018 23:12
Watch nvidia-smi and show associated users
#!/usr/bin/bash
watch "nvidia-smi | tail -n +7; nvidia-smi | nvidia-smi | sed -n '/| Processes:/,\$p' | tail -n +4 | head -n -1 | awk '{print \$3}' | while read line; do ps a -o pid,user | grep \$line | grep -v grep; done
"
@MInner
MInner / gpu_profile.py
Created September 12, 2017 16:11
A script to generate per-line GPU memory usage trace. For more meaningful results set `CUDA_LAUNCH_BLOCKING=1`.
import datetime
import linecache
import os
import pynvml3
import torch
print_tensor_sizes = True
last_tensor_sizes = set()
gpu_profile_fn = f'{datetime.datetime.now():%d-%b-%y-%H:%M:%S}-gpu_mem_prof.txt'
backtick 0 0 0 whoami
caption always "%{-b ..}%-w%{+b ..}[[%n%f*%t]]%{-}%+w %= %0`@%H | %l"
bind ` focus prev
@MInner
MInner / top_k_seq2seq.py
Last active October 25, 2017 02:46
This snipped extracts top k beams from the beam search output of github.com/google/seq2seq.
# based on https://github.com/google/seq2seq/blob/master/bin/tools/generate_beam_viz.py
# extracts probabilities and sequences from .npz file generated during beam search.
# and pickles a list of the length n_samples that has beam_width most probable tuples
# (path, logprob, prob)
# where probs are scaled to 1.
import numpy as np
import networkx as nx
import pickle