dmitrysarov

import pandas as pd
import os
import json

def flatten_dict(d):
    """
    transform dict of dicts to single level dict
    """
    fd = {}
    for k, v in d.items():

dmitrysarov

devops

Segmentation fault

can you do:

gdb python

and in gdb:

r main.py

dmitrysarov

import subprocess, os
import re
import click

def find_free_gpu(amount_of_space):
    #in Mb
    nvidia_ouput = subprocess.check_output(['nvidia-smi'])
    pid_memory = re.findall('(\d+)MiB\s/\s(\d+)MiB', str(nvidia_ouput), re.DOTALL)
    print(pid_memory)
    for gpu_num, (used, whole) in enumerate(pid_memory):

dmitrysarov

function FindProxyForURL(url, host) {
    PROXY = "PROXY 10.211.55.4:3128"

    // Apple.com via proxy
    if (shExpMatch(host,"*philips.com*")) {
        return PROXY;
    }
    if (shExpMatch(host,"https://confluence.atlas.philips.com")) {
        return PROXY;
    }

dmitrysarov

Python hacks

jupyter profiling https://jakevdp.github.io/PythonDataScienceHandbook/01.07-timing-and-profiling.html
line profiler https://github.com/rkern/line_profiler

Jupyter animation

%%capture
%matplotlib inline
import matplotlib.pyplot as plt

dmitrysarov

import subprocess, os                                                                                                                                                                                                         
import re                                                                                                                                                                                                                     
import click                                                                                                                                                                                                                  
                                                                                                                                                                                                                              
def find_free_gpu(amount_of_space):                                                                                                 

dmitrysarov

def write_avi(frames, path):
    '''
    frames - 3D numpy array (seq, height, width)
    path - result avi file path. Extantion of file have to be .avi
    '''
    def norm(x):
        x = x - x.min()
        x = x / x.max()
        return (x*255).astype(np.uint8)
    frames = norm(frames)

dmitrysarov

import subprocess
import re
import pprint
nvidia_ouput = subprocess.check_output(['nvidia-smi'])
part = re.findall('Usage(.+)', str(nvidia_ouput), re.DOTALL)
pid_memory = re.findall('\s(\d)\s+(\d+).+?\s(\d+MiB)', str(part), re.DOTALL)
user_dict = {}
print('='*50)
for gpu_num, pid, memory in pid_memory:
    print('GPU number: ', gpu_num)

dmitrysarov

import numpy as np
import matplotlib.pyplot as plt

class Iterative_hist(object):
    '''
    Collect distribution information in iterative manner. 
    Useful when whole data not fit memory.
    Usage:
        if min and max data value is not known, call set_min_max() method, iterative, on hall dataset
        call add_data() method, for data addition

dmitrysarov

# %matplotlib inline
%matplotlib notebook

from ipywidgets import interact, widgets, Layout
import matplotlib.pyplot as plt
from matplotlib.image import AxesImage
from IPython.display import display
from numpy.fft import fftshift, fft2, ifft2, ifftshift
import time
from skimage import transform