dmitrysarov

devops

Segmentation fault

can you do:

gdb python

and in gdb:

r main.py

dmitrysarov

Python hacks

jupyter profiling https://jakevdp.github.io/PythonDataScienceHandbook/01.07-timing-and-profiling.html
line profiler https://github.com/rkern/line_profiler

Jupyter animation

%%capture
%matplotlib inline
import matplotlib.pyplot as plt

dmitrysarov

import subprocess
import re
import pprint
nvidia_ouput = subprocess.check_output(['nvidia-smi'])
part = re.findall('Usage(.+)', str(nvidia_ouput), re.DOTALL)
pid_memory = re.findall('\s(\d)\s+(\d+).+?\s(\d+MiB)', str(part), re.DOTALL)
user_dict = {}
print('='*50)
for gpu_num, pid, memory in pid_memory:
    print('GPU number: ', gpu_num)

dmitrysarov

def write_avi(frames, path):
    '''
    frames - 3D numpy array (seq, height, width)
    path - result avi file path. Extantion of file have to be .avi
    '''
    def norm(x):
        x = x - x.min()
        x = x / x.max()
        return (x*255).astype(np.uint8)
    frames = norm(frames)

dmitrysarov

function FindProxyForURL(url, host) {
    PROXY = "PROXY 10.211.55.4:3128"

    // Apple.com via proxy
    if (shExpMatch(host,"*philips.com*")) {
        return PROXY;
    }
    if (shExpMatch(host,"https://confluence.atlas.philips.com")) {
        return PROXY;
    }

dmitrysarov

import subprocess, os                                                                                                                                                                                                         
import re                                                                                                                                                                                                                     
import click                                                                                                                                                                                                                  
                                                                                                                                                                                                                              
def find_free_gpu(amount_of_space):                                                                                                 

dmitrysarov

import subprocess, os
import re
import click

def find_free_gpu(amount_of_space):
    #in Mb
    nvidia_ouput = subprocess.check_output(['nvidia-smi'])
    pid_memory = re.findall('(\d+)MiB\s/\s(\d+)MiB', str(nvidia_ouput), re.DOTALL)
    print(pid_memory)
    for gpu_num, (used, whole) in enumerate(pid_memory):

dmitrysarov

import pandas as pd
import os
import json

def flatten_dict(d):
    """
    transform dict of dicts to single level dict
    """
    fd = {}
    for k, v in d.items():