andrei-pokrovsky

import datetime
import linecache
import os

import pynvml3
import torch

print_tensor_sizes = True
last_tensor_sizes = set()
gpu_profile_fn = f'{datetime.datetime.now():%d-%b-%y-%H:%M:%S}-gpu_mem_prof.txt'

andrei-pokrovsky

import torch
import torch.nn as nn
from torch.nn import Parameter
from torch.autograd import Variable, Function
from collections import defaultdict
import graphviz

"""
This is a rather distorted implementation of graph visualization in PyTorch.

andrei-pokrovsky

from graphviz import Digraph
import torch
from torch.autograd import Variable, Function

def iter_graph(root, callback):
    queue = [root]
    seen = set()
    while queue:
        fn = queue.pop()
        if fn in seen:

andrei-pokrovsky

#include <sys/time.h>
#include <cuda_runtime.h>
#include <stdio.h>

inline double seconds()
{
    struct timeval tp;
    struct timezone tzp;
    int i = gettimeofday(&tp, &tzp);
    return ((double)tp.tv_sec + (double)tp.tv_usec * 1.e-6);

andrei-pokrovsky

Do a pickaxe style search through all your stashes

git stash list -S "the thing" -p

A treasure trove of git scripts

John Wiegley has a ton of useful git scripts which he's collected from around the net: https://github.com/jwiegley/git-scripts

andrei-pokrovsky

# This isn't supposed to run as a bash script, i named it with ".sh" for syntax highlighting.

# https://developer.nvidia.com/nsight-systems
# https://docs.nvidia.com/nsight-systems/profiling/index.html

# My preferred nsys (command line executable used to create profiles) commands
#
# In your script, write
# torch.cuda.nvtx.range_push("region name")
# ...

andrei-pokrovsky

/*

VC++ Directories
----------------
  Include Directories: C:\Anaconda3\include
  Library Directories: C:\Anaconda3\libs

*/

#include <iostream>

andrei-pokrovsky

#!/usr/bin/env python

"""
Demonstrate how to pass IPC handles to GPU data between processes in Python.
"""

import ctypes
import numpy as np
import multiprocessing as mp
import zmq

andrei-pokrovsky

# Make mouse useful in copy mode
setw -g mode-mouse on

# Allow mouse to select which pane to use
set -g mouse-select-pane on

# Allow mouse dragging to resize panes
set -g mouse-resize-pane on

# Allow mouse to select windows

andrei-pokrovsky

Ordinary launch commands (no profiling):

Single-process:

python main_amp.py -a resnet50 --b 224 --deterministic --workers 4 --opt-level O1 ./bare_metal_train_val/

Multi-process:

python -m torch.distributed.launch  --nproc_per_node=2 main_amp.py -a resnet50 --b 224 --deterministic --workers 4 --opt-level O1 ./bare_metal_train_val/