Emin Orhan eminorhan

## pytorch_multinode_slurm.md

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              0 stars
            
          
                eminorhan
                / pytorch_multinode_slurm.md
            
            
              Last active
              May 7, 2021 09:19
            
              
                A minimal example demonstrating how to do multi-node distributed training with pytorch on a slurm cluster
              
          
    The following code is intentionally skeletal. Please feel free to flesh out the details according to your own needs.
import os
import builtins
import argparse
import torch
import torch.distributed as dist
from torchvision.datasets import ImageFolder
from torch.utils.data import DataLoader

  
## untar_each_to_its_own.sh
for a in `ls -1 *.tar`; do mkdir ${a%.*}; tar -xf $a -C ${a%.*}; rm -f $a; done &

## wget_txt.sh
wget -i filename.txt

## cocoapi.py
from pycocotools.coco import COCO
import numpy as np

dataDir = '/home/emin/coco'
dataType = 'val2017'
annFile = '{}/annotations/instances_{}.json'.format(dataDir, dataType)

coco = COCO(annFile)  # build COCO object

catIds = coco.getCatIds()  # category ids (80)

## read_videos.sh
nohup python -u /misc/vlgscratch5/LakeGroup/emin/sayavakepicut/read_video.py --save-dir '/misc/vlgscratch5/LakeGroup/emin/sayavakepicut/5fps_300s/ava_trainval' --fps 5 --seg-len 300 '/misc/vlgscratch5/LakeGroup/shared_data/ava_v2/trainval' &

## del_checkpoint.py
checkpoint = torch.load(CHECKPOINT_PATH)
model.module.load_state_dict(checkpoint['model_state_dict'])
del checkpoint
torch.cuda.empty_cache()

## count_files.sh
ls -1 | wc -l

## git_repo.sh
git init
git add . && git commit -m "first commit"
git remote add origin REMOTE_GITHUB_REPO_URL
git remote -v
git push origin master

## nohup2.sh
nohup python -u script.py &> nohup2.out &

## count_in_tar.sh
tar -tf archive.tar | wc -l
	from pycocotools.coco import COCO
	import numpy as np

	dataDir = '/home/emin/coco'
	dataType = 'val2017'
	annFile = '{}/annotations/instances_{}.json'.format(dataDir, dataType)

	coco = COCO(annFile) # build COCO object

	catIds = coco.getCatIds() # category ids (80)
	checkpoint = torch.load(CHECKPOINT_PATH)
	model.module.load_state_dict(checkpoint['model_state_dict'])
	del checkpoint
	torch.cuda.empty_cache()
	git init
	git add . && git commit -m "first commit"
	git remote add origin REMOTE_GITHUB_REPO_URL
	git remote -v
	git push origin master