shubhamagarwal92

import torch
import logging
from pytorch_transformers import BertModel, BertTokenizer
from pytorch_transformers import *
from typing import List

CLS_TOKEN = "[CLS]"
SEP_TOKEN = "[SEP]"

logger = logging.getLogger(__name__)

shubhamagarwal92

# Fixed https://github.com/alexandre01/deepsvg/blob/master/dataset/download.sh

#!/usr/bin/env bash

function gdrive_download () {
  CONFIRM=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate "https://docs.google.com/uc?export=download&id=$1" -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')
  wget --no-check-certificate --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$CONFIRM&id=$1" -O $2
  rm -rf /tmp/cookies.txt
}

shubhamagarwal92

# Following https://stackoverflow.com/questions/38080407/how-can-i-install-the-latest-anaconda-with-wget

latest=$(wget -qO- https://repo.anaconda.com/archive/  | 
grep -Eo "(href=\")(Anaconda3-.*-Linux-x86_64.sh)*\"" | 
sed 's/href=//g' | sed 's/\"//g' | head -n 1); wget "https://repo.anaconda.com/archive/$latest"

bash $latest -b

rm $latest
echo 'export PATH="~/anaconda3/bin:$PATH"' >> ~/.bashrc 

shubhamagarwal92

# Refer https://fizzylogic.nl/2017/11/06/edit-jupyter-notebooks-over-ssh/

# On server
jupyter notebook --no-browser --port=8080

# On local machine
ssh -N -L 8080:localhost:8080 <remote_user>@<remote_host>

shubhamagarwal92

tail -f logs.txt
watch -n1 nvidia-smi
htop
watch -n 1 free -h

shubhamagarwal92

import pandas as pd


def read_json_to_df(file_path):
    # df = pd.read_json(path_or_buf=file_path,orient='records',lines=True)
    df = pd.read_json(path_or_buf=file_path, orient='records')
    return df

def read_json_list_to_df(json_list):
    df = pd.DataFrame.from_records(json_list)

shubhamagarwal92

# Check number of cpus
grep -c ^processor /proc/cpuinfo     

sudo apt-get install tmux 
sudo apt-get install htop
sudo apt install git

# Conda
sudo mkdir -p /scratch/shubham/packages
sudo chown -R shubham /scratch/shubham

shubhamagarwal92

## Adding submodules 
git submodule add https://github.com/path/to/the_submodule
git submodule init
git submodule update

## Removing submodules
# https://stackoverflow.com/a/21211232/3776827
git rm the_submodule
rm -rf .git/modules/the_submodule

shubhamagarwal92

Pre-requisite: You should be able to do ssh without entering password. See this

Steps (Fig 1 and 2)

1. Open project in PyCharm
2. Make directory on server
3. Test you can ssh without entering password. (Save your rsa key on server)
4. Preferences -> Deployment
5. Enable automatic upload https://www.jetbrains.com/help/pycharm/deployment-in-pycharm.html#upload-to-default-server
6. First time upload your directory. Right click project folder -> deployment -> deploy to <machine_name>
7. Add multiple directories if want to deploy on different servers.

shubhamagarwal92

# https://stackoverflow.com/a/52076633/3776827
# Run as download_drive_file.sh file_id

# where file_id is google drive file id
# https://drive.google.com/file/d/0B-u9nH58139bTy1XRFdqaVEzUGs/view
# Eg. file_id here is  0B-u9nH58139bTy1XRFdqaVEzUGs
curl gdrive.sh | bash -s $1