README.md

#IPython Remote Notebook

##Installation and setup

1. Download or clone this gist:

    git clone https://gist.github.com/6600757.git remote_ipython
   

2. Add or edit your default notebook configuration file (.ipython/profile_default/ipython_notebook_config.py) on Janus so that it looks like the following:

    c = get_config()
    c.NotebookApp.ip = '*'
    c.NotebookApp.port = ipythonport
    c.NotebookApp.open_browser = False
   

   If you do note have an .ipython/profile_default directory, just run the notebook on Janus once. Create the ipython_notebook_config.py file if it does not exist.

3. Copy the notebook_janus.py file onto Janus. Please note the location.

4. Copy the notebook_local.py file onto your local machine.

5. Make sure you have the control sockets specified in your .ssh/config file locally. For example, here is what my file looks like:

    Host login
        HostName login.rc.colorado.edu
        User molu8455
        ControlMaster auto
        ControlPath ~/.ssh/sockets/%r@%h:%p
   

   Of coures, you'll want to replace my user name, molu8455, with your own.

6. Edit the configuration settings at the top of the notebook_local.py file.

    login_base = 'login'
    remote_dir = '/home/molu8455/bin'
    remote_default_dir = '/projects/molu8455'
    port = 8055
   

   Again please modify and replace molu8455 where appropriate. If you use a gateway machine, like bumba you can bounce:

    login_base = 'ssh login'
   

##Running

1. Login to Janus with your one time password device using the .ssh/config specification.

    ssh login
   

2. In a terminal session on your local machine, run the local notebook command.

    python notebook_local.py
   

   You can change the defaults if you would like. For example,

    python notebook_local.py -d . -n 2 -p 8765
   

   will run the notebook in my $HOME directory with 2 nodes on port 8765.

3. When you are done with the notebook session, hit control-c to cancel the script, which will also cancel the job on Janus.

##Reuse

You can put a custom python command in a bash script for quick access to certain directories and resource levels. For example, I have a datascience.sh script that contains the following information.

python notebook_local.py -n 2 -d /home/molu8455/projects/data_science_meetup_2013

This allows me to have a shortcut.

./datascience.sh

notebook_janus.py

#!/curc/tools/x_86_64/rh6/anaconda/1.6.1/bin/python
import sys
import argparse
import subprocess
import time
import jinja2 as jin
import xml.etree.ElementTree as et

template = jin.Template('''
#!/bin/bash
#PBS -N ipython_notebook
#PBS -q janus-admin
#PBS -l walltime=01:00:00
#PBS -l nodes={{n}}:ppn=12
#PBS -j oe
#PBS -o output-$PBS_JOBNAME-$PBS_JOBID

echo $HOSTNAME > $HOME/node.$PBS_JOBNAME.$PBS_JOBID

cd {{d}}

echo 'ssh -L '{{p}}':'$HOSTNAME':'{{p}} '-f -N' \
$USER'@admin04.rc.colorado.edu -p 53770' > $HOME/notebook_cmd

ipython notebook --pylab=inline --port={{p}}

''')

def get_args(argv):
    parser = argparse.ArgumentParser()  
    parser.add_argument('-n', help='number of nodes',default=1)
    parser.add_argument('-d', help='directory', default='.')
    parser.add_argument('-p', help='port', default=8055)
    return parser.parse_args(argv)


def create_pbs_script(args):
    pbs_script = template.render({'n': args.n,
                                  'd': args.d,
                                  'p': args.p })

    with open('ipython_notebook.pbs', 'w') as outfile:
        outfile.write(pbs_script)


def submit_job():
    cmd = ['/curc/tools/free/redhat_6_x86_64/torque-4.2.3/bin/qsub',
           'ipython_notebook.pbs']
    pid = subprocess.Popen(cmd,
                        stdout=subprocess.PIPE, 
                        stderr=subprocess.PIPE)
    pid.wait()
    stdout, stderr = pid.communicate()

    job_id = stdout.split('.')[0]
    return job_id


def get_momsup(job_id):

    filename = 'node.ipython_notebook.{0}.moab.rc.colorado.edu'.format(job_id)
    with open(filename) as infile:
        data = infile.read()
        nodename = data.strip()
    return nodename


if __name__ == '__main__':

    args = get_args(sys.argv[1:])

    create_pbs_script(args)
    
    job_id = submit_job()

    nodename = None
    #TODO add timeout?
    while not nodename:
        try:
            nodename = get_momsup(job_id)
            print 'jobid='+str(job_id)
            print 'nodename='+str(nodename)
        except IOError, e:
            time.sleep(2)

notebook_local.py

#!/usr/bin/env python
import sys
import os
import time
import argparse
import subprocess

login_base = 'login'
login_base = '-t bumba ssh login'
remote_dir = '/home/molu8455/bin'
remote_default_dir = '/projects/molu8455'
port = 8055

def get_args(argv):
    parser = argparse.ArgumentParser()  
    parser.add_argument('-n', help='number of nodes',default=1)
    parser.add_argument('-d', help='directory', default=remote_default_dir)
    parser.add_argument('-p', help='port', default=port)
    return parser.parse_args(argv)

def create_pbs_job(args):
    cmd = 'python '
    cmd += os.path.join(remote_dir,'notebook_janus.py')
    cmd += ' -n {0} -d {1} -p {2}'.format(args.n, args.d, args.p)

    cmd = 'ssh ' + login_base +' "'+ cmd +'"'
    print 'waiting for job to start'
    print cmd
    pid = subprocess.Popen(cmd, shell=True,
                        stdout=subprocess.PIPE, 
                        stderr=subprocess.PIPE)
    pid.wait()
    stdout, stderr = pid.communicate()

    for line in stdout.split('\n'):
        if line.startswith('jobid='):
            jobid=line.split('=')[1].strip()
        if line.startswith('nodename='):
            nodename=line.split('=')[1].strip()

    print 'job {0} is running on {1}'.format(jobid,nodename)

    return nodename, jobid


def delete_tunnels():

    print 'deleting tunnels'
    pro = subprocess.Popen("ps -fe | grep 'ssh -L'", 
                           shell=True, 
                           stdout=subprocess.PIPE, 
                           stderr=subprocess.PIPE)
    pro.wait()
    stdout, stderr = pro.communicate()

    for line in stdout.split('\n'):
        if len(line) > 0:
            cmd = 'kill -9 ' + str(line.split()[1])
            tmp = subprocess.Popen(cmd, 
                                   shell=True, 
                                   stdout=subprocess.PIPE, 
                                   stderr=subprocess.PIPE)
            tmp.wait()


def create_tunnel(nodename, port):

    print 'creating new tunnel'
    cmd = 'ssh -L {0}:{1}:{0} -f -N {2}'.format(port,nodename,login_base)
    print cmd
    pro = subprocess.Popen(cmd, 
                           shell=True, 
                           stdout=subprocess.PIPE, 
                           stderr=subprocess.PIPE)
    pro.wait()


def wait_quit(jobid):
    print 'CNTR-C to quit job and exit'
    try:
        while(True):
            time.sleep(60)
    except KeyboardInterrupt:
        cmd = '/curc/tools/free/redhat_6_x86_64/torque-4.2.3/bin/qdel ' + str(jobid)
        cmd = 'ssh ' + login_base +' "'+ cmd +'"'
        print 'killing job', jobid
        pid = subprocess.Popen(cmd, shell=True,
                        stdout=subprocess.PIPE, 
                        stderr=subprocess.PIPE)
        pid.wait()
        sys.exit()


if __name__ == '__main__':

    args = get_args(sys.argv[1:])

    print ''
    nodename, jobid = create_pbs_job(args)

    delete_tunnels()

    create_tunnel(nodename, args.p)

    print 'notebook is ready'
    print ''
    print 'localhost:'+str(args.p)
    print ''
   
    wait_quit(jobid)