Load python notebook from another python notebook, all hosted on s3 using s3contents

load_notebook_from_s3.py

import json
import io, os, sys, types
import boto3
from IPython import get_ipython
import nbformat
from IPython.core.interactiveshell import InteractiveShell

S3_BUCKET_NAME = 'my-very-cool-bucket'

'''
If you are running juypter notebooks off instances with ephemeral storage (e.g. Heroku) you probably want to save them on S3.
Thankfully you can use: https://github.com/danielfrg/s3contents

The code here allows you to import modules from notebooks into other notebooks while running jupyter while using S3 to store all your notebooks.

Most of the code was copied from here: 
https://jupyter-notebook.readthedocs.io/en/stable/examples/Notebook/Importing%20Notebooks.html

to figure out how to load a notebook from a string, see: 
https://nbformat.readthedocs.io/en/latest/api.html#reading-and-writing
'''

class NotebookLoader(object):
    """Module Loader for Jupyter Notebooks"""
    def __init__(self, path=None):
        self.shell = InteractiveShell.instance()
        self.path = path

    def load_module(self, fullname):
        """import a notebook as a module"""        
        s3_key = "{}.ipynb".format(fullname.strip().replace('.','/'))
        client = boto3.client('s3')
        obj = client.get_object(Bucket=S3_BUCKET_NAME, Key=s3_key)
        nb_str = obj['Body'].read()
        nb = nbformat.from_dict(json.loads(nb_str))
        
        # create the module and add it to sys.modules
        # if name in sys.modules:
        #    return sys.modules[name]
        mod = types.ModuleType(fullname)
#         mod.__file__ = path
        mod.__loader__ = self
        mod.__dict__['get_ipython'] = get_ipython
        sys.modules[fullname] = mod
        
        # extra work to ensure that magics that would affect the user_ns
        # actually affect the notebook module's ns
        save_user_ns = self.shell.user_ns
        self.shell.user_ns = mod.__dict__
        
        try:
            for cell in nb.cells:
                if cell.cell_type == 'code':
                    # transform the input to executable Python
                    code = self.shell.input_transformer_manager.transform_cell(cell.source)
                    # run the code in themodule
                    exec(code, mod.__dict__)
        finally:
            self.shell.user_ns = save_user_ns
        return mod


class NotebookFinder(object):
    """Module finder that locates Jupyter Notebooks"""
    def __init__(self):
        self.loaders = {}

    def find_module(self, fullname, path=None):
        key = path
        if path:
            # lists aren't hashable
            key = os.path.sep.join(path)

        if key not in self.loaders:
            self.loaders[key] = NotebookLoader(path)
        return self.loaders[key]

sys.meta_path.append(NotebookFinder())

'''
if you have a file in your s3 bucket named 
tools.ipynb
you can import it by running: 
'''
import tools
'''
if you have a file named tools.ipynb under a directory /stuff
you can import it by running: 
'''
import stuff.tools